diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2a89fc19ee..004bffd29c 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -76,7 +76,7 @@ jobs: performance=true rules_performance=true ;; - src/sqlbuild/cli/commands/_helpers/compile/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|tests/e2e/src/sqlbuild/cli/commands/main/compile/*) + src/sqlbuild/cli/commands/_helpers/compile/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*) broad=true performance=true ;; @@ -90,7 +90,7 @@ jobs: duckdb=true format_performance=true ;; - tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/virtual/state/classes/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*) + tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*) postgres=true ;; tests/e2e/*) @@ -195,14 +195,12 @@ jobs: target: test-e2e-duckdb-build-core - name: build incremental target: test-e2e-duckdb-build-incremental - - name: build virtual - target: test-e2e-duckdb-build-virtual - name: CLI data target: test-e2e-duckdb-cli-data - name: CLI commands target: test-e2e-duckdb-cli - - name: virtual environments - target: test-e2e-duckdb-virtual + - name: CLI inspection + target: test-e2e-duckdb-inspection - name: integrations target: test-e2e-duckdb-integrations env: @@ -390,6 +388,62 @@ jobs: make test-e2e-dense-compile-performance \ SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}" + - name: Run diagnostic-heavy compile guards + if: matrix.models == 1000 + run: make test-e2e-diagnostic-compile-performance + + compile_performance_ratio: + name: Same-runner compile ratio (${{ matrix.kind }} 3000 models) + needs: [changes, lockfile] + if: needs.changes.outputs.performance == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - kind: dense + max_ratio: "1.10" + - kind: fresh + max_ratio: "1.30" + steps: + - uses: actions/checkout@v4 + - name: Check out the comparison base + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha || 'main' }} + path: base + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + base/uv.lock + base/Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install head dependencies + run: uv sync --locked --all-groups --all-extras + - name: Install base dependencies + working-directory: base + run: uv sync --locked --all-extras + - name: Compare base and head cold compiles + env: + KIND: ${{ matrix.kind }} + MAX_RATIO: ${{ matrix.max_ratio }} + run: | + uv run python -m scripts.compare_compile_performance \ + --kind "${KIND}" \ + --max-ratio "${MAX_RATIO}" \ + --models 3000 \ + --base-python base/.venv/bin/python \ + --head-python .venv/bin/python + cache_compile_performance: name: Fresh-process compile cache (${{ matrix.models }} models) needs: [changes, lockfile] @@ -551,6 +605,7 @@ jobs: performance, cold_compile_performance, dense_compile_performance, + compile_performance_ratio, cache_compile_performance, rules_performance, postgres, @@ -568,6 +623,7 @@ jobs: PERFORMANCE_RESULT: ${{ needs.performance.result }} COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }} DENSE_COMPILE_PERFORMANCE_RESULT: ${{ needs.dense_compile_performance.result }} + COMPILE_PERFORMANCE_RATIO_RESULT: ${{ needs.compile_performance_ratio.result }} CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }} RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }} POSTGRES_RESULT: ${{ needs.postgres.result }} @@ -587,6 +643,7 @@ jobs: "${PERFORMANCE_RESULT}" \ "${COLD_COMPILE_PERFORMANCE_RESULT}" \ "${DENSE_COMPILE_PERFORMANCE_RESULT}" \ + "${COMPILE_PERFORMANCE_RATIO_RESULT}" \ "${CACHE_COMPILE_PERFORMANCE_RESULT}" \ "${RULES_PERFORMANCE_RESULT}" \ "${POSTGRES_RESULT}"; do diff --git a/AGENTS.md b/AGENTS.md index 0c114aa6bb..001f20099e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,6 +18,56 @@ explicitly requests it; rely on CI for those suites. - Before pushing, run the exact static CI target with all optional dependencies available: `uv sync --all-extras` followed by `make check-ci`. +## Real-Project Verification and Performance Guards + +- Synthetic fixtures and clean benchmark projects do not exercise every path. Before releasing + changes to compilation, SQL analysis, diagnostics, formatting, Rules, or planning, run the built + branch against at least one large real project available locally outside this repository, and + compare it with the latest release. Compare diagnostic counts by code, cold compile wall time and + peak memory, and, for formatter changes, that formatting preserves compiled dependencies, lineage, + and query semantics. +- Treat every new diagnostic on a real project as a false positive until warehouse evidence proves + otherwise, for example a model that has built successfully since its SQL last changed. +- A material slowdown or new false positive is a release blocker, not a follow-up. +- Real-project names, SQL, identifiers, file paths, and results never enter this repository, commits, + pull requests, or CI output. Record only neutral aggregate conclusions, and rebuild every + reproduction synthetically. +- Performance guards must cover worst-case paths, not only clean projects: very large models, + diagnostic-heavy compiles with many errors and warnings, wide queries, and deep macro expansion. + Any per-diagnostic, per-reference, or per-token work must be bounded by a test with a strict time + limit. A path that is only slow when something is wrong is still a regression. + +## Performance Work + +- Never compare timings across different machines, runners, or sessions. Every performance claim + must come from a same-machine comparison: base and candidate built from the same toolchain, run + alternately at least three times each, reporting medians of both wall time and CPU time + (user+sys) plus the per-phase `compile_timings`. +- On shared local machines, confirm no concurrent builds or benchmarks are running and record the + load average. If load exceeds the core count, treat wall times as unreliable and report CPU time. +- Hosted CI runners vary by up to ~60%. Do not diagnose regressions from absolute CI budgets or + from runs on different runners; use the same-runner ratio guard or a same-job paired comparison. +- Before attributing a regression to a cause, prove it with an on/off experiment (a temporary + kill switch or a revert) on the same machine. Profiles identify suspects; experiments convict. + Remove experiment switches before committing. +- Measure the modes users run: cold without cache, cold writing the cache, and warm. A change that + affects analysis, validation, or caching must report all three. +- Performance fixes must not reduce coverage. Show that diagnostics are unchanged or a strict, + validated superset on the dense benchmark and a real-project run. +- Do not raise performance budgets or tolerances to make a regression pass. + +## Native SQL (Polyglot) Changes + +- Treat every Polyglot change as a compiler change: before adopting a new Polyglot version, run the + same-machine dense and fresh 3,000-model comparisons against the current version and report the + ratio and per-phase timings. +- Polyglot diagnostics adopted by SQLBuild are enforceable errors or dropped. Never surface a new + native finding as a non-blocking warning; map it to a SQLBuild error code with a mechanical remedy. +- Coverage conditions must not depend on incidental query shape (for example, only checking + statements with a set operation). Checks apply uniformly or the gap is documented and ticketed. +- Run the Polyglot feature-gate checks (`make test-rust-feature-gates`) before pushing Polyglot + changes; cross-module helpers must respect feature gates. + ## Public Repository Hygiene - Treat every tracked file, generated artifact, fixture, benchmark, filename, commit, branch, pull @@ -68,6 +118,9 @@ requests local verification or CI is unavailable. - State the expected verification scope in subagent prompts and explicitly prohibit unnecessary full-suite runs. - Do not delay committing and pushing a focused fix solely to repeat checks already completed successfully by another agent or CI. +- Subagents reporting performance results must include wall time, CPU time, run count, machine + load, and whether base and candidate ran on the same machine. Results lacking these are + hypotheses, not evidence. ## State and Source of Truth diff --git a/Cargo.lock b/Cargo.lock index 7ff26b6b74..e9a33b13bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,10 +310,12 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "polyglot-sql" -version = "0.12.3" -source = "git+https://github.com/chio-labs/polyglot.git?rev=f3774102f518f716cd648e4fdebb9d8558a0b63a#f3774102f518f716cd648e4fdebb9d8558a0b63a" +version = "0.13.8" +source = "git+https://github.com/chio-labs/polyglot.git?rev=25ad011c49f2c7cd48f4427abb8fc0a6af2e4120#25ad011c49f2c7cd48f4427abb8fc0a6af2e4120" dependencies = [ "polyglot-sql-ast-derive", + "polyglot-sql-function-catalogs", + "regex", "serde", "serde_json", "stacker", @@ -323,14 +325,19 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" -version = "0.12.3" -source = "git+https://github.com/chio-labs/polyglot.git?rev=f3774102f518f716cd648e4fdebb9d8558a0b63a#f3774102f518f716cd648e4fdebb9d8558a0b63a" +version = "0.13.8" +source = "git+https://github.com/chio-labs/polyglot.git?rev=25ad011c49f2c7cd48f4427abb8fc0a6af2e4120#25ad011c49f2c7cd48f4427abb8fc0a6af2e4120" dependencies = [ "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "polyglot-sql-function-catalogs" +version = "0.13.8" +source = "git+https://github.com/chio-labs/polyglot.git?rev=25ad011c49f2c7cd48f4427abb8fc0a6af2e4120#25ad011c49f2c7cd48f4427abb8fc0a6af2e4120" + [[package]] name = "portable-atomic" version = "1.15.0" diff --git a/Cargo.toml b/Cargo.toml index a991f4e1b1..53d956f36e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ fensu-policy = "=0.22.0" globset = "0.4" proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "f3774102f518f716cd648e4fdebb9d8558a0b63a", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "25ad011c49f2c7cd48f4427abb8fc0a6af2e4120", default-features = false, features = [ "ast-tools", "builder", "dialect-bigquery", diff --git a/Makefile b/Makefile index d15fc69ca5..0b58e690ba 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ SHELL := /bin/bash .PHONY: verify verify-quick verify-pg coverage cli-preview rust-check \ check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \ - test-e2e-duckdb-build-incremental test-e2e-duckdb-build-virtual \ - test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-virtual \ + test-e2e-duckdb-build-incremental \ + test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ test-e2e-duckdb-integrations test-e2e-performance \ test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ test-e2e-dense-compile-performance test-e2e-varied-cache-performance @@ -102,24 +102,17 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py -E2E_DUCKDB_BUILD_VIRTUAL_PATHS := \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_build_state.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_incremental_build.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_microbatch_lifecycle.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_mode_guard.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_promote.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_rollback.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_seed_build.py \ - tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py E2E_DUCKDB_CLI_DATA_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/load \ tests/e2e/src/sqlbuild/cli/commands/main/providers \ - tests/e2e/src/sqlbuild/cli/commands/main/scenario + tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py \ + tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py \ + tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ + tests/e2e/scripts/compile_performance_ratio \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ tests/e2e/src/sqlbuild/cli/commands/main/bigquery \ @@ -144,23 +137,20 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \ tests/e2e/src/sqlbuild/cli/commands/main/test -E2E_DUCKDB_VIRTUAL_PATHS := \ +E2E_DUCKDB_INSPECTION_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/clone \ tests/e2e/src/sqlbuild/cli/commands/main/diff \ tests/e2e/src/sqlbuild/cli/commands/main/janitor \ - tests/e2e/src/sqlbuild/cli/commands/main/plan \ - tests/e2e/src/sqlbuild/cli/commands/main/reconcile \ - tests/e2e/src/sqlbuild/cli/commands/main/state + tests/e2e/src/sqlbuild/cli/commands/main/plan E2E_DUCKDB_INTEGRATIONS_PATHS := tests/e2e/src/sqlbuild/integrations E2E_DUCKDB_PATHS := \ $(E2E_DUCKDB_BUILD_CORE_PATHS) \ $(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS) \ - $(E2E_DUCKDB_BUILD_VIRTUAL_PATHS) \ $(E2E_DUCKDB_CLI_DATA_PATHS) \ $(E2E_DUCKDB_CLI_PATHS) \ - $(E2E_DUCKDB_VIRTUAL_PATHS) \ + $(E2E_DUCKDB_INSPECTION_PATHS) \ $(E2E_DUCKDB_INTEGRATIONS_PATHS) define run_e2e_duckdb @@ -199,8 +189,6 @@ test-e2e-duckdb-build-core: test-e2e-duckdb-build-incremental: $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS)) -test-e2e-duckdb-build-virtual: - $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_BUILD_VIRTUAL_PATHS)) test-e2e-duckdb-cli-data: $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_CLI_DATA_PATHS)) @@ -208,8 +196,8 @@ test-e2e-duckdb-cli-data: test-e2e-duckdb-cli: $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_CLI_PATHS)) -test-e2e-duckdb-virtual: - $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_VIRTUAL_PATHS)) +test-e2e-duckdb-inspection: + $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_INSPECTION_PATHS)) test-e2e-duckdb-integrations: $(call run_e2e_duckdb,$@,$(E2E_DUCKDB_INTEGRATIONS_PATHS)) @@ -218,10 +206,9 @@ check-e2e-shards: bash scripts/check_e2e_shards.sh \ --shard build-core $(E2E_DUCKDB_BUILD_CORE_PATHS) \ --shard build-incremental $(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS) \ - --shard build-virtual $(E2E_DUCKDB_BUILD_VIRTUAL_PATHS) \ --shard cli-data $(E2E_DUCKDB_CLI_DATA_PATHS) \ --shard cli $(E2E_DUCKDB_CLI_PATHS) \ - --shard virtual $(E2E_DUCKDB_VIRTUAL_PATHS) \ + --shard inspection $(E2E_DUCKDB_INSPECTION_PATHS) \ --shard integrations $(E2E_DUCKDB_INTEGRATIONS_PATHS) @@ -244,6 +231,13 @@ test-e2e-cold-compile-performance: -m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \ -vv --log-cli-level=INFO --color=yes +.PHONY: test-e2e-diagnostic-compile-performance +test-e2e-diagnostic-compile-performance: + env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \ + tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py \ + -n auto --dist loadfile -m cold_compile_performance \ + -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes + test-e2e-dense-compile-performance: test -n "$(SQLBUILD_BENCHMARK_MODELS)" env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \ @@ -266,17 +260,6 @@ test-e2e-varied-cache-performance: -n auto --dist loadfile -m cache_compile_performance \ -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes -test-virtual: - @mkdir -p /tmp/opencode - @log="/tmp/opencode/test-virtual-$$(date +%Y%m%d-%H%M%S).log"; \ - echo "Logging to $$log"; \ - env PYTHONUNBUFFERED=1 TESTCONTAINERS_RYUK_DISABLED=true SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest \ - $(VIRTUAL_TEST_ROOTS) \ - -k "$(VIRTUAL_TEST_KEYWORD)" \ - -m "not dbt and not performance" -vv --color=yes -n auto --dist loadfile 2>&1 | tee "$$log"; \ - status=$${PIPESTATUS[0]}; \ - echo "TEST_VIRTUAL_EXIT=$$status (log: $$log)" | tee -a "$$log"; \ - exit $$status skills: @@ -290,8 +273,6 @@ export DBT_EXECUTABLE SQLBUILD_CONCURRENCY ?= 8 -VIRTUAL_TEST_ROOTS ?= tests/e2e/src/sqlbuild/cli/commands/main tests/e2e/src/sqlbuild/integrations/dagster -VIRTUAL_TEST_KEYWORD ?= virtual or reconcile or diff or janitor or snapshot or dagster test-dbt: diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs index 2e4535d748..186c8ab787 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/functions.rs @@ -1,6 +1,5 @@ //! Register the versioned Python boundary for the native Rules engine. -use pyo3::exceptions::PyValueError; use pyo3::prelude::{ Bound, IntoPyObject, Py, PyAny, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -14,27 +13,75 @@ use crate::models::CatalogueResponse; use crate::rules::main::{catalogue, selected_codes}; const SKILL_OWNER: &str = "sqlbuild"; +use crate::bindings::_helpers::panics::compiler_guard; +use crate::bindings::types::CompilerDetach; const SKILL_IDENTITY: &str = "sqlbuild-rules"; +#[pyfunction] +fn normalize_analysis_sql( + py: Python<'_>, + request: crate::semantic_validation::models::NormalizationInput, +) -> PyResult { + py.compiler_detach(|| { + crate::semantic_validation::main::normalize::normalize_analysis_sql(request) + }) + .map_err(value_error) +} + +#[pyfunction] +fn normalize_dialect_sql(py: Python<'_>, sql: &str, dialect: &str) -> PyResult { + py.compiler_detach(|| { + crate::semantic_validation::main::normalize_dialect::normalize_dialect_sql(sql, dialect) + }) + .map_err(value_error) +} + +#[pyfunction] +fn normalize_analysis_sqls( + py: Python<'_>, + dialect: &str, + requests: Vec<(String, std::collections::HashMap)>, +) -> PyResult> { + py.compiler_detach(|| { + crate::semantic_validation::main::normalize_batch::normalize_analysis_sqls( + dialect, requests, + ) + }) + .map_err(value_error) +} + +#[pyfunction] +fn binding_diagnostics( + py: Python<'_>, + sql: &str, + dialect: &str, + rows: Vec, +) -> PyResult> { + py.compiler_detach(|| { + crate::semantic_validation::main::diagnostics::binding_diagnostics(sql, dialect, rows) + }) + .map_err(value_error) +} + fn value_error(error: impl std::fmt::Display) -> PyErr { - PyValueError::new_err(error.to_string()) + crate::bindings::_helpers::panics::compiler_error(error) } #[pyfunction] fn evaluate_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| evaluate::evaluate_json(request_json)) + py.compiler_detach(|| evaluate::evaluate_json(request_json)) .map_err(value_error) } #[pyfunction] fn lint_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::sql_lint::main::engine::lint_json(request_json)) + py.compiler_detach(|| crate::sql_lint::main::engine::lint_json(request_json)) .map_err(value_error) } #[pyfunction] fn finalize_rule_findings_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::engine::main::finalize::finalize_findings_json(request_json)) + py.compiler_detach(|| crate::engine::main::finalize::finalize_findings_json(request_json)) .map_err(value_error) } @@ -53,7 +100,7 @@ fn prepare_lint_sql( py: Python<'_>, request: LintPreparationRequest, ) -> PyResult> { - py.detach(|| { + py.compiler_detach(|| { crate::sql_lint::main::preparation::prepare( &request.expanded, &request.before_expansion, @@ -65,61 +112,65 @@ fn prepare_lint_sql( } #[pyfunction] -fn lint_backtick_identifiers(dialect: &str) -> bool { - crate::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect) +fn lint_backtick_identifiers(dialect: &str) -> PyResult { + compiler_guard(|| { + Ok(crate::sql_lint::main::backtick_identifiers::backtick_identifiers(dialect)) + }) } #[pyfunction] fn lint_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::sql_lint::main::batch_engine::lint_batch_json(request_json)) - .map_err(pyo3::exceptions::PyValueError::new_err) + py.compiler_detach(|| crate::sql_lint::main::batch_engine::lint_batch_json(request_json)) + .map_err(value_error) } #[pyfunction] fn format_sql_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::sql_lint::main::formatter::format_json(request_json)) + py.compiler_detach(|| crate::sql_lint::main::formatter::format_json(request_json)) .map_err(value_error) } #[pyfunction] fn format_sql_batch_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::sql_lint::main::batch_formatter::format_batch_json(request_json)) + py.compiler_detach(|| crate::sql_lint::main::batch_formatter::format_batch_json(request_json)) .map_err(value_error) } #[pyfunction(name = "validate_sql_with_schema_json")] fn schema_validation_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::semantic_validation::main::validation_json(request_json)) + py.compiler_detach(|| crate::semantic_validation::main::validation_json(request_json)) .map_err(value_error) } #[pyfunction(name = "validate_sql_with_schemas_json")] fn schema_validations_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::semantic_validation::main::validations_json(request_json)) + py.compiler_detach(|| crate::semantic_validation::main::validations_json(request_json)) .map_err(value_error) } #[pyfunction] fn analyze_sql_uses_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::semantic_usage::main::analyze_json(request_json)) + py.compiler_detach(|| crate::semantic_usage::main::analyze_json(request_json)) .map_err(value_error) } #[pyfunction] fn analyze_queries_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::query_analysis::main::analyze::analyze_json(request_json)) + py.compiler_detach(|| crate::query_analysis::main::analyze::analyze_json(request_json)) .map_err(value_error) } #[pyfunction] fn analyze_project_queries_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::query_analysis::main::analyze_project::analyze_project_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + crate::query_analysis::main::analyze_project::analyze_project_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| { + py.compiler_detach(|| { crate::query_analysis::main::analyze_project_compact::analyze_project_compact_json( request_json, ) @@ -129,19 +180,21 @@ fn analyze_project_queries_compact_json(py: Python<'_>, request_json: &str) -> P #[pyfunction] fn render_sql_test_comparisons_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::compiler::main::sql_test_rendering::render_json(request_json)) + py.compiler_detach(|| crate::compiler::main::sql_test_rendering::render_json(request_json)) .map_err(value_error) } #[pyfunction] fn plan_and_render_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::compiler::main::sql_test_planning::plan_and_render_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + crate::compiler::main::sql_test_planning::plan_and_render_json(request_json) + }) + .map_err(value_error) } #[pyfunction] fn resolve_sql_test_chains_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| { + py.compiler_detach(|| { crate::compiler::main::sql_test_chain_resolution::resolve_chains_json(request_json) }) .map_err(value_error) @@ -149,7 +202,7 @@ fn resolve_sql_test_chains_json(py: Python<'_>, request_json: &str) -> PyResult< #[pyfunction] fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| { + py.compiler_detach(|| { crate::compiler::main::sql_test_difference_sampling::render_difference_sample_json( request_json, ) @@ -159,8 +212,10 @@ fn render_sql_test_difference_sample_json(py: Python<'_>, request_json: &str) -> #[pyfunction] fn extract_sql_tests_json(py: Python<'_>, request_json: &str) -> PyResult { - py.detach(|| crate::compiler::main::sql_test_extraction::extract_batch_json(request_json)) - .map_err(value_error) + py.compiler_detach(|| { + crate::compiler::main::sql_test_extraction::extract_batch_json(request_json) + }) + .map_err(value_error) } fn authored_value_to_python( @@ -262,7 +317,7 @@ type ParsedModelHeader = ( #[pyfunction] fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult> { let parsed = py - .detach(|| crate::compiler::main::model_header_parsing::parse_batch(&headers)) + .compiler_detach(|| crate::compiler::main::model_header_parsing::parse_batch(&headers)) .map_err(value_error)?; parsed .into_iter() @@ -278,53 +333,65 @@ fn parse_model_headers(py: Python<'_>, headers: Vec) -> PyResult PyResult> { - crate::compiler::main::model_header_tokenizing::tokenize_one(header).map_err(value_error) + compiler_guard(|| { + crate::compiler::main::model_header_tokenizing::tokenize_one(header).map_err(value_error) + }) } #[pyfunction] fn substitute_static_project_vars( sqls: Vec, variables: Vec<(String, String)>, -) -> Vec<(u8, Option)> { - crate::compiler::main::sql_interpolation::substitute_batch(&sqls, &variables) +) -> PyResult)>> { + compiler_guard(|| { + Ok(crate::compiler::main::sql_interpolation::substitute_batch( + &sqls, &variables, + )) + }) } #[pyfunction] fn extract_static_sql_references( sql: &str, -) -> Option> { - crate::compiler::main::sql_references::extract(sql) +) -> PyResult>> { + compiler_guard(|| Ok(crate::compiler::main::sql_references::extract(sql))) } #[pyfunction] fn load_config_json(project_dir: &str) -> PyResult { - load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error) + compiler_guard(|| { + load::load_config_json(std::path::Path::new(project_dir)).map_err(value_error) + }) } #[pyfunction] fn catalogue_json() -> PyResult { - serde_json::to_string(&CatalogueResponse { - version: API_VERSION, - rules: catalogue::catalogue(), + compiler_guard(|| { + serde_json::to_string(&CatalogueResponse { + version: API_VERSION, + rules: catalogue::catalogue(), + }) + .map_err(value_error) }) - .map_err(value_error) } #[pyfunction] fn selected_codes_json(request_json: &str) -> PyResult { - selected_codes::selected_codes_json(request_json).map_err(value_error) + compiler_guard(|| selected_codes::selected_codes_json(request_json).map_err(value_error)) } #[pyfunction] fn render_owned_skill(content: &str, input_fingerprint: &str) -> PyResult { - fensu_policy::render_owned_skill( - SKILL_OWNER, - SKILL_IDENTITY, - input_fingerprint, - content.as_bytes(), - ) - .map_err(value_error) - .and_then(|value| String::from_utf8(value).map_err(value_error)) + compiler_guard(|| { + fensu_policy::render_owned_skill( + SKILL_OWNER, + SKILL_IDENTITY, + input_fingerprint, + content.as_bytes(), + ) + .map_err(value_error) + .and_then(|value| String::from_utf8(value).map_err(value_error)) + }) } #[pyfunction] @@ -343,6 +410,18 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(finalize_rule_findings_json, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_json, module)?)?; module.add_function(wrap_pyfunction!(prepare_lint_sql, module)?)?; + module.add( + "NativeCompilerError", + module + .py() + .get_type::(), + )?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(normalize_analysis_sql, module)?)?; + module.add_function(wrap_pyfunction!(normalize_analysis_sqls, module)?)?; + module.add_function(wrap_pyfunction!(normalize_dialect_sql, module)?)?; + module.add_function(wrap_pyfunction!(binding_diagnostics, module)?)?; module.add_function(wrap_pyfunction!(lint_backtick_identifiers, module)?)?; module.add_function(wrap_pyfunction!(lint_sql_batch_json, module)?)?; module.add_function(wrap_pyfunction!(format_sql_json, module)?)?; diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs index 8f03d97c29..579b19c7b5 100644 --- a/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/mod.rs @@ -1 +1,2 @@ pub(crate) mod functions; +pub(crate) mod panics; diff --git a/crates/sqlbuild-rules-native/src/bindings/_helpers/panics.rs b/crates/sqlbuild-rules-native/src/bindings/_helpers/panics.rs new file mode 100644 index 0000000000..1c0a23fd9a --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/_helpers/panics.rs @@ -0,0 +1,51 @@ +//! Convert unwinding native failures into ordinary named compiler errors. + +use pyo3::Python; +use pyo3::exceptions::{PyRuntimeError, PyValueError}; +use pyo3::{PyErr, create_exception}; + +create_exception!(_native, NativeCompilerError, PyRuntimeError); +const PANIC_MESSAGE: &str = "NativeCompilerError: native SQL compilation panicked"; + +pub(crate) fn compiler_error(error: impl std::fmt::Display) -> PyErr { + let message = error.to_string(); + if is_compiler_panic(&message) { + NativeCompilerError::new_err(message) + } else { + PyValueError::new_err(message) + } +} + +pub(crate) fn compiler_guard( + operation: impl FnOnce() -> pyo3::PyResult, +) -> pyo3::PyResult { + match catch_unwind(AssertUnwindSafe(operation)) { + Ok(result) => result, + Err(_) => Err(NativeCompilerError::new_err(PANIC_MESSAGE)), + } +} +use std::panic::{AssertUnwindSafe, catch_unwind}; + +use crate::bindings::types::CompilerDetach; + +impl CompilerDetach for Python<'_> { + fn compiler_detach Result + Send>( + self, + operation: F, + ) -> Result { + self.detach(|| catch_compiler_panic(operation)) + } +} + +pub(crate) fn catch_compiler_panic( + operation: impl FnOnce() -> Result, +) -> Result { + match catch_unwind(AssertUnwindSafe(operation)) { + Ok(result) => result, + Err(_) => Err(PANIC_MESSAGE.to_owned()), + } +} + +pub(crate) fn is_compiler_panic(message: &str) -> bool { + message == PANIC_MESSAGE +} diff --git a/crates/sqlbuild-rules-native/src/bindings/main/compiler_error.rs b/crates/sqlbuild-rules-native/src/bindings/main/compiler_error.rs new file mode 100644 index 0000000000..bfc30b985b --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/main/compiler_error.rs @@ -0,0 +1,3 @@ +pub(crate) fn compiler_error(error: impl std::fmt::Display) -> pyo3::PyErr { + crate::bindings::_helpers::panics::compiler_error(error) +} diff --git a/crates/sqlbuild-rules-native/src/bindings/main/compiler_guard.rs b/crates/sqlbuild-rules-native/src/bindings/main/compiler_guard.rs new file mode 100644 index 0000000000..eff844d185 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/main/compiler_guard.rs @@ -0,0 +1,5 @@ +pub(crate) fn compiler_guard( + operation: impl FnOnce() -> pyo3::PyResult, +) -> pyo3::PyResult { + crate::bindings::_helpers::panics::compiler_guard(operation) +} diff --git a/crates/sqlbuild-rules-native/src/bindings/main/mod.rs b/crates/sqlbuild-rules-native/src/bindings/main/mod.rs index f862bee245..3b8691d5be 100644 --- a/crates/sqlbuild-rules-native/src/bindings/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/bindings/main/mod.rs @@ -1 +1,3 @@ +pub(crate) mod compiler_error; +pub(crate) mod compiler_guard; pub mod register; diff --git a/crates/sqlbuild-rules-native/src/bindings/mod.rs b/crates/sqlbuild-rules-native/src/bindings/mod.rs index 893c6f5c06..5d64841a74 100644 --- a/crates/sqlbuild-rules-native/src/bindings/mod.rs +++ b/crates/sqlbuild-rules-native/src/bindings/mod.rs @@ -1,2 +1,5 @@ pub(crate) mod _helpers; pub mod main; +#[cfg(test)] +mod tests; +pub(crate) mod types; diff --git a/crates/sqlbuild-rules-native/src/bindings/tests.rs b/crates/sqlbuild-rules-native/src/bindings/tests.rs new file mode 100644 index 0000000000..b7e6cb4341 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/tests.rs @@ -0,0 +1,4 @@ +#[path = "tests/test_panics.rs"] +mod panics; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-rules-native/src/bindings/tests/test_panics.rs b/crates/sqlbuild-rules-native/src/bindings/tests/test_panics.rs new file mode 100644 index 0000000000..b25d9d08b1 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/tests/test_panics.rs @@ -0,0 +1,29 @@ +use crate::bindings::_helpers::panics::{catch_compiler_panic, is_compiler_panic}; +use crate::bindings::tests::test_types::PanicTestCase; + +#[test] +fn given_native_failure_when_crossing_python_boundary_then_only_unwinds_become_named_compiler_errors() + { + let test_cases = [ + PanicTestCase { + description: "panic becomes a recoverable compiler exception", + operation: || std::panic::panic_any("synthetic compiler failure"), + expected_named_error: true, + }, + PanicTestCase { + description: "ordinary compiler error keeps its own message", + operation: || Err("unknown column 'customer_id'".to_owned()), + expected_named_error: false, + }, + ]; + for test_case in test_cases { + let error = catch_compiler_panic(test_case.operation) + .expect_err("native failure must become an error"); + assert_eq!( + is_compiler_panic(&error), + test_case.expected_named_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules-native/src/bindings/tests/test_types.rs b/crates/sqlbuild-rules-native/src/bindings/tests/test_types.rs new file mode 100644 index 0000000000..2457330ad1 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/tests/test_types.rs @@ -0,0 +1,5 @@ +pub(super) struct PanicTestCase { + pub(super) description: &'static str, + pub(super) operation: fn() -> Result<(), String>, + pub(super) expected_named_error: bool, +} diff --git a/crates/sqlbuild-rules-native/src/bindings/types.rs b/crates/sqlbuild-rules-native/src/bindings/types.rs new file mode 100644 index 0000000000..f0038d008c --- /dev/null +++ b/crates/sqlbuild-rules-native/src/bindings/types.rs @@ -0,0 +1,6 @@ +pub(crate) trait CompilerDetach { + fn compiler_detach Result + Send>( + self, + operation: F, + ) -> Result; +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_namespace.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_namespace.rs new file mode 100644 index 0000000000..73d729193d --- /dev/null +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/cte_namespace.rs @@ -0,0 +1,106 @@ +//! Lexically scoped AST renaming before model CTEs are lifted into a shared WITH. + +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; + +use polyglot_sql::expressions::{Identifier, Null}; +use polyglot_sql::traversal::transform_map; +use polyglot_sql::{DialectType, Expression}; + +const CTE_NAME_PREFIX: &str = "__sqb_cte_"; + +/// Allocate bounded, collision-free ordinal names within one rendered query. +#[derive(Default)] +pub(crate) struct CteNamespace { + occupied: HashSet, + next: usize, +} + +impl CteNamespace { + /// Reserve generated-prefix names from all steps; no other token can collide. + pub(crate) fn reserve(&mut self, sql: &str) { + self.occupied.extend( + sql.split(|c: char| !c.is_alphanumeric() && c != '_') + .filter(|token| { + token + .as_bytes() + .get(..CTE_NAME_PREFIX.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(CTE_NAME_PREFIX.as_bytes())) + }) + .map(str::to_ascii_lowercase), + ); + } + + fn allocate(&mut self) -> String { + loop { + let name = format!("{CTE_NAME_PREFIX}{:x}", self.next); + self.next += 1; + if self.occupied.insert(name.clone()) { + return name; + } + } + } + + pub(crate) fn rewrite( + &mut self, + expression: Expression, + dialect: DialectType, + ) -> polyglot_sql::Result { + let allocator = RefCell::new(self); + transform_map(expression, &|mut expression| { + let Expression::Select(select) = &mut expression else { + return Ok(expression); + }; + let Some(mut with) = select.with.take() else { + return Ok(expression); + }; + let mut names: HashMap = HashMap::new(); + for cte in &mut with.ctes { + let original = identifier_key(&cte.alias, dialect); + let replacement = allocator.borrow_mut().allocate(); + if with.recursive { + names.insert(original.clone(), replacement.clone()); + } + let body = std::mem::replace(&mut cte.this, Expression::Null(Null)); + cte.this = rewrite_relations(body, &names, dialect)?; + names.insert(original, replacement.clone()); + cte.alias = Identifier::new(replacement); + } + expression = rewrite_relations(expression, &names, dialect)?; + if let Expression::Select(select) = &mut expression { + select.with = Some(with); + } + Ok(expression) + }) + } +} + +fn rewrite_relations( + expression: Expression, + names: &HashMap, + dialect: DialectType, +) -> polyglot_sql::Result { + transform_map(expression, &|mut expression| { + if let Expression::Table(table) = &mut expression + && table.schema.is_none() + && table.catalog.is_none() + && let Some(name) = names.get(&identifier_key(&table.name, dialect)) + { + if table.alias.is_none() { + table.alias = Some(table.name.clone()); + } + table.name = Identifier::new(name); + } + Ok(expression) + }) +} + +fn identifier_key(identifier: &Identifier, dialect: DialectType) -> String { + match dialect { + DialectType::Snowflake if !identifier.quoted => identifier.name.to_uppercase(), + DialectType::Snowflake | DialectType::PostgreSQL if identifier.quoted => { + identifier.name.clone() + } + _ => identifier.name.to_lowercase(), + } +} diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs index ca1796ea9f..360665ac7e 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod cte_namespace; pub(crate) mod cte_sql; pub(crate) mod expected_columns; pub(crate) mod extraction; diff --git a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs index d6d787066b..d10eabed1e 100644 --- a/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs +++ b/crates/sqlbuild-rules-native/src/compiler/_helpers/sql_tests/rendering.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use polyglot_sql::{Dialect, DialectType, Expression}; +use crate::compiler::_helpers::sql_tests::cte_namespace::CteNamespace; use crate::compiler::_helpers::sql_tests::cte_sql::{ cte_definition_sql, leading_with_prefix_end, with_leading_ctes, }; @@ -100,22 +101,44 @@ struct RenderCteState<'a> { dialect: &'a Dialect, lifted: Vec<(String, String)>, name_counts: CteSuffixCounter, + namespace: CteNamespace, } impl<'a> RenderCteState<'a> { - fn new(dialect: &'a Dialect) -> Self { + fn new(dialect: &'a Dialect, chain: &[ChainStep], assertions: &[AssertionStep]) -> Self { + let mut namespace = CteNamespace::default(); + for step in chain { + namespace.reserve(&step.resolved_sql); + namespace.reserve(step.comparison_body_sql.as_deref().unwrap_or_default()); + namespace.reserve(step.expected_cte_sql.as_deref().unwrap_or_default()); + for (name, sql) in step.lifted_ctes.iter().chain(&step.expected_lifted_ctes) { + namespace.reserve(name); + namespace.reserve(sql); + } + } + for assertion in assertions { + namespace.reserve(&assertion.resolved_sql); + namespace.reserve(assertion.comparison_body_sql.as_deref().unwrap_or_default()); + for (name, sql) in &assertion.lifted_ctes { + namespace.reserve(name); + namespace.reserve(sql); + } + } Self { dialect, lifted: Vec::new(), name_counts: CteSuffixCounter::default(), + namespace, } } - fn lift(&mut self, sql: &str, enabled: bool) -> String { + fn lift(&mut self, sql: &str, enabled: bool, namespace: bool) -> String { if !enabled || leading_with_prefix_end(sql).is_none() { return sql.to_string(); } - let Some((step_ctes, body_sql)) = split_top_level_with(sql, self.dialect) else { + let Some((step_ctes, body_sql)) = + split_top_level_with(sql, self.dialect, namespace.then_some(&mut self.namespace)) + else { return sql.to_string(); }; if step_ctes.iter().any(|(name, body)| { @@ -132,7 +155,7 @@ impl<'a> RenderCteState<'a> { /// Place a chain step's generated CTEs at top level and return its comparison body. fn actual_step_sql(&mut self, step: &ChainStep, enabled: bool) -> String { if step.lifted_ctes.is_empty() { - return self.lift(&step.resolved_sql, enabled); + return self.lift(&step.resolved_sql, enabled, true); } if !self.merge(&step.lifted_ctes) { return step.resolved_sql.clone(); @@ -142,18 +165,19 @@ impl<'a> RenderCteState<'a> { .as_deref() .unwrap_or(&step.resolved_sql), enabled, + true, ) } /// Place an expected step's helper CTEs at top level and return its comparison body. fn expected_step_sql(&mut self, step: &ChainStep, expected_sql: &str, enabled: bool) -> String { if step.expected_lifted_ctes.is_empty() { - return self.lift(expected_sql, enabled); + return self.lift(expected_sql, enabled, false); } if !self.merge(&step.expected_lifted_ctes) { return with_leading_ctes(&step.expected_lifted_ctes, expected_sql); } - self.lift(expected_sql, enabled) + self.lift(expected_sql, enabled, false) } fn merge(&mut self, ctes: &[(String, String)]) -> bool { @@ -173,6 +197,56 @@ impl<'a> RenderCteState<'a> { fn unique_suffix(&mut self, model_name: &str) -> String { self.name_counts.next(model_name) } + + /// Nested model bodies must also be isolated from the shared outer scope. + fn definition(&mut self, name: &str, sql: &str, enabled: bool) -> String { + let scoped = if enabled && leading_with_prefix_end(sql).is_some() { + let (protected, identifiers) = protect_backtick_identifiers(sql); + match self.dialect.parse(&protected) { + Ok(mut expressions) => (|| { + if expressions.len() != 1 { + return None; + } + let expression = match self + .namespace + .rewrite(expressions.pop()?, self.dialect.dialect_type()) + { + Ok(expression) => expression, + Err(_) => return None, + }; + let sql = match self.dialect.generate(&expression) { + Ok(sql) => sql, + Err(_) => return None, + }; + Some(restore_backtick_identifiers(&sql, &identifiers)) + })(), + Err(_) => None, + } + } else { + None + }; + cte_definition_sql(name, scoped.as_deref().unwrap_or(sql)) + } + + fn definitions(&mut self, enabled: bool) -> Vec { + let mut definitions: Vec = Vec::new(); + for (name, sql) in std::mem::take(&mut self.lifted) { + if enabled + && leading_with_prefix_end(&sql).is_some() + && let Some((ctes, body)) = + split_top_level_with(&sql, self.dialect, Some(&mut self.namespace)) + { + definitions.extend( + ctes.iter() + .map(|(name, sql)| self.definition(name, sql, enabled)), + ); + definitions.push(self.definition(&name, &body, enabled)); + } else { + definitions.push(self.definition(&name, &sql, enabled)); + } + } + definitions + } } /// Readable, collision-free comparison CTE suffixes in step order. @@ -259,17 +333,13 @@ fn render_difference_sample_sql(request: &DifferenceSampleRequest, dialect: &Dia let Some(expected_input) = step.expected_cte_sql.as_deref() else { return String::new(); }; - let mut cte_state = RenderCteState::new(dialect); + let mut cte_state = RenderCteState::new(dialect, std::slice::from_ref(step), &[]); let actual_sql = cte_state.actual_step_sql(step, request.sql_analysis_enabled); let expected_sql = cte_state.expected_step_sql(step, expected_input, request.sql_analysis_enabled); - let mut cte_parts: Vec = cte_state - .lifted - .iter() - .map(|(name, sql)| cte_definition_sql(name, sql)) - .collect(); - cte_parts.push(cte_definition_sql("__actual", &actual_sql)); - cte_parts.push(cte_definition_sql("__expected", &expected_sql)); + let mut cte_parts = cte_state.definitions(request.sql_analysis_enabled); + cte_parts.push(cte_state.definition("__actual", &actual_sql, request.sql_analysis_enabled)); + cte_parts.push(cte_state.definition("__expected", &expected_sql, request.sql_analysis_enabled)); let (left, right) = match request.direction { DifferenceDirection::Unexpected => ("__actual", "__expected"), DifferenceDirection::Missing => ("__expected", "__actual"), @@ -304,7 +374,7 @@ pub(crate) fn render_comparison_sql(request: &RenderRequest, dialect: &Dialect) if request.chain.is_empty() && request.assertions.is_empty() { return String::new(); } - let mut cte_state = RenderCteState::new(dialect); + let mut cte_state = RenderCteState::new(dialect, &request.chain, &request.assertions); let mut comparison_ctes: Vec = Vec::new(); let mut select_parts: Vec = Vec::new(); let rendered_steps = rendered_chain_steps(&request.chain, &request.assertions); @@ -318,7 +388,11 @@ pub(crate) fn render_comparison_sql(request: &RenderRequest, dialect: &Dialect) continue; } let actual_sql = cte_state.actual_step_sql(step, request.sql_analysis_enabled); - comparison_ctes.push(cte_definition_sql(&actual_cte, &actual_sql)); + comparison_ctes.push(cte_state.definition( + &actual_cte, + &actual_sql, + request.sql_analysis_enabled, + )); if request.probe_step_index == Some(step_index) { probe_actual_cte = Some(actual_cte.clone()); } @@ -327,7 +401,11 @@ pub(crate) fn render_comparison_sql(request: &RenderRequest, dialect: &Dialect) }; let expected_sql = cte_state.expected_step_sql(step, expected_input, request.sql_analysis_enabled); - comparison_ctes.push(cte_definition_sql(&expected_cte, &expected_sql)); + comparison_ctes.push(cte_state.definition( + &expected_cte, + &expected_sql, + request.sql_analysis_enabled, + )); let projection = compared_projection(step); select_parts.push(format!( "SELECT {step_index} AS step_index, '{}' AS model_name, \ @@ -357,11 +435,16 @@ pub(crate) fn render_comparison_sql(request: &RenderRequest, dialect: &Dialect) .as_deref() .unwrap_or(&assertion.resolved_sql), request.sql_analysis_enabled, + true, ) } else { assertion.resolved_sql.clone() }; - comparison_ctes.push(cte_definition_sql(&assertion_cte, &assertion_sql)); + comparison_ctes.push(cte_state.definition( + &assertion_cte, + &assertion_sql, + request.sql_analysis_enabled, + )); select_parts.push(format!( "SELECT {assertion_index} AS step_index, 'assertion {}' AS model_name, \ (SELECT COUNT(*) FROM {assertion_cte}) AS actual_count, \ @@ -375,11 +458,7 @@ pub(crate) fn render_comparison_sql(request: &RenderRequest, dialect: &Dialect) if select_parts.is_empty() { return String::new(); } - let mut cte_parts: Vec = cte_state - .lifted - .iter() - .map(|(name, sql)| cte_definition_sql(name, sql)) - .collect(); + let mut cte_parts = cte_state.definitions(request.sql_analysis_enabled); cte_parts.extend(comparison_ctes); if request.probe_step_index.is_some() { return probe_actual_cte.map_or_else(String::new, |actual_cte| { @@ -402,7 +481,11 @@ fn existing_cte<'a>(lifted: &'a [(String, String)], name: &str) -> Option<&'a (S .find(|(existing, _)| existing.eq_ignore_ascii_case(name)) } -fn split_top_level_with(sql: &str, dialect: &Dialect) -> Option<(Vec<(String, String)>, String)> { +fn split_top_level_with( + sql: &str, + dialect: &Dialect, + namespace: Option<&mut CteNamespace>, +) -> Option<(Vec<(String, String)>, String)> { let (protected, identifiers) = protect_backtick_identifiers(sql); let mut statements = match dialect.parse(&protected) { Ok(statements) => statements, @@ -412,10 +495,24 @@ fn split_top_level_with(sql: &str, dialect: &Dialect) -> Option<(Vec<(String, St return None; } let mut expression = statements.pop()?; + if let Some(namespace) = namespace { + expression = match namespace.rewrite(expression, dialect.dialect_type()) { + Ok(expression) => expression, + Err(_) => return None, + }; + } let Expression::Select(select) = &mut expression else { return None; }; let with = select.with.take()?; + if with.recursive + || with.search.is_some() + || with.ctes.iter().any(|cte| { + !cte.columns.is_empty() || cte.materialized.is_some() || !cte.key_expressions.is_empty() + }) + { + return None; + } let mut ctes = Vec::with_capacity(with.ctes.len()); for cte in with.ctes { let sql = match dialect.generate(&cte.this) { @@ -464,8 +561,19 @@ fn protect_backtick_identifiers(sql: &str) -> (String, Vec<(String, String)>) { } cursor = next_identifier + 1; } - let placeholder = format!("SQB_PROTECTED_IDENTIFIER_{}", identifiers.len()); - identifiers.push((placeholder.clone(), sql[start - 1..cursor].to_string())); + let identifier = &sql[start - 1..cursor]; + if !identifier[1..identifier.len() - 1].contains(['`', '.']) { + protected.push_str(identifier); + continue; + } + let placeholder = match identifiers.iter().find(|(_, value)| value == identifier) { + Some((placeholder, _)) => placeholder.clone(), + None => { + let placeholder = format!("SQB_PROTECTED_IDENTIFIER_{}", identifiers.len()); + identifiers.push((placeholder.clone(), identifier.to_string())); + placeholder + } + }; protected.push_str(&placeholder); } protected.push_str(&sql[cursor..]); diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs index f62533ed44..5673f32d9d 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/helpers.rs @@ -2,6 +2,8 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Barrier, Mutex}; +use crate::compiler::_helpers::sql_tests::cte_namespace::CteNamespace; +use polyglot_sql::{Dialect, DialectType, Expression}; use serde_json::{Value, json}; use crate::compiler::_helpers::model_headers::tokenization::{ @@ -1377,3 +1379,87 @@ pub(crate) fn mock_read_through_helper_brings_its_mock_dependencies_into_scope() } true } +pub(super) fn bounded_names_are_unique() -> bool { + let dialect = Dialect::get(DialectType::DuckDB); + let long = "orders_".repeat(40); + let sql = format!( + "WITH {long} AS (SELECT 1 AS order_id), __sqb_cte_0 AS (SELECT * FROM {long}), \ + final AS (SELECT * FROM __sqb_cte_0) SELECT final.* FROM final" + ); + let reserved = "__sqb_cte_1 __SQB_CTE_2 __ref__orders __source__orders __expected__orders __actual__orders"; + let mut first = CteNamespace::default(); + first.reserve(&sql); + first.reserve(reserved); + let parsed = dialect.parse(&sql).expect("valid CTE query").remove(0); + let rewritten = first + .rewrite(parsed.clone(), dialect.dialect_type()) + .expect("valid rename"); + let Expression::Select(select) = &rewritten else { + return false; + }; + let names: Vec<_> = select + .with + .as_ref() + .expect("WITH retained") + .ctes + .iter() + .map(|cte| cte.alias.name.as_str()) + .collect(); + let second_step = first + .rewrite(parsed.clone(), dialect.dialect_type()) + .expect("second rename"); + let mut replay = CteNamespace::default(); + replay.reserve(&sql); + replay.reserve(reserved); + names.len() == 3 + && names + .iter() + .all(|name| name.len() <= 30 && name.starts_with("__sqb_cte_")) + && !names.contains(&"__sqb_cte_0") + && !names.contains(&"__sqb_cte_1") + && !names.contains(&"__sqb_cte_2") + && rewritten != second_step + && rewritten + == replay + .rewrite(parsed, dialect.dialect_type()) + .expect("deterministic rename") +} + +pub(super) fn quoted_names_keep_bindings() -> bool { + let dialect = Dialect::get(DialectType::Snowflake); + let sql = "WITH final AS (SELECT 1 AS order_id), \"final\" AS (SELECT 2 AS order_id) \ + SELECT a.order_id, b.order_id, 'final' AS label FROM FINAL a CROSS JOIN \"final\" b"; + let mut namespace = CteNamespace::default(); + namespace.reserve(sql); + let rewritten = namespace + .rewrite( + dialect.parse(sql).expect("valid quoted CTEs").remove(0), + dialect.dialect_type(), + ) + .expect("quoted rename"); + let rendered = dialect.generate(&rewritten).expect("render renamed CTEs"); + rendered.contains("FROM __sqb_cte_0 AS a") + && rendered.contains("JOIN __sqb_cte_1 AS b") + && rendered.contains("'final' AS label") +} +pub(super) fn model_cte_names_are_isolated_across_dialects() -> bool { + [("duckdb", "final"), ("snowflake", "final"), ("postgres", "final"), ("tsql", "final"), ("bigquery", "`final`"), ("databricks", "`final`")].iter().all(|(dialect, name)| { + let request = json!({ + "requests": [{ + "sqlAnalysisDialect": dialect, + "chain": [{ + "modelName": "orders", + "resolvedSql": format!("WITH {name} AS (SELECT 2 AS order_id) SELECT final.order_id FROM final"), + "comparisonBodySql": format!("WITH {name} AS (SELECT order_id + 1 AS order_id FROM __ref__stg_orders) SELECT final.order_id FROM final"), + "liftedCtes": [["__ref__stg_orders", format!("WITH {name} AS (SELECT 1 AS order_id) SELECT * FROM {name}")]], + "expectedCteSql": "SELECT 2 AS order_id" + }] + }] + }); + let response = crate::compiler::_helpers::sql_tests::rendering::render_json(&request.to_string()).expect("supported render dialect"); + let response: Value = serde_json::from_str(&response).expect("render response"); + let sql = response[0]["sql"].as_str().expect("rendered query"); + sql.contains("__sqb_cte_0 AS") && sql.contains("FROM __sqb_cte_0 ") + && sql.contains("__sqb_cte_1 AS") && !sql.contains("AS (WITH") + }) +} diff --git a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs index ffa81a4d68..ae54b27aea 100644 --- a/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs +++ b/crates/sqlbuild-rules-native/src/compiler/tests/test_sql_test_rendering.rs @@ -5,11 +5,30 @@ use crate::compiler::tests::helpers::{ snowflake_plan_keeps_quoted_expected_columns, sqlserver_difference_sample_projects_bracketed_columns, }; +use crate::compiler::tests::helpers::{ + bounded_names_are_unique, model_cte_names_are_isolated_across_dialects, + quoted_names_keep_bindings, +}; use crate::compiler::tests::test_types::SqlTestRenderingTestCase; #[test] fn given_sql_rendering_cases_when_rendering_native_batches_then_expected_behavior_holds() { let test_cases = [ + SqlTestRenderingTestCase { + description: "model CTEs are isolated across every first-class dialect", + run: model_cte_names_are_isolated_across_dialects, + expected_success: true, + }, + SqlTestRenderingTestCase { + description: "long and reserved CTE names are bounded, unique and deterministic", + run: bounded_names_are_unique, + expected_success: true, + }, + SqlTestRenderingTestCase { + description: "quoted CTEs and literals retain their bindings", + run: quoted_names_keep_bindings, + expected_success: true, + }, SqlTestRenderingTestCase { description: "ordered comparison batches preserve SQL order", run: ordered_comparison_batch_preserves_order, diff --git a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs index 8dff5e1504..a1a8f3a8db 100644 --- a/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/_helpers/evaluation.rs @@ -831,6 +831,17 @@ fn apply_fault_policy( evaluated_codes: &[String], faults: Vec, ) -> Result, String> { + let unevaluated: BTreeSet<_> = faults + .iter() + .filter(|fault| fault.unevaluated) + .map(|fault| { + ( + fault.code.clone(), + fault.path.clone(), + fault.message.clone(), + ) + }) + .collect(); let findings = faults .into_iter() .map(fault_to_finding) @@ -857,15 +868,27 @@ fn apply_fault_policy( }) .collect::>(); let grammar = RulesCodeGrammar; - apply_suppressions(ApplySuppressionsRequest { + let result = apply_suppressions(ApplySuppressionsRequest { findings, evaluated_codes, suppressions: &suppressions, scoped_ignores: &scoped_ignores, grammar: &grammar, }) - .map(|result| result.findings.into_iter().map(finding_to_fault).collect()) - .map_err(lifecycle_error) + .map_err(lifecycle_error)?; + Ok(result + .findings + .into_iter() + .map(|finding| { + let mut fault = finding_to_fault(finding); + fault.unevaluated = unevaluated.contains(&( + fault.code.clone(), + fault.path.clone(), + fault.message.clone(), + )); + fault + }) + .collect()) } fn fault_to_finding(fault: Fault) -> Result { @@ -883,6 +906,7 @@ fn fault_to_finding(fault: Fault) -> Result { fn finding_to_fault(finding: Finding) -> Fault { Fault { + unevaluated: false, code: finding.code, path: finding.path, line: u64::from(finding.line.unwrap_or(1)), diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs index 96ee59552f..98e029fabd 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_empty_input_tests.rs @@ -190,7 +190,7 @@ fn given_sql_test_shapes_when_evaluating_empty_input_rule_then_flags_only_filler ], }, test_types::EmptyInputTestRuleTestCase { - description: "an unparseable mock is not flagged", + description: "an unparseable mock reports missing evaluation evidence", test: helpers::empty_input_test_fact( "orders__empty_inputs_produce_no_rows", &[( @@ -201,7 +201,9 @@ fn given_sql_test_shapes_when_evaluating_empty_input_rule_then_flags_only_filler &[NO_ROWS_ASSERTION], ), allowed_tests: json!([]), - expected_messages: &[], + expected_messages: &[ + "SQBRTEST203 tests/unit/test_orders__empty_inputs_produce_no_rows.sql: Rule SQBRTEST203 could not be evaluated for test block 1: could not parse SQL-test fixture: sql parser error: sql parser error: Expected: an expression, found: EOF; generic fallback also failed: sql parser error: Expected: an expression, found: EOF", + ], }, test_types::EmptyInputTestRuleTestCase { description: "a macro-mode test is not flagged", diff --git a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs index 579941bbeb..09ec3df1ec 100644 --- a/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs +++ b/crates/sqlbuild-rules-native/src/engine/tests/test_evaluation.rs @@ -479,6 +479,49 @@ fn given_dialect_quoting_around_table_functions_when_evaluating_rules_then_model Ok(()) } +#[test] +fn given_unparseable_model_when_evaluating_then_retains_named_rule_fault() -> Result<(), String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let test_cases = [test_types::DialectEvaluationTestCase { + description: "invalid query is missing evidence rather than a passing contract", + dialect: "duckdb", + query_sql: "SELECT (", + expected_code: "SQBRCONTRACT101", + }]; + for test_case in test_cases { + let config = json!({"select": ["SQBRCONTRACT101"], "cache": {"enabled": false}}); + let mut request: Value = serde_json::from_str(&helpers::request(&project_dir, &config)) + .map_err(|error| error.to_string())?; + request["models"][0]["query_sql"] = json!(test_case.query_sql); + request["dialect"] = json!(test_case.dialect); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + assert_eq!( + result["faults"][0]["code"], test_case.expected_code, + "{}", + test_case.description + ); + assert_eq!( + result["faults"][0]["unevaluated"], true, + "{}", + test_case.description + ); + assert_eq!( + result["faults"][0]["path"], request["models"][0]["relative_path"], + "{}", + test_case.description + ); + assert!( + result["faults"][0]["message"] + .as_str() + .is_some_and(|message| message.contains("could not be evaluated")), + "{}", + test_case.description + ); + } + Ok(()) +} + #[test] fn given_comment_apostrophes_when_generic_fallback_normalizes_rules_sql_then_model_parses() -> Result<(), String> { diff --git a/crates/sqlbuild-rules-native/src/models.rs b/crates/sqlbuild-rules-native/src/models.rs index e720b25b28..eb9a699ee0 100644 --- a/crates/sqlbuild-rules-native/src/models.rs +++ b/crates/sqlbuild-rules-native/src/models.rs @@ -540,6 +540,8 @@ impl Default for ScopeIndexFacts { #[derive(Clone, Debug, Default, Deserialize, Serialize)] #[serde(default, deny_unknown_fields)] pub(crate) struct Model { + #[serde(default)] + pub sql_analysis_disabled: bool, pub name: String, pub relative_path: String, pub query_sql: String, @@ -697,6 +699,8 @@ pub(crate) struct CustomRule { #[derive(Clone, Debug, Deserialize, Serialize, Eq, PartialEq)] #[serde(deny_unknown_fields)] pub(crate) struct Fault { + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub unevaluated: bool, pub code: String, pub path: String, pub line: u64, diff --git a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs index 021a098b98..19c0d2fcfb 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs @@ -34,6 +34,58 @@ struct AnalysisRequest { schema: Option, #[serde(default)] binding_schema: Option, + #[serde(default)] + binding_options: Option, + #[serde(default)] + binding_references: Option>, + #[serde(default)] + binding_override: Option, + /// Relation stubs in `sql` that rename the catalog relations of the binding schema. + #[serde(default)] + binding_aliases: Option>, + #[serde(default)] + analysis_references: Option>, + #[serde(default)] + sqlbuild_diagnostics: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct BindingOptions { + #[serde(default)] + check_types: bool, + #[serde(default)] + semantic: bool, + #[serde(default)] + known_functions: Vec, + #[serde(default)] + known_types: Vec, +} + +impl AnalysisRequest { + fn validation_options(&self) -> SchemaValidationOptions { + SchemaValidationOptions { + check_types: self + .binding_options + .as_ref() + .is_some_and(|options| options.check_types), + semantic: self + .binding_options + .as_ref() + .is_some_and(|options| options.semantic), + known_functions: self + .binding_options + .as_ref() + .map_or_else(Vec::new, |options| options.known_functions.clone()), + known_types: self + .binding_options + .as_ref() + .map_or_else(Vec::new, |options| options.known_types.clone()), + check_references: true, + strict: Some(true), + strict_syntax: false, + ..Default::default() + } + } } #[derive(Debug, Serialize)] @@ -262,8 +314,39 @@ pub(crate) fn analyze_project_json(request_json: &str) -> Result } pub(crate) fn analyze_project_compact_json(request_json: &str) -> Result { - let request: CompactProjectAnalysisBatchRequest = + analyze_project_compact_with_catalog(request_json, None) +} + +pub(crate) fn analyze_project_compact_with_catalog( + request_json: &str, + catalog: Option<&crate::semantic_validation::models::ProjectCatalog>, +) -> Result { + let mut request: CompactProjectAnalysisBatchRequest = serde_json::from_str(request_json).map_err(|error| error.to_string())?; + for query in &mut request.queries { + query.sqlbuild_diagnostics = catalog.is_some(); + if let Some(references) = &query.analysis_references { + query.schema = catalog + .ok_or("analysis references require a native project catalog")? + .analysis_schema(references); + } + if let Some(references) = &query.binding_references { + let catalog = catalog.ok_or("binding references require a native project catalog")?; + let mut schema = catalog.reference_schema(references, query.binding_override)?; + if let Some(aliases) = &query.binding_aliases { + catalog.alias_reference_schema(&mut schema, references, aliases); + } + query.binding_schema = Some(schema); + if !catalog.quoted_ignore_case || !query.sql.contains('"') { + query.binding_options = Some(BindingOptions { + check_types: catalog.options.check_types, + semantic: catalog.options.semantic, + known_functions: catalog.options.known_functions.clone(), + known_types: catalog.options.known_types.clone(), + }); + } + } + } let workers = request.workers.clamp(1, MAX_WORKERS); if request .templates @@ -350,7 +433,7 @@ pub(crate) fn analyze_project_compact_json(request_json: &str) -> Result = pool.install(|| { batch .into_par_iter() - .map(analyze_compact_query_work) + .map(|work| analyze_compact_query_work(work, catalog)) .collect() }); for group in analysis_groups { @@ -371,12 +454,50 @@ pub(crate) fn analyze_project_compact_json(request_json: &str) -> Result CompiledQueryWorkResult { - let work = match try_borrowed_query(work) { +fn analyze_compact_query_work( + work: CompactQueryWork, + catalog: Option<&crate::semantic_validation::models::ProjectCatalog>, +) -> CompiledQueryWorkResult { + let diagnostic_sql = work + .query + .sqlbuild_diagnostics + .then(|| work.query.sql.clone()); + let dialect = work.query.dialect.parse::(); + let mut result = analyze_compact_query_work_inner(work, catalog); + if let (Some(sql), Ok(dialect)) = (diagnostic_sql, dialect) + && let Some(validation) = result.validation.take() + { + result.validation = Some(validation.and_then(|value| { + crate::semantic_validation::main::map_diagnostics::map_diagnostics(&sql, dialect, value) + })); + } + result +} + +fn analyze_compact_query_work_inner( + work: CompactQueryWork, + catalog: Option<&crate::semantic_validation::models::ProjectCatalog>, +) -> CompiledQueryWorkResult { + let exact_catalog = catalog.filter(|catalog| { + work.query + .binding_schema + .as_ref() + .is_some_and(|schema| catalog.needs_identifier_encoding(&work.query.sql, schema)) + }); + let work = match try_borrowed_query(work, exact_catalog) { Ok(result) => return result, Err(work) => *work, }; - let (query_result, validation, _) = compile_query(work.query, work.project_projections); + let exact_schema = exact_catalog.and_then(|_| work.query.binding_schema.clone()); + let sql = exact_catalog.map(|_| work.query.sql.clone()); + let (query_result, mut validation, expression) = + compile_query(work.query, work.project_projections); + if let (Some(catalog), Some(schema), Some(sql)) = (exact_catalog, exact_schema, sql) { + validation = Some(match expression { + Some(expression) => catalog.expression_validation(&sql, &schema, expression), + None => catalog.validate(&sql, &schema), + }); + } CompiledQueryWorkResult { projections: project_query_templates(&query_result, work.projections), validation, @@ -385,7 +506,8 @@ fn analyze_compact_query_work(work: CompactQueryWork) -> CompiledQueryWorkResult /// Fold fully bound lexical query graphs; unsupported shapes retain the existing resolver. fn try_borrowed_query( - work: CompactQueryWork, + mut work: CompactQueryWork, + exact_catalog: Option<&crate::semantic_validation::models::ProjectCatalog>, ) -> Result> { if work.query.schema.is_none() || (work.query.binding_schema.is_none() @@ -429,8 +551,14 @@ fn try_borrowed_query( return Err(Box::new(work)); } if matches!(&expression, polyglot_sql::Expression::Select(select) - if select.with.is_none() && select.expressions.iter().any(|projection| super::borrowed_facts::projection_star(projection).is_some())) + if select.with.is_none() + && select.expressions.iter().any(|projection| super::borrowed_facts::projection_star(projection).is_some())) { + if matches!(&expression, polyglot_sql::Expression::Select(select) + if select.from.as_ref().is_some_and(|from| matches!(from.expressions.as_slice(), [polyglot_sql::Expression::Subquery(_)]))) + { + work.project_projections = false; + } return Err(Box::new(work)); } if !matches!( @@ -452,14 +580,10 @@ fn try_borrowed_query( || matches!(node, polyglot_sql::Expression::Select(select) if select.with.as_ref().is_some_and(|with| with.recursive)) || matches!(node, polyglot_sql::Expression::Union(union) if union.with.as_ref().is_some_and(|with| with.recursive)) || matches!(node, polyglot_sql::Expression::Select(select) if !borrowed_sources_supported(select, dialect)) - }) { - return Err(Box::new(work)); - } - if expression.dfs().any(|node| { - matches!(node, polyglot_sql::Expression::Select(select) - if select.expressions.iter().any(|projection| projection.dfs().any(|child| matches!(child, - polyglot_sql::Expression::Select(_) | polyglot_sql::Expression::Union(_) - | polyglot_sql::Expression::Intersect(_) | polyglot_sql::Expression::Except(_))))) + || matches!(node, polyglot_sql::Expression::Select(select) + if select.expressions.iter().any(|projection| projection.dfs().any(|child| matches!(child, + polyglot_sql::Expression::Select(_) | polyglot_sql::Expression::Union(_) + | polyglot_sql::Expression::Intersect(_) | polyglot_sql::Expression::Except(_))))) }) { return Err(Box::new(work)); } @@ -474,41 +598,9 @@ fn try_borrowed_query( }) }) .and_then(|schema| super::borrowed_facts::infer_bound(&expression, Some(schema), dialect)); - let validation = work - .query - .binding_schema - .as_ref() - .or(work.query.schema.as_ref()) - .map(|schema| { - if bound_facts.is_some() { - return Ok(ValidationResult::with_errors(Vec::new())); - } - let result = polyglot_sql::validation::validate_parsed_with_schema( - vec![expression.clone()], - dialect, - schema, - &SchemaValidationOptions { - check_types: false, - check_references: true, - strict: Some(true), - semantic: false, - strict_syntax: false, - ..Default::default() - }, - ); - crate::semantic_validation::main::complete_parsed_validation( - crate::semantic_validation::main::ParsedValidationRequest { - sql: &work.query.sql, - dialect, - schema, - result, - expression: Some(&expression), - }, - ) - }); - if !matches!(validation, Some(Ok(ref result)) if result.valid) { - return Err(Box::new(work)); - } + let catalog_expression = exact_catalog.map(|_| expression.clone()); + let needs_validation = bound_facts.is_none() || work.query.binding_options.is_some(); + let mut validation_expression = None; let unannotated_outputs: Option> = work .projections .iter() @@ -522,6 +614,9 @@ fn try_borrowed_query( }) .collect(); if unannotated_outputs.is_none() { + if needs_validation { + validation_expression = Some(expression.clone()); + } let schema = work.query.schema.as_ref().map(|schema| { polyglot_sql::validation::mapping_schema_from_validation_schema_with_dialect( schema, dialect, @@ -619,12 +714,60 @@ fn try_borrowed_query( Ok(ProjectAnalysis { columns, lineage_columns, - has_star: false, + has_star: matches!(&expression, polyglot_sql::Expression::Select(select) + if select.expressions.iter().any(|projection| super::borrowed_facts::projection_star(projection).is_some())), requires_legacy_fallback, preserve_fallback_lineage: true, }), )); } + let mut validation = work + .query + .binding_schema + .as_ref() + .or(work.query.schema.as_ref()) + .map(|schema| { + if !needs_validation { + return Ok(ValidationResult::with_errors(Vec::new())); + } + let retain_expression = + crate::semantic_validation::main::may_have_extra_clause_checks(&work.query.sql); + let validation_expression = validation_expression.take().unwrap_or_else(|| { + if retain_expression { + expression.clone() + } else { + std::mem::replace( + &mut expression, + polyglot_sql::Expression::Null(polyglot_sql::expressions::Null), + ) + } + }); + let result = polyglot_sql::validation::validate_parsed_with_schema( + vec![validation_expression], + dialect, + schema, + &work.query.validation_options(), + ); + crate::semantic_validation::main::complete_parsed_validation( + crate::semantic_validation::main::ParsedValidationRequest { + sql: &work.query.sql, + dialect, + schema, + result, + expression: retain_expression.then_some(&expression), + }, + ) + }); + if !matches!(validation, Some(Ok(ref result)) if result.valid) { + return Err(Box::new(work)); + } + if let (Some(catalog), Some(expression), Some(schema)) = ( + exact_catalog, + catalog_expression, + work.query.binding_schema.as_ref(), + ) { + validation = Some(catalog.expression_validation(&work.query.sql, schema, expression)); + } Ok(CompiledQueryWorkResult { projections, validation: if work.query.binding_schema.is_some() { @@ -799,6 +942,7 @@ type CompiledQueryResult = ( ); fn compile_query(mut request: AnalysisRequest, project_projections: bool) -> CompiledQueryResult { + let validation_options = request.validation_options(); let Some(schema) = request.binding_schema.take() else { return (query_analysis(request, project_projections), None, None); }; @@ -814,14 +958,7 @@ fn compile_query(mut request: AnalysisRequest, project_projections: bool) -> Com schema: request.schema, }, &schema, - &SchemaValidationOptions { - check_types: false, - check_references: true, - strict: Some(true), - semantic: false, - strict_syntax: false, - ..Default::default() - }, + &validation_options, project_projections, ); let validation = crate::semantic_validation::main::complete_parsed_validation( @@ -933,7 +1070,7 @@ struct CompactProjectBatch { #[derive(Debug, Serialize)] #[serde(untagged)] enum CompactProjectResponse { - Success((Vec, bool)), + Success((Vec, bool, bool)), LegacyTypeRecovery((Vec, bool, &'static str)), Failure(String), } @@ -1033,7 +1170,11 @@ impl CompactProjectAccumulator { "native project type recovery requires legacy fallback", )) } else { - CompactProjectResponse::Success((columns, analysis.has_star)) + CompactProjectResponse::Success(( + columns, + analysis.has_star, + analysis.preserve_fallback_lineage, + )) }, ) } diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project_catalog.rs b/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project_catalog.rs new file mode 100644 index 0000000000..d01c6710a2 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/query_analysis/main/analyze_project_catalog.rs @@ -0,0 +1,6 @@ +pub(crate) fn analyze_project_compact_with_catalog( + request_json: &str, + catalog: &crate::semantic_validation::models::ProjectCatalog, +) -> Result { + crate::query_analysis::engine::analyze_project_compact_with_catalog(request_json, Some(catalog)) +} diff --git a/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs b/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs index dfdba3210b..9a89ac6347 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/main/mod.rs @@ -1,3 +1,4 @@ pub(crate) mod analyze; pub(crate) mod analyze_project; +pub(crate) mod analyze_project_catalog; pub(crate) mod analyze_project_compact; diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs b/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs index bbc44b8051..0641838a6e 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs @@ -255,7 +255,7 @@ pub(crate) fn compact_project_query_interns_repeated_lineage_strings() -> bool { json!(["model", "orders", "order_id", "BIGINT"]) ); assert_eq!(response["facts"][0], json!([2, 3, 1, 0, 1, [[0, 1, 2]]])); - assert_eq!(response["templates"][0], json!([[0], false])); + assert_eq!(response["templates"][0], json!([[0], false, false])); assert_eq!(response["analyses"][0], json!([0, [[1, 1]]])); true } diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs index be9623a11c..ce3d420736 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/domain_layout.rs @@ -1,4 +1,5 @@ use crate::models::{DeclarationKind, Fault, Model, RuleMetadata, ScopeDeclaration, ScopeKind}; +use crate::rules::_helpers::evaluation::path_fault; use crate::rules::models::ProjectEvaluationRequest; use std::collections::{BTreeMap, BTreeSet}; @@ -666,14 +667,3 @@ fn threshold(evaluation: &ProjectEvaluationRequest<'_>, name: &str, default: u32 .copied() .unwrap_or(default) } - -fn path_fault(rule: &RuleMetadata, path: &str, message: String, remediation: String) -> Fault { - Fault { - code: rule.code.clone(), - path: path.into(), - line: 1, - column: 1, - message, - remediation, - } -} diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs index 2c32435cfc..3bd6fefdc0 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/dynamic_contracts.rs @@ -35,6 +35,7 @@ pub(crate) fn evaluate_without_query( && let Some(rule) = selected.get(code) { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, @@ -110,6 +111,7 @@ fn rules_need_no_query(model: &Model, selected: &BTreeMap fn fault(model: &Model, rule: &RuleMetadata) -> Fault { Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs index 881c4f9658..5ee489c8fb 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/evaluation.rs @@ -115,7 +115,23 @@ enum ImportRejection { } pub(crate) fn evaluate_model(request: ModelEvaluationRequest<'_>) -> Result, String> { - stacker::grow(8 * 1024 * 1024, || evaluate_model_inner(request)) + if request.model.sql_analysis_disabled { + return Ok(Vec::new()); + } + let model = request.model; + let selected = request.selected; + match stacker::grow(8 * 1024 * 1024, || evaluate_model_inner(request)) { + Ok(faults) => Ok(faults), + Err(reason) => Ok(selected.values().filter(|rule| requires_model_query(rule)).map(|rule| Fault { + unevaluated: true, + code: rule.code.clone(), + path: model.relative_path.clone(), + line: 1, + column: 1, + message: format!("Rule {} could not be evaluated: {reason}", rule.code), + remediation: "Use supported SQL, disable SQL analysis, or explicitly ignore the affected Rule.".to_owned(), + }).collect()), + } } fn evaluate_model_inner(request: ModelEvaluationRequest<'_>) -> Result, String> { @@ -239,14 +255,27 @@ pub(crate) fn evaluate_project_rules(request: &ProjectEvaluationRequest<'_>) -> } fn requires_parsed_model(selected: &BTreeMap) -> bool { - selected.values().any(|rule| { - !rule.custom - && !rule.code.starts_with("SQBRSQL") - && !matches!( + selected.values().any(|rule| requires_model_query(rule)) +} + +fn requires_model_query(rule: &RuleMetadata) -> bool { + !rule.custom + && (rule.code.starts_with("SQBRMODEL") + || rule.code.starts_with("SQBRCONTRACT") + || matches!( rule.code.as_str(), - "SQBRDECLARATION201" | "SQBRDECLARATION301" | "SQBRTEST301" - ) - }) + "SQBRGRAPH102" + | "SQBRPROJECT101" + | "SQBRPROJECT102" + | "SQBRPROJECT103" + | "SQBRPROJECT104" + | "SQBRPROJECT105" + | "SQBRPROJECT106" + | "SQBRDECLARATION101" + | "SQBRDECLARATION102" + | "SQBRTEST201" + | "SQBRTEST202" + )) } pub(super) fn parse_rule_statements( @@ -522,6 +551,7 @@ fn is_lambda_parameter_type(word: &[u8]) -> bool { fn fault(model: &Model, rule: &RuleMetadata, position: Option<&Position>) -> Fault { Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: position.map_or(1, |value| value.line), @@ -552,8 +582,14 @@ macro_rules! custom_fault { }; } -fn path_fault(path: &str, rule: &RuleMetadata, message: String, remediation: String) -> Fault { +pub(super) fn path_fault( + rule: &RuleMetadata, + path: &str, + message: String, + remediation: String, +) -> Fault { Fault { + unevaluated: false, code: rule.code.clone(), path: path.into(), line: 1, @@ -1371,8 +1407,8 @@ fn duplicate_enums(request: &EvaluateRequest, rule: &RuleMetadata, faults: &Faul let signature = serde_json::to_string(&members).unwrap_or_default(); if let Some(previous) = signatures.get(&signature) { faults.push(path_fault( - &declaration.relative_path, rule, + &declaration.relative_path, format!("enum {:?} duplicates {:?}", declaration.name, previous.name), rule.remediation.clone(), )); @@ -1430,8 +1466,8 @@ fn declaration_domain_placement( }; let root = parts.first().copied().unwrap_or("declarations"); faults.push(path_fault( - path, rule, + path, format!( "public declaration {:?} has no configured domain folder", name @@ -1462,6 +1498,7 @@ fn custom_rule_test_coverage( } if custom.test_case_count < minimum { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: custom.source.clone().unwrap_or_else(|| "rules".into()), line: custom.source_line.max(1), diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs index 338be8342a..b16fb6e14f 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/explicit_output_types.rs @@ -70,6 +70,7 @@ pub(crate) fn evaluate_proven_dynamic(model: &Model, rule: &RuleMetadata, faults .filter(|column| !column.data_type.is_empty() && !column.type_proven) { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, @@ -90,6 +91,7 @@ pub(crate) fn evaluate_proven_dynamic(model: &Model, rule: &RuleMetadata, faults .filter(|family| !family.type_proven) { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, @@ -219,6 +221,7 @@ impl OutputRule<'_> { fn push(&self, position: Option<(u64, u64)>, message: String, remediation: String) { let (line, column) = position.unwrap_or((1, 1)); self.faults.push(Fault { + unevaluated: false, code: self.rule.code.clone(), path: self.model.relative_path.clone(), line, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/model_layers.rs index 9f9a42372d..8a9bf4039f 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/model_layers.rs @@ -83,6 +83,7 @@ pub(crate) fn internal_forward_refs( } for exception in configured.difference(&used) { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: "sqlbuild_project.toml".into(), line: 1, @@ -193,6 +194,7 @@ fn model_fault( remediation: Option, ) -> Fault { Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs index cccdbd2368..a68c859612 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/sql_test_rules.rs @@ -4,14 +4,15 @@ use crate::models::{ ScopeResource, SqlScenarioFact, SqlTestCteFact, SqlTestFact, SqlTestMode, }; use crate::rules::_helpers::evaluation::{ - dependency_name, group_by_empty, normalize_rules_sql, parse_rule_statements, unwrap_nested, + dependency_name, group_by_empty, normalize_rules_sql, parse_rule_statements, path_fault, + unwrap_nested, }; use crate::rules::models::ProjectEvaluationRequest; use sqlparser::ast::{ BinaryOperator, Expr, FunctionArg, FunctionArgExpr, LimitClause, Query, Select, SetExpr, Statement, TableFactor, Value, Visit, Visitor, }; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::ops::ControlFlow; use std::path::Path; @@ -31,6 +32,51 @@ pub(crate) fn evaluate_project( }); let mut scenarios: Vec<&SqlScenarioFact> = evaluation.request.sql_scenarios.iter().collect(); scenarios.sort_by(|left, right| left.source_path.cmp(&right.source_path)); + let mut fixture_eligibility: HashMap<&str, Result<(), String>> = HashMap::new(); + + for rule in evaluation.selected.values().filter(|rule| { + matches!( + rule.code.as_str(), + "SQBRTEST201" | "SQBRTEST202" | EMPTY_INPUT_RULE_CODE + ) + }) { + for test in &tests { + if !matches!(test.mode, SqlTestMode::Model) + || test.has_macro_mocks + || test.has_model_query_overrides + { + continue; + } + for cte in test + .authored_ctes + .iter() + .filter(|cte| fixture_facts(&cte.name, &cte.sql).mock) + .chain(test.expected_ctes.iter()) + .chain(test.assertion_ctes.iter()) + { + if fixture_facts(&cte.name, &cte.sql).empty_fixture_marker { + continue; + } + if let Err(reason) = fixture_eligibility.entry(&cte.sql).or_insert_with(|| { + fixture_query(&cte.sql, &evaluation.request.dialect).map(|_| ()) + }) { + let mut fault = path_fault( + rule, + &test.source_path, + format!( + "Rule {} could not be evaluated for test block {}: {reason}", + rule.code, test.block_index + ), + "Use supported fixture SQL or explicitly ignore the affected Rule." + .to_owned(), + ); + fault.unevaluated = true; + faults.push(fault); + break; + } + } + } + } if let Some(rule) = evaluation.selected.get("SQBRTEST101") { faults.extend(canonical_roots(rule, &tests, &scenarios)); @@ -434,17 +480,6 @@ fn name_fault( ) } -fn path_fault(rule: &RuleMetadata, path: &str, message: String, remediation: String) -> Fault { - Fault { - code: rule.code.clone(), - path: path.to_owned(), - line: 1, - column: 1, - message, - remediation, - } -} - fn is_beneath(path: &str, root: &str) -> bool { path.strip_prefix(root) .is_some_and(|suffix| suffix.starts_with('/')) @@ -681,7 +716,7 @@ fn empty_relation(cte: &SqlTestCteFact, dialect: &str) -> bool { if fixture_facts(&cte.name, &cte.sql).empty_fixture_marker { return true; } - let Some(query) = parse_fixture_query(&cte.sql, dialect) else { + let Ok(query) = fixture_query(&cte.sql, dialect) else { return false; }; let SetExpr::Select(select) = query.body.as_ref() else { @@ -692,14 +727,12 @@ fn empty_relation(cte: &SqlTestCteFact, dialect: &str) -> bool { || (filtered_to_zero_rows(select) && !contains_function(&query.order_by))) } -fn parse_fixture_query(sql: &str, dialect: &str) -> Option { - let mut statements = match parse_rule_statements(&normalize_rules_sql(dialect, sql), dialect) { - Ok(statements) => statements, - Err(_) => return None, - }; +fn fixture_query(sql: &str, dialect: &str) -> Result { + let mut statements = parse_rule_statements(&normalize_rules_sql(dialect, sql), dialect) + .map_err(|error| format!("could not parse SQL-test fixture: {error}"))?; match (statements.pop(), statements.is_empty()) { - (Some(Statement::Query(query)), true) => Some(*query), - _ => None, + (Some(Statement::Query(query)), true) => Ok(*query), + _ => Err("SQL-test fixture must contain exactly one query".to_owned()), } } @@ -750,7 +783,7 @@ fn numeric_literal(expression: &Expr) -> Option { } fn bare_row_existence(sql: &str, targets: &[String], dialect: &str) -> bool { - let Some(query) = parse_fixture_query(sql, dialect) else { + let Ok(query) = fixture_query(sql, dialect) else { return false; }; let SetExpr::Select(select) = query.body.as_ref() else { diff --git a/crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs b/crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs index 583740a43c..9dfa14ecb0 100644 --- a/crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs +++ b/crates/sqlbuild-rules-native/src/rules/_helpers/typed_contract_columns.rs @@ -14,6 +14,7 @@ pub(crate) fn evaluate(model: &Model, rule: &RuleMetadata, faults: &FaultCollect for column in &model.columns { if column.data_type.trim().is_empty() { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, @@ -26,6 +27,7 @@ pub(crate) fn evaluate(model: &Model, rule: &RuleMetadata, faults: &FaultCollect for family in &model.dynamic_columns { if family.data_type.trim().is_empty() { faults.push(Fault { + unevaluated: false, code: rule.code.clone(), path: model.relative_path.clone(), line: 1, diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs new file mode 100644 index 0000000000..5b88c77840 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/catalog.rs @@ -0,0 +1,532 @@ +//! Compile-owned schema catalog and native, batched binding requests. + +use crate::bindings::main::compiler_error::compiler_error; +use crate::bindings::types::CompilerDetach; +use std::collections::HashMap; + +use crate::semantic_validation::models::{CatalogInput, Columns, ProjectCatalog}; +use crate::semantic_validation::types::{BindingRequest, DiagnosticRow, Relations}; +use crate::semantic_validation::{ + _helpers::{diagnostics, identifiers}, + main as validation, +}; +use polyglot_sql::validation::{SchemaColumn, SchemaTable}; +use polyglot_sql::{ + Dialect, DialectType, SchemaValidationOptions, ValidationError, ValidationResult, + ValidationSchema, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, FromPyObject, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::pymethods; +use pyo3::types::{PyBytes, PyDict, PyDictMethods}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +impl<'py> FromPyObject<'py> for Columns { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + value + .downcast::()? + .iter() + .map(|(name, data_type)| Ok((name.extract()?, data_type.extract()?))) + .collect::>>() + .map(Self) + } +} +#[pymethods] +impl ProjectCatalog { + fn inferred_schema( + &self, + py: Python<'_>, + sql: &str, + columns: Columns, + inputs: Relations, + ) -> PyResult>> { + py.compiler_detach(|| { + use polyglot_sql::{Expression, ExpressionWalk}; + let parsed = Dialect::get(self.dialect) + .parse(sql) + .map_err(|error| error.to_string())?; + let mut identifiers: HashMap = HashMap::new(); + for expression in &parsed { + if let Some(select) = expression.dfs().find_map(|node| match node { + Expression::Select(select) => Some(select), + _ => None, + }) { + for projection in &select.expressions { + let identifier = match projection { + Expression::Alias(alias) => Some(&alias.alias), + Expression::Column(column) => Some(&column.name), + _ => None, + }; + if let Some(identifier) = identifier { + let name = if identifier.quoted { + format!("\"{}\"", identifier.name.replace('"', "\"\"")) + } else { + identifier.name.clone() + }; + identifiers.insert(identifier.name.clone(), name); + } + } + break; + } + } + let mut exact_inputs: HashMap = HashMap::new(); + for shape in inputs.values() { + for (name, _) in &shape.0 { + if name.len() > 1 && name.starts_with('"') && name.ends_with('"') { + exact_inputs + .insert(name[1..name.len() - 1].replace("\"\"", "\""), name.clone()); + } + } + } + let mut result: HashMap> = HashMap::new(); + for (name, column_type) in columns.0 { + let binding_name = identifiers + .get(&name) + .or_else(|| exact_inputs.get(&name)) + .cloned() + .unwrap_or(name); + result.insert(binding_name, column_type); + } + Ok(result) + }) + .map_err(compiler_error) + } + + #[new] + fn new(request: CatalogInput) -> PyResult { + let CatalogInput { + dialect, + quoted_ignore_case, + known_functions, + known_types, + relations, + } = request; + let dialect = dialect + .parse::() + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let mut catalog = Self { + dialect, + quoted_ignore_case, + options: SchemaValidationOptions { + known_functions, + known_types, + check_types: matches!( + dialect, + DialectType::DuckDB + | DialectType::PostgreSQL + | DialectType::Snowflake + | DialectType::BigQuery + ), + check_references: true, + semantic: true, + strict: Some(true), + complexity_guard: Some(polyglot_sql::ComplexityGuardOptions { + max_function_call_depth: Some(512), + ..Default::default() + }), + ..Default::default() + }, + tables: HashMap::new(), + overrides: Vec::new(), + analysis_tables: HashMap::new(), + }; + catalog.update_relations(relations); + Ok(catalog) + } + + fn update_relations(&mut self, relations: Relations) { + for (name, columns) in relations { + self.tables.insert(name.clone(), self.table(&name, columns)); + } + } + + fn with_relations(&self, relations: Relations) -> Self { + let mut catalog = Self { + dialect: self.dialect, + options: self.options.clone(), + quoted_ignore_case: self.quoted_ignore_case, + tables: self.tables.clone(), + overrides: Vec::new(), + analysis_tables: self.analysis_tables.clone(), + }; + catalog.update_relations(relations); + catalog + } + + fn update_analysis(&mut self, relations: HashMap) { + for (name, (types, nullability)) in relations { + let types: HashMap<_, _> = types.0.into_iter().collect(); + let mut table = self.table(&name, Columns::default()); + table.name = name.clone(); + for (column, nullable) in nullability.0 { + table.columns.push(SchemaColumn { + data_type: types + .get(&column) + .and_then(Clone::clone) + .unwrap_or_else(|| "UNKNOWN".to_owned()), + name: column, + nullable: match nullable.as_deref() { + Some("non_null") => Some(false), + Some("nullable") => Some(true), + _ => None, + }, + primary_key: false, + unique: false, + references: None, + }); + } + self.analysis_tables.insert(name, table); + } + } + + fn register_override(&mut self, relations: Relations) -> usize { + let tables: HashMap = relations + .into_iter() + .map(|(name, columns)| (name.clone(), self.table(&name, columns))) + .collect(); + self.overrides.push(tables); + self.overrides.len() - 1 + } + + fn validation_payload(&self, requests: Vec) -> PyResult { + let mut payloads = Vec::with_capacity(requests.len()); + for (sql, references, overrides) in requests { + let schema = self + .schema(&references, overrides) + .map_err(compiler_error)?; + payloads.push(serde_json::json!({"sql": sql, "dialect": self.dialect.to_string(), "schema": schema, + "options": self.options, "quoted_ignore_case": self.quoted_ignore_case})); + } + serde_json::to_string(&payloads).map_err(|error| PyValueError::new_err(error.to_string())) + } + + fn analyze_compact<'py>( + &self, + py: Python<'py>, + payload: &[u8], + ) -> PyResult> { + let text = std::str::from_utf8(payload) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + let result = py.compiler_detach(|| crate::query_analysis::main::analyze_project_catalog::analyze_project_compact_with_catalog(text, self)) + .map_err(compiler_error)?; + Ok(PyBytes::new(py, result.as_bytes())) + } + + fn binding_results( + &self, + py: Python<'_>, + requests: Vec, + ) -> PyResult>> { + py.compiler_detach(|| { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4.min(requests.len().max(1))) + .stack_size(16 * 1024 * 1024) + .build() + .map_err(|error| error.to_string())?; + pool.install(|| { + requests + .into_par_iter() + .map(|(sql, references, overrides)| { + let schema = self.schema(&references, overrides)?; + let result = self.validate(&sql, &schema)?; + let result = diagnostics::map_diagnostics(&sql, self.dialect, result)?; + Ok(result.errors.into_iter().map(diagnostic_row).collect()) + }) + .collect::, String>>() + }) + }) + .map_err(compiler_error) + } +} + +impl ProjectCatalog { + pub(crate) fn with_options( + dialect: DialectType, + options: SchemaValidationOptions, + quoted_ignore_case: bool, + ) -> Self { + Self { + dialect, + options, + quoted_ignore_case, + tables: HashMap::new(), + overrides: Vec::new(), + analysis_tables: HashMap::new(), + } + } + pub(crate) fn analysis_schema( + &self, + references: &[(String, String)], + ) -> Option { + let mut tables: Vec = Vec::new(); + for (name, alias) in references { + if let Some(table) = self + .analysis_tables + .get(name) + .filter(|table| !table.columns.is_empty()) + { + let mut table = table.clone(); + table.name = alias.clone(); + tables.push(table); + } + } + if tables.is_empty() { + None + } else { + Some(ValidationSchema { + tables, + strict: None, + }) + } + } + pub(crate) fn reference_schema( + &self, + references: &[(String, bool)], + override_id: Option, + ) -> Result { + let mut schema = self.schema(references, HashMap::new())?; + if let Some(id) = override_id { + let overrides = self + .overrides + .get(id) + .ok_or("invalid native binding override")?; + for ((name, _), table) in references.iter().zip(&mut schema.tables) { + if let Some(replacement) = overrides.get(name) { + *table = replacement.clone(); + } + } + } + Ok(schema) + } + /// Rename `reference_schema` tables to the relation stubs of canonical SQL. + pub(crate) fn alias_reference_schema( + &self, + schema: &mut ValidationSchema, + references: &[(String, bool)], + aliases: &HashMap, + ) { + for ((name, _), table) in references.iter().zip(&mut schema.tables) { + if let Some(alias) = aliases.get(name) { + table.name = self.schema_name(alias.clone()); + } + } + } + + fn table(&self, name: &str, columns: Columns) -> SchemaTable { + let columns: Vec<_> = columns + .0 + .into_iter() + .map(|(name, data_type)| SchemaColumn { + name: self.schema_name(name), + data_type: data_type.unwrap_or_else(|| "UNKNOWN".to_owned()), + nullable: None, + primary_key: false, + unique: false, + references: None, + }) + .collect(); + SchemaTable { + name: self.schema_name(name.to_owned()), + schema: None, + columns, + aliases: Vec::new(), + primary_key: Vec::new(), + unique_keys: Vec::new(), + foreign_keys: Vec::new(), + } + } + + fn schema_name(&self, name: String) -> String { + if self.quoted_ignore_case { + name.to_ascii_uppercase() + } else { + name + } + } + + pub(crate) fn schema( + &self, + references: &[(String, bool)], + mut overrides: Relations, + ) -> Result { + let mut tables = Vec::with_capacity(references.len()); + for (name, closed) in references { + let table = if let Some(columns) = overrides.remove(name) { + self.table(name, columns) + } else if !closed { + self.table(name, Columns::default()) + } else { + self.tables + .get(name) + .cloned() + .ok_or_else(|| format!("binding catalog has no relation {name}"))? + }; + tables.push(table); + } + Ok(ValidationSchema { + tables, + strict: Some(true), + }) + } + + pub(crate) fn validate( + &self, + sql: &str, + schema: &ValidationSchema, + ) -> Result { + self.statement_validation(sql, schema, None) + } + + pub(crate) fn expression_validation( + &self, + sql: &str, + schema: &ValidationSchema, + expression: polyglot_sql::Expression, + ) -> Result { + self.statement_validation(sql, schema, Some(vec![expression])) + } + + pub(crate) fn needs_identifier_encoding(&self, sql: &str, schema: &ValidationSchema) -> bool { + if self.quoted_ignore_case { + sql.contains('"') + } else { + schema.tables.iter().any(has_exact_columns) + } + } + + fn statement_validation( + &self, + sql: &str, + schema: &ValidationSchema, + parsed: Option>, + ) -> Result { + let exact_names = !self.quoted_ignore_case && schema.tables.iter().any(has_exact_columns); + let mut encoded_schema; + let schema = if exact_names { + encoded_schema = schema.clone(); + for table in &mut encoded_schema.tables { + table.name = identifiers::encoded_name(&table.name, self.dialect); + if let Some(name) = &mut table.schema { + *name = identifiers::encoded_name(name, self.dialect); + } + for name in &mut table.aliases { + *name = identifiers::encoded_name(name, self.dialect); + } + for column in &mut table.columns { + column.name = identifiers::encoded_name(&column.name, self.dialect); + } + } + &encoded_schema + } else { + schema + }; + let statements = match parsed.map_or_else( + || { + Dialect::get(self.dialect).parse_with_options( + sql, + &polyglot_sql::ParseOptions { + complexity_guard: self.options.complexity_guard, + }, + ) + }, + Ok, + ) { + Ok(statements) => statements, + Err(_) => { + return Ok(polyglot_sql::validate_with_schema( + sql, + self.dialect, + schema, + &self.options, + )); + } + }; + if statements.iter().any(|statement| !statement.is_statement()) { + return Ok(polyglot_sql::validate_with_schema( + sql, + self.dialect, + schema, + &self.options, + )); + } + let (statements, authored) = if exact_names { + identifiers::transform_statements(statements, Some(self.dialect))? + } else if self.quoted_ignore_case && sql.contains('"') { + identifiers::fold_statements(statements)? + } else { + (statements, Vec::new()) + }; + let mut clause_errors: Vec = Vec::new(); + if validation::may_have_extra_clause_checks(sql) { + let mapping = polyglot_sql::mapping_schema_from_validation_schema_with_dialect( + schema, + self.dialect, + ); + for statement in &statements { + clause_errors.extend(validation::missing_clause_columns( + &polyglot_sql::build_scope(statement), + &mapping, + )); + } + } + let mut result = polyglot_sql::validation::validate_parsed_with_schema( + statements, + self.dialect, + schema, + &self.options, + ); + if result.valid { + result.errors.extend(clause_errors); + } + for error in &mut result.errors { + if exact_names { + error.message = + identifiers::restore_names(&error.message, &authored, error.start, error.end)?; + continue; + } + for (normalized, spelling, span) in &authored { + if error.start.is_none_or(|start| { + span.is_none_or(|span| { + start <= span.end && error.end.unwrap_or(start) >= span.start + }) + }) { + error.message = error + .message + .replace(&format!("'{normalized}'"), &format!("'{spelling}'")) + .replace( + &format!("'{}'", normalized.to_ascii_lowercase()), + &format!("'{spelling}'"), + ); + } + } + } + result.valid = !result + .errors + .iter() + .any(|error| error.severity == polyglot_sql::ValidationSeverity::Error); + Ok(result) + } +} + +pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { + let severity = match error.severity { + polyglot_sql::ValidationSeverity::Error => "error", + _ => "warning", + }; + ( + error.code, + error.message, + error.line, + error.column, + error.start, + error.end, + severity.to_owned(), + ) +} + +fn has_exact_columns(table: &SchemaTable) -> bool { + table + .columns + .iter() + .any(|column| column.name.starts_with('"')) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs new file mode 100644 index 0000000000..2dae13a1f8 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/diagnostics.rs @@ -0,0 +1,410 @@ +//! SQLBuild codes and dialect-specific diagnostic evidence. + +use crate::semantic_validation::_helpers::catalog::diagnostic_row; +use crate::semantic_validation::types::DiagnosticRow; +use polyglot_sql::expressions::Select; +use polyglot_sql::{ + Dialect, DialectType, Expression, ExpressionWalk, ValidationError, ValidationResult, + ValidationSeverity, +}; +use regex::Regex; +use std::collections::HashSet; +use std::sync::LazyLock; + +const UNKNOWN_COLUMN: &str = "E201"; +const UNKNOWN_FUNCTION: &str = "E202"; +const DUCKDB_KEYWORD_FUNCTIONS: [&str; 4] = [ + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "LOCALTIME", + "LOCALTIMESTAMP", +]; +const ORDER_BY: &str = "ORDER BY"; +const ERROR: &str = "error"; +static COLUMN: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?:Unknown column|Ambiguous column reference|JOIN USING column) '([^']+)'") + .map_err(|error| error.to_string()) +}); +static ALIAS: LazyLock> = LazyLock::new(|| { + Regex::new(r"^(?:Unknown column '([^']+)'(?: in table '[^']+'| \(not found in any referenced table\))|Ambiguous unqualified column '([^']+)' found in [0-9]+ referenced tables)$").map_err(|error| error.to_string()) +}); +static FUNCTION: LazyLock> = LazyLock::new(|| { + Regex::new(r"^Unknown function '([^']+)' for dialect ").map_err(|error| error.to_string()) +}); +static VALUES: LazyLock> = LazyLock::new(|| { + Regex::new(r"(?i)^Unknown column '(column([1-9][0-9]*))'$").map_err(|error| error.to_string()) +}); +static CONTEXTS: LazyLock, String>> = LazyLock::new(|| { + [ + ("JOIN USING", r"\bUSING\s*\("), + ("QUALIFY", r"\bQUALIFY\b"), + ("HAVING", r"\bHAVING\b"), + ("GROUP BY", r"\bGROUP\s+BY\b"), + (ORDER_BY, r"\bORDER\s+BY\b"), + ("WINDOW PARTITION BY", r"\bPARTITION\s+BY\b"), + ("JOIN ON", r"\bON\b"), + ("WHERE", r"\bWHERE\b"), + ("SELECT", r"\bSELECT\b"), + ] + .into_iter() + .map(compile_context) + .collect() +}); + +fn compile_context( + (label, pattern): (&'static str, &'static str), +) -> Result<(&'static str, Regex), String> { + Ok(( + label, + Regex::new(&format!("(?i){pattern}")).map_err(|error| error.to_string())?, + )) +} + +pub(crate) fn column_pattern() -> Result<&'static Regex, String> { + COLUMN.as_ref().map_err(Clone::clone) +} + +struct DiagnosticContext<'a> { + sql: &'a str, + dialect: DialectType, + statements: Option>>, +} + +impl DiagnosticContext<'_> { + fn parsed(&mut self) -> &polyglot_sql::Result> { + self.statements + .get_or_insert_with(|| Dialect::get(self.dialect).parse(self.sql)) + } + + fn map(&mut self, mut error: ValidationError) -> Result, String> { + if error.code.starts_with('B') { + return Ok(Some(error)); + } + let Some(code) = sqlbuild_code(&error.code) else { + return Ok(None); + }; + if let Some(start) = error.start { + (error.line, error.column) = line_column(self.sql, start); + } + if self.dialect == DialectType::Snowflake + && (VALUES + .as_ref() + .map_err(Clone::clone)? + .is_match(&error.message) + || ALIAS + .as_ref() + .map_err(Clone::clone)? + .is_match(&error.message)) + { + for expression in self.parsed().iter().flatten() { + for node in expression.dfs() { + if let Expression::Select(select) = node + && select_proves(select, &error)? + { + return Ok(None); + } + } + } + } + if self.dialect == DialectType::DuckDB + && error.code == UNKNOWN_FUNCTION + && let Some(captures) = FUNCTION + .as_ref() + .map_err(Clone::clone)? + .captures(&error.message) + && DUCKDB_KEYWORD_FUNCTIONS + .iter() + .any(|keyword| keyword.eq_ignore_ascii_case(&captures[1])) + && keyword_calls_only(self.parsed(), &captures[1]) + { + return Ok(None); + } + if let Some(captures) = column_pattern()?.captures(&error.message) { + let name = captures[1].rsplit('.').next().unwrap_or(&captures[1]); + let occurrences = occurrences(self.sql, name); + if let Some(&first) = occurrences.first() { + if error.line.is_none() && occurrences.len() == 1 { + (error.line, error.column) = + line_column(self.sql, self.sql[..first].chars().count()); + } + if let Some(context) = context(&self.sql[..first])? { + error.message.push_str(&format!(" (context: {context})")); + } + } + } + if error.code.starts_with('W') { + error.severity = ValidationSeverity::Error; + } + error.code = code; + Ok(Some(error)) + } +} + +pub(crate) fn map_diagnostics( + sql: &str, + dialect: DialectType, + result: ValidationResult, +) -> Result { + let mut context = DiagnosticContext { + sql, + dialect, + statements: None, + }; + let mut errors: Vec = Vec::with_capacity(result.errors.len()); + for error in result.errors { + if let Some(error) = context.map(error)? { + errors.push(error); + } + } + let valid = !errors + .iter() + .any(|error| error.severity == ValidationSeverity::Error); + Ok(ValidationResult { valid, errors }) +} + +fn sqlbuild_code(code: &str) -> Option { + let direct = match code { + "E001" | "E002" | "E003" | "E004" | "E200" => Some("B000"), + "E201" => Some("B002"), + "E221" => Some("B003"), + "E222" => Some("B004"), + "E223" => Some("B005"), + "E202" => Some("B101"), + "E203" => Some("B102"), + _ => None, + }; + if let Some(code) = direct { + return Some(code.to_owned()); + } + let number = match code.get(1..)?.parse::() { + Ok(number) => number, + Err(_) => return None, + }; + if code.starts_with('E') && ((210..=219).contains(&number) || (230..=234).contains(&number)) { + return Some(format!("B{number}")); + } + if code.starts_with('W') { + return implicit_conversion_code(number).map(str::to_owned); + } + None +} + +/// Maps implicit-conversion findings to blocking codes; each has an explicit-CAST remedy. +fn implicit_conversion_code(number: usize) -> Option<&'static str> { + match number { + 210 => Some("B217"), + 211 => Some("B212"), + 212 => Some("B214"), + 213 => Some("B218"), + 214 => Some("B215"), + 215 => Some("B211"), + 216 => Some("B213"), + _ => None, + } +} + +fn occurrences(sql: &str, name: &str) -> Vec { + sql.match_indices(name) + .filter_map(|(offset, _)| { + let word = |ch: char| ch.is_ascii_alphanumeric() || ch == '_'; + (!sql[..offset].chars().next_back().is_some_and(word) + && !sql[offset + name.len()..].chars().next().is_some_and(word)) + .then_some(offset) + }) + .collect() +} + +fn context(prefix: &str) -> Result, String> { + let mut latest = None; + for (label, pattern) in CONTEXTS.as_ref().map_err(Clone::clone)? { + if let Some(found) = pattern.find_iter(prefix).last() + && latest.is_none_or(|(start, _)| found.start() > start) + { + latest = Some((found.start(), *label)); + } + } + let Some((start, label)) = latest else { + return Ok(None); + }; + if label == ORDER_BY + && let Some(over) = prefix.to_ascii_uppercase().rfind("OVER") + && (over > start + || prefix[over..].matches('(').count() > prefix[over..].matches(')').count()) + { + return Ok(Some("WINDOW ORDER BY")); + } + Ok(Some(label)) +} + +fn line_column(sql: &str, offset: usize) -> (Option, Option) { + let mut line = 1; + let mut column = 1; + for ch in sql.chars().take(offset) { + if ch == '\n' { + line += 1; + column = 1; + } else { + column += 1; + } + } + (Some(line), Some(column)) +} + +fn unqualified(expression: &Expression, name: &str) -> bool { + if let Expression::Column(column) = expression { + return column.table.is_none() + && !column.name.quoted + && column.name.name.eq_ignore_ascii_case(name); + } + if matches!(expression, Expression::Select(_)) { + return false; + } + expression + .children() + .iter() + .any(|child| unqualified(child, name)) +} + +fn keyword_calls_only(parsed: &polyglot_sql::Result>, name: &str) -> bool { + let mut calls = parsed + .iter() + .flatten() + .flat_map(|expression| expression.dfs()) + .filter_map(|node| match node { + Expression::Function(function) if function.name.eq_ignore_ascii_case(name) => { + Some(function.no_parens) + } + _ => None, + }) + .peekable(); + calls.peek().is_some() && calls.all(|no_parens| no_parens) +} + +fn select_proves(select: &Select, error: &ValidationError) -> Result { + if let Some(captures) = VALUES + .as_ref() + .map_err(Clone::clone)? + .captures(&error.message) + { + let ordinal = match captures[2].parse::() { + Ok(ordinal) => ordinal, + Err(_) => return Ok(false), + }; + if !select.joins.is_empty() { + return Ok(false); + } + let Some(from) = &select.from else { + return Ok(false); + }; + let [Expression::Values(values)] = from.expressions.as_slice() else { + return Ok(false); + }; + if values.alias.is_some() + || !values.column_aliases.is_empty() + || values.expressions.is_empty() + || values + .expressions + .iter() + .any(|row| row.expressions.len() < ordinal) + { + return Ok(false); + } + for projection in &select.expressions { + let value = match projection { + Expression::Alias(alias) => &alias.this, + other => other, + }; + if let Expression::Column(column) = value + && column.table.is_none() + && !column.name.quoted + && column.name.name.eq_ignore_ascii_case(&captures[1]) + && column.span.is_some_and(|span| { + Some(span.line) == error.line && Some(span.column) == error.column + }) + { + return Ok(true); + } + } + return Ok(false); + } + let Some(captures) = ALIAS + .as_ref() + .map_err(Clone::clone)? + .captures(&error.message) + else { + return Ok(false); + }; + let Some(name) = captures + .get(1) + .or_else(|| captures.get(2)) + .map(|name| name.as_str().to_ascii_lowercase()) + else { + return Ok(false); + }; + let mut aliases: HashSet = HashSet::new(); + for projection in &select.expressions { + let (expression, alias) = match projection { + Expression::Alias(alias) => ( + &alias.this, + (!alias.alias.quoted).then_some(alias.alias.name.to_ascii_lowercase()), + ), + other => (other, None), + }; + if error.code == UNKNOWN_COLUMN && aliases.contains(&name) && unqualified(expression, &name) + { + return Ok(true); + } + if let Some(alias) = alias { + aliases.insert(alias); + } + } + if !aliases.contains(&name) { + return Ok(false); + } + if error.code == UNKNOWN_COLUMN { + return Ok(select + .where_clause + .as_ref() + .is_some_and(|clause| unqualified(&clause.this, &name))); + } + Ok(select.order_by.as_ref().is_some_and(|order| { + order + .expressions + .iter() + .any(|ordered| unqualified(&ordered.this, &name)) + })) +} + +pub(crate) fn binding_diagnostics( + sql: &str, + dialect: &str, + rows: Vec, +) -> Result, String> { + let errors: Vec = rows + .into_iter() + .map( + |(code, message, line, column, start, end, severity)| ValidationError { + code, + message, + line, + column, + start, + end, + severity: if severity == ERROR { + ValidationSeverity::Error + } else { + ValidationSeverity::Warning + }, + }, + ) + .collect(); + let result = map_diagnostics( + sql, + dialect.parse().map_err(|error| format!("{error}"))?, + ValidationResult { + valid: false, + errors, + }, + )?; + Ok(result.errors.into_iter().map(diagnostic_row).collect()) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs new file mode 100644 index 0000000000..0d15bb9956 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/identifiers.rs @@ -0,0 +1,284 @@ +//! Fold only binding identifiers in the AST, retaining SQL and authored spans. + +use polyglot_sql::expressions::{Cte, Identifier}; +use polyglot_sql::optimizer::normalize_identifiers::{ + get_normalization_strategy, normalize_identifier, +}; +use polyglot_sql::tokens::Span; +use polyglot_sql::{DialectType, Expression, ExpressionWalk}; +use regex::{Captures, Regex}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::LazyLock; + +pub(super) type AuthoredNames = Vec<(String, String, Option)>; +const MIN_QUOTED_NAME_BYTES: usize = 2; +static ENCODED_BINDING_NAME: LazyLock> = + LazyLock::new(|| Regex::new(r"(?i)\b__SQB_ID_[0-9A-F]+\b").map_err(|error| error.to_string())); + +pub(super) fn restore_names( + message: &str, + authored: &AuthoredNames, + start: Option, + end: Option, +) -> Result { + let mut names: HashMap = HashMap::new(); + for (normalized, spelling, span) in authored { + if start.is_some_and(|start| { + span.is_some_and(|span| start <= span.end && end.unwrap_or(start) >= span.start) + }) { + names.insert(normalized.to_ascii_uppercase(), spelling); + } else { + names + .entry(normalized.to_ascii_uppercase()) + .or_insert(spelling); + } + } + Ok(ENCODED_BINDING_NAME + .as_ref() + .map_err(Clone::clone)? + .replace_all(message, |captures: &Captures<'_>| { + names + .get(&captures[0].to_ascii_uppercase()) + .copied() + .unwrap_or(&captures[0]) + .to_owned() + }) + .into_owned()) +} + +struct IdentifierBindings { + names: RefCell, + exact_dialect: Option, +} + +impl IdentifierBindings { + fn borrow_mut(&self) -> std::cell::RefMut<'_, AuthoredNames> { + self.names.borrow_mut() + } + fn into_inner(self) -> AuthoredNames { + self.names.into_inner() + } +} + +pub(super) fn encoded_name(name: &str, dialect: DialectType) -> String { + let identifier = + if name.len() >= MIN_QUOTED_NAME_BYTES && name.starts_with('"') && name.ends_with('"') { + Identifier::quoted(name[1..name.len() - 1].replace("\"\"", "\"")) + } else { + Identifier::new(name) + }; + encode_identifier(identifier, dialect) +} + +fn encode_identifier(identifier: Identifier, dialect: DialectType) -> String { + let identity = normalize_identifier(identifier, get_normalization_strategy(Some(dialect))).name; + let mut encoded = String::from("__SQB_ID_"); + for byte in identity.as_bytes() { + encoded.push_str(&format!("{byte:02X}")); + } + encoded +} + +fn fold(mut identifier: Identifier, authored: &IdentifierBindings) -> Identifier { + if !identifier.quoted && authored.exact_dialect.is_none() { + return identifier; + } + let original = identifier.name.clone(); + if let Some(dialect) = authored.exact_dialect { + identifier.name = encode_identifier(identifier.clone(), dialect); + identifier.quoted = false; + } else { + identifier.name.make_ascii_uppercase(); + } + authored + .borrow_mut() + .push((identifier.name.clone(), original, identifier.span)); + identifier +} + +fn fold_ctes(mut ctes: Vec, authored: &IdentifierBindings) -> Vec { + for cte in &mut ctes { + cte.alias = fold(cte.alias.clone(), authored); + for identifier in &mut cte.columns { + *identifier = fold(identifier.clone(), authored); + } + } + ctes +} + +pub(super) fn fold_statements( + statements: Vec, +) -> Result<(Vec, AuthoredNames), String> { + transform_statements(statements, None) +} + +pub(super) fn transform_statements( + statements: Vec, + exact_dialect: Option, +) -> Result<(Vec, AuthoredNames), String> { + let authored = IdentifierBindings { + names: RefCell::new(Vec::new()), + exact_dialect, + }; + let mut output = Vec::with_capacity(statements.len()); + for statement in statements { + output.push( + statement + .transform_owned(|mut expression| { + match &mut expression { + Expression::Identifier(identifier) => { + *identifier = fold(identifier.clone(), &authored); + } + Expression::Column(column) => { + column.name = fold(column.name.clone(), &authored); + if let Some(table) = &mut column.table { + *table = fold(table.clone(), &authored); + } + } + Expression::Alias(alias) => { + alias.alias = fold(alias.alias.clone(), &authored); + } + Expression::Table(table) => { + table.name = fold(table.name.clone(), &authored); + for identifier in + [&mut table.schema, &mut table.catalog, &mut table.alias] + .into_iter() + .flatten() + { + *identifier = fold(identifier.clone(), &authored); + } + for identifier in &mut table.column_aliases { + *identifier = fold(identifier.clone(), &authored); + } + } + Expression::Subquery(query) => { + if let Some(alias) = &mut query.alias { + *alias = fold(alias.clone(), &authored); + } + for identifier in &mut query.column_aliases { + *identifier = fold(identifier.clone(), &authored); + } + } + Expression::Values(values) => { + if let Some(alias) = &mut values.alias { + *alias = fold(alias.clone(), &authored); + } + for identifier in &mut values.column_aliases { + *identifier = fold(identifier.clone(), &authored); + } + } + Expression::Select(select) => { + if let Some(with) = &mut select.with { + with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); + } + for join in &mut select.joins { + for identifier in &mut join.using { + *identifier = fold(identifier.clone(), &authored); + } + } + if let Some(windows) = &mut select.windows { + for window in windows { + window.name = fold(window.name.clone(), &authored); + } + } + } + Expression::With(with) => { + with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); + } + Expression::Union(query) => { + if let Some(with) = &mut query.with { + with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); + } + } + Expression::Intersect(query) => { + if let Some(with) = &mut query.with { + with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); + } + } + Expression::Except(query) => { + if let Some(with) = &mut query.with { + with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); + } + } + Expression::Join(join) => { + for identifier in &mut join.using { + *identifier = fold(identifier.clone(), &authored); + } + } + Expression::Star(star) => { + if let Some(table) = &mut star.table { + *table = fold(table.clone(), &authored); + } + if let Some(columns) = &mut star.except { + for column in columns { + *column = fold(column.clone(), &authored); + } + } + if let Some(columns) = &mut star.rename { + for (old, new) in columns { + *old = fold(old.clone(), &authored); + *new = fold(new.clone(), &authored); + } + } + } + Expression::Pivot(pivot) => { + if let Some(alias) = &mut pivot.alias { + *alias = fold(alias.clone(), &authored); + } + for column in &mut pivot.alias_columns { + *column = fold(column.clone(), &authored); + } + if let Some(with) = &mut pivot.with { + with.ctes = fold_ctes(std::mem::take(&mut with.ctes), &authored); + } + } + Expression::Unpivot(pivot) => { + pivot.value_column = fold(pivot.value_column.clone(), &authored); + pivot.name_column = fold(pivot.name_column.clone(), &authored); + if let Some(alias) = &mut pivot.alias { + *alias = fold(alias.clone(), &authored); + } + for column in pivot + .alias_columns + .iter_mut() + .chain(&mut pivot.extra_value_columns) + { + *column = fold(column.clone(), &authored); + } + } + Expression::Unnest(unnest) => { + if let Some(alias) = &mut unnest.alias { + *alias = fold(alias.clone(), &authored); + } + if let Some(alias) = &mut unnest.offset_alias { + *alias = fold(alias.clone(), &authored); + } + } + Expression::Lateral(lateral) => { + if lateral.alias_quoted + && let Some(alias) = &mut lateral.alias + { + let original = alias.clone(); + alias.make_ascii_uppercase(); + authored.borrow_mut().push((alias.clone(), original, None)); + } + for alias in &mut lateral.column_aliases { + alias.make_ascii_uppercase(); + } + } + Expression::Cte(cte) => { + cte.alias = fold(cte.alias.clone(), &authored); + for identifier in &mut cte.columns { + *identifier = fold(identifier.clone(), &authored); + } + } + _ => {} + } + Ok(Some(expression)) + }) + .map_err(|error| error.to_string())?, + ); + } + Ok((output, authored.into_inner())) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs new file mode 100644 index 0000000000..a524e55b94 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod catalog; +pub(crate) mod diagnostics; +pub(crate) mod identifiers; +pub(crate) mod normalization; +pub(crate) mod positions; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/normalization.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/normalization.rs new file mode 100644 index 0000000000..1d60228f59 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/normalization.rs @@ -0,0 +1,290 @@ +//! Analysis normalization with a character-offset provenance map. + +use crate::sql_scan::main::matching_paren::matching_paren; +use crate::sql_scan::main::non_code_end::non_code_end; +use crate::sql_scan::models::QuotePolicy; +use regex::Regex; +use std::collections::HashMap; +use std::sync::LazyLock; + +const DBT_REF: &str = "dbt_ref"; +const UDF: &str = "udf"; +type Edit = (usize, usize, MappedSql); +static REFERENCES: LazyLock> = LazyLock::new(|| { + Regex::new(r#"__(ref|seed|source|dbt_ref)\((?:"[^"]+"\s*,\s*)?"([^"]+)"\)"#) + .map_err(|error| error.to_string()) +}); +static FUNCTIONS: LazyLock> = LazyLock::new(|| { + Regex::new(r#"__(udf|table_fn)\("([A-Za-z_][A-Za-z0-9_]*)"\)\s*"#) + .map_err(|error| error.to_string()) +}); +static PLACEHOLDERS: LazyLock> = + LazyLock::new(|| Regex::new(r"@@@(\w+)").map_err(|error| error.to_string())); +static VARIANT_BASE: LazyLock> = LazyLock::new(|| { + Regex::new(r#"(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:""|[^"])+")(?:\.(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:""|[^"])+"))*:(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:""|[^"])+")(?:(?::|\.)(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:""|[^"])+"))*$"#).map_err(|error| error.to_string()) +}); +static SIMPLE_KEY: LazyLock> = LazyLock::new(|| { + Regex::new(r#"^\s*(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:""|[^"])+"|[0-9]+|'(?:''|[^'])*')\s*$"#) + .map_err(|error| error.to_string()) +}); + +#[derive(Clone)] +pub(super) struct MappedSql { + pub(super) sql: String, + pub(super) offsets: Vec, +} + +impl MappedSql { + pub(super) fn new(sql: &str) -> Self { + Self { + sql: sql.to_owned(), + offsets: (0..=sql.chars().count()).collect(), + } + } + + fn slice(&self, start: usize, end: usize) -> Self { + let first = self.sql[..start].chars().count(); + let last = first + self.sql[start..end].chars().count(); + Self { + sql: self.sql[start..end].to_owned(), + offsets: self.offsets[first..=last].to_vec(), + } + } + + fn generated(text: String, anchor: usize) -> Self { + Self { + offsets: vec![anchor; text.chars().count() + 1], + sql: text, + } + } + + fn append(&mut self, other: Self) { + self.sql.push_str(&other.sql); + self.offsets.pop(); + self.offsets.extend(other.offsets); + } + + fn edits(self, edits: Vec<(usize, usize, Self)>) -> Self { + if edits.is_empty() { + return self; + } + let mut result = self.slice(0, 0); + let mut previous = 0; + for (start, end, replacement) in edits { + result.append(self.slice(previous, start)); + result.append(replacement); + previous = end; + } + result.append(self.slice(previous, self.sql.len())); + result + } +} + +pub(super) fn normalize( + sql: &str, + dialect: &str, + stubs: &HashMap, + placeholders: &HashMap, +) -> Result { + let mut result = MappedSql::new(sql); + let mut edits: Vec = Vec::new(); + for captures in REFERENCES + .as_ref() + .map_err(Clone::clone)? + .captures_iter(&result.sql) + { + let whole = captures.get(0).ok_or("missing reference match")?; + let name = captures.get(2).ok_or("missing reference name")?; + let replacement = if &captures[1] != DBT_REF && stubs.contains_key(name.as_str()) { + MappedSql::generated( + stubs[name.as_str()].clone(), + result.offsets[result.sql[..name.start()].chars().count()], + ) + } else { + result.slice(name.start(), name.end()) + }; + edits.push((whole.start(), whole.end(), replacement)); + } + result = result.edits(edits); + let mut edits: Vec = Vec::new(); + let mut consumed_until = 0; + for captures in FUNCTIONS + .as_ref() + .map_err(Clone::clone)? + .captures_iter(&result.sql) + { + let whole = captures.get(0).ok_or("missing function match")?; + if whole.start() < consumed_until { + continue; + } + if result.sql.as_bytes().get(whole.end()) != Some(&b'(') { + continue; + } + let name = captures.get(2).ok_or("missing function name")?; + let mut end = whole.end(); + let stub = if &captures[1] == UDF { + format!("__sqlbuild_udf_{}", name.as_str()) + } else { + end = matching_paren(result.sql.as_bytes(), whole.end(), QuotePolicy::COMPILER) + .map_err(|error| format!("invalid table-function analysis input: {error:?}"))? + + 1; + let default = format!("__sqlbuild_table_function_{}", name.as_str()); + stubs.get(&default).cloned().unwrap_or(default) + }; + edits.push(( + whole.start(), + end, + MappedSql::generated( + stub, + result.offsets[result.sql[..whole.start()].chars().count()], + ), + )); + consumed_until = end; + } + result = result.edits(edits); + if dialect.eq_ignore_ascii_case("snowflake") { + result = normalize_variant(result)?; + } + let mut edits: Vec = Vec::new(); + for captures in PLACEHOLDERS + .as_ref() + .map_err(Clone::clone)? + .captures_iter(&result.sql) + { + if let Some(value) = placeholders.get(&captures[1]) { + let whole = captures.get(0).ok_or("missing placeholder match")?; + edits.push(( + whole.start(), + whole.end(), + MappedSql::generated( + value.clone(), + result.offsets[result.sql[..whole.start()].chars().count()], + ), + )); + } + } + Ok(result.edits(edits)) +} + +pub(crate) fn normalize_analysis_sql( + sql: &str, + dialect: &str, + stubs: HashMap, + placeholders: HashMap, +) -> Result { + normalize(sql, dialect, &stubs, &placeholders).map(|mapped| mapped.sql) +} + +pub(crate) fn normalize_dialect_sql(sql: &str, dialect: &str) -> Result { + if !dialect.eq_ignore_ascii_case("snowflake") { + return Ok(sql.to_owned()); + } + normalize_variant(MappedSql::new(sql)).map(|mapped| mapped.sql) +} + +fn skip_non_code(sql: &str, index: usize) -> Result, String> { + if sql[index..].starts_with("$$") { + return Ok(Some( + sql[index + 2..] + .find("$$") + .map_or(sql.len(), |end| index + 4 + end), + )); + } + non_code_end( + sql.as_bytes(), + index, + QuotePolicy { + backtick_identifiers: true, + single_quote_backslash_escapes: true, + double_quote_backslash_escapes: false, + dollar_quotes: true, + }, + ) + .map_err(|error| format!("invalid SQL analysis input: {error:?}")) +} + +fn matching_bracket(sql: &str, start: usize) -> Result, String> { + let mut depth = 1; + let mut index = start + 1; + while index < sql.len() { + if let Some(end) = skip_non_code(sql, index)? { + index = end; + continue; + } + match sql.as_bytes()[index] { + b'[' => depth += 1, + b']' => { + depth -= 1; + if depth == 0 { + return Ok(Some(index)); + } + } + _ => {} + } + index += sql[index..].chars().next().map_or(1, char::len_utf8); + } + Ok(None) +} + +fn normalize_variant(input: MappedSql) -> Result { + let sql = &input.sql; + let mut edits: Vec = Vec::new(); + let mut index = 0; + while index < sql.len() { + if let Some(end) = skip_non_code(sql, index)? { + index = end; + continue; + } + if sql.as_bytes()[index] != b'[' { + index += sql[index..].chars().next().map_or(1, char::len_utf8); + continue; + } + let Some(close) = matching_bracket(sql, index)? else { + index += 1; + continue; + }; + if SIMPLE_KEY + .as_ref() + .map_err(Clone::clone)? + .is_match(&sql[index + 1..close]) + { + index = close + 1; + continue; + } + let key = normalize_variant(input.slice(index + 1, close))?; + let mut base_end = index; + loop { + base_end = sql[..base_end].trim_end().len(); + if !sql[..base_end].ends_with("*/") { + break; + } + let Some(open) = sql[..base_end - 2].rfind("/*") else { + break; + }; + base_end = open; + } + if let Some(base) = VARIANT_BASE + .as_ref() + .map_err(Clone::clone)? + .find(&sql[..base_end]) + { + let anchor = input.offsets[sql[..base.start()].chars().count()]; + let mut replacement = MappedSql::generated("GET(".to_owned(), anchor); + replacement.append(input.slice(base.start(), index)); + replacement.append(MappedSql::generated( + ", ".to_owned(), + input.offsets[sql[..index].chars().count()], + )); + replacement.append(key); + replacement.append(MappedSql::generated( + ")".to_owned(), + input.offsets[sql[..close].chars().count()], + )); + edits.push((base.start(), close + 1, replacement)); + } else if key.sql != sql[index + 1..close] { + edits.push((index + 1, close, key)); + } + index = close + 1; + } + Ok(input.edits(edits)) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/positions.rs b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/positions.rs new file mode 100644 index 0000000000..cc2dabb355 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/_helpers/positions.rs @@ -0,0 +1,242 @@ +//! Native authored positions, including macro-expansion passes. + +use crate::bindings::main::compiler_guard::compiler_guard; +use crate::semantic_validation::_helpers::diagnostics::column_pattern; +use crate::semantic_validation::_helpers::normalization::normalize; +use crate::semantic_validation::models::{BindingPositions, PositionInput}; +use pyo3::exceptions::PyValueError; +use pyo3::{PyResult, pymethods}; +use regex::Regex; +use std::collections::HashMap; +use std::sync::LazyLock; + +const POPULAR_TOKEN_THRESHOLD: usize = 200; +static TOKENS: LazyLock> = + LazyLock::new(|| Regex::new(r"\w+|[^\w\s]").map_err(|error| error.to_string())); +type Mapping = (usize, usize, usize, usize); + +#[pymethods] +impl BindingPositions { + #[new] + fn new(request: PositionInput) -> PyResult { + compiler_guard(|| { + let PositionInput { + authored, + query, + expanded, + cleaned, + passes, + } = request; + let mapped = normalize(&expanded, "snowflake", &HashMap::new(), &HashMap::new()); + let offsets = match mapped { + Ok(mapped) if mapped.sql == cleaned => mapped.offsets, + _ if expanded == cleaned => (0..=expanded.chars().count()).collect(), + _ => align_offsets(&expanded, &cleaned).map_err(PyValueError::new_err)?, + }; + Ok(Self { + query_start: authored + .find(&query) + .map(|offset| authored[..offset].chars().count()), + lines: line_starts(&authored), + cleaned_lines: line_starts(&cleaned), + authored, + offsets, + passes, + }) + }) + } + + fn position( + &self, + start: Option, + line: Option, + column: Option, + message: &str, + ) -> PyResult<(Option, Option)> { + compiler_guard(|| { + if let Some(captures) = column_pattern() + .map_err(PyValueError::new_err)? + .captures(message) + { + let identifier = captures[1].rsplit('.').next().unwrap_or(&captures[1]); + let mut occurrences = + self.authored + .match_indices(identifier) + .filter(|(offset, _)| { + let before = self.authored[..*offset].chars().next_back(); + let after = self.authored[*offset + identifier.len()..].chars().next(); + !before.is_some_and(word) && !after.is_some_and(word) + }); + if let Some((offset, _)) = occurrences.next() + && occurrences.next().is_none() + { + return Ok(position( + &self.lines, + self.authored[..offset].chars().count(), + )); + } + } + let Some(query_start) = self.query_start else { + return Ok((None, None)); + }; + let offset = start.or_else(|| { + line.zip(column).map(|(line, column)| { + self.cleaned_lines + .get(line.saturating_sub(1)) + .copied() + .unwrap_or(0) + + column.saturating_sub(1) + }) + }); + let Some(offset) = offset else { + return Ok(position(&self.lines, query_start)); + }; + let mut mapped = self + .offsets + .get(offset) + .copied() + .unwrap_or_else(|| *self.offsets.last().unwrap_or(&0)); + for pass in self.passes.iter().rev() { + let mut adjusted = mapped as isize; + for &(source_start, source_end, output_start, output_end) in pass { + if mapped < output_start { + break; + } + if mapped < output_end { + adjusted = source_start as isize; + break; + } + adjusted += + (source_end - source_start) as isize - (output_end - output_start) as isize; + } + mapped = adjusted.max(0) as usize; + } + Ok(position(&self.lines, query_start + mapped)) + }) + } +} + +fn word(character: char) -> bool { + character == '_' || character.is_alphanumeric() +} + +fn line_starts(sql: &str) -> Vec { + let mut starts = vec![0]; + starts.extend( + sql.chars() + .enumerate() + .filter_map(|(index, ch)| (ch == '\n').then_some(index + 1)), + ); + starts +} + +fn position(lines: &[usize], offset: usize) -> (Option, Option) { + let line = lines.partition_point(|start| *start <= offset).max(1); + (Some(line), Some(offset - lines[line - 1] + 1)) +} + +fn align_offsets(original: &str, normalized: &str) -> Result, String> { + let tokens = TOKENS.as_ref().map_err(Clone::clone)?; + let a: Vec<_> = tokens.find_iter(original).collect(); + let b: Vec<_> = tokens.find_iter(normalized).collect(); + let av: Vec<_> = a + .iter() + .map(|token| token.as_str().to_lowercase()) + .collect(); + let bv: Vec<_> = b + .iter() + .map(|token| token.as_str().to_lowercase()) + .collect(); + let mut index: HashMap<&str, Vec> = HashMap::new(); + for (position, token) in bv.iter().enumerate() { + index.entry(token).or_default().push(position); + } + if b.len() >= POPULAR_TOKEN_THRESHOLD { + index.retain(|_, positions| positions.len() <= b.len() / 100 + 1); + } + let mut pending = vec![(0, a.len(), 0, b.len())]; + let mut blocks: Vec<(usize, usize, usize)> = Vec::new(); + while let Some((alo, ahi, blo, bhi)) = pending.pop() { + let (mut ai, mut bi, mut size) = (alo, blo, 0); + let mut previous: HashMap = HashMap::new(); + for (i, token) in av.iter().enumerate().take(ahi).skip(alo) { + let mut current: HashMap = HashMap::new(); + for &j in index.get(token.as_str()).into_iter().flatten() { + if j < blo { + continue; + } + if j >= bhi { + break; + } + let length = j + .checked_sub(1) + .and_then(|j| previous.get(&j)) + .copied() + .unwrap_or(0) + + 1; + current.insert(j, length); + if length > size { + ai = i + 1 - length; + bi = j + 1 - length; + size = length; + } + } + previous = current; + } + while ai > alo && bi > blo && av[ai - 1] == bv[bi - 1] { + ai -= 1; + bi -= 1; + size += 1; + } + while ai + size < ahi && bi + size < bhi && av[ai + size] == bv[bi + size] { + size += 1; + } + if size > 0 { + blocks.push((ai, bi, size)); + if alo < ai && blo < bi { + pending.push((alo, ai, blo, bi)); + } + if ai + size < ahi && bi + size < bhi { + pending.push((ai + size, ahi, bi + size, bhi)); + } + } + } + blocks.sort(); + let ac = byte_char_map(original); + let bc = byte_char_map(normalized); + let mut mappings: Vec = Vec::new(); + for (ai, bi, size) in blocks { + for i in 0..size { + mappings.push(( + ac[a[ai + i].start()], + ac[a[ai + i].end()], + bc[b[bi + i].start()], + bc[b[bi + i].end()], + )); + } + } + let mut offsets = Vec::with_capacity(normalized.chars().count() + 1); + let original_len = original.chars().count(); + let mut index = 0; + for offset in 0..=normalized.chars().count() { + while index < mappings.len() && offset > mappings[index].3 { + index += 1; + } + offsets.push(match mappings.get(index) { + Some(&(start, end, target, _)) => { + start + offset.saturating_sub(target).min(end - start) + } + None => original_len, + }); + } + Ok(offsets) +} + +fn byte_char_map(sql: &str) -> Vec { + let mut map = vec![0; sql.len() + 1]; + for (index, (byte, ch)) in sql.char_indices().enumerate() { + map[byte..byte + ch.len_utf8()].fill(index); + map[byte + ch.len_utf8()] = index + 1; + } + map +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main.rs index fde57ff38d..cc9d6f58cf 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/main.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main.rs @@ -1,5 +1,12 @@ //! Stable SQLBuild boundary around Polyglot schema-aware validation. +pub(crate) mod diagnostics; +pub(crate) mod map_diagnostics; +pub(crate) mod normalize; +pub(crate) mod normalize_batch; +pub(crate) mod normalize_dialect; +use crate::semantic_validation::models::ProjectCatalog; + use polyglot_sql::{ Dialect, DialectType, Expression, ExpressionWalk, Resolver, SchemaValidationOptions, ValidationError, ValidationResult, ValidationSchema, build_scope, @@ -13,13 +20,15 @@ const MAX_VALIDATION_WORKERS: usize = 4; const VALIDATION_WORKER_STACK_BYTES: usize = 16 * 1024 * 1024; #[derive(Deserialize)] -struct ValidationRequest { - sql: String, +pub(super) struct ValidationRequest { + pub(super) sql: String, #[serde(default = "default_dialect")] - dialect: String, - schema: ValidationSchema, + pub(super) dialect: String, + pub(super) schema: ValidationSchema, + #[serde(default)] + pub(super) options: SchemaValidationOptions, #[serde(default)] - options: SchemaValidationOptions, + pub(super) quoted_ignore_case: bool, } #[derive(Debug)] @@ -63,13 +72,17 @@ pub(crate) fn validations_json(request_json: &str) -> Result { serde_json::to_string(&results?).map_err(|error| error.to_string()) } -fn validation_result(request: ValidationRequest) -> Result { +pub(super) fn validation_result(request: ValidationRequest) -> Result { let dialect: DialectType = request.dialect.parse().map_err(|error| { format!( "unsupported Polyglot dialect '{}': {error}", request.dialect ) })?; + if request.quoted_ignore_case { + return ProjectCatalog::with_options(dialect, request.options, true) + .validate(&request.sql, &request.schema); + } let mut result = validate_with_schema( request.sql.as_str(), dialect, @@ -78,7 +91,12 @@ fn validation_result(request: ValidationRequest) -> Result Result bool { +pub(crate) fn may_have_extra_clause_checks(sql: &str) -> bool { sql.as_bytes() .windows(5) .any(|word| word.eq_ignore_ascii_case(b"using")) @@ -146,7 +164,7 @@ pub(crate) fn complete_parsed_validation( Ok(result) } -fn missing_clause_columns( +pub(super) fn missing_clause_columns( scope: &polyglot_sql::Scope, schema: &polyglot_sql::MappingSchema, ) -> Vec { @@ -237,8 +255,15 @@ impl ClauseResolver<'_, '_> { fn source_has_column(&mut self, source_name: &str, column_name: &str) -> bool { let Ok(columns) = self.resolver.get_source_columns(source_name) else { - return false; + return self + .scope + .sources + .keys() + .any(|name| name.eq_ignore_ascii_case(source_name)); }; + if columns.is_empty() { + return true; + } columns .iter() .any(|name| name.eq_ignore_ascii_case(column_name)) diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main/diagnostics.rs new file mode 100644 index 0000000000..792630ca23 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main/diagnostics.rs @@ -0,0 +1,9 @@ +use crate::semantic_validation::types::DiagnosticRow; + +pub(crate) fn binding_diagnostics( + sql: &str, + dialect: &str, + rows: Vec, +) -> Result, String> { + crate::semantic_validation::_helpers::diagnostics::binding_diagnostics(sql, dialect, rows) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs new file mode 100644 index 0000000000..75bfd83774 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main/map_diagnostics.rs @@ -0,0 +1,9 @@ +use polyglot_sql::{DialectType, ValidationResult}; + +pub(crate) fn map_diagnostics( + sql: &str, + dialect: DialectType, + result: ValidationResult, +) -> Result { + crate::semantic_validation::_helpers::diagnostics::map_diagnostics(sql, dialect, result) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize.rs new file mode 100644 index 0000000000..add0711ac1 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize.rs @@ -0,0 +1,10 @@ +use crate::semantic_validation::models::NormalizationInput; + +pub(crate) fn normalize_analysis_sql(request: NormalizationInput) -> Result { + crate::semantic_validation::_helpers::normalization::normalize_analysis_sql( + &request.sql, + &request.dialect, + request.stubs, + request.placeholders, + ) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_batch.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_batch.rs new file mode 100644 index 0000000000..5337ae4090 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_batch.rs @@ -0,0 +1,19 @@ +use std::collections::HashMap; + +pub(crate) fn normalize_analysis_sqls( + dialect: &str, + requests: Vec<(String, HashMap)>, +) -> Result, String> { + let mut results: Vec = Vec::with_capacity(requests.len()); + for (sql, placeholders) in requests { + results.push( + crate::semantic_validation::_helpers::normalization::normalize_analysis_sql( + &sql, + dialect, + HashMap::new(), + placeholders, + )?, + ); + } + Ok(results) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_dialect.rs b/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_dialect.rs new file mode 100644 index 0000000000..f492ec101e --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/main/normalize_dialect.rs @@ -0,0 +1,3 @@ +pub(crate) fn normalize_dialect_sql(sql: &str, dialect: &str) -> Result { + crate::semantic_validation::_helpers::normalization::normalize_dialect_sql(sql, dialect) +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs b/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs index c90d047f0b..faf043af1c 100644 --- a/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs +++ b/crates/sqlbuild-rules-native/src/semantic_validation/mod.rs @@ -1,4 +1,7 @@ +pub(crate) mod _helpers; pub(crate) mod main; +pub(crate) mod models; +pub(crate) mod types; #[cfg(test)] mod tests; diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/models.rs b/crates/sqlbuild-rules-native/src/semantic_validation/models.rs new file mode 100644 index 0000000000..07e407c815 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/models.rs @@ -0,0 +1,71 @@ +//! Compile-owned native catalog, mapping state and Python-boundary requests. + +use crate::semantic_validation::types::{Expansion, Relations}; +use polyglot_sql::validation::SchemaTable; +use polyglot_sql::{DialectType, SchemaValidationOptions}; +use pyo3::{FromPyObject, pyclass}; +use std::collections::HashMap; + +#[derive(Default, Debug)] +pub(crate) struct Columns(pub(crate) Vec<(String, Option)>); + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct CatalogInput { + pub(crate) dialect: String, + pub(crate) quoted_ignore_case: bool, + pub(crate) known_functions: Vec, + pub(crate) known_types: Vec, + pub(crate) relations: Relations, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct NormalizationInput { + pub(crate) sql: String, + pub(crate) dialect: String, + pub(crate) stubs: HashMap, + pub(crate) placeholders: HashMap, +} + +#[derive(FromPyObject, Debug)] +#[pyo3(from_item_all)] +pub(crate) struct PositionInput { + pub(crate) authored: String, + pub(crate) query: String, + pub(crate) expanded: String, + pub(crate) cleaned: String, + pub(crate) passes: Vec>, +} + +#[pyclass(module = "sqlbuild._native")] +pub(crate) struct ProjectCatalog { + pub(crate) dialect: DialectType, + pub(crate) options: SchemaValidationOptions, + pub(crate) quoted_ignore_case: bool, + pub(super) tables: HashMap, + pub(super) overrides: Vec>, + pub(super) analysis_tables: HashMap, +} + +#[pyclass(module = "sqlbuild._native")] +#[derive(Debug)] +pub(crate) struct BindingPositions { + pub(super) authored: String, + pub(super) query_start: Option, + pub(super) lines: Vec, + pub(super) cleaned_lines: Vec, + pub(super) offsets: Vec, + pub(super) passes: Vec>, +} + +impl std::fmt::Debug for ProjectCatalog { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ProjectCatalog") + .field("dialect", &self.dialect) + .field("quoted_ignore_case", &self.quoted_ignore_case) + .field("relation_count", &self.tables.len()) + .finish_non_exhaustive() + } +} diff --git a/crates/sqlbuild-rules-native/src/semantic_validation/types.rs b/crates/sqlbuild-rules-native/src/semantic_validation/types.rs new file mode 100644 index 0000000000..d3c9798d08 --- /dev/null +++ b/crates/sqlbuild-rules-native/src/semantic_validation/types.rs @@ -0,0 +1,15 @@ +use crate::semantic_validation::models::Columns; +use std::collections::HashMap; + +pub(crate) type Relations = HashMap; +pub(crate) type BindingRequest = (String, Vec<(String, bool)>, Relations); +pub(crate) type DiagnosticRow = ( + String, + String, + Option, + Option, + Option, + Option, + String, +); +pub(crate) type Expansion = (usize, usize, usize, usize); diff --git a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs index 904148eebb..0824ad481c 100644 --- a/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules-native/src/sql_lint/_helpers/engine.rs @@ -375,6 +375,14 @@ pub(crate) fn lint(request: LintRequest) -> Result { }, ); let statements = parser.parse().map_err(|error| error.to_string())?; + for statement in &statements { + if statement + .dfs() + .any(|node| matches!(node, Expression::Command(_) | Expression::Raw(_))) + { + return Err("unsupported syntax: native parser retained an opaque SQL node".to_owned()); + } + } let external_identifiers: HashSet = request .external_identifiers .iter() diff --git a/fensu.toml b/fensu.toml index a5409ee41b..6c84a5a076 100644 --- a/fensu.toml +++ b/fensu.toml @@ -223,11 +223,6 @@ rule = "FFR601" path = "src/sqlbuild/executor/run/_helpers/materializations/microbatch.py" reason = "Microbatch execution is one ordered failure-sensitive lifecycle; splitting reconciliation, DML, completion publication, and aggregate gates would hide the required phase ordering across module boundaries." -[[targets.python.rule_exceptions]] -rule = "FFR601" -path = "src/sqlbuild/virtual/executor/_helpers/build.py" -reason = "Virtual build orchestration intentionally keeps resolution, immutable physical-version execution, lease fencing, and state publication in one auditable lifecycle module." - [targets.rust] analyzer = "rust" roots = ["crates"] @@ -251,6 +246,11 @@ rule = "FPRSR307" path = "crates/sqlbuild-rules-native/src/semantic_validation/main.rs" reason = "This module owns the existing coarse native schema-validation boundary exposed through PyO3." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-rules-native/src/semantic_validation/models.rs" +reason = "ProjectCatalog implements Debug manually because Polyglot SchemaValidationOptions does not implement Debug; other models derive it." + [targets.rust.rule_options.FPRSL301] forbidden_packages = [ "sqlbuild-structure-checker", diff --git a/pyproject.toml b/pyproject.toml index 6e6c13445b..4842b204f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "duckdb>=1.3.2", "filelock>=3.20.3", "orjson>=3.10.0", - "polyglot-sql-chio==0.12.3", + "polyglot-sql-chio==0.13.8", "pydantic-settings>=2,<3", "PyYAML>=6.0.2", "rich>=15.0.0", diff --git a/scripts/cli_preview/_helpers/workflow.py b/scripts/cli_preview/_helpers/workflow.py index 20afa082ee..b4a678c918 100644 --- a/scripts/cli_preview/_helpers/workflow.py +++ b/scripts/cli_preview/_helpers/workflow.py @@ -16,15 +16,11 @@ from sqlbuild.presentation.main.supports_color import supports_color _PAYMENTS_MODEL_PATH: Path = Path("models/staging/stg_payments.sql") -_VIRTUAL_MODEL_PATH: Path = Path("models/fact_waffle_orders.sql") _QUERY_MARKER: str = " amount_cents,\n" _CHANGED_QUERY_MARKER: str = ( " CAST(CASE WHEN payment_method = 'card' THEN 'preview'" " ELSE CAST(amount_cents AS VARCHAR) END AS INTEGER) AS amount_cents,\n" ) -_VIRTUAL_QUERY_FILTER: str = "WHERE o.order_id IS NOT NULL" -_JANITOR_SECTION_MARKER: str = "[janitor]" -_JANITOR_CONFIG: str = "\n[janitor]\nenabled = true\nretention_days = 0\n" type _SignalHandler = Callable[[int, FrameType | None], object] | int | None @@ -92,100 +88,6 @@ def preview_scenes() -> tuple[PreviewScene, ...]: description="model lineage tree", command=("lineage", "fact_orders", "--direction", "both"), ), - *_virtual_preview_scenes(), - ) - - -def _virtual_preview_scenes() -> tuple[PreviewScene, ...]: - return ( - PreviewScene( - name="virtual-plan", - description="virtual first-run plan and state summary", - template="virtual", - setup_commands=(("state", "init"),), - command=("plan",), - ), - PreviewScene( - name="virtual-build", - description="local DuckDB-backed virtual build", - template="virtual", - setup_commands=(("state", "init"),), - command=("build",), - ), - PreviewScene( - name="virtual-plan-changed", - description="changed virtual plan with direct and downstream impact", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - mutate_virtual_model=True, - command=("plan", "--changes-only"), - ), - PreviewScene( - name="virtual-branch-build", - description="branch-like virtual environment build", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - mutate_virtual_model=True, - command=("build", "--virtual-env", "pr"), - ), - PreviewScene( - name="virtual-diff", - description="schema comparison between finalized virtual environments", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - mutate_virtual_model=True, - post_mutation_commands=(("build", "--virtual-env", "pr"),), - command=("diff", "dev:pr", "--schema-only"), - ), - PreviewScene( - name="virtual-promote", - description="promote a branch virtual environment into dev", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - mutate_virtual_model=True, - post_mutation_commands=(("build", "--virtual-env", "pr"),), - command=("promote", "--from", "pr", "--to", "dev"), - ), - PreviewScene( - name="virtual-rollback", - description="restore the previous finalized virtual checkpoint", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - mutate_virtual_model=True, - post_mutation_commands=(("build",),), - command=("rollback",), - ), - PreviewScene( - name="virtual-checkpoints", - description="list virtual environment build checkpoints", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - mutate_virtual_model=True, - post_mutation_commands=(("build",),), - command=("state", "checkpoints", "list", "--virtual-env", "dev"), - ), - PreviewScene( - name="virtual-reconcile", - description="inspect virtual state and warehouse consistency", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - command=("reconcile", "--virtual-env", "dev"), - ), - PreviewScene( - name="virtual-freshness", - description="inspect persisted virtual source freshness", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - command=("freshness", "--state", "--virtual-env", "dev"), - ), - PreviewScene( - name="virtual-janitor", - description="virtual state and warehouse cleanup summary", - template="virtual", - setup_commands=(("state", "init"), ("build",)), - enable_janitor=True, - command=("janitor", "--auto-approve"), - ), ) @@ -239,8 +141,6 @@ def _prepare_project( ) if scaffold_return_code != 0: raise PreviewSetupError(f"failed to create {scene.template} playground") - if scene.enable_janitor: - _enable_janitor(project_dir=project_dir) setup_command: tuple[str, ...] for setup_command in scene.setup_commands: return_code: int = _run_sqb( @@ -255,19 +155,6 @@ def _prepare_project( raise PreviewSetupError(f"preview setup command failed: sqb {' '.join(setup_command)}") if scene.mutate_payments: _mutate_payments_model(project_dir=project_dir) - if scene.mutate_virtual_model: - _mutate_virtual_model(project_dir=project_dir) - for setup_command in scene.post_mutation_commands: - return_code = _run_sqb( - sqb=sqb, - project_dir=project_dir, - command=setup_command, - environment=environment, - no_color=True, - quiet=True, - ) - if return_code != 0: - raise PreviewSetupError(f"preview setup command failed: sqb {' '.join(setup_command)}") def _run_sqb( @@ -322,22 +209,6 @@ def _mutate_payments_model(*, project_dir: Path) -> None: model_path.write_text(changed, encoding="utf-8") -def _mutate_virtual_model(*, project_dir: Path) -> None: - model_path: Path = project_dir / _VIRTUAL_MODEL_PATH - source: str = model_path.read_text(encoding="utf-8") - if _VIRTUAL_QUERY_FILTER in source: - raise PreviewSetupError(f"virtual playground model is already mutated: {model_path}") - model_path.write_text(f"{source.rstrip()}\n{_VIRTUAL_QUERY_FILTER}\n", encoding="utf-8") - - -def _enable_janitor(*, project_dir: Path) -> None: - project_path: Path = project_dir / "sqlbuild_project.toml" - source: str = project_path.read_text(encoding="utf-8") - if _JANITOR_SECTION_MARKER in source: - raise PreviewSetupError(f"virtual playground already configures janitor: {project_path}") - project_path.write_text(f"{source.rstrip()}\n{_JANITOR_CONFIG}", encoding="utf-8") - - def _resolve_sqb_executable() -> str: executable: str | None = shutil.which("sqb") if executable is None: diff --git a/scripts/cli_preview/constants.py b/scripts/cli_preview/constants.py index 35078d05af..745156fdb5 100644 --- a/scripts/cli_preview/constants.py +++ b/scripts/cli_preview/constants.py @@ -3,5 +3,3 @@ from __future__ import annotations ALL_SCENES: str = "all" -ALL_VIRTUAL_SCENES: str = "virtual-all" -VIRTUAL_TEMPLATE: str = "virtual" diff --git a/scripts/cli_preview/main/run.py b/scripts/cli_preview/main/run.py index 039f1f05d8..abbc31ade3 100644 --- a/scripts/cli_preview/main/run.py +++ b/scripts/cli_preview/main/run.py @@ -5,7 +5,7 @@ import argparse from scripts.cli_preview._helpers.workflow import execute_preview_scene, preview_scenes -from scripts.cli_preview.constants import ALL_SCENES, ALL_VIRTUAL_SCENES, VIRTUAL_TEMPLATE +from scripts.cli_preview.constants import ALL_SCENES from scripts.cli_preview.exceptions import PreviewSetupError from scripts.cli_preview.models import PreviewScene @@ -47,7 +47,7 @@ def _build_parser(*, scenes: tuple[PreviewScene, ...]) -> argparse.ArgumentParse parser.add_argument( "scene", nargs="?", - choices=(*names, ALL_VIRTUAL_SCENES, ALL_SCENES), + choices=(*names, ALL_SCENES), default=ALL_SCENES, ) parser.add_argument("--list", action="store_true", help="list available preview scenes") @@ -61,6 +61,4 @@ def _select_scenes( ) -> tuple[PreviewScene, ...]: if requested_name == ALL_SCENES: return scenes - if requested_name == ALL_VIRTUAL_SCENES: - return tuple(scene for scene in scenes if scene.template == VIRTUAL_TEMPLATE) return tuple(scene for scene in scenes if scene.name == requested_name) diff --git a/scripts/cli_preview/models.py b/scripts/cli_preview/models.py index 6e1dd375fc..c8142db61f 100644 --- a/scripts/cli_preview/models.py +++ b/scripts/cli_preview/models.py @@ -14,8 +14,5 @@ class PreviewScene: command: tuple[str, ...] template: str = "waffle_shop" setup_commands: tuple[tuple[str, ...], ...] = () - post_mutation_commands: tuple[tuple[str, ...], ...] = () mutate_payments: bool = False - mutate_virtual_model: bool = False - enable_janitor: bool = False expected_return_codes: tuple[int, ...] = (0,) diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py new file mode 100644 index 0000000000..d333d5c1e3 --- /dev/null +++ b/scripts/cold_compile_performance/_helpers/diagnostic_measurement.py @@ -0,0 +1,77 @@ +"""Fresh-process and location-only measurements for diagnostic workloads.""" + +import json +import subprocess +import sys +from pathlib import Path +from time import perf_counter +from typing import Any + +from scripts.cold_compile_performance.exceptions import CompileBenchmarkFixtureError +from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( + get_authored_binding_location, +) +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.spec.contracts.models import SourceLocation + + +def measure_diagnostic_compile( + *, project_dir: Path, timeout: float +) -> tuple[float, dict[str, Any]]: + """Keep invalid-project exit codes and diagnostics part of the performance contract.""" + started: float = perf_counter() + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-c", + "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())", + "--project-dir", + str(project_dir), + "compile", + "--no-cache", + "--json", + ], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + elapsed: float = perf_counter() - started + (project_dir / "measurement.json").write_text( + json.dumps({"elapsed": elapsed, "returncode": result.returncode}) + ) + (project_dir / "compile.stderr").write_text(result.stderr) + (project_dir / "compile.json").write_text(result.stdout) + if result.returncode != 1: + raise CompileBenchmarkFixtureError( + f"expected diagnostic failure, got {result.returncode}: {result.stderr}" + ) + return elapsed, json.loads(result.stdout) + + +def measure_position_mapping(*, count: int, lines: int = 4000) -> float: + """Measure many spans against one large normalized macro-expanded SQL body.""" + original: str = ( + "SELECT\n" + + ",\n".join(f" amount + {index} AS amount_{index}" for index in range(lines)) + + '\nFROM __ref("orders") WHERE amount > 5' + ) + normalized: str = original.replace('__ref("orders")', '"orders"').replace("\n", " ") + offset: int = normalized.index("amount > 5") + diagnostic: SqlBindingDiagnostic = SqlBindingDiagnostic( + code="B217", message="Incompatible comparison", start=offset, end=offset + len("amount > 5") + ) + started: float = perf_counter() + for _ in range(count): + location: SourceLocation | None = get_authored_binding_location( + path=Path("models/orders.sql"), + authored_sql=original, + authored_query_sql=original, + cleaned_sql=normalized, + diagnostic=diagnostic, + ) + if location is None or location.line != lines + 2: + raise CompileBenchmarkFixtureError( + "mapping must preserve the authored comparison location" + ) + return perf_counter() - started diff --git a/scripts/cold_compile_performance/_helpers/diagnostic_project.py b/scripts/cold_compile_performance/_helpers/diagnostic_project.py new file mode 100644 index 0000000000..3c83eed60f --- /dev/null +++ b/scripts/cold_compile_performance/_helpers/diagnostic_project.py @@ -0,0 +1,75 @@ +"""Synthetic workloads for error-only compile paths and deep poisoned lineage.""" + +from pathlib import Path + + +def write_diagnostic_project( + *, project_dir: Path, model_count: int = 3, width: int = 32, depth: int = 100 +) -> None: + """Generate large macro-backed CTE models with independent reference and conversion errors.""" + project_dir.mkdir(parents=True, exist_ok=True) + models: Path = project_dir / "models" + models.mkdir(exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "diagnostic_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + columns: list[str] = [f"amount_{index:03}" for index in range(width)] + declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns) + (models / "orders.sql").write_text( + f"MODEL (materialized view, contract enforced, columns ({declarations}));\nSELECT " + + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns) + ) + for model_index in range(model_count): + ctes: list[str] = ['input_orders AS (SELECT * FROM __ref("orders"))'] + previous: str = "input_orders" + for level in range(depth): + name: str = f"stage_{level:03}" + projection: str = ",\n".join(f" {column} + 1 AS {column}" for column in columns) + ctes.append(f"{name} AS (\nSELECT\n{projection}\nFROM {previous}\n)") + previous = name + errors: list[str] = [f"missing_{index:03} AS broken_{index:03}" for index in range(50)] + conversions: list[str] = [ + f"{columns[index % width]} = 'shipped' AS risky_{index:03}" for index in range(50) + ] + sql: str = "MODEL (materialized view);\nWITH\n" + ",\n".join(ctes) + sql += "\nSELECT\n" + ",\n".join((*errors, *conversions)) + f"\nFROM {previous}\n" + name = f"large_orders_{model_index}" + (models / f"{name}.sql").write_text(sql) + (models / f"downstream_orders_{model_index}.sql").write_text( + "MODEL (materialized view);\nSELECT " + + ", ".join(f"broken_{index:03} + 1 AS result_{index:03}" for index in range(50)) + + f' FROM __ref("{name}")' + ) + + +def write_cascade_project(*, project_dir: Path, depth: int, width: int = 50) -> None: + """Generate a reverse-named DAG with many independently poisoned output columns.""" + project_dir.mkdir(parents=True, exist_ok=True) + models: Path = project_dir / "models" + models.mkdir(exist_ok=True) + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "cascade_orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + columns: list[str] = [f"amount_{index:03}" for index in range(width)] + declarations: str = ", ".join(f"{column} (type INTEGER)" for column in columns) + header: str = f"MODEL (materialized view, contract enforced, columns ({declarations}));\n" + (models / "input_orders.sql").write_text( + f"MODEL (materialized view, contract enforced, columns ({declarations}, " + "ordered_at (type TIMESTAMP)));\nSELECT " + + ", ".join(f"CAST(1 AS INTEGER) AS {column}" for column in columns) + + ", CAST('2026-04-01' AS TIMESTAMP) AS ordered_at" + ) + root: str = ", ".join(f"{column} + ordered_at AS {column}" for column in columns) + previous: str = f"orders_{depth:03}" + (models / f"{previous}.sql").write_text( + header + "SELECT " + root + ' FROM __ref("input_orders")' + ) + for level in reversed(range(depth)): + name: str = f"orders_{level:03}" + (models / f"{name}.sql").write_text( + header + + "SELECT " + + ", ".join(f"{column} > TIMESTAMP '2026-04-01' AS {column}" for column in columns) + + f' FROM __ref("{previous}")' + ) + previous = name diff --git a/scripts/cold_compile_performance/constants.py b/scripts/cold_compile_performance/constants.py index 2e7ef3ed24..ab5452b22a 100644 --- a/scripts/cold_compile_performance/constants.py +++ b/scripts/cold_compile_performance/constants.py @@ -1,6 +1,7 @@ """Constants for fresh-process compile performance guards.""" UNLIMITED_CGROUP_MEMORY_VALUE: str = "max" +COMPILE_MEASUREMENT_FIELD_COUNT: int = 6 DENSE_JOIN_THRESHOLDS: tuple[int, ...] = (55, 75, 90, 99) DENSE_JOIN_COUNTS: tuple[int, ...] = (0, 1, 4, 8, 20) diff --git a/scripts/cold_compile_performance/main/read_compile_measurement.py b/scripts/cold_compile_performance/main/read_compile_measurement.py new file mode 100644 index 0000000000..111cba63df --- /dev/null +++ b/scripts/cold_compile_performance/main/read_compile_measurement.py @@ -0,0 +1,50 @@ +"""Report completed or interrupted benchmark measurements before guard assertions.""" + +import json +from pathlib import Path + +from scripts.cold_compile_performance.constants import COMPILE_MEASUREMENT_FIELD_COUNT + + +def read_compile_measurement( + *, label: str, measurement_path: Path, output_path: Path, elapsed_seconds: float +) -> tuple[object, list[str]]: + """Read one timed CLI result and report unavailable measurements explicitly.""" + measurement: list[str] = [] + if measurement_path.exists(): + lines: list[str] = measurement_path.read_text(encoding="utf-8").splitlines() + if lines: + measurement = lines[-1].split() + payload: object = None + if output_path.exists(): + try: + payload = json.loads(output_path.read_bytes()) + except json.JSONDecodeError: + payload = None + report: dict[str, object] = { + "label": label, + "wall_seconds": elapsed_seconds, + "cpu_seconds": None, + "cpu_utilization": None, + "peak_rss_bytes": None, + "major_page_faults": None, + "minor_page_faults": None, + "compile_timings": payload.get("compile_timings") if isinstance(payload, dict) else None, + } + if len(measurement) == COMPILE_MEASUREMENT_FIELD_COUNT: + wall, rss, user, system, major, minor = measurement + try: + report.update( + wall_seconds=float(wall), + cpu_seconds=float(user) + float(system), + cpu_utilization=(float(user) + float(system)) / float(wall) + if float(wall) + else None, + peak_rss_bytes=int(rss) * 1024, + major_page_faults=int(major), + minor_page_faults=int(minor), + ) + except ValueError: + measurement = [] + print("compile measurement " + json.dumps(report, sort_keys=True), flush=True) + return payload, measurement diff --git a/scripts/compare_compile_performance.py b/scripts/compare_compile_performance.py new file mode 100644 index 0000000000..ef851a0327 --- /dev/null +++ b/scripts/compare_compile_performance.py @@ -0,0 +1,15 @@ +"""Direct wrapper for same-runner compile performance comparison.""" + +from __future__ import annotations + +from scripts.compile_performance_ratio.main.compare import compare_compile_performance + + +def main(argv: list[str] | None = None) -> int: + """Compare base and head cold compiles on the same machine.""" + + return compare_compile_performance(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sqlbuild/cli/commands/_helpers/build_virtual/__init__.py b/scripts/compile_performance_ratio/__init__.py similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/build_virtual/__init__.py rename to scripts/compile_performance_ratio/__init__.py diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_retention/__init__.py b/scripts/compile_performance_ratio/_helpers/__init__.py similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/janitor_retention/__init__.py rename to scripts/compile_performance_ratio/_helpers/__init__.py diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py new file mode 100644 index 0000000000..5e92b48b82 --- /dev/null +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -0,0 +1,140 @@ +"""Generate a benchmark project once and time alternating base and head compiles.""" + +from __future__ import annotations + +import json +import os +import resource +import shutil +import statistics +import subprocess +import time +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.compile_performance_ratio.constants import ( + BASE_LABEL, + COMPILE_ENTRY, + DENSE_KIND, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + FRESH_AUDIT_SHARE, + FRESH_FUNCTION_SHARE, + FRESH_MACRO_SHARE, + FRESH_SEED_SHARE, + FRESH_SOURCE_SHARE, + FRESH_TEST_SHARE, + HEAD_LABEL, + REPORTED_PHASES, +) +from scripts.compile_performance_ratio.exceptions import CompileComparisonError +from scripts.compile_performance_ratio.models import CompileComparison, CompileRun +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + write_semantic_compile_project, +) + + +def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> None: + """Write the dense or fresh benchmark project used by both builds.""" + + if kind == DENSE_KIND: + write_dense_compile_project(project_dir=project_dir, model_count=models) + return + write_semantic_compile_project( + project_dir=project_dir, + model_count=models, + source_count=max(1, round(models * FRESH_SOURCE_SHARE)), + seed_count=max(1, round(models * FRESH_SEED_SHARE)), + function_count=max(1, round(models * FRESH_FUNCTION_SHARE)), + macro_count=max(1, round(models * FRESH_MACRO_SHARE)), + test_count=max(1, round(models * FRESH_TEST_SHARE)), + audit_count=max(1, round(models * FRESH_AUDIT_SHARE)), + ) + + +def compare_builds( + *, + kind: str, + models: int, + project_dir: Path, + base_python: Path, + head_python: Path, + runs: int, +) -> CompileComparison: + """Alternate base and head cold compiles after one untimed warm-up compile per build.""" + + builds: tuple[tuple[str, Path], ...] = ((BASE_LABEL, base_python), (HEAD_LABEL, head_python)) + for label, python in builds: + _compile_once(label=label, python=python, project_dir=project_dir) + results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []} + for _ in range(runs): + for label, python in builds: + results[label].append( + _compile_once(label=label, python=python, project_dir=project_dir) + ) + base: list[CompileRun] = results[BASE_LABEL] + head: list[CompileRun] = results[HEAD_LABEL] + return CompileComparison( + kind=kind, + models=models, + base_wall_seconds=statistics.median(run.wall_seconds for run in base), + head_wall_seconds=statistics.median(run.wall_seconds for run in head), + base_cpu_seconds=statistics.median(run.cpu_seconds for run in base), + head_cpu_seconds=statistics.median(run.cpu_seconds for run in head), + base_timings_ms=_median_phases(runs=base), + head_timings_ms=_median_phases(runs=head), + ) + + +def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: + shutil.rmtree(project_dir / "target", ignore_errors=True) + environment: dict[str, str] = { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } + before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) + started: float = time.perf_counter() + completed: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(python), + "-c", + COMPILE_ENTRY, + "--project-dir", + str(project_dir), + "--no-color", + "compile", + "--json", + "--no-cache", + ], + capture_output=True, + text=True, + env=environment, + check=False, + ) + wall_seconds: float = time.perf_counter() - started + after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) + if completed.returncode != 0: + raise CompileComparisonError( + f"{label} compile failed with exit {completed.returncode}: {completed.stderr[-2000:]}" + ) + payload: dict[str, object] = json.loads(completed.stdout) + timings: object = payload.get("compile_timings", {}) + return CompileRun( + label=label, + wall_seconds=wall_seconds, + cpu_seconds=(after.ru_utime - before.ru_utime) + (after.ru_stime - before.ru_stime), + timings_ms={ + name: int(value) + for name, value in (timings.items() if isinstance(timings, dict) else ()) + if name in REPORTED_PHASES and isinstance(value, (int, float)) + }, + ) + + +def _median_phases(*, runs: list[CompileRun]) -> dict[str, float]: + medians: dict[str, float] = {} + for phase in REPORTED_PHASES: + values: list[int] = [run.timings_ms.get(phase, 0) for run in runs] + medians[phase] = statistics.median(values) + return medians diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py new file mode 100644 index 0000000000..586f218f68 --- /dev/null +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -0,0 +1,50 @@ +"""Render same-runner comparison results for logs and job summaries.""" + +from __future__ import annotations + +import platform +from pathlib import Path + +from scripts.compile_performance_ratio.constants import REPORTED_PHASES +from scripts.compile_performance_ratio.models import CompileComparison + + +def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: float) -> str: + """Summarize medians, ratios and per-phase timings as a Markdown table.""" + + lines: list[str] = [ + f"### {comparison.kind} {comparison.models} models: head vs base (same runner)", + "", + f"Runner CPU: {_cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", + "", + "| Metric | Base | Head | Ratio |", + "|---|---:|---:|---:|", + f"| Wall (s) | {comparison.base_wall_seconds:.2f} | " + f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |", + f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | " + f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |", + ] + for phase in REPORTED_PHASES: + base: float = comparison.base_timings_ms.get(phase, 0) + head: float = comparison.head_timings_ms.get(phase, 0) + ratio: str = f"{head / base:.3f}" if base else "n/a" + lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |") + return "\n".join(lines) + "\n" + + +def append_summary(*, path: Path | None, markdown: str) -> None: + """Append to the CI job summary when one is configured.""" + + if path is None: + return + with path.open("a", encoding="utf-8") as summary: + summary.write(markdown + "\n") + + +def _cpu_model() -> str: + cpuinfo: Path = Path("/proc/cpuinfo") + if cpuinfo.exists(): + for line in cpuinfo.read_text(encoding="utf-8").splitlines(): + if line.startswith("model name"): + return line.split(":", 1)[1].strip() + return platform.processor() or "unknown" diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py new file mode 100644 index 0000000000..3bd0a44012 --- /dev/null +++ b/scripts/compile_performance_ratio/constants.py @@ -0,0 +1,28 @@ +"""Stable constants for same-runner compile performance comparisons.""" + +DEFAULT_RUNS: int = 3 +DEFAULT_MAX_RATIO: float = 1.10 +DENSE_KIND: str = "dense" +FRESH_KIND: str = "fresh" +PROJECT_KINDS: tuple[str, ...] = (DENSE_KIND, FRESH_KIND) +BASE_LABEL: str = "base" +HEAD_LABEL: str = "head" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"}) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +COMPILE_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +FRESH_SOURCE_SHARE: float = 713 / 3000 +FRESH_SEED_SHARE: float = 141 / 3000 +FRESH_FUNCTION_SHARE: float = 71 / 3000 +FRESH_MACRO_SHARE: float = 37 / 3000 +FRESH_TEST_SHARE: float = 2945 / 3000 +FRESH_AUDIT_SHARE: float = 5056 / 3000 +REPORTED_PHASES: tuple[str, ...] = ( + "total_ms", + "graph_ms", + "model_analysis_ms", + "analysis_native_ms", + "built_in_rules_ms", + "custom_rules_ms", + "test_planning_ms", + "write_ms", +) diff --git a/scripts/compile_performance_ratio/exceptions.py b/scripts/compile_performance_ratio/exceptions.py new file mode 100644 index 0000000000..9b5b24f93e --- /dev/null +++ b/scripts/compile_performance_ratio/exceptions.py @@ -0,0 +1,5 @@ +"""Same-runner compile performance comparison errors.""" + + +class CompileComparisonError(RuntimeError): + """Raised when a compared build fails to compile the benchmark project.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/__init__.py b/scripts/compile_performance_ratio/main/__init__.py similarity index 100% rename from tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/__init__.py rename to scripts/compile_performance_ratio/main/__init__.py diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py new file mode 100644 index 0000000000..c1542bbaa9 --- /dev/null +++ b/scripts/compile_performance_ratio/main/compare.py @@ -0,0 +1,69 @@ +"""Fail when head compiles slower than base by more than a ratio on the same runner.""" + +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +from pathlib import Path + +from scripts.compile_performance_ratio._helpers.measure import ( + compare_builds, + write_benchmark_project, +) +from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown +from scripts.compile_performance_ratio.constants import ( + DEFAULT_MAX_RATIO, + DEFAULT_RUNS, + PROJECT_KINDS, +) +from scripts.compile_performance_ratio.models import CompileComparison + + +def compare_compile_performance(argv: list[str] | None = None) -> int: + """Compare base and head cold compiles and enforce the wall and CPU ratio limit.""" + + args: argparse.Namespace = _parse_args(argv) + summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY") + with tempfile.TemporaryDirectory(prefix="sqlbuild-ratio-") as root: + project_dir: Path = Path(root) / f"{args.kind}_{args.models}" + print(f"Generating {args.kind} project with {args.models} models", file=sys.stderr) + write_benchmark_project(kind=args.kind, project_dir=project_dir, models=args.models) + print(f"Comparing base and head over {args.runs} alternating runs", file=sys.stderr) + comparison: CompileComparison = compare_builds( + kind=args.kind, + models=args.models, + project_dir=project_dir, + base_python=args.base_python, + head_python=args.head_python, + runs=args.runs, + ) + markdown: str = comparison_markdown( + comparison=comparison, runs=args.runs, max_ratio=args.max_ratio + ) + print(markdown) + _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) + exceeded: list[str] = [ + f"{name} ratio {ratio:.3f} exceeds {args.max_ratio:.2f}" + for name, ratio in (("wall", comparison.wall_ratio), ("CPU", comparison.cpu_ratio)) + if ratio > args.max_ratio + ] + if exceeded: + print("Compile performance regression: " + "; ".join(exceeded), file=sys.stderr) + return 1 + print("Compile performance within the same-runner limit", file=sys.stderr) + return 0 + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description="Compare base and head cold compiles on the same machine." + ) + parser.add_argument("--kind", choices=PROJECT_KINDS, required=True) + parser.add_argument("--models", type=int, required=True) + parser.add_argument("--base-python", type=Path, required=True) + parser.add_argument("--head-python", type=Path, required=True) + parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) + parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) + return parser.parse_args(argv) diff --git a/scripts/compile_performance_ratio/models.py b/scripts/compile_performance_ratio/models.py new file mode 100644 index 0000000000..b42166aaee --- /dev/null +++ b/scripts/compile_performance_ratio/models.py @@ -0,0 +1,33 @@ +"""Typed records for same-runner compile performance comparisons.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CompileRun: + label: str + wall_seconds: float + cpu_seconds: float + timings_ms: dict[str, int] + + +@dataclass(frozen=True) +class CompileComparison: + kind: str + models: int + base_wall_seconds: float + head_wall_seconds: float + base_cpu_seconds: float + head_cpu_seconds: float + base_timings_ms: dict[str, float] + head_timings_ms: dict[str, float] + + @property + def wall_ratio(self) -> float: + return self.head_wall_seconds / self.base_wall_seconds + + @property + def cpu_ratio(self) -> float: + return self.head_cpu_seconds / self.base_cpu_seconds diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index 264aefef52..b3fe3bae74 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -29,7 +29,6 @@ "scripts/fensu_policy/", "src/sqlbuild/adapter/", "src/sqlbuild/adapters/", - "src/sqlbuild/virtual/state/classes/", ) ALLOWED_DBT_REF_SCAN_PATHS: frozenset[str] = frozenset( { @@ -41,12 +40,12 @@ { "src/sqlbuild/adapter/relations/main/relation_lookup.py", "src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py", + "src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py", "src/sqlbuild/executor/janitor/_helpers/plan.py", "src/sqlbuild/executor/pipeline/_helpers/testing.py", "src/sqlbuild/executor/run/_helpers/materializations/microbatch.py", "src/sqlbuild/integrations/dbt/_helpers/lineage/columns.py", "src/sqlbuild/integrations/dbt/_helpers/planning/model_planning.py", - "src/sqlbuild/virtual/executor/_helpers/clone.py", } ) ALLOWED_SELECTOR_PARSE_PATH: str = ( @@ -100,12 +99,6 @@ "source_target_name": frozenset( {"src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py"} ), - "source_connection": frozenset( - { - "src/sqlbuild/virtual/executor/_helpers/build.py", - "src/sqlbuild/virtual/planner/main/plan.py", - } - ), } SELECTOR_STRING_METHOD_NAMES: frozenset[str] = frozenset( {"startswith", "endswith", "lstrip", "rstrip"} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index 2e102a1ab1..19c6124540 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -17,8 +17,15 @@ plan, and ships tools for proving a change is correct. Use those tools; do not s - Macros are plain Python functions called as `@macro_name("arg")`. Constants and enums are `@const("name")` and `@enum("name").MEMBER`. - Config, column schemas, contracts and audits live in the `MODEL()` header, not YAML. -- `sqb compile` is offline and strict: syntax, binding, types, contracts, column lineage and Rules - all run before anything touches the warehouse. Fix compile errors; do not bypass them. +- `sqb compile` is offline: syntax, per-input binding, grouping semantics, contracts, metadata, + column lineage and Rules run before warehouse execution. Explicit projections close output names + even over open inputs. DuckDB/MotherDuck, PostgreSQL, Snowflake, and BigQuery enable expression + type checks: proven bind-time rejections and implicit conversions are errors; add an explicit CAST. + Unknown types do not block compilation; catalogue coverage is partial. + Open table sources need an enforced contract or plan/build warehouse inspection for complete + column checking. `--json` reports partial-check reasons. Intentional escape hatches are + `MODEL (sql_analysis false)`, path defaults, and `--no-sql-analysis` (also disabling inference + and lineage). - Declarations are scoped by folder. A macro, enum or constant must live in the narrowest folder that contains every use, or compile rejects it. See `sqb scope` below. - Plans record why each model runs (first run, query change, config change, upstream change). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index 34057c0f06..e9704c74f2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -22,6 +22,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Project Configuration](concepts/project-configuration.md) (`concepts/project-configuration`) - Configure your SQLBuild project with sqlbuild_project.toml and sqlbuild_local.toml. - [Resource Identities](concepts/resource-identities.md) (`concepts/resource-identities`) - Canonical names for SQLBuild resources, selectors, state, and integrations. +- [Semantic Compilation](concepts/semantic-compilation.md) (`concepts/semantic-compilation`) - Offline reference, semantic and type checks that sqb compile runs before any warehouse work. - [Overview](concepts/adapters.md) (`concepts/adapters`) - Supported database engines and their connection configuration. - [Snowflake](concepts/adapters/snowflake.md) (`concepts/adapters/snowflake`) - Snowflake adapter configuration for SQLBuild. - [DuckDB](concepts/adapters/duckdb.md) (`concepts/adapters/duckdb`) - DuckDB adapter configuration for SQLBuild. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md index 1e55f722d0..401f7e9aef 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/scenario.md @@ -29,6 +29,7 @@ sqb scenario test [flags] | `--select`, `-s` | Select scenarios to run | | `--exclude` | Exclude scenarios from the selection | | `--retain` | Keep scenario-owned warehouse artifacts for inspection | +| `--scenario-namespace NAMESPACE` | Isolate warehouse artifacts for this run; overrides environment and config | | `--local` | Run locally against DuckDB using captured JSONL snapshots | | `--strict` | Treat missing/stale local snapshots as errors instead of skips | | `--sync-snapshots` | Capture missing/stale snapshots before local run (requires `--local`) | @@ -122,6 +123,7 @@ sqb scenario capture [flags] | `--select`, `-s` | Select scenarios to capture | | `--exclude` | Exclude scenarios from the selection | | `--retain` | Keep warehouse fixture artifacts after capture | +| `--scenario-namespace NAMESPACE` | Isolate warehouse artifacts for this capture; overrides environment and config | | `--force` | Bypass snapshot capture safety limits | | `--max-snapshot-rows` | Override per-relation row limit | | `--max-snapshot-total-rows` | Override total row limit | @@ -146,6 +148,11 @@ Snapshots are written to `tests/_scenario_snapshots//` with a `sc ## Runtime artifacts +Both subcommands report the effective namespace and its source. `scenario test --json` includes +`execution.scenario_namespace` (null when unset) and `execution.scenario_namespace_source` +(`cli`, `env`, `local config`, `project config`, or `unset`). Progress goes to stderr in JSON mode. +See [parallel CI runs](../concepts/scenarios.md#parallel-ci-runs) for namespace configuration and cleanup. + Both remote and local scenario runs write runtime artifacts to `target/run/scenarios//`: ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index 5c524be8ff..58a486a7a4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -54,7 +54,7 @@ configuration. | Field | Description | |-------|-------------| -| `run_despite_unchanged` | Table-only change-aware policy. `always` rebuilds whenever selected. A duration such as `30d`, `12h`, or `90m` rebuilds while the newest timestamp-based upstream source observation is within that age; it is not a periodic schedule. | +| `run_despite_unchanged` | Removed with virtual environments. It is rejected in MODEL headers, project defaults, and path defaults. | Table promotion mode is a project setting rather than a `MODEL()` field. Staged promotion is the default. Immediate promotion is incompatible with model type enforcement and exact contracts; see [Materializations](materializations.md#table). diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 9322d66e68..4699a4ad94 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -464,9 +464,12 @@ See [janitor](../cli/janitor.md) for the archive and delete lifecycle. ## Scenario -Configuration for scenario snapshot capture safety limits and local type overrides: +Configuration for scenario run isolation, snapshot capture safety limits, and local type overrides: ```toml +[scenario] +run_namespace = "orders-job-123" # Optional; omit to preserve existing scenario relation names + [scenario.snapshot_limits] max_rows_per_relation = 10000 max_total_rows = 50000 @@ -477,6 +480,7 @@ max_total_bytes = 52428800 | Field | Default | Description | |-------|---------|-------------| | `max_rows_per_relation` | none | Maximum rows per captured relation | +| `scenario.run_namespace` | unset | Optional 1–128-character namespace using ASCII letters, digits, `-`, `_`, or `.`. CLI `--scenario-namespace` overrides `SQLBUILD_SCENARIO_NAMESPACE`, then local config, then project config. | | `max_total_rows` | none | Maximum total rows across all relations in one scenario | | `max_bytes_per_relation` | none | Maximum JSONL bytes per relation file | | `max_total_bytes` | none | Maximum total JSONL bytes per scenario | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md index 2afbe938fc..9e1727b616 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/scenarios.md @@ -145,13 +145,54 @@ Scenario artifacts are physically isolated from production: ### Execution flow -1. Clean any existing scenario artifacts from a previous run +1. Clean any existing scenario artifacts from a previous run in the effective run namespace 2. Materialize source, ref, and seed fixtures as physical tables 3. Load required project seeds 4. Build required models in dependency order (incremental models run as full-refresh in scenarios) 5. Run expected-output comparisons (order-insensitive) 6. Run zero-row assertions -7. Clean up all scenario-owned artifacts (unless `--retain`) +7. Clean up this namespace's scenario-owned artifacts (unless `--retain`) + +### Parallel CI runs + +Give each concurrent CI job a unique scenario run namespace when jobs share a warehouse schema: + +```bash +export SQLBUILD_SCENARIO_NAMESPACE="ci-${CI_RUN_ID}-${CI_JOB_ID}" +sqb scenario test +# The same option applies to warehouse fixture capture: +sqb scenario capture --scenario-namespace "ci-${CI_RUN_ID}-${CI_JOB_ID}" +``` + +Use a unique job/run identifier, including an attempt or matrix identifier when necessary. +A pull-request identifier is suitable only when there is at most one active run for that identifier. +Concurrent runs with the **same namespace** are unsupported; namespaces do not add locking. +Keep separate working directories for concurrent jobs: namespaces isolate warehouse relation names, +not local snapshot files or `target/run/scenarios/` output paths. + +The precedence is `--scenario-namespace` > `SQLBUILD_SCENARIO_NAMESPACE` > +`sqlbuild_local.toml` > `sqlbuild_project.toml`. Both config files accept: + +```toml +[scenario] +run_namespace = "orders-job-123" +``` + +Values must contain 1–128 ASCII letters, digits, hyphens, underscores, or periods. +Empty or invalid values fail before warehouse work; values are case-sensitive and are not trimmed. +The effective namespace and its source appear in run output, including JSON output. + +With a namespace, the artifact prefix is the first 12 hexadecimal characters of +SHA-256 of `project_name:namespace:scenario_name`. Only the hash enters identifiers, so identifier +length limits are unchanged. Without a namespace, SQLBuild keeps the existing hash of +`project_name:scenario_name` and all existing physical names. Prefix collisions are checked among +the project's scenarios within the effective namespace. + +Preparation, normal teardown, and failure cleanup affect only the current namespace's relations. +`--retain` keeps that namespace's artifacts for inspection. The janitor still recognizes artifacts +from every namespace, including crashed CI jobs, and applies the existing age/retention policy +without needing their namespace values. Choose retention longer than active scenario jobs. +For SQLBuild versions without namespace support, use a separate warehouse schema per CI job. ### Inspecting with `--retain` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md new file mode 100644 index 0000000000..f53d9d1d1f --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/semantic-compilation.md @@ -0,0 +1,191 @@ + + +# Semantic Compilation + +> Offline reference, semantic and type checks that sqb compile runs before any warehouse work. + +Online: https://sqlbuild.com/docs/concepts/semantic-compilation/ + +## Contents + +- Authoritative column names +- Checks and limits +- Reading diagnostics +- Diagnostic codes + +`sqb compile` stays offline. It validates SQL references against each input separately, including +models with no relation inputs. A join to an open source does not disable checks on a closed input. +An unqualified name that could belong to an open input cannot be rejected offline. + +## Authoritative column names + +- Enforced model and source contracts, seeds, and declared table-function outputs are closed shapes. +- Expression sources have the column names inferred from their SQL expression. +- An explicit model projection defines its output names even over an open input. Types propagate + through dependencies where they can be inferred; an unknown type does not make the names unknown. +- A star over closed inputs expands; a star over an open input remains open. +- Declaring some columns on a table source does **not** declare its complete shape. Use + `contract: enforced` when those declarations are complete. + +Adopt a physical source schema with: + +```bash +sqb contract generate --from --select source:raw_orders --write +``` + +`sqb plan` and `sqb build` recheck source readers against inspected physical columns before model +execution. Existing bulk metadata is reused. Selected table sources omitted from that metadata +are inspected in database-grouped batches: one relation-list call and one bulk-column call per +database, using the adapter's normal metadata batching. Missing managed sources and ambiguous +unqualified physical names stay open. + +## Checks and limits + +Binding checks apply to SQL clauses, CTEs, subqueries, and expanded macros. Semantic grouping, +aggregate and window-placement diagnostics are errors. Project UDF arity and known argument types +are checked separately; declared return types contribute to inference. SQL and Python UDFs and +table functions are registered with the native catalogue, along with types named in project +contracts and function signatures. Registration does not replace SQLBuild's signature checks. + +Known output shapes also validate `unique_key`, `cursor`, `cursor_type`, `cursor_inputs`, custom +`config.partition_column`, `row_diff_exclude_columns`, `row_diff_tolerances.by_column`, source +`cursor_column`, audit expressions/relationships/accepted values, and SQL-test mock/expected names. + +Expression type checks are enabled for **DuckDB (including MotherDuck), PostgreSQL, Snowflake, +and BigQuery**. Other adapters retain reference and semantic checks without expression type checks. + +Proven bind-time rejections and implicit conversions are both errors. For example, DuckDB rejects +`ordered_at > 5` for a TIMESTAMP column. Snowflake accepts `order_id = customer_code` for an +INTEGER and a VARCHAR column, but the conversion fails on non-numeric values, so SQLBuild reports +it as a blocking B217 error; write the conversion explicitly. Typed string literals such as +`ordered_at > '2026-01-01'` remain valid. Columns from `SELECT *` are type-checked through their +upstream shapes in every query. Unknown input types never cause type errors. Dialect coercions +differ: numeric predicates are accepted by DuckDB but rejected by PostgreSQL, Snowflake, and +BigQuery. +Snowflake's function catalogue is partial and deliberately does not reject unknown function names. +Function signatures and overload coverage remain partial; compilation is not a replacement for +executing SQL unit tests. + +Derived-table column aliases, UNPIVOT, and Snowflake GROUPING SETS are checked. DuckDB and Snowflake +reject excess relation aliases; fewer aliases than output columns remain valid. Missing UNPIVOT +input columns are errors. + +Project function registration prevents unknown-function errors but does not declare return types to +the validator: some expressions consuming a project function result remain unchecked. Raw external +relations and scope-ordering cases also remain +partially checked. A cast to a type SQLBuild does not know is a B218 error. Types named in project +contracts and function signatures are registered, so declare an extension-defined type there. + +One notice summarizes partial semantic checks. `sqb compile --json` includes per-model reasons in +`semantic_checks_partial`; progress and notices go to stderr. The notice does not increment the +diagnostic warning count. + +Escape hatches are `MODEL (sql_analysis false)`, the corresponding path default, and +`--no-sql-analysis`. Disabling SQL analysis also disables inference and lineage for that model. + +## Reading diagnostics + +Diagnostics use one layout: headline, resource and location, source snippet with a span underline, +notes, then code-specific help. SQL-test diagnostics include their authored fixture snippet and +use `sql test: name`, just as model diagnostics use `model: name`. JSON retains the location's +exclusive `end_line`/`end_column` boundary, notes, and help. + +An unknown column identifies its input and authored alias. Suggestions use edit distance with a +bounded typo cutoff and recognize short abbreviations such as `qty`. No suggestion is printed when +none is close. The available-column note puts the closest names first and shows at most ten names. + +```text +error[B002]: Unknown column 'qty' in stg_orders (as o) + model: fact_orders + --> models/marts/fact_orders.sql:16:5 + | + 16 | o.qty, + | ^^^ + = note: stg_orders has: quantity, status, waffle_type_id, customer_id, order_id, ordered_at + = note: 2 downstream uses of fact_orders.quantity were not checked because of this error + = help: did you mean 'quantity'? +``` + +Type errors report uppercase types and available operand evidence, for example +`o.ordered_at is TIMESTAMP, 5 is INTEGER`. Comparison help recommends a literal appropriate to the +type family and dialect. Native spans are mapped back through normalization and macro expansion; +simple binary expressions are underlined in full. Diagnostics without usable spans keep the +authored query-start fallback. + +Error recovery treats poisoned output names and their downstream lineage as provisional. A +downstream error traceable only to that root is suppressed and counted in a root-error note; +independent errors on other columns and other inputs remain errors. Recovery does not silence a +whole dependent query or make its unchecked uses valid. Affected models remain in +`semantic_checks_partial`, including on warm/cache-hit runs. + +Located binding and type errors also make the affected projection's type unknown. That uncertainty +propagates through column lineage. Failed dependants are checked again with only those input types +unknown; an independent error on another column or input remains visible. Root notes count unchecked +downstream output uses, including transitive uses. Contract type checks do not emit a secondary +unknown-type warning for those same poisoned outputs. If native evidence cannot locate the failing +projection, recovery retains the diagnostic rather than opening the entire input. + +Temporal cursor metadata accepts DATE, DATETIME, and TIMESTAMP families interchangeably, including +Snowflake timestamp variants. A timestamp cursor on a string, numeric, or Boolean column still +produces B301. + +Snowflake binding honours `QUOTED_IDENTIFIERS_IGNORE_CASE = true` from the effective target's +connection `session_parameters`, for both offline declared schemas and warehouse-backed rebinding. +False or absent settings preserve the dialect's existing case rules. For example: + +```toml +[targets.analytics] +schema = "preserve" + +[targets.analytics.connection] +session_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true } +``` + +The setting participates in the analysis-cache identity. Identifier folding is token-based and +does not alter string literals, comments, or authored SQL. Diagnostic offset mapping is lazy and +uses a bounded cache of token alignments, shared by all diagnostics on the same normalized query. + +Snowflake set-operation checks are directional: each branch is compared with the accumulated +result type, following the query's operator precedence (`INTERSECT` binds more tightly than +`UNION` and `EXCEPT`). Proven incompatibilities and implicit conversions whose success depends on +runtime values both produce blocking B215 errors. For example, combining a VARCHAR branch with a +numeric branch requires a value-dependent numeric conversion; cast the branch column explicitly. + +An unmodelled Snowflake function has an unknown result type rather than inheriting its first +argument's type. SQLBuild retains the output name and lineage, reports partial type coverage, and +uses explicit declared function signatures where available. This prevents a guessed return type +from creating downstream type errors. + +The single human partial-check notice groups model names by reason, gives the relevant repair or +rerun command, and truncates long lists. `sqb compile --json` contains the full selected-model +reason map. Unknown output types and unresolved stars are reported as explicitly as open sources; +the notice never reduces them to a count alone. + +## Diagnostic codes + +| SQLBuild | Native | Meaning | +|---|---|---| +| B002 | E201 | Unknown column | +| B003 | E221 | Ambiguous column | +| B004–B005 | E222–E223 | Reference/scope errors | +| B101 | E202 | Unknown function | +| B102 | E203 or project declaration | Invalid function arity | +| B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | +| B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | +| B216 | E216 | Set-operation, subquery, or row arity | +| B230–B232 | E230–E232 | Grouping, aggregate, and window semantics | +| B233 | E233 | Duplicate scope names, according to dialect rules | +| B234 | E234 | Invalid LIMIT/OFFSET bounds | +| B300 | SQLBuild | Unknown metadata/audit column | +| B301 | SQLBuild | Incompatible declared metadata, audit, or UDF argument type | +| B302 | SQLBuild | Unknown SQL-test fixture/expected column | + +Implicit conversions are enforced rather than reported as warnings. Each native W210–W216 +finding maps to the SQLBuild code of the incompatibility it risks: comparisons to B217, arithmetic +to B212, assignments to B214, casts and coercible literals to B218, set-operation branches to B215, +predicates to B211, and function arguments to B213. The remedy is always an explicit conversion, +for example `order_id = CAST(customer_code AS INTEGER)`, or an explicit comparison such as +`quantity <> 0`. There is no per-model suppression. + +B002–B005 predate these changes and overlap the executor's separately phased build codes. New +semantic codes use distinct ranges. Authored model diagnostics retain source paths and positions. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md b/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md index 9d6fc76bf5..5d8a775d1b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/verifying-changes.md @@ -3,7 +3,7 @@ A model that compiles, builds and passes tests can still return the wrong numbers. `sqb diff` proves what changed. It has two modes: -- **Model diff** compares built models between two targets (or two virtual environments). +- **Model diff** compares built models between two targets. - **Query diff** compares any two SQL queries on one connection. No build is required. ## Contents @@ -30,7 +30,7 @@ proves what changed. It has two modes: sqb diff : (--full | --schema-only | --bounded ) --select [flags] ``` -- `FROM` and `TO` are configured target names (virtual environment names in virtual mode). The +- `FROM` and `TO` are configured target names. The `TO` target's connection runs the comparison and must be able to read both namespaces. - `--select` is required. Exactly one mode is required. - `--full` and `--bounded` join rows on the model's `unique_key`; models without one support only diff --git a/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py b/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py index c13d30bcd5..0eb81c97e8 100644 --- a/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py +++ b/src/sqlbuild/adapter/contract/_helpers/raw_statement_audit.py @@ -5,7 +5,7 @@ import ast from pathlib import Path -_AUDITED_ROOTS: tuple[str, ...] = ("adapter", "adapters", "virtual/state") +_AUDITED_ROOTS: tuple[str, ...] = ("adapter", "adapters") _RAW_RECEIVERS: frozenset[str] = frozenset({"client", "raw_connection", "raw_cursor"}) _STATEMENT_METHODS: frozenset[str] = frozenset({"execute", "executemany", "query"}) _APPROVED_BOUNDARIES: dict[str, frozenset[tuple[str, str]]] = { diff --git a/src/sqlbuild/adapter/contract/classes/base_adapter.py b/src/sqlbuild/adapter/contract/classes/base_adapter.py index d63e1249c6..9c253c3119 100644 --- a/src/sqlbuild/adapter/contract/classes/base_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/base_adapter.py @@ -62,6 +62,7 @@ PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.get_columns_for_relations import ( @@ -247,10 +248,10 @@ def physical_relation_generation( return None return relations[0].created_at.isoformat() - def recommended_max_sql_length(self) -> int | None: - """Return the recommended maximum SQL length for lightweight unit-test queries.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """Return a hard SQL text limit and unit (bytes mean UTF-8), or None for no static limit.""" - return 256_000 + return None def maximum_identifier_length(self) -> int: """Return the maximum unqualified identifier length supported by the adapter.""" diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 95623ea398..5638d35122 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -64,6 +64,7 @@ LoaderLogicalType, MigrationTransfer, PromotionStrategy, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.get_columns_for_relations import ( @@ -626,10 +627,10 @@ def render_apply_historical_check_snapshot_changes( invalidate_hard_deletes=invalidate_hard_deletes, ) - def recommended_max_sql_length(self) -> int | None: - """DuckDB uses the framework default recommendation for lightweight unit-test SQL.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """No practical SQL text cap: https://duckdb.org/docs/stable/operations_manual/limits.""" - return 256_000 + return None def default_schema(self) -> str: """DuckDB uses 'main' as its default schema.""" diff --git a/src/sqlbuild/adapter/contract/classes/strict_adapter.py b/src/sqlbuild/adapter/contract/classes/strict_adapter.py index a58341aa7f..1017aaa377 100644 --- a/src/sqlbuild/adapter/contract/classes/strict_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/strict_adapter.py @@ -26,6 +26,7 @@ FrameworkType, LoaderLogicalType, PromotionStrategy, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.compiler.compile.types import FunctionLanguage @@ -89,8 +90,8 @@ def supports_table_functions(self) -> bool: ... @abstractmethod - def recommended_max_sql_length(self) -> int | None: - """Return the recommended maximum SQL length for lightweight queries.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """Return the hard SQL text limit and unit, or None; bytes mean UTF-8.""" ... @abstractmethod diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index 81426da31b..72cc84fa63 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -62,6 +62,10 @@ class ExpressionInferenceProfile: sql_analysis_dialect: str | None = None function_nullability_rules: Mapping[str, FunctionNullabilityRule] = field(default_factory=dict) function_return_types: Mapping[str, str] = field(default_factory=dict) + quoted_identifiers_ignore_case: bool = False + semantic_known_functions: tuple[str, ...] = () + semantic_known_types: tuple[str, ...] = () + binding_catalog: Any | None = field(default=None, repr=False, compare=False) def function_nullability_rule(self, function_name: str) -> FunctionNullabilityRule | None: """Return the adapter rule for a function name, if one is registered.""" diff --git a/src/sqlbuild/adapter/contract/types.py b/src/sqlbuild/adapter/contract/types.py index 2025669fc2..6ce905ee68 100644 --- a/src/sqlbuild/adapter/contract/types.py +++ b/src/sqlbuild/adapter/contract/types.py @@ -4,11 +4,12 @@ from collections.abc import Callable from enum import StrEnum -from typing import Protocol +from typing import Literal, Protocol from sqlbuild.compiler.lineage.types import InferredNullability type FunctionNullabilityRule = Callable[[tuple[InferredNullability, ...]], InferredNullability] +type StatementSizeLimit = tuple[int, Literal["bytes", "characters"]] class AdapterExecute[ConnectionT, ResultT](Protocol): diff --git a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py index 7965c8c81c..5a6e057a9c 100644 --- a/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py +++ b/src/sqlbuild/adapters/bigquery/classes/bigquery_adapter.py @@ -80,6 +80,7 @@ RetentionScope, SnapshotLatestVersionStyle, SnapshotUpdateStyle, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.get_columns_for_relations import ( @@ -553,10 +554,10 @@ def python_functions_inherit_default_namespace(self) -> bool: def supports_unqualified_function_fingerprints(self) -> bool: return False - def recommended_max_sql_length(self) -> int | None: - """Return the recommended maximum SQL length for lightweight unit-test queries.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """Unresolved GoogleSQL has a 1 MB cap: https://cloud.google.com/bigquery/quotas#query_jobs.""" - return 256_000 + return (1_048_576, "bytes") def maximum_identifier_length(self) -> int: """Return the maximum unqualified identifier length supported by the adapter.""" diff --git a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py index 592bbccdfe..65af1de664 100644 --- a/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py +++ b/src/sqlbuild/adapters/databricks/classes/databricks_adapter.py @@ -79,6 +79,7 @@ RetentionScope, SnapshotLatestVersionStyle, SnapshotUpdateStyle, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.get_columns_for_relations import ( @@ -1089,8 +1090,10 @@ def supports_durable_clone(self) -> bool: def supports_relation_age_metadata(self) -> bool: return True - def recommended_max_sql_length(self) -> int | None: - return 256_000 + def max_statement_size(self) -> StatementSizeLimit | None: + """16 MiB query text cap: https://docs.databricks.com/api/statement-execution/v1/execute-statement.""" + + return (16 * 1_048_576, "bytes") def connect(self, config: dict[str, Any]) -> _DatabricksConnection: """Open a Databricks SQL connection from the resolved config.""" diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index ed09d3e30a..57b6d091b7 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -75,6 +75,7 @@ PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.get_columns_for_relations import ( @@ -172,10 +173,10 @@ def supports_unqualified_function_fingerprints(self) -> bool: def supports_table_functions(self) -> bool: return False - def recommended_max_sql_length(self) -> int | None: - """Return the recommended maximum SQL length for lightweight unit-test queries.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """No documented practical SQL text cap: https://www.postgresql.org/docs/current/limits.html.""" - return 256_000 + return None def maximum_identifier_length(self) -> int: """Return the maximum unqualified identifier length supported by the adapter.""" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index fff2fc38cb..7afd631c09 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -75,6 +75,7 @@ RetentionScope, SnapshotLatestVersionStyle, SnapshotUpdateStyle, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc @@ -739,10 +740,10 @@ def python_functions_inherit_default_namespace(self) -> bool: def supports_unqualified_function_fingerprints(self) -> bool: return False - def recommended_max_sql_length(self) -> int | None: - """Return the recommended maximum SQL length for lightweight unit-test queries.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """1 MB truncates metadata, not execution: https://docs.snowflake.com/en/user-guide/query-size-limits.""" - return 256_000 + return None def maximum_identifier_length(self) -> int: """Return the maximum unqualified identifier length supported by the adapter.""" diff --git a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py index f39656e348..527ef2eb58 100644 --- a/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py +++ b/src/sqlbuild/adapters/sqlserver/classes/sqlserver_adapter.py @@ -77,6 +77,7 @@ PromotionStrategy, SnapshotLatestVersionStyle, SnapshotUpdateStyle, + StatementSizeLimit, TablePromotionMode, ) from sqlbuild.adapter.relations.main.get_columns_for_relations import ( @@ -1575,10 +1576,10 @@ def persists_python_functions(self) -> bool: def python_functions_inherit_default_namespace(self) -> bool: return True - def recommended_max_sql_length(self) -> int | None: - """Return the recommended maximum SQL length for lightweight unit-test queries.""" + def max_statement_size(self) -> StatementSizeLimit | None: + """Defer 65,536 * negotiated packet bytes to SQL Server: https://learn.microsoft.com/en-us/sql/sql-server/maximum-capacity-specifications-for-sql-server.""" - return 256_000 + return None def relation_exists( self, diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py index 802b44b58a..200bc8e58c 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/outputs.py @@ -54,9 +54,7 @@ def write_build_plan_text( full_refresh=request.full_refresh, use_color=invocation.use_color, python_plan_entries=pipeline_result.python_plan_entries, - include_direct_freshness_diagnostics=( - invocation.virtual_mode or invocation.effective_changes_only - ), + include_direct_freshness_diagnostics=(False), ) plan_stream.write("\n" + plan_text + "\n\n") plan_stream.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py b/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py index 57befccd0a..1f75fa5a4e 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/phase_timings.py @@ -16,15 +16,12 @@ BuildCommandRequest, BuildInvocation, BuildPhaseTimings, - VirtualBuildCliRequest, - VirtualBuildExecution, ) from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker from sqlbuild.diagnostics.models import PartialBuildPhaseTimings from sqlbuild.presentation.classes.cli_document import CliDocument from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.virtual.executor.models import VirtualBuildPipelineResult def write_build_phase_timings( @@ -130,62 +127,6 @@ def finalize_exceptional_with_timings( _ = finish_cost_collection_timing(started_at=cost_started_at) -def write_virtual_build_phase_timings( - *, - stream: TextIO, - request: VirtualBuildCliRequest, - execution: VirtualBuildExecution, - result: VirtualBuildPipelineResult, - cost_collection_seconds: float, -) -> None: - """Write completed virtual-build timings from typed pipeline results.""" - - if not request.verbose and not request.debug: - return - total_seconds: float = ( - time.monotonic() - request.command_started_at - if request.command_started_at is not None - else execution.elapsed - ) - write_build_phase_timings( - stream=stream, - timings=BuildPhaseTimings( - compile_seconds=result.compile_seconds, - planning_seconds=result.planning_seconds, - connection_preparation_seconds=( - result.execution_result.timings.connection_preparation_seconds - ), - schema_preparation_seconds=result.execution_result.timings.schema_preparation_seconds, - execution_seconds=result.execution_result.timings.execution_seconds, - cost_collection_seconds=cost_collection_seconds, - total_seconds=total_seconds, - ), - use_color=request.use_color, - ) - - -def record_and_write_virtual_build_phase_timings( - *, - stream: TextIO, - request: VirtualBuildCliRequest, - execution: VirtualBuildExecution, - result: VirtualBuildPipelineResult, - cost_collection_seconds: float, -) -> None: - """Record and write completed virtual-build timing diagnostics.""" - - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - if timing_tracker is not None: - timing_tracker.cost_collection_seconds = cost_collection_seconds - write_virtual_build_phase_timings( - stream=stream, - request=request, - execution=execution, - result=result, - cost_collection_seconds=cost_collection_seconds, - ) - - def _timing_rows(timings: BuildPhaseTimings) -> tuple[tuple[str, str], ...]: values: tuple[tuple[str, float | None], ...] = ( ("compile", timings.compile_seconds), diff --git a/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py b/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py index ee095140e8..c2062b0a6f 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py +++ b/src/sqlbuild/cli/commands/_helpers/build_execution/run_context.py @@ -45,13 +45,7 @@ def _format_build_run_context(*, context: BuildRunContext, use_color: bool) -> s + len(context.discovered_inputs.asset_functions) ) placement_rows: tuple[tuple[str, str], ...] - if context.virtual_logical_schema is not None or context.virtual_physical_schema is not None: - placement_rows = ( - ("logical schema", _display_value(context.virtual_logical_schema)), - ("physical schema", _display_value(context.virtual_physical_schema)), - ) - else: - placement_rows = (("schema", _display_value(context.project.effective_target_schema)),) + placement_rows = (("schema", _display_value(context.project.effective_target_schema)),) document.header(text="Execution") document.fields( rows=( diff --git a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py index 47d3d73ef8..d53eb30a41 100644 --- a/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/build_planning/invocation.py @@ -9,10 +9,6 @@ from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) -from sqlbuild.cli.commands._helpers.runtime.mode_policy import ( - enforce_no_defer_to_in_virtual_mode, - enforce_virtual_only_flags_in_virtual_mode, -) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( AdapterConnectionContext, @@ -27,9 +23,6 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_changes_only import ( - resolve_effective_changes_only, -) from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, TargetConfig @@ -51,37 +44,6 @@ def resolve_build_invocation(*, request: BuildCommandRequest) -> BuildInvocation ) if request.defer_to is not None and effective_defer_clone_from is not None: raise CliUserError("--defer-clone-from cannot be used with --defer-to", code="C408") - if ( - discovered_inputs.project_config.settings.virtual_environments - and effective_defer_clone_from is not None - ): - raise CliUserError( - "build does not support --defer-clone-from when virtual_environments = true", - code="C412", - ) - if discovered_inputs.project_config.settings.virtual_environments and request.manifest: - raise CliUserError( - "build does not support --manifest when virtual_environments = true", - code="C264", - ) - effective_changes_only: bool = resolve_effective_changes_only( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=request.selected_target, - cli_changes_only=request.changes_only, - ) - enforce_no_defer_to_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name="build", - defer_to=request.defer_to, - ) - enforce_virtual_only_flags_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name="build", - virtual_env=request.virtual_env, - include_stale_upstreams=request.include_stale_upstreams, - changes_only=effective_changes_only, - ) adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( discovered_inputs=discovered_inputs, effective_project_dir=effective_project_dir, @@ -143,7 +105,6 @@ def resolve_build_invocation(*, request: BuildCommandRequest) -> BuildInvocation effective_project_dir=effective_project_dir, discovered_inputs=discovered_inputs, effective_defer_clone_from=effective_defer_clone_from, - effective_changes_only=effective_changes_only, adapter_name=adapter_context.adapter_name, adapter=adapter_context.adapter, connection_config=adapter_context.connection_config, @@ -152,7 +113,6 @@ def resolve_build_invocation(*, request: BuildCommandRequest) -> BuildInvocation connection_progress=reporters.connection, planning_progress=reporters.planning, should_load_sources=should_load_sources, - virtual_mode=bool(discovered_inputs.project_config.settings.virtual_environments), effective_target_name=effective_target_name, execution_limits=execution_limits, ) diff --git a/src/sqlbuild/cli/commands/_helpers/build_virtual/virtual_checks.py b/src/sqlbuild/cli/commands/_helpers/build_virtual/virtual_checks.py deleted file mode 100644 index 19a7ccb78e..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_virtual/virtual_checks.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Virtual build post-build python checks phase.""" - -from __future__ import annotations - -from pathlib import Path -from typing import TextIO - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.check.core import ( - load_results_by_loader_name, - record_python_run_state_results, - relevant_check_functions, - write_check_results, -) -from sqlbuild.compiler.discovery.models import DiscoveredCheckFunction, DiscoveredProjectInputs -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.python_nodes.main.checks import execute_python_checks -from sqlbuild.executor.python_nodes.models import ( - PythonCheckExecutionResult, - PythonNodeRunState, - PythonNodeRuntime, -) -from sqlbuild.provider.main.runtime import ProviderContainer -from sqlbuild.virtual.executor.classes.node_result_store import VirtualNodeResultStore -from sqlbuild.virtual.executor.models import VirtualBuildPipelineResult -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime - - -def run_post_virtual_build_checks( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - result: VirtualBuildPipelineResult, - exclude: tuple[str, ...], - reload_sources: bool, - providers: ProviderContainer | None, - stream: TextIO, - use_color: bool, -) -> tuple[PythonCheckExecutionResult, ...]: - """Execute relevant python checks after a successful virtual build.""" - - if result.execution_result.status != BuildStatus.SUCCESS: - return () - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - check_functions: tuple[DiscoveredCheckFunction, ...] = relevant_check_functions( - discovered_inputs=discovered_inputs, - python_graph=python_graph, - exclude=exclude, - selected_dependency_names=frozenset( - python_result.node_name for python_result in result.python_node_results - ) - | frozenset( - load_results_by_loader_name( - source_map=result.display_plan_output.source_map, - load_results=result.execution_result.load_results, - ) - ), - ) - if not check_functions: - return () - check_results: tuple[PythonCheckExecutionResult, ...] - check_connection: object = adapter.connect(connection_config) - try: - state_config, state_backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - check_state_connection: object = state_backend.connect(state_config.connection) - try: - check_result_store: VirtualNodeResultStore = VirtualNodeResultStore( - backend=state_backend, - state_connection=check_state_connection, - state_schema=state_config.schema, - virtual_environment_name=result.virtual_environment_name, - target_database=adapter.default_database(), - target_schema=adapter.default_schema(), - ) - check_run_state: PythonNodeRunState = PythonNodeRunState() - _ = record_python_run_state_results( - discovered_inputs=discovered_inputs, - run_state=check_run_state, - python_results=result.python_node_results, - load_results=result.execution_result.load_results, - source_map=result.display_plan_output.source_map, - ) - check_results = execute_python_checks( - check_functions=check_functions, - python_graph=python_graph, - upstream_python_results=result.python_node_results, - upstream_load_results=result.execution_result.load_results, - upstream_load_results_by_loader_name=load_results_by_loader_name( - source_map=result.display_plan_output.source_map, - load_results=result.execution_result.load_results, - ), - runtime=PythonNodeRuntime( - adapter=adapter, - connection_config=connection_config, - connection=check_connection, - run_id=result.project.run_id, - target=result.project.effective_target_name, - vars=result.project.effective_vars, - is_reload=reload_sources, - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - providers=providers, - result_store=check_result_store, - ), - run_state=check_run_state, - ) - finally: - state_backend.close(check_state_connection) - finally: - adapter.close(check_connection) - write_check_results( - stream=stream, - results=check_results, - use_color=use_color, - check_functions=check_functions, - python_graph=python_graph, - ) - return check_results diff --git a/src/sqlbuild/cli/commands/_helpers/build_virtual/virtual_execution.py b/src/sqlbuild/cli/commands/_helpers/build_virtual/virtual_execution.py deleted file mode 100644 index 965b60e741..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/build_virtual/virtual_execution.py +++ /dev/null @@ -1,195 +0,0 @@ -"""Virtual build CLI execution phase.""" - -from __future__ import annotations - -import sys -from collections.abc import Callable -from pathlib import Path -from typing import TextIO - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.input.parsing import ( - parse_cursor_integer, - parse_cursor_timestamp, -) -from sqlbuild.cli.commands.classes.virtual_build_plan_hook import VirtualBuildPlanHook -from sqlbuild.cli.commands.models import ( - VirtualBuildCliRequest, - VirtualBuildExecution, - VirtualBuildPlanHookConfig, -) -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, -) -from sqlbuild.cli.progress.classes.planning_progress_reporter import ( - PlanningProgressReporter, -) -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import ExecutionLimitsConfig, TargetConfig -from sqlbuild.virtual.executor.main.build import run_virtual_build as run_virtual_build_pipeline -from sqlbuild.virtual.executor.models import ( - VirtualBuildExecutionHooks, - VirtualBuildHooks, - VirtualBuildOptions, - VirtualBuildPipelineResult, -) -from sqlbuild.virtual.planner.models import VirtualPlanOptions - - -def execute_virtual_build( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - adapter_name: str, - connection_config: dict[str, object], - request: VirtualBuildCliRequest, - progress_stream: TextIO | None, - on_project_ready: Callable[[CompiledProject], None] | None = None, -) -> VirtualBuildExecution: - """Prepare virtual build reporting and execute the pipeline.""" - - stream: TextIO = progress_stream or ( - sys.stderr if request.debug or request.json_output else sys.stdout - ) - stream.write("\n") - stream.flush() - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - stream=stream, - use_color=request.use_color, - ) - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=stream, - use_color=request.use_color, - ) - physical_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=request.selected_target, - ) - physical_target_config: TargetConfig | None = ( - resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=physical_target_name, - ) - if physical_target_name is not None - else None - ) - unsuffixed_virtual_environment_name: str | None = ( - physical_target_config.state.unsuffixed_virtual_env - if physical_target_config is not None - else None - ) - plan_hook: VirtualBuildPlanHook = VirtualBuildPlanHook( - stream=stream, - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - config=VirtualBuildPlanHookConfig( - full_refresh=request.full_refresh, - allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, - allow_table_type_downgrade=request.allow_table_type_downgrade, - allow_retention_decrease=request.allow_retention_decrease, - use_color=request.use_color, - verbose=request.verbose, - debug=request.debug, - json_output=request.json_output, - execution_command=request.execution_command, - concurrency=request.concurrency, - connection_config=connection_config, - selector_files=request.selector_files, - virtual_environment_name=( - request.virtual_environment_name or physical_target_name or "default" - ), - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - effective_target_name=physical_target_name, - execution_limits=( - physical_target_config.execution_limits - if physical_target_config is not None - else ExecutionLimitsConfig() - ), - ), - ) - cursor_overrides: CursorOverrides = request.cursor_overrides or CursorOverrides() - - def handle_plan_ready( - *, - project: CompiledProject, - plan_output: PlanOutput, - python_plan_entries: tuple[PythonPlanEntry, ...], - ) -> VirtualBuildExecutionHooks: - if on_project_ready is not None: - on_project_ready(project) - return plan_hook.on_plan_ready( - project=project, - plan_output=plan_output, - python_plan_entries=python_plan_entries, - ) - - try: - result: VirtualBuildPipelineResult = run_virtual_build_pipeline( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=VirtualBuildOptions( - planning=VirtualPlanOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - defer_sources_to=request.defer_sources_to, - cursor_overrides=request.cursor_overrides, - max_microbatches=request.max_microbatches, - full_refresh=request.full_refresh, - virtual_environment_name=request.virtual_environment_name, - include_stale_upstreams=request.include_stale_upstreams, - changes_only=request.changes_only, - auto_load_sources=request.auto_load_sources, - reload_sources=request.reload_sources, - include_python=request.include_python, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - external_sql_reference_resolver=request.external_sql_reference_resolver, - ), - seed_only=request.seed_only, - fail_fast=request.fail_fast, - allow_snapshot_schema_change=request.allow_snapshot_schema_change, - concurrency=request.concurrency, - run_tests=request.run_tests, - run_audits=request.run_audits, - snapshots=discovered_inputs.project_config.snapshots, - start_cursor_ts=parse_cursor_timestamp(cursor_overrides.start_ts), - end_cursor_ts=parse_cursor_timestamp(cursor_overrides.end_ts), - start_cursor_int=parse_cursor_integer(cursor_overrides.start_int), - end_cursor_int=parse_cursor_integer(cursor_overrides.end_int), - providers=request.providers, - ), - hooks=VirtualBuildHooks( - on_plan_ready=handle_plan_ready, - on_progress=planning_progress.on_progress, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, - elapsed_seconds=elapsed_seconds, - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, - elapsed_seconds=elapsed_seconds, - ) - ), - ), - ) - finally: - _ = plan_hook.close() - return VirtualBuildExecution(result=result, stream=stream, elapsed=plan_hook.elapsed) diff --git a/src/sqlbuild/cli/commands/_helpers/clone/virtual.py b/src/sqlbuild/cli/commands/_helpers/clone/virtual.py deleted file mode 100644 index 3ddeff79b1..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/virtual.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Virtual clone command execution phase.""" - -from __future__ import annotations - -import time - -from sqlbuild.cli.commands._helpers.clone.virtual_output import ( - is_virtual_clone_success, - render_virtual_clone_output, -) -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_target_connection_config, -) -from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation -from sqlbuild.cli.output.main._virtual_clone_execution_json import ( - format_virtual_clone_execution_json, -) -from sqlbuild.cli.output.main._write_execution_json_output import write_execution_json_output -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.virtual.executor.main.clone import run_virtual_clone -from sqlbuild.virtual.executor.models import CloneOptions, VirtualCloneResult - - -def execute_virtual_clone(*, request: CloneCommandRequest, invocation: CloneInvocation) -> int: - """Run virtual clone and render its output.""" - - progress: PlanningProgressReporter = PlanningProgressReporter( - stream=invocation.progress_stream, - use_color=invocation.use_color, - ) - progress.start("Cloning virtual environment...") - clone_start: float = time.monotonic() - result: VirtualCloneResult = run_virtual_clone( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - origin_target_name=request.origin_target_name, - destination_target_name=invocation.destination_target_name, - destination_connection_config=resolve_target_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - target_name=invocation.destination_target_name, - cli_vars=request.cli_vars, - ), - options=CloneOptions( - virtual_environment_name=request.virtual_env, - skip_locked=request.skip_locked, - no_sql_validation=request.no_sql_validation, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ), - ) - elapsed: float = time.monotonic() - clone_start - if is_virtual_clone_success(result): - progress.complete(f"Cloned virtual environment. ({elapsed:.2f}s)") - else: - progress.error(f"Clone finished with errors. ({elapsed:.2f}s)") - progress.finish(blank_line_after=True) - render_virtual_clone_output( - result=result, - use_color=invocation.use_color, - verbose=request.verbose, - ) - write_execution_json_output( - payload=format_virtual_clone_execution_json(result=result), - json_output=False, - json_output_path=request.json_output_path, - ) - return 0 if is_virtual_clone_success(result) else 1 diff --git a/src/sqlbuild/cli/commands/_helpers/clone/virtual_output.py b/src/sqlbuild/cli/commands/_helpers/clone/virtual_output.py deleted file mode 100644 index bc6c0e9a0f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/clone/virtual_output.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Virtual clone output helpers.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.virtual.executor.models import VirtualCloneResult - - -def render_virtual_clone_output( - *, result: VirtualCloneResult, use_color: bool, verbose: bool -) -> None: - """Render virtual physical-version hydration output.""" - - style: CliStyle = CliStyle(use_color=use_color) - doc: CliDocument = CliDocument(style) - origin: str = style.object_name(result.origin_environment) - destination: str = style.object_name(result.destination_environment) - doc.blank() - doc.header(text="Virtual clone", suffix=f"{origin} -> {destination}") - doc.line(f" mode {result.mode}") - if result.destination_virtual_environment is not None: - doc.line(f" destination VDE {result.destination_virtual_environment}") - doc.line(f" origin state {'used' if result.origin_state_used else 'not used'}") - doc.line(" destination refs unchanged") - doc.line(f" selected artifacts {_count(count=result.selected_count, style=style)}") - doc.line(f" found in origin {_count(count=result.found_count, style=style)}") - doc.line(f" hydrated {_count(count=result.hydrated_count, style=style)}") - doc.line(f" already present {_count(count=result.reused_count, style=style)}") - missing: str = _warn_count(count=result.missing_count, style=style) - skipped: str = _warn_count(count=result.skipped_locked_count, style=style) - doc.line(f" missing in origin {missing}") - doc.line(f" skipped locked {skipped}") - _append_set( - doc=doc, result=result, action="missing", label="missing", style=style, verbose=verbose - ) - _append_set( - doc=doc, - result=result, - action="skipped_locked", - label="skipped locked", - style=style, - verbose=verbose, - ) - print(doc.render(), end="") - - -def is_virtual_clone_success(result: VirtualCloneResult) -> bool: - return result.missing_count == 0 - - -def _count(*, count: int, style: CliStyle) -> str: - value: str = f"{count:,}" - return style.value(value) - - -def _warn_count(*, count: int, style: CliStyle) -> str: - value: str = f"{count:,}" - return style.warning_strong(value) if count else value - - -def _append_set( - *, - doc: CliDocument, - result: VirtualCloneResult, - action: str, - label: str, - style: CliStyle, - verbose: bool, -) -> None: - names: tuple[str, ...] = tuple( - item.artifact_name for item in result.item_results if item.action == action - ) - if not names: - return - limit: int = len(names) if verbose else 20 - rendered_label: str = style.muted(label) - doc.line(f" {rendered_label}: " + ", ".join(names[:limit])) - if len(names) > limit: - suffix: str = f" ... {len(names) - limit:,} more; use --verbose to show all" - doc.line(style.muted(suffix)) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 75944bcc6d..80ecaede2b 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -3,11 +3,16 @@ from __future__ import annotations import json +from collections import Counter from importlib.metadata import PackageNotFoundError, version from pathlib import Path import orjson +from sqlbuild.cli.commands._helpers.compile.semantic_notice import ( + selected_semantic_coverage, + semantic_coverage_notice, +) from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.output.models import ( @@ -56,6 +61,7 @@ def format_compile_text( lineage_mode: CompileLineageMode = CompileLineageMode.FAST, use_color: bool, selected_keys: frozenset[CompiledObjectKey] | None = None, + sql_validation_enabled: bool = True, ) -> str: """Format human-readable compile output.""" @@ -94,11 +100,29 @@ def format_compile_text( ) lines.append(" " + style.muted("Use --json for the full compile report.")) lines.append("") + notice: str | None = semantic_coverage_notice( + project=graph.project, + selected_keys=selected_keys, + sql_validation_enabled=sql_validation_enabled, + ) + if notice: + lines.append(notice) + lines.append("") if diagnostics: lines.append( _format_diagnostics_text( diagnostics=diagnostics, - source_texts=_model_source_texts(graph.project.models), + source_texts={ + **_model_source_texts(graph.project.models), + **{ + source.source_file.relative_path: source.source_file.contents + for source in graph.project.sources + }, + **{ + test.test_file.relative_path: test.test_file.contents + for test in graph.project.sql_tests + }, + }, style=style, ) ) @@ -159,6 +183,7 @@ def format_compile_json( diagnostics: tuple[CompilerDiagnostic, ...], lineage_mode: CompileLineageMode = CompileLineageMode.FAST, selected_keys: frozenset[CompiledObjectKey] | None = None, + sql_validation_enabled: bool = True, ) -> str: """Serialize the offline compile report as JSON.""" @@ -173,6 +198,11 @@ def format_compile_json( selected_keys=selected_keys, ), "diagnostics": [_diagnostic_to_json(diagnostic) for diagnostic in diagnostics], + "semantic_checks_partial": selected_semantic_coverage( + project=graph.project, + selected_keys=selected_keys, + sql_validation_enabled=sql_validation_enabled, + ), "compile_timings": timings_ms, "lineage_mode": lineage_mode.value, "resources": _resources(graph=graph, lineage=lineage), @@ -503,6 +533,10 @@ def _diagnostic_to_json(diagnostic: CompilerDiagnostic) -> dict[str, object]: ] if diagnostic.help is not None: payload["help"] = diagnostic.help + if diagnostic.notes: + payload["notes"] = list(diagnostic.notes) + if diagnostic.affected_rules: + payload["affected_rules"] = list(diagnostic.affected_rules) return payload @@ -523,7 +557,17 @@ def _format_diagnostics_text( *, diagnostics: tuple[CompilerDiagnostic, ...], source_texts: dict[Path, str], style: CliStyle ) -> str: lines: list[str] = [] + warning_counts: Counter[CompilerDiagnostic] = Counter( + diagnostic + for diagnostic in diagnostics + if diagnostic.severity == DiagnosticSeverity.WARNING + ) + shown_warnings: set[CompilerDiagnostic] = set() for diagnostic in diagnostics: + if diagnostic in shown_warnings: + continue + if diagnostic.severity == DiagnosticSeverity.WARNING: + shown_warnings.add(diagnostic) lines.extend( _format_diagnostic_text( diagnostic=diagnostic, @@ -531,6 +575,9 @@ def _format_diagnostics_text( style=style, ) ) + count: int = warning_counts[diagnostic] + if count > 1: + lines.append(f" = {count} occurrences; all retained in JSON and the warning count") lines.append("") if lines: lines.pop() @@ -549,8 +596,7 @@ def _format_diagnostic_text( diagnostic.resource_type is not None and str(diagnostic.resource_type) != CompiledResourceType.MODEL ): - label = "resource" - resource = f"{diagnostic.resource_type}: {resource}" + label = str(diagnostic.resource_type).replace("_", " ") lines.append(f" {label}: {style.object_name(resource)}") if diagnostic.location is not None: lines.extend( @@ -571,6 +617,9 @@ def _format_diagnostic_text( source_texts=source_texts, ) ) + lines.extend(f" {style.muted('= note:')} {note}" for note in diagnostic.notes) + if diagnostic.affected_rules: + lines.append(f" {style.muted('= affected Rules:')} {', '.join(diagnostic.affected_rules)}") if diagnostic.help is not None: lines.append(f" {style.muted('= help:')} {diagnostic.help}") return lines diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index cb4bf124ed..e497a1f14a 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -229,6 +229,7 @@ def _analyze_compile_project( line=fault.line, column=fault.column, help=fault.remediation, + affected_rules=fault.affected_rules, ) for fault in rules_result.findings ) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py b/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py new file mode 100644 index 0000000000..1ce65484c7 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/compile/semantic_notice.py @@ -0,0 +1,82 @@ +"""One actionable semantic-check notice for human and machine commands.""" + +from sqlbuild.compiler.compile.main.semantic_coverage import get_semantic_coverage +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.compile.types import CompiledResourceType + +_DISPLAY_LIMIT: int = 10 +_ACTIONS: dict[str, str] = { + "some output types are unknown": "declare output types, then run `sqb compile --no-cache`", + "output shape could not be inferred": "declare the output schema; run `sqb compile --no-cache`", + "unresolved star over open inputs": ( + "declare complete input contracts, then run `sqb compile --no-cache`" + ), + "SQL analysis disabled": "enable sql_analysis, then run `sqb compile --no-cache`", + "type checking awaits dialect coercion support": ( + "use a supported dialect for type checks; run `sqb compile --json` to inspect coverage" + ), +} + + +def selected_semantic_coverage( + *, + project: CompiledProject, + selected_keys: frozenset[CompiledObjectKey] | None, + sql_validation_enabled: bool = True, +) -> dict[str, tuple[str, ...]]: + """Report coverage for the compile selection rather than unanalyzed sibling models.""" + if not sql_validation_enabled: + return {} + reasons: dict[str, tuple[str, ...]] = get_semantic_coverage(project) + if selected_keys is None: + return reasons + names: set[str] = { + key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL + } + return {name: values for name, values in reasons.items() if name in names} + + +def semantic_coverage_notice( + *, + project: CompiledProject, + selected_keys: frozenset[CompiledObjectKey] | None = None, + sql_validation_enabled: bool = True, +) -> str | None: + """Summarize partial checks without increasing the diagnostic warning count.""" + reasons: dict[str, tuple[str, ...]] = selected_semantic_coverage( + project=project, + selected_keys=selected_keys, + sql_validation_enabled=sql_validation_enabled, + ) + if not reasons: + return None + groups: dict[str, list[str]] = {} + for model, values in sorted(reasons.items()): + for reason in values: + groups.setdefault(reason, []).append(model) + lines: list[str] = [f"Semantic checks were partial for {len(reasons)} models:"] + for reason, models in sorted(groups.items())[:10]: + suffix: str = ( + f", and {len(models) - _DISPLAY_LIMIT} more" if len(models) > _DISPLAY_LIMIT else "" + ) + detail: str = reason.replace("open source: ", "reads open source ").replace( + "open input: ", "reads open input " + ) + action: str = _ACTIONS.get( + reason, "declare the input shape, then run `sqb compile --no-cache`" + ) + if reason.startswith("open source: "): + source: str = reason.removeprefix("open source: ") + action = ( + "run `sqb contract generate --from " + f"--select source:{source} --write` and enforce the complete source contract" + ) + elif reason.startswith("depends on invalid model "): + action = "fix the upstream error, then run `sqb compile --no-cache`" + lines.append(f" {', '.join(models[:10])}{suffix}: {detail}; {action}.") + if len(groups) > _DISPLAY_LIMIT: + lines.append( + f" and {len(groups) - 10} more reasons; " + "run `sqb compile --json` for the full model/reason list." + ) + return "\n".join(lines) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/execution.py b/src/sqlbuild/cli/commands/_helpers/diff/execution.py index b8463e48c4..468a0e7ef1 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/execution.py @@ -6,7 +6,7 @@ import time from datetime import UTC, datetime from pathlib import Path -from typing import Any, TextIO +from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationLookup @@ -28,11 +28,7 @@ DirectDiffPreparation, QueryDiffPreparation, QueryDiffRunOutcome, - VirtualDiffPreparation, - VirtualDiffRunOutcome, ) -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter from sqlbuild.compiler.compile.main.effective_runtime import build_effective_runtime_config from sqlbuild.compiler.compile.main.effective_target_namespace import ( build_effective_target_namespace, @@ -54,13 +50,9 @@ QueryDiffArtifactCleanupResult, RowDiffSamplingOverride, ) -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) -from sqlbuild.virtual.diff.main.diff import run_virtual_diff -from sqlbuild.virtual.diff.models import VirtualDiffOptions def prepare_direct_diff( @@ -670,121 +662,6 @@ def _preflight_query_columns( ) -def prepare_virtual_diff( - *, request: DiffCommandRequest, invocation: DiffInvocation -) -> VirtualDiffPreparation: - """Resolve virtual diff adapter, connection, and sample limits.""" - - if request.from_name is None or request.to_name is None: - raise CliUserError("virtual diff requires FROM:TO", code="C236") - effective_adapter_name: str = resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ) - return VirtualDiffPreparation( - from_virtual_environment=request.from_name, - to_virtual_environment=request.to_name, - adapter=resolve_adapter( - adapter_name=effective_adapter_name, - project_dir=invocation.effective_project_dir, - ), - connection_config=resolve_project_connection_config( - discovered_inputs=invocation.discovered_inputs, - project_dir=invocation.effective_project_dir, - cli_vars=request.cli_vars, - ), - effective_max_column_examples=_effective_max_examples( - explicit_value=request.max_column_examples, verbose=request.verbose - ), - effective_max_row_only_examples=_effective_max_examples( - explicit_value=request.max_row_only_examples, verbose=request.verbose - ), - use_color=not request.no_color and supports_color(), - ) - - -def execute_virtual_diff( - *, request: DiffCommandRequest, invocation: DiffInvocation, preparation: VirtualDiffPreparation -) -> VirtualDiffRunOutcome: - """Execute a virtual environment diff.""" - - progress_stream: TextIO = sys.stderr if request.json_output else sys.stdout - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=progress_stream, - use_color=preparation.use_color, - ) - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=resolve_effective_adapter_name( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - ), - stream=progress_stream, - use_color=preparation.use_color, - ) - ( - result, - selected_names, - skipped_names, - from_stale, - to_stale, - from_working, - to_working, - ) = run_virtual_diff( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - adapter=preparation.adapter, - connection_config=preparation.connection_config, - from_virtual_environment_name=preparation.from_virtual_environment, - to_virtual_environment_name=preparation.to_virtual_environment, - options=VirtualDiffOptions( - no_sql_validation=request.no_sql_validation, - select=request.select, - exclude=request.exclude, - schema_only=request.schema_only, - bounded=request.bounded, - collect_samples=not request.schema_only, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - max_models=request.max_models, - max_columns=request.max_columns, - sampling_override=_sampling_override(request=request), - unique_key_override=request.unique_key_override, - unkeyed=request.unkeyed, - excluded_columns_override=request.excluded_columns_override, - tolerance_overrides=parse_cli_tolerance_overrides(values=request.tolerance_overrides), - allow_partial_diff=request.allow_partial_diff, - cli_vars=request.cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - ), - hooks=ConnectionHooks( - on_progress=planning_progress.on_progress, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - return VirtualDiffRunOutcome( - result=result, - selected_names=selected_names, - skipped_names=skipped_names, - from_stale=from_stale, - to_stale=to_stale, - from_working=from_working, - to_working=to_working, - ) - - def _effective_max_examples(*, explicit_value: int | None, verbose: bool) -> int: return explicit_value if explicit_value is not None else (10 if verbose else 3) diff --git a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py index 2b5c50a41b..fe8e300213 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/invocation.py @@ -34,13 +34,11 @@ def resolve_diff_invocation(*, request: DiffCommandRequest) -> DiffInvocation: if is_query_diff_request(request=request) else discover_project_inputs(project_dir=effective_project_dir) ) - is_virtual_mode: bool = discovered_inputs.project_config.settings.virtual_environments - if not request.select and not is_virtual_mode and not is_query_diff_request(request=request): + if not request.select and not is_query_diff_request(request=request): raise CliUserError("diff requires --select in v1", code="C204") return DiffInvocation( effective_project_dir=effective_project_dir, discovered_inputs=discovered_inputs, - is_virtual_mode=is_virtual_mode, ) @@ -119,8 +117,6 @@ def _validate_query_diff_request(*, request: DiffCommandRequest) -> None: "raw-query diff does not support --bounded; filter both raw queries explicitly", code="C214", ) - if request.allow_partial_diff: - raise CliUserError("raw-query diff does not use --allow-partial-diff", code="C215") if request.max_models is not None: raise CliUserError("raw-query diff does not accept --max-models", code="C243") if request.left_label is not None and not request.left_label.strip(): diff --git a/src/sqlbuild/cli/commands/_helpers/diff/outputs.py b/src/sqlbuild/cli/commands/_helpers/diff/outputs.py index 63cc070018..05e8c73ccd 100644 --- a/src/sqlbuild/cli/commands/_helpers/diff/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/diff/outputs.py @@ -7,14 +7,11 @@ write_diff_json_output, ) from sqlbuild.cli.commands._helpers.diff.output import has_diff_failures, render_diff_output -from sqlbuild.cli.commands._helpers.diff.virtual_output import format_virtual_diff_header from sqlbuild.cli.commands.constants import DEFAULT_DIFF_MAX_VALUE_LENGTH from sqlbuild.cli.commands.models import ( DiffCommandRequest, DiffExampleRenderOptions, DirectDiffPreparation, - VirtualDiffPreparation, - VirtualDiffRunOutcome, ) from sqlbuild.executor.diff.models import DiffExecutionResult from sqlbuild.presentation.main.supports_color import supports_color @@ -68,61 +65,6 @@ def write_direct_diff_output( ) -def write_virtual_diff_output( - *, - request: DiffCommandRequest, - preparation: VirtualDiffPreparation, - outcome: VirtualDiffRunOutcome, -) -> None: - """Write virtual diff header and optional diff body.""" - - example_render_options: DiffExampleRenderOptions = _example_render_options(request=request) - write_diff_json_output( - path=request.json_output_path, - result=outcome.result, - from_label=preparation.from_virtual_environment, - to_label=preparation.to_virtual_environment, - example_render_options=example_render_options, - ) - if request.json_output: - print( - render_diff_json_output( - result=outcome.result, - from_label=preparation.from_virtual_environment, - to_label=preparation.to_virtual_environment, - example_render_options=example_render_options, - ) - ) - return - header: str = format_virtual_diff_header( - from_virtual_environment=preparation.from_virtual_environment, - to_virtual_environment=preparation.to_virtual_environment, - outcome=outcome, - allow_partial_diff=request.allow_partial_diff, - verbose=request.verbose, - use_color=preparation.use_color, - ) - print() - print(header) - print() - if outcome.result.model_results: - print( - render_diff_output( - result=outcome.result, - from_label=preparation.from_virtual_environment, - to_label=preparation.to_virtual_environment, - mode_label=_mode_label(request=request), - use_color=preparation.use_color, - verbose=request.verbose, - max_column_examples=preparation.effective_max_column_examples, - max_row_only_examples=preparation.effective_max_row_only_examples, - example_render_options=example_render_options, - ) - ) - else: - print("No VDE ref differences in selected scope.") - - def resolve_diff_exit_code(result: DiffExecutionResult) -> int: """Resolve the diff exit code from failure state.""" diff --git a/src/sqlbuild/cli/commands/_helpers/diff/virtual_output.py b/src/sqlbuild/cli/commands/_helpers/diff/virtual_output.py deleted file mode 100644 index 3529469cd9..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/diff/virtual_output.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Virtual diff CLI output helpers.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.models import VirtualDiffRunOutcome -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle - - -def format_virtual_diff_header( - *, - from_virtual_environment: str, - to_virtual_environment: str, - outcome: VirtualDiffRunOutcome, - allow_partial_diff: bool, - verbose: bool, - use_color: bool, -) -> str: - """Format the virtual diff summary header.""" - - style: CliStyle = CliStyle(use_color=use_color) - from_label: str = style.object_name(from_virtual_environment) - to_label: str = style.object_name(to_virtual_environment) - selected_count: str = style.value(f"{len(outcome.selected_names):,}") - compared_count: str = style.value( - f"{len(outcome.selected_names) - len(outcome.skipped_names):,}" - ) - skipped_count: str = style.muted(f"{len(outcome.skipped_names):,}") - has_working_vde: bool = outcome.from_working or outcome.to_working - working_label: str = "yes" if has_working_vde else "no" - if has_working_vde: - working_value: str = style.warning_strong(working_label) - if allow_partial_diff: - working_value += style.muted(" (partial allowed)") - else: - working_value = working_label - doc: CliDocument = CliDocument(style) - doc.header(text="Virtual diff", suffix=f"{from_label} -> {to_label}") - doc.line(f" selected models {selected_count}") - doc.line(f" compared models {compared_count}") - doc.line(f" unchanged refs skipped {skipped_count}") - doc.line(f" working VDEs {working_value}") - if verbose and (outcome.from_stale or outcome.to_stale): - if outcome.from_stale: - doc.line( - f" {from_virtual_environment} not current with workspace: " - + ", ".join(outcome.from_stale) - ) - if outcome.to_stale: - doc.line( - f" {to_virtual_environment} not current with workspace: " - + ", ".join(outcome.to_stale) - ) - return doc.render(trailing_newline=False) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 9f68afdacd..02eb54d4d8 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -59,8 +59,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) LineageCommandRequest, LoadCommandRequest, PlanCommandRequest, - PromoteCommandRequest, - RollbackCommandRequest, ScopeCommandRequest, SeedCommandRequest, TestCommandRequest, @@ -121,7 +119,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) ), json_output=args.json, full_refresh=args.full_refresh, - virtual_env=args.virtual_env, load_sources=args.load_sources, include_python=args.include_python, no_color=args.no_color, @@ -129,8 +126,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) exclude=tuple(args.exclude), verbose=args.verbose, cli_vars=args.vars, - include_stale_upstreams=args.include_stale_upstreams, - changes_only=args.changes_only, max_microbatches=args.max_microbatches, selection_diagnostics=args.selection_diagnostics, ) @@ -163,7 +158,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) no_color=args.no_color, fail_fast=args.fail_fast, full_refresh=args.full_refresh, - virtual_env=args.virtual_env, load_sources=args.load_sources, reload_sources=args.reload, include_python=args.include_python, @@ -178,8 +172,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) verbose=args.verbose, debug=args.debug, cli_vars=args.vars, - include_stale_upstreams=args.include_stale_upstreams, - changes_only=args.changes_only, run_tests=args.run_tests, run_audits=args.run_audits, manifest=args.manifest, @@ -205,7 +197,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) fail_on_error=args.fail_on_error, compare_state=args.state, fail_on_stale=args.fail_on_stale, - virtual_environment_name=args.virtual_env, ) ) if args.command == CliCommand.TEST: @@ -319,8 +310,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) origin_target_name=args.from_target, destination_target_name=args.to_target, hard_copy=args.hard_copy, - virtual_env=args.virtual_env, - skip_locked=args.skip_locked, select=select, exclude=tuple(args.exclude), verbose=args.verbose, @@ -357,7 +346,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) exclude=tuple(args.exclude), verbose=args.verbose, cli_vars=args.vars, - allow_partial_diff=args.allow_partial_diff, selected_target=args.target, left_query=args.left_query, left_query_file=args.left_query_file, @@ -374,52 +362,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) full_example_values=args.full_example_values, ) ) - if args.command == CliCommand.RECONCILE: - return handlers.run_reconcile( - project_dir=project_dir, - no_color=args.no_color, - virtual_environment=args.virtual_env, - reconcile_command=args.reconcile_command, - model_name=getattr(args, "reconcile_model", None), - seed_name=getattr(args, "reconcile_seed", None), - physical_relation_name=getattr(args, "reconcile_physical_relation", None), - auto_approve=getattr(args, "auto_approve", False), - cli_vars=args.vars, - ) - if args.command == CliCommand.PROMOTE: - if args.from_virtual_environment is None or args.to_virtual_environment is None: - raise CliUserError("promote requires --from and --to", code="C244") - return handlers.run_promote( - PromoteCommandRequest( - project_dir=project_dir, - no_color=args.no_color, - no_sql_validation=args.no_sql_validation, - from_virtual_environment=args.from_virtual_environment, - to_virtual_environment=args.to_virtual_environment, - select=select, - exclude=tuple(args.exclude), - allow_partial_promotion=args.allow_partial_promotion, - include_stale_upstreams=args.include_stale_upstreams, - verbose=args.verbose, - cli_vars=args.vars, - ) - ) - if args.command == CliCommand.ROLLBACK: - return handlers.run_rollback( - RollbackCommandRequest( - project_dir=project_dir, - no_color=args.no_color, - no_sql_validation=args.no_sql_validation, - virtual_environment=args.virtual_env, - verbose=args.verbose, - checkpoint_id=args.rollback_checkpoint_id, - select=select, - exclude=tuple(args.exclude), - allow_partial_rollback=args.allow_partial_rollback, - include_stale_upstreams=args.include_stale_upstreams, - cli_vars=args.vars, - ) - ) if args.command == CliCommand.QUERY: query_limit: int | None = None if args.query_no_limit else args.query_limit return handlers.run_query( @@ -464,20 +406,6 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) direct_state_history_versions=args.direct_state_history_versions, ) ) - if args.command == CliCommand.STATE: - if args.state_command is None: - raise CliUserError("state requires a subcommand such as 'init'", code="C901") - return handlers.run_state( - project_dir=project_dir, - state_command=args.state_command, - backup_id=args.state_backup_id, - auto_approve=args.auto_approve, - no_color=args.no_color, - checkpoint_command=args.state_checkpoint_command, - checkpoint_id=args.state_checkpoint_id, - virtual_environment=args.virtual_env, - allow_copy=getattr(args, "allow_copy", False), - ) return _dispatch_local_command( args=args, handlers=handlers, @@ -695,6 +623,7 @@ def _dispatch_scenario_command( ) return handlers.run_scenario( ScenarioTestCommandRequest( + scenario_namespace=args.scenario_namespace, project_dir=project_dir, no_sql_validation=False, no_color=args.no_color, @@ -719,6 +648,7 @@ def _dispatch_scenario_command( if args.scenario_command == SCENARIO_CAPTURE_COMMAND: return handlers.run_scenario_capture( ScenarioCaptureCommandRequest( + scenario_namespace=args.scenario_namespace, project_dir=project_dir, no_sql_validation=False, no_color=args.no_color, diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py index 5ae4ad1e03..07fea40c29 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch_errors.py @@ -81,14 +81,12 @@ def _expected_command_failure_types() -> tuple[type[Exception], ...]: from sqlbuild.executor.pipeline.exceptions import AuditExecutionError from sqlbuild.runtime.execution_limits.exceptions import ExecutionDurationLimitError from sqlbuild.spec.contracts.exceptions import SpecConfigError - from sqlbuild.virtual.state.exceptions import StateBackendError return ( AuditExecutionError, DiscoveryError, ExecutionDurationLimitError, SpecConfigError, - StateBackendError, ValueError, ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py index 011924dc14..261ede787a 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/lazy_handlers.py @@ -117,22 +117,6 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: module_name="sqlbuild.cli.commands.main.state._janitor", function_name="run_janitor", ), - "promote": _lazy_handler( - module_name="sqlbuild.cli.commands.main.state._promote", - function_name="run_promote", - ), - "reconcile": _lazy_handler( - module_name="sqlbuild.cli.commands.main.state._reconcile", - function_name="run_reconcile", - ), - "rollback": _lazy_handler( - module_name="sqlbuild.cli.commands.main.state._rollback", - function_name="run_rollback", - ), - "state": _lazy_handler( - module_name="sqlbuild.cli.commands.main.state._state", - function_name="run_state", - ), "init": _lazy_handler( module_name="sqlbuild.cli.commands.main.workspace._init", function_name="run_init", @@ -190,21 +174,6 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: run_load=lazy["load"], run_clone=lazy["clone"], run_diff=lazy["diff"], - run_reconcile=lambda project_dir, no_color, virtual_environment, reconcile_command, model_name, seed_name, physical_relation_name, auto_approve, cli_vars: ( # noqa: E501 - lazy["reconcile"]( - project_dir=project_dir, - no_color=no_color, - virtual_environment=virtual_environment, - reconcile_command=reconcile_command, - model_name=model_name, - seed_name=seed_name, - physical_relation_name=physical_relation_name, - auto_approve=auto_approve, - cli_vars=cli_vars, - ) - ), - run_promote=lazy["promote"], - run_rollback=lazy["rollback"], run_query=lambda project_dir, sql, query_file, selected_target, output_format, limit: lazy[ "query" ]( @@ -226,19 +195,6 @@ def build_lazy_cli_handlers() -> CliEntrypointHandlers: ), run_lineage=lazy["lineage"], run_janitor=lazy["janitor"], - run_state=lambda project_dir, state_command, backup_id, auto_approve, no_color, checkpoint_command, checkpoint_id, virtual_environment, allow_copy: ( # noqa: E501 - lazy["state"]( - project_dir=project_dir, - state_command=state_command, - backup_id=backup_id, - auto_approve=auto_approve, - no_color=no_color, - checkpoint_command=checkpoint_command, - checkpoint_id=checkpoint_id, - virtual_environment=virtual_environment, - allow_copy=allow_copy, - ) - ), run_init=lazy["init"], run_playground=lazy["playground"], run_skills_update=lambda project_dir, global_install, targets, force: lazy["skills"]( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index adfbfcf97f..519fef5eb7 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -26,7 +26,6 @@ from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode from sqlbuild.compiler.contract_adoption.types import ContractAction from sqlbuild.compiler.lineage.types import ColumnLineageMode -from sqlbuild.virtual.state.types import StateCommand def _installed_version() -> str: @@ -66,7 +65,6 @@ def build_cli_parser(*, use_color: bool = False) -> argparse.ArgumentParser: _add_plan_and_build_parsers(subparsers) _add_quality_parsers(subparsers) _add_data_parsers(subparsers) - _add_virtual_parsers(subparsers) _add_inspection_parsers(subparsers) _add_contract_parser(subparsers) _add_maintenance_parsers(subparsers) @@ -190,14 +188,6 @@ def _add_plan_and_build_parsers( ) plan_parser.add_argument("--json", action="store_true", default=False) plan_parser.add_argument("--full-refresh", action="store_true", default=False) - plan_parser.add_argument("--virtual-env", default=None) - plan_parser.add_argument("--include-stale-upstreams", action="store_true", default=False) - plan_parser.add_argument( - "--changes-only", - action="store_true", - default=False, - help="Only include resources that require execution.", - ) plan_parser.add_argument( "--selection-diagnostics", action="store_true", @@ -230,14 +220,6 @@ def _add_plan_and_build_parsers( build_parser.add_argument("--defer-sources-to", default=None) build_parser.add_argument("--target", default=None) build_parser.add_argument("--json", action="store_true", default=False) - build_parser.add_argument("--virtual-env", default=None) - build_parser.add_argument("--include-stale-upstreams", action="store_true", default=False) - build_parser.add_argument( - "--changes-only", - action="store_true", - default=False, - help="Only include resources that require execution.", - ) build_parser.add_argument( "--selection-diagnostics", action="store_true", @@ -272,7 +254,6 @@ def _add_quality_parsers( freshness_parser.add_argument("--json", action="store_true", default=False) freshness_parser.add_argument("--state", action="store_true", default=False) freshness_parser.add_argument("--target", default=None) - freshness_parser.add_argument("--virtual-env", default=None) freshness_parser.add_argument("--fail-on-error", action="store_true", default=False) freshness_parser.add_argument("--fail-on-stale", action="store_true", default=False) _ = add_execution_json_output_arg(freshness_parser) @@ -360,8 +341,6 @@ def _add_data_parsers( clone_parser.add_argument("--from", dest="from_target", required=True) clone_parser.add_argument("--to", dest="to_target", default=None) clone_parser.add_argument("--hard-copy", action="store_true", default=False) - clone_parser.add_argument("--virtual-env", default=None) - clone_parser.add_argument("--skip-locked", action="store_true", default=False) clone_parser.add_argument("--verbose", "-v", action="store_true", default=False) _ = add_select_args(clone_parser) _ = add_vars_args(clone_parser) @@ -383,7 +362,6 @@ def _add_data_parsers( diff_parser.add_argument("--exhaustive", action="store_true", default=False) diff_parser.add_argument("--max-models", type=int, default=None) diff_parser.add_argument("--max-columns", type=int, default=None) - diff_parser.add_argument("--allow-partial-diff", action="store_true", default=False) diff_parser.add_argument( "--target", default=None, @@ -448,50 +426,6 @@ def _add_data_parsers( _ = add_vars_args(diff_parser) -def _add_virtual_parsers( - subparsers: argparse._SubParsersAction[argparse.ArgumentParser], -) -> None: - reconcile_parser: argparse.ArgumentParser = subparsers.add_parser(CliCommand.RECONCILE) - reconcile_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - reconcile_parser.add_argument("--model", dest="reconcile_model", default=None) - reconcile_parser.add_argument("--seed", dest="reconcile_seed", default=None) - reconcile_subparsers: argparse._SubParsersAction[argparse.ArgumentParser] - reconcile_subparsers = reconcile_parser.add_subparsers(dest="reconcile_command") - reconcile_repair_view_parser: argparse.ArgumentParser = reconcile_subparsers.add_parser( - "repair-view" - ) - reconcile_repair_view_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - reconcile_repair_view_parser.add_argument("--model", dest="reconcile_model", default=None) - reconcile_repair_view_parser.add_argument("--seed", dest="reconcile_seed", default=None) - reconcile_attach_parser: argparse.ArgumentParser = reconcile_subparsers.add_parser("attach") - reconcile_attach_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - reconcile_attach_parser.add_argument("--model", dest="reconcile_model", required=True) - reconcile_attach_parser.add_argument( - "--physical-relation", dest="reconcile_physical_relation", required=True - ) - reconcile_attach_parser.add_argument("--auto-approve", action="store_true", default=False) - - promote_parser: argparse.ArgumentParser = subparsers.add_parser(CliCommand.PROMOTE) - _add_sql_analysis_override(promote_parser) - promote_parser.add_argument("--from", dest="from_virtual_environment", required=True) - promote_parser.add_argument("--to", dest="to_virtual_environment", required=True) - promote_parser.add_argument("--allow-partial-promotion", action="store_true", default=False) - promote_parser.add_argument("--include-stale-upstreams", action="store_true", default=False) - promote_parser.add_argument("--verbose", "-v", action="store_true", default=False) - _ = add_select_args(promote_parser) - _ = add_vars_args(promote_parser) - - rollback_parser: argparse.ArgumentParser = subparsers.add_parser(CliCommand.ROLLBACK) - _add_sql_analysis_override(rollback_parser) - rollback_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - rollback_parser.add_argument("--checkpoint-id", dest="rollback_checkpoint_id", default=None) - rollback_parser.add_argument("--allow-partial-rollback", action="store_true", default=False) - rollback_parser.add_argument("--include-stale-upstreams", action="store_true", default=False) - rollback_parser.add_argument("--verbose", "-v", action="store_true", default=False) - _ = add_select_args(rollback_parser) - _ = add_vars_args(rollback_parser) - - def _add_inspection_parsers( subparsers: argparse._SubParsersAction[argparse.ArgumentParser], ) -> None: @@ -647,47 +581,6 @@ def _add_maintenance_parsers( janitor_parser.add_argument("--retention-days", type=int, default=None) janitor_parser.add_argument("--direct-state-history-versions", type=int, default=None) - state_parser: argparse.ArgumentParser = subparsers.add_parser(CliCommand.STATE) - state_subparsers: argparse._SubParsersAction[argparse.ArgumentParser] - state_subparsers = state_parser.add_subparsers(dest="state_command") - state_subparsers.add_parser(StateCommand.INIT.value) - state_subparsers.add_parser(StateCommand.MIGRATE.value) - state_adopt_parser: argparse.ArgumentParser = state_subparsers.add_parser( - StateCommand.ADOPT.value - ) - state_adopt_parser.add_argument("--allow-copy", action="store_true", default=False) - state_detach_parser: argparse.ArgumentParser = state_subparsers.add_parser( - StateCommand.DETACH.value - ) - state_detach_parser.add_argument("--allow-copy", action="store_true", default=False) - state_rollback_parser: argparse.ArgumentParser = state_subparsers.add_parser( - StateCommand.ROLLBACK.value - ) - state_rollback_parser.add_argument("--backup-id", dest="state_backup_id", default=None) - state_reset_parser: argparse.ArgumentParser = state_subparsers.add_parser( - StateCommand.RESET.value - ) - state_reset_parser.add_argument("--auto-approve", action="store_true", default=False) - state_checkpoints_parser: argparse.ArgumentParser = state_subparsers.add_parser("checkpoints") - state_checkpoints_subparsers: argparse._SubParsersAction[argparse.ArgumentParser] - state_checkpoints_subparsers = state_checkpoints_parser.add_subparsers( - dest="state_checkpoint_command" - ) - state_checkpoints_list_parser: argparse.ArgumentParser = ( - state_checkpoints_subparsers.add_parser("list") - ) - state_checkpoints_list_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - state_checkpoints_show_parser: argparse.ArgumentParser = ( - state_checkpoints_subparsers.add_parser("show") - ) - state_checkpoints_show_parser.add_argument("state_checkpoint_id", metavar="checkpoint_id") - state_checkpoints_show_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - state_checkpoints_diff_parser: argparse.ArgumentParser = ( - state_checkpoints_subparsers.add_parser("diff") - ) - state_checkpoints_diff_parser.add_argument("state_checkpoint_id", metavar="checkpoint_id") - state_checkpoints_diff_parser.add_argument("--virtual-env", dest="virtual_env", default=None) - def _add_workspace_parsers( subparsers: argparse._SubParsersAction[argparse.ArgumentParser], @@ -711,6 +604,7 @@ def _add_workspace_parsers( scenario_subparsers: argparse._SubParsersAction[argparse.ArgumentParser] scenario_subparsers = scenario_parser.add_subparsers(dest="scenario_command") scenario_test_parser: argparse.ArgumentParser = scenario_subparsers.add_parser("test") + scenario_test_parser.add_argument("--scenario-namespace", metavar="NAMESPACE") scenario_test_parser.add_argument("scenario_selector", nargs="*", metavar="scenario") scenario_test_parser.add_argument("--retain", dest="scenario_retain", action="store_true") scenario_test_parser.add_argument("--local", dest="scenario_local", action="store_true") @@ -727,6 +621,7 @@ def _add_workspace_parsers( scenario_snapshot_group.add_argument("--refresh", dest="scenario_refresh", action="store_true") _ = add_scenario_snapshot_safety_args(scenario_test_parser) scenario_capture_parser: argparse.ArgumentParser = scenario_subparsers.add_parser("capture") + scenario_capture_parser.add_argument("--scenario-namespace", metavar="NAMESPACE") scenario_capture_parser.add_argument("scenario_selector", nargs="*", metavar="scenario") scenario_capture_parser.add_argument("--retain", dest="scenario_retain", action="store_true") _ = add_select_args(scenario_capture_parser) diff --git a/src/sqlbuild/cli/commands/_helpers/freshness/state.py b/src/sqlbuild/cli/commands/_helpers/freshness/state.py index fa7b837eb6..c072959494 100644 --- a/src/sqlbuild/cli/commands/_helpers/freshness/state.py +++ b/src/sqlbuild/cli/commands/_helpers/freshness/state.py @@ -7,13 +7,9 @@ from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter from sqlbuild.adapter.contract.models import RelationLookup from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.compiler.source_freshness.main.read import read_latest_source_freshness from sqlbuild.compiler.source_freshness.models import SourceFreshnessIdentity, SourceFreshnessRecord -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.models import SourceFreshnessRecord as VirtualSourceFreshnessRecord def read_direct_freshness_state_for_command( @@ -52,60 +48,6 @@ def read_direct_freshness_state_for_command( return records -def read_virtual_freshness_state_for_command( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Any, - virtual_environment_name: str | None, -) -> dict[str, SourceFreshnessRecord]: - """Read virtual source freshness state by source name for one virtual environment.""" - - config: Any - backend: Any - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - state_connection: Any = backend.connect(config.connection) - try: - target_name: str | None = virtual_environment_name or resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if target_name is None: - return {} - records: tuple[VirtualSourceFreshnessRecord, ...] = ( - backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=config.schema, - virtual_environment_name=target_name, - ) - ) - return { - record.source_name: _direct_record_from_virtual_record(record) for record in records - } - finally: - backend.close(state_connection) - - -def _direct_record_from_virtual_record( - record: VirtualSourceFreshnessRecord, -) -> SourceFreshnessRecord: - return SourceFreshnessRecord( - source_name=record.source_name, - target_database=None, - target_schema=None, - target_name=None, - run_id=record.virtual_environment_name, - strategy=record.strategy, - value_kind=record.value_kind, - data_version=record.data_version, - data_version_hash=record.data_version_hash, - observed_at=record.observed_at, - ) - - def _resolve_state_database(*, project: Any) -> str | None: destination: Any for destination in _iter_state_destinations(project=project): diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py index a516b1629c..de4d7cce9e 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/output.py @@ -8,16 +8,11 @@ from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, JanitorArchivedRelation, - JanitorCheckpointCandidate, JanitorDeleteCandidate, - JanitorDetachedVirtualEnvironmentCandidate, - JanitorExpiredLockCandidate, - JanitorExpiredVirtualEnvironmentCandidate, JanitorPlan, JanitorQueryDiffArtifactCandidate, JanitorSkippedRelation, JanitorSkippedSchema, - JanitorStateBackupCandidate, ) from sqlbuild.presentation.classes.cli_style import CliStyle @@ -83,48 +78,12 @@ def _write_plan_summary(*, plan: JanitorPlan, stream: TextIO, style: CliStyle) - label="query artifacts", items=plan.query_diff_artifact_candidates, ) - _write_count_row( - stream=stream, - style=style, - label="checkpoints pruned", - items=plan.checkpoint_candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="detached VDEs pruned", - items=plan.detached_virtual_environment_candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="expired VDEs pruned", - items=plan.expired_virtual_environment_candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="state backups pruned", - items=plan.state_backup_candidates, - ) - _write_count_row( - stream=stream, - style=style, - label="expired locks pruned", - items=plan.expired_lock_candidates, - ) _write_count_row( stream=stream, style=style, label="direct state pruned", items=plan.direct_state_prune_candidates, ) - _write_count_row( - stream=stream, - style=style, - label="virtual state pruned", - items=plan.virtual_state_prune_candidates, - ) skipped_count: str = style.accent(str(len(plan.skipped_relations))) stream.write(f" {'objects skipped':<22} {skipped_count}\n") @@ -248,52 +207,6 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> _write_query_artifact_candidates(plan=plan, stream=stream, style=style) - if plan.checkpoint_candidates: - stream.write(f"\n{style.success('Eligible checkpoints')}\n") - checkpoint_candidate: JanitorCheckpointCandidate - for checkpoint_candidate in plan.checkpoint_candidates: - stream.write( - f" {style.object_name(checkpoint_candidate.checkpoint_id)} " - f"{style.muted(checkpoint_candidate.virtual_environment_name)}\n" - ) - - if plan.detached_virtual_environment_candidates: - stream.write(f"\n{style.success('Eligible detached VDEs')}\n") - detached_candidate: JanitorDetachedVirtualEnvironmentCandidate - for detached_candidate in plan.detached_virtual_environment_candidates: - stream.write( - f" {style.object_name(detached_candidate.virtual_environment_name)} " - f"{style.muted('detached virtual environment')}\n" - ) - - if plan.expired_virtual_environment_candidates: - stream.write(f"\n{style.success('Eligible expired VDEs')}\n") - expired_environment_candidate: JanitorExpiredVirtualEnvironmentCandidate - for expired_environment_candidate in plan.expired_virtual_environment_candidates: - target_name: str = expired_environment_candidate.virtual_environment_name - stream.write( - f" {style.object_name(target_name)} " - f"{style.muted('expired virtual environment')}\n" - ) - - if plan.state_backup_candidates: - stream.write(f"\n{style.success('Eligible state backups')}\n") - state_backup_candidate: JanitorStateBackupCandidate - for state_backup_candidate in plan.state_backup_candidates: - stream.write( - f" {style.object_name(state_backup_candidate.backup_id)} " - f"{style.muted(state_backup_candidate.schema_name)}\n" - ) - - if plan.expired_lock_candidates: - stream.write(f"\n{style.success('Eligible expired locks')}\n") - expired_lock_candidate: JanitorExpiredLockCandidate - for expired_lock_candidate in plan.expired_lock_candidates: - stream.write( - f" {style.object_name(expired_lock_candidate.lock_key)} " - f"{style.muted(expired_lock_candidate.owner_id)}\n" - ) - if plan.direct_state_prune_candidates: stream.write(f"\n{style.success('Eligible direct state pruning')}\n") for direct_state_candidate in plan.direct_state_prune_candidates: @@ -302,14 +215,6 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> f"{style.muted(f'keep latest {direct_state_candidate.retain_versions}')}\n" ) - if plan.virtual_state_prune_candidates: - stream.write(f"\n{style.success('Eligible virtual state pruning')}\n") - for virtual_state_candidate in plan.virtual_state_prune_candidates: - stream.write( - f" {style.object_name(virtual_state_candidate.display_name())} " - f"{style.muted(virtual_state_candidate.reason)}\n" - ) - if plan.skipped_relations: stream.write(f"\n{style.success('Skipped objects')}\n") skipped: JanitorSkippedRelation @@ -324,15 +229,7 @@ def write_plan(*, plan: JanitorPlan, stream: TextIO, use_color: bool = False) -> def confirmation_text(plan: JanitorPlan) -> str: """Build the exact confirmation phrase for a janitor plan.""" - state_candidate_count: int = ( - len(plan.checkpoint_candidates) - + len(plan.detached_virtual_environment_candidates) - + len(plan.expired_virtual_environment_candidates) - + len(plan.state_backup_candidates) - + len(plan.expired_lock_candidates) - + len(plan.direct_state_prune_candidates) - + len(plan.virtual_state_prune_candidates) - ) + state_candidate_count: int = len(plan.direct_state_prune_candidates) archive_prefix: str = ( f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py index 5126d0d108..f7d2a47eee 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_output/outputs.py @@ -43,13 +43,7 @@ def janitor_plan_has_work(planning_result: JanitorPlanningResult) -> bool: or plan.archive_candidates or plan.archive_deletion_candidates or plan.query_diff_artifact_candidates - or plan.checkpoint_candidates - or plan.detached_virtual_environment_candidates - or plan.expired_virtual_environment_candidates - or plan.state_backup_candidates - or plan.expired_lock_candidates or plan.direct_state_prune_candidates - or plan.virtual_state_prune_candidates ) @@ -58,23 +52,13 @@ def confirm_janitor_plan(*, planning_result: JanitorPlanningResult) -> bool: plan: JanitorPlan = planning_result.plan expected: str = confirmation_text(plan) - state_candidate_count: int = ( - len(plan.checkpoint_candidates) - + len(plan.detached_virtual_environment_candidates) - + len(plan.expired_virtual_environment_candidates) - + len(plan.state_backup_candidates) - + len(plan.expired_lock_candidates) - + len(plan.virtual_state_prune_candidates) - ) - prune_count: int = len(plan.direct_state_prune_candidates) + len( - plan.virtual_state_prune_candidates - ) + prune_count: int = len(plan.direct_state_prune_candidates) archive_prefix: str = ( f"archive {len(plan.archive_candidates)} and " if plan.archive_candidates else "" ) - if state_candidate_count or prune_count: + if prune_count: physical_deletion_count: int = physical_janitor_deletion_count(plan) - deletion_count: int = physical_deletion_count + state_candidate_count + prune_count + deletion_count: int = physical_deletion_count + prune_count sys.stdout.write( f"Janitor will {archive_prefix}delete {deletion_count} items " f"from {environment_label(plan)}.\n" @@ -131,33 +115,14 @@ def _deleted_message(*, result: JanitorExecutionResult) -> str: def _deletion_summary(*, result: JanitorExecutionResult) -> str: - deleted_state_count: int = ( - len(result.deleted_checkpoints) - + len(result.deleted_detached_virtual_environments) - + len(result.deleted_expired_virtual_environments) - + len(result.deleted_state_backups) - + len(result.deleted_expired_locks) - ) deleted_object_count: int = ( len(result.deleted) + len(result.deleted_archives) + len(result.deleted_query_diff_artifacts) ) objects: str = format_count_noun(count=deleted_object_count, singular="object") - pruned_state_count: int = len(result.pruned_direct_state) + len(result.pruned_virtual_state) - non_checkpoint_state_count: int = deleted_state_count - len(result.deleted_checkpoints) - if non_checkpoint_state_count or pruned_state_count: - pruned_state_label: str = ( - "state table" if result.pruned_virtual_state else "direct state table" - ) - return ( - f"Deleted {objects}, deleted " - f"{format_count_noun(count=deleted_state_count, singular='state item')}, and pruned " - f"{format_count_noun(count=pruned_state_count, singular=pruned_state_label)}." - ) - if result.deleted_checkpoints: - checkpoints: str = format_count_noun( - count=len(result.deleted_checkpoints), singular="checkpoint" - ) - return f"Deleted {objects} and {checkpoints}." + pruned_state_count: int = len(result.pruned_direct_state) + if pruned_state_count: + pruned: str = format_count_noun(count=pruned_state_count, singular="direct state table") + return f"Deleted {objects} and pruned {pruned}." return f"Deleted {objects}." diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_retention/checkpoints.py b/src/sqlbuild/cli/commands/_helpers/janitor_retention/checkpoints.py deleted file mode 100644 index 9f3529f1f2..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_retention/checkpoints.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Janitor checkpoint planning helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.janitor.models import JanitorCheckpointCandidate, JanitorRelationKey -from sqlbuild.virtual.state.main.checkpoints.checkpoint_retention import ( - inspect_checkpoint_retention, -) -from sqlbuild.virtual.state.models import CheckpointRetentionInspection, PhysicalRelationRecord - - -def checkpoint_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str | None, -) -> CheckpointRetentionInspection | None: - """Inspect virtual checkpoint retention when janitor runs in virtual mode.""" - - if not discovered_inputs.project_config.settings.virtual_environments: - return None - if virtual_environment_name is None: - return None - return inspect_checkpoint_retention( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - virtual_environment_name=virtual_environment_name, - max_checkpoints=discovered_inputs.project_config.janitor.max_checkpoints, - ) - - -def checkpoint_protected_relation_keys( - *, - retention: CheckpointRetentionInspection | None, -) -> frozenset[JanitorRelationKey]: - """Build janitor relation keys protected by retained checkpoints.""" - - if retention is None: - return frozenset() - relations: tuple[PhysicalRelationRecord, ...] = retention.retained_physical_relations - return frozenset( - JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ) - for relation in relations - ) - - -def checkpoint_protected_relation_reasons( - *, - retention: CheckpointRetentionInspection | None, -) -> dict[JanitorRelationKey, str]: - """Build protection reasons for retained checkpoint physical relations.""" - - if retention is None: - return {} - return { - JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ): "relation is referenced by a retained virtual checkpoint" - for relation in retention.retained_physical_relations - } - - -def checkpoint_candidates( - *, - retention: CheckpointRetentionInspection | None, -) -> tuple[JanitorCheckpointCandidate, ...]: - """Build janitor checkpoint pruning candidates.""" - - if retention is None: - return () - return tuple( - JanitorCheckpointCandidate( - checkpoint_id=checkpoint.checkpoint_id, - virtual_environment_name=checkpoint.virtual_environment_name, - created_at=checkpoint.created_at, - ) - for checkpoint in retention.prune_checkpoints - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_retention/detached_environments.py b/src/sqlbuild/cli/commands/_helpers/janitor_retention/detached_environments.py deleted file mode 100644 index 32e7f3dc81..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_retention/detached_environments.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Janitor detached VDE planning helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.janitor.models import ( - JanitorDetachedVirtualEnvironmentCandidate, - JanitorRelationKey, -) -from sqlbuild.virtual.state.main.retention.detached_environment_retention import ( - inspect_detached_environment_retention, -) -from sqlbuild.virtual.state.models import ( - DetachedVirtualEnvironmentInspection, - PhysicalRelationRecord, -) - - -def detached_environment_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - retention_days: int, -) -> DetachedVirtualEnvironmentInspection | None: - """Inspect detached VDE cleanup when janitor runs in virtual mode.""" - - if not discovered_inputs.project_config.settings.virtual_environments: - return None - return inspect_detached_environment_retention( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - retention_days=retention_days, - ) - - -def detached_environment_protected_relation_keys( - *, - retention: DetachedVirtualEnvironmentInspection | None, -) -> frozenset[JanitorRelationKey]: - """Build janitor relation keys protected by retained VDE current refs.""" - - if retention is None: - return frozenset() - relations: tuple[PhysicalRelationRecord, ...] = retention.retained_physical_relations - return frozenset( - JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ) - for relation in relations - ) - - -def detached_environment_scan_relation_keys( - *, - retention: DetachedVirtualEnvironmentInspection | None, -) -> frozenset[JanitorRelationKey]: - """Build relation keys whose schemas must be scanned for detached VDE cleanup.""" - - if retention is None: - return frozenset() - return frozenset( - JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ) - for relation in retention.cleanup_physical_relations - ) - - -def detached_environment_protected_relation_reasons( - *, - retention: DetachedVirtualEnvironmentInspection | None, -) -> dict[JanitorRelationKey, str]: - """Build protection reasons for retained VDE current-ref physical relations.""" - - if retention is None: - return {} - return { - JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ): "relation is referenced by an active or retained virtual environment" - for relation in retention.retained_physical_relations - } - - -def detached_environment_candidates( - *, - retention: DetachedVirtualEnvironmentInspection | None, -) -> tuple[JanitorDetachedVirtualEnvironmentCandidate, ...]: - """Build janitor detached VDE cleanup candidates.""" - - if retention is None: - return () - return tuple( - JanitorDetachedVirtualEnvironmentCandidate( - virtual_environment_name=environment.virtual_environment_name, - updated_at=environment.updated_at, - ) - for environment in retention.cleanup_virtual_environments - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_retention/expired_environments.py b/src/sqlbuild/cli/commands/_helpers/janitor_retention/expired_environments.py deleted file mode 100644 index cb539adb4c..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_retention/expired_environments.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Janitor expired VDE planning helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.janitor.models import ( - JanitorExpiredVirtualEnvironmentCandidate, - JanitorRelationKey, -) -from sqlbuild.virtual.state.main.retention.expired_environment_retention import ( - inspect_expired_environment_retention, -) -from sqlbuild.virtual.state.models import ( - ExpiredVirtualEnvironmentInspection, - PhysicalRelationRecord, -) - - -def expired_environment_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - active_virtual_environment_name: str | None, - retention_days: int, -) -> ExpiredVirtualEnvironmentInspection | None: - """Inspect non-active VDE cleanup when janitor runs in virtual mode.""" - - if not discovered_inputs.project_config.settings.virtual_environments: - return None - return inspect_expired_environment_retention( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - active_virtual_environment_name=active_virtual_environment_name, - retention_days=retention_days, - ) - - -def expired_environment_protected_relation_keys( - *, - retention: ExpiredVirtualEnvironmentInspection | None, -) -> frozenset[JanitorRelationKey]: - """Build relation keys protected by retained VDE current refs.""" - - if retention is None: - return frozenset() - return frozenset(_relation_key(relation) for relation in retention.retained_physical_relations) - - -def expired_environment_scan_relation_keys( - *, - retention: ExpiredVirtualEnvironmentInspection | None, -) -> frozenset[JanitorRelationKey]: - """Build relation keys whose schemas must be scanned for VDE cleanup.""" - - if retention is None: - return frozenset() - return frozenset(_relation_key(relation) for relation in retention.cleanup_physical_relations) - - -def expired_environment_protected_relation_reasons( - *, - retention: ExpiredVirtualEnvironmentInspection | None, -) -> dict[JanitorRelationKey, str]: - """Build protection reasons for retained VDE current-ref physical relations.""" - - if retention is None: - return {} - return { - _relation_key(relation): ( - "relation is referenced by an active or retained virtual environment" - ) - for relation in retention.retained_physical_relations - } - - -def expired_environment_candidates( - *, - retention: ExpiredVirtualEnvironmentInspection | None, -) -> tuple[JanitorExpiredVirtualEnvironmentCandidate, ...]: - """Build janitor expired VDE cleanup candidates.""" - - if retention is None: - return () - return tuple( - JanitorExpiredVirtualEnvironmentCandidate( - virtual_environment_name=environment.virtual_environment_name, - updated_at=environment.updated_at, - ) - for environment in retention.cleanup_virtual_environments - ) - - -def _relation_key(relation: PhysicalRelationRecord) -> JanitorRelationKey: - return JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_retention/microbatch_replays.py b/src/sqlbuild/cli/commands/_helpers/janitor_retention/microbatch_replays.py deleted file mode 100644 index 6549f7f3a3..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_retention/microbatch_replays.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Janitor protection helpers for active virtual microbatch replays.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.janitor.models import JanitorRelationKey -from sqlbuild.virtual.state.main.retention.microbatch_replay_retention import ( - inspect_active_microbatch_replay_relations, -) -from sqlbuild.virtual.state.models import PhysicalRelationRecord - - -def active_microbatch_replay_relations( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs -) -> tuple[PhysicalRelationRecord, ...]: - """Inspect replay roots when janitor runs in virtual mode.""" - - if not discovered_inputs.project_config.settings.virtual_environments: - return () - return inspect_active_microbatch_replay_relations( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - ) - - -def active_microbatch_replay_protected_relation_keys( - *, relations: tuple[PhysicalRelationRecord, ...] -) -> frozenset[JanitorRelationKey]: - """Build relation keys protected by incomplete replay requirements.""" - - return frozenset(_relation_key(relation) for relation in relations) - - -def active_microbatch_replay_protected_relation_reasons( - *, relations: tuple[PhysicalRelationRecord, ...] -) -> dict[JanitorRelationKey, str]: - """Build protection reasons for incomplete replay physical versions.""" - - return { - _relation_key(relation): "relation has an active microbatch replay requirement" - for relation in relations - } - - -def _relation_key(relation: PhysicalRelationRecord) -> JanitorRelationKey: - return JanitorRelationKey( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_retention/state_cleanup.py b/src/sqlbuild/cli/commands/_helpers/janitor_retention/state_cleanup.py deleted file mode 100644 index 2e8397aec2..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/janitor_retention/state_cleanup.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Janitor state-only cleanup helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.janitor.models import ( - JanitorExpiredLockCandidate, - JanitorStateBackupCandidate, - JanitorVirtualStatePruneCandidate, -) -from sqlbuild.virtual.state.constants import PYTHON_NODE_VERSION_TABLE -from sqlbuild.virtual.state.main.retention.state_janitor_retention import ( - inspect_state_janitor_retention, -) -from sqlbuild.virtual.state.models import StateJanitorInspection - - -def state_janitor_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - retention_days: int, -) -> StateJanitorInspection | None: - """Inspect state-only cleanup when janitor runs in virtual mode.""" - - if not discovered_inputs.project_config.settings.virtual_environments: - return None - return inspect_state_janitor_retention( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - retention_days=retention_days, - ) - - -def state_backup_candidates( - *, - retention: StateJanitorInspection | None, -) -> tuple[JanitorStateBackupCandidate, ...]: - """Build janitor state backup cleanup candidates.""" - - if retention is None: - return () - return tuple( - JanitorStateBackupCandidate( - backup_id=backup.backup_id, - schema_name=backup.schema_name, - created_at=backup.created_at, - ) - for backup in retention.state_backups - ) - - -def expired_lock_candidates( - *, - retention: StateJanitorInspection | None, -) -> tuple[JanitorExpiredLockCandidate, ...]: - """Build janitor expired lock cleanup candidates.""" - - if retention is None: - return () - return tuple( - JanitorExpiredLockCandidate( - lock_key=lock.lock_key, - owner_id=lock.owner_id, - expires_at=lock.expires_at, - ) - for lock in retention.expired_locks - ) - - -def virtual_state_prune_candidates( - *, - retention: StateJanitorInspection | None, -) -> tuple[JanitorVirtualStatePruneCandidate, ...]: - """Build janitor virtual-state orphan cleanup candidates.""" - - if retention is None or retention.unreferenced_python_node_versions == 0: - return () - return ( - JanitorVirtualStatePruneCandidate( - schema=retention.schema, - table_name=PYTHON_NODE_VERSION_TABLE, - reason=( - f"{retention.unreferenced_python_node_versions} unreferenced Python identity " - "version(s)" - ), - ), - ) diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py index 66b41629b0..06820c93fd 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/execution.py @@ -1,108 +1,26 @@ -"""Janitor command execution phase.""" +"""Janitor command warehouse cleanup execution phase.""" from __future__ import annotations -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) from sqlbuild.cli.commands.models import ( JanitorCompileContext, JanitorConnectionContext, - JanitorInvocation, JanitorPlanningResult, - JanitorRetentionInspection, ) -from sqlbuild.compiler.compile.models import CompiledRelationLocation from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.models import JanitorExecutionResult -from sqlbuild.virtual.executor.main.virtual_target import build_virtual_destination -from sqlbuild.virtual.state.main.checkpoints.delete_checkpoint import ( - delete_virtual_environment_checkpoint, -) -from sqlbuild.virtual.state.main.environments.delete_virtual_environment import ( - delete_virtual_environment, -) -from sqlbuild.virtual.state.main.locks.delete_lock import delete_lock -from sqlbuild.virtual.state.main.python_identities.prune_python_node_identities import ( - prune_unreferenced_python_node_versions, -) -from sqlbuild.virtual.state.main.retention.delete_state_backup import delete_state_backup def execute_janitor_cleanup( *, - invocation: JanitorInvocation, compile_context: JanitorCompileContext, connection_context: JanitorConnectionContext, - inspection: JanitorRetentionInspection, planning_result: JanitorPlanningResult, ) -> JanitorExecutionResult: - """Execute the janitor plan and virtual-state callbacks.""" + """Execute the warehouse cleanup plan.""" return execute_janitor_plan( plan=planning_result.plan, adapter=compile_context.adapter, connection=connection_context.connection, - delete_checkpoint=lambda candidate: delete_virtual_environment_checkpoint( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - checkpoint_id=candidate.checkpoint_id, - ), - delete_detached_virtual_environment=lambda candidate: delete_virtual_environment( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - virtual_environment_name=candidate.virtual_environment_name, - ), - delete_expired_virtual_environment=lambda candidate: delete_virtual_environment( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - virtual_environment_name=_drop_logical_vde_views( - compile_context=compile_context, - connection_context=connection_context, - virtual_environment_name=candidate.virtual_environment_name, - unsuffixed_virtual_environment_name=inspection.unsuffixed_virtual_environment_name, - ), - ), - delete_state_backup=lambda candidate: delete_state_backup( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - backup_id=candidate.backup_id, - ), - delete_expired_lock=lambda candidate: delete_lock( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - lock_key=candidate.lock_key, - ), - prune_virtual_state=lambda candidate: prune_unreferenced_python_node_versions( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), ) - - -def _drop_logical_vde_views( - *, - compile_context: JanitorCompileContext, - connection_context: JanitorConnectionContext, - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, -) -> str: - recorder: StatementRecorder = StatementRecorder() - for model in compile_context.project.models: - virtual_target: CompiledRelationLocation = build_virtual_destination( - adapter=compile_context.adapter, - target=model.destination, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - compile_context.adapter.drop_view( - connection=connection_context.connection, - destination=resolve_relation_location_qualified_name( - adapter=compile_context.adapter, - location=virtual_target, - ), - if_exists=True, - statement_recorder=recorder, - ) - return virtual_environment_name diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py index 53e5bee982..25e3cdfcaa 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/invocation.py @@ -40,7 +40,6 @@ def resolve_janitor_settings( retention_days: int = resolve_janitor_retention_days( override=request.retention_days, configured=invocation.discovered_inputs.project_config.janitor.retention_days, - virtual_environments=invocation.discovered_inputs.project_config.settings.virtual_environments, ) if retention_days < 0: raise CliUserError("janitor --retention-days must be >= 0", code="C501") diff --git a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py index fa126f10e7..329c57f9a5 100644 --- a/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/janitor_runtime/planning.py @@ -1,119 +1,21 @@ -"""Janitor command inspection and planning phases.""" +"""Janitor command warehouse inspection and planning phases.""" from __future__ import annotations import sys import time -from sqlbuild.cli.commands._helpers.janitor_retention.checkpoints import ( - checkpoint_candidates, - checkpoint_protected_relation_keys, - checkpoint_protected_relation_reasons, - checkpoint_retention, -) -from sqlbuild.cli.commands._helpers.janitor_retention.detached_environments import ( - detached_environment_candidates, - detached_environment_protected_relation_keys, - detached_environment_protected_relation_reasons, - detached_environment_retention, - detached_environment_scan_relation_keys, -) -from sqlbuild.cli.commands._helpers.janitor_retention.expired_environments import ( - expired_environment_candidates, - expired_environment_protected_relation_keys, - expired_environment_protected_relation_reasons, - expired_environment_retention, - expired_environment_scan_relation_keys, -) -from sqlbuild.cli.commands._helpers.janitor_retention.microbatch_replays import ( - active_microbatch_replay_protected_relation_keys, - active_microbatch_replay_protected_relation_reasons, - active_microbatch_replay_relations, -) -from sqlbuild.cli.commands._helpers.janitor_retention.state_cleanup import ( - expired_lock_candidates, - state_backup_candidates, - state_janitor_retention, - virtual_state_prune_candidates, -) from sqlbuild.cli.commands.models import ( JanitorCompileContext, JanitorConnectionContext, JanitorInvocation, JanitorPlanningResult, - JanitorRetentionInspection, JanitorSettings, ) from sqlbuild.executor.janitor.main.plan import build_janitor_plan -from sqlbuild.executor.janitor.models import ( - JanitorDirectModeSettings, - JanitorPlan, - JanitorRelationKey, - JanitorRelationScope, - JanitorStateCandidates, -) +from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.virtual.state.models import ( - CheckpointRetentionInspection, - DetachedVirtualEnvironmentInspection, - ExpiredVirtualEnvironmentInspection, - PhysicalRelationRecord, - StateJanitorInspection, -) - - -def inspect_janitor_retention( - *, - invocation: JanitorInvocation, - settings: JanitorSettings, - compile_context: JanitorCompileContext, -) -> JanitorRetentionInspection: - """Inspect virtual state retention candidates.""" - - with OperationLifecycle(operation_kind="janitor", operation_name="janitor_state_inspection"): - checkpoint: CheckpointRetentionInspection | None = checkpoint_retention( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - virtual_environment_name=compile_context.project.effective_target_name, - ) - detached_environment: DetachedVirtualEnvironmentInspection | None = ( - detached_environment_retention( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - retention_days=settings.retention_days, - ) - ) - expired_environment: ExpiredVirtualEnvironmentInspection | None = ( - expired_environment_retention( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - active_virtual_environment_name=compile_context.project.effective_target_name, - retention_days=settings.retention_days, - ) - ) - unsuffixed_virtual_environment_name: str | None = _unsuffixed_virtual_environment_name( - invocation=invocation, - compile_context=compile_context, - ) - state: StateJanitorInspection | None = state_janitor_retention( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - retention_days=settings.retention_days, - ) - replay_relations: tuple[PhysicalRelationRecord, ...] = active_microbatch_replay_relations( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ) - return JanitorRetentionInspection( - checkpoint=checkpoint, - detached_environment=detached_environment, - expired_environment=expired_environment, - state=state, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - active_microbatch_replay_relations=replay_relations, - ) def build_janitor_execution_plan( @@ -122,9 +24,8 @@ def build_janitor_execution_plan( settings: JanitorSettings, compile_context: JanitorCompileContext, connection_context: JanitorConnectionContext, - inspection: JanitorRetentionInspection, ) -> JanitorPlanningResult: - """Build the janitor execution plan from retention inspections.""" + """Build the janitor execution plan from warehouse facts.""" inspect_start: float = time.perf_counter() status: TransientStatusReporter = TransientStatusReporter( @@ -132,28 +33,6 @@ def build_janitor_execution_plan( use_color=invocation.use_color, ) status.start("Inspecting warehouse state...") - protected_relation_keys: frozenset[JanitorRelationKey] = ( - checkpoint_protected_relation_keys(retention=inspection.checkpoint) - | detached_environment_protected_relation_keys(retention=inspection.detached_environment) - | expired_environment_protected_relation_keys(retention=inspection.expired_environment) - | active_microbatch_replay_protected_relation_keys( - relations=inspection.active_microbatch_replay_relations - ) - ) - protected_relation_reasons: dict[JanitorRelationKey, str] = ( - detached_environment_protected_relation_reasons(retention=inspection.detached_environment) - ) - protected_relation_reasons.update( - expired_environment_protected_relation_reasons(retention=inspection.expired_environment) - ) - protected_relation_reasons.update( - checkpoint_protected_relation_reasons(retention=inspection.checkpoint) - ) - protected_relation_reasons.update( - active_microbatch_replay_protected_relation_reasons( - relations=inspection.active_microbatch_replay_relations - ) - ) with OperationLifecycle( operation_kind="janitor", operation_name="janitor_candidate_planning" ) as lifecycle: @@ -164,32 +43,8 @@ def build_janitor_execution_plan( retention_days=settings.retention_days, delete_tracked_only=invocation.discovered_inputs.project_config.janitor.delete_tracked_only, exclude_patterns=invocation.discovered_inputs.project_config.janitor.exclude_patterns, - relation_scope=JanitorRelationScope( - scan_relation_keys=detached_environment_scan_relation_keys( - retention=inspection.detached_environment - ) - | expired_environment_scan_relation_keys(retention=inspection.expired_environment), - protected_relation_keys=protected_relation_keys, - protected_relation_reasons=protected_relation_reasons, - ), - state_candidates=JanitorStateCandidates( - checkpoint_candidates=checkpoint_candidates(retention=inspection.checkpoint), - detached_virtual_environment_candidates=detached_environment_candidates( - retention=inspection.detached_environment - ), - expired_virtual_environment_candidates=expired_environment_candidates( - retention=inspection.expired_environment - ), - state_backup_candidates=state_backup_candidates(retention=inspection.state), - expired_lock_candidates=expired_lock_candidates(retention=inspection.state), - virtual_state_prune_candidates=virtual_state_prune_candidates( - retention=inspection.state - ), - ), direct_settings=JanitorDirectModeSettings( - enabled=( - not invocation.discovered_inputs.project_config.settings.virtual_environments - ), + enabled=True, state_history_versions=settings.direct_state_history_versions, archive_retention_days=settings.archive_retention_days, ), @@ -209,24 +64,6 @@ def _janitor_candidate_count(plan: JanitorPlan) -> int: plan.candidates, plan.archive_deletion_candidates, plan.query_diff_artifact_candidates, - plan.checkpoint_candidates, - plan.detached_virtual_environment_candidates, - plan.expired_virtual_environment_candidates, - plan.state_backup_candidates, - plan.expired_lock_candidates, plan.direct_state_prune_candidates, - plan.virtual_state_prune_candidates, ) ) - - -def _unsuffixed_virtual_environment_name( - *, invocation: JanitorInvocation, compile_context: JanitorCompileContext -) -> str | None: - if compile_context.project.effective_target_name is None: - return None - return resolve_target_config( - project_config=invocation.discovered_inputs.project_config, - local_config=invocation.discovered_inputs.local_config, - target_name=compile_context.project.effective_target_name, - ).state.unsuffixed_virtual_env diff --git a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py index 2a1677f1d9..26479888f1 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/invocation.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/invocation.py @@ -9,10 +9,6 @@ from sqlbuild.cli.commands._helpers.runtime.adapter_context import ( resolve_adapter_connection_context, ) -from sqlbuild.cli.commands._helpers.runtime.mode_policy import ( - enforce_no_defer_to_in_virtual_mode, - enforce_virtual_only_flags_in_virtual_mode, -) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.models import ( AdapterConnectionContext, @@ -27,9 +23,6 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_changes_only import ( - resolve_effective_changes_only, -) def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: @@ -41,25 +34,7 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=effective_project_dir ) - effective_changes_only: bool = resolve_effective_changes_only( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=request.selected_target, - cli_changes_only=request.changes_only, - ) _validate_preview_target(discovered_inputs=discovered_inputs, as_target=request.as_target) - enforce_no_defer_to_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name="plan", - defer_to=request.defer_to, - ) - enforce_virtual_only_flags_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name="plan", - virtual_env=request.virtual_env, - include_stale_upstreams=request.include_stale_upstreams, - changes_only=effective_changes_only, - ) adapter_context: AdapterConnectionContext = resolve_adapter_connection_context( discovered_inputs=discovered_inputs, effective_project_dir=effective_project_dir, @@ -81,7 +56,6 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: return PlanInvocation( effective_project_dir=effective_project_dir, discovered_inputs=discovered_inputs, - effective_changes_only=effective_changes_only, adapter=adapter_context.adapter, connection_config=adapter_context.connection_config, use_color=use_color, @@ -89,7 +63,6 @@ def resolve_plan_invocation(*, request: PlanCommandRequest) -> PlanInvocation: connection_progress=reporters.connection, planning_progress=reporters.planning, should_load_sources=should_load_sources, - virtual_mode=bool(discovered_inputs.project_config.settings.virtual_environments), ) @@ -98,11 +71,6 @@ def _validate_preview_target( ) -> None: if as_target is None: return - if discovered_inputs.project_config.settings.virtual_environments: - raise CliUserError( - "plan --as is supported only in direct mode", - help="Virtual environments preview other environments with --virtual-env.", - ) configured: set[str] = set(discovered_inputs.project_config.targets) | set( discovered_inputs.local_config.targets ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py index 38700c175c..8044b5a2cc 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/outputs.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/outputs.py @@ -38,9 +38,7 @@ def write_plan_command_output( use_color=invocation.use_color, display_options=display_options, python_plan_entries=pipeline_result.python_plan_entries, - include_direct_freshness_diagnostics=( - invocation.virtual_mode or invocation.effective_changes_only - ), + include_direct_freshness_diagnostics=(False), ) + "\n" ) diff --git a/src/sqlbuild/cli/commands/_helpers/plan/planning.py b/src/sqlbuild/cli/commands/_helpers/plan/planning.py index e760773b14..58fa03ed69 100644 --- a/src/sqlbuild/cli/commands/_helpers/plan/planning.py +++ b/src/sqlbuild/cli/commands/_helpers/plan/planning.py @@ -2,11 +2,6 @@ from __future__ import annotations -from io import StringIO - -from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( - enforce_snapshot_full_refresh_policy, -) from sqlbuild.cli.commands._helpers.planning.external_refs import ( resolve_external_sql_reference_resolver, ) @@ -17,8 +12,6 @@ CompilePipelineResult, ) from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.virtual.planner.main.plan import run_virtual_plan_pipeline -from sqlbuild.virtual.planner.models import VirtualPlanOptions def compile_plan_pipeline( @@ -26,59 +19,12 @@ def compile_plan_pipeline( request: PlanCommandRequest, invocation: PlanInvocation, ) -> CompilePipelineResult: - """Compile the plan pipeline in virtual or direct mode.""" + """Compile the plan pipeline.""" external_sql_reference_resolver: object | None = resolve_external_sql_reference_resolver( project_dir=invocation.effective_project_dir, discovered_inputs=invocation.discovered_inputs, ) - if invocation.virtual_mode: - virtual_result: CompilePipelineResult = run_virtual_plan_pipeline( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - connection_config=invocation.connection_config, - options=VirtualPlanOptions( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - defer_sources_to=request.defer_sources_to, - cursor_overrides=request.cursor_overrides, - full_refresh=request.full_refresh, - virtual_environment_name=request.virtual_env, - include_stale_upstreams=request.include_stale_upstreams, - changes_only=invocation.effective_changes_only, - auto_load_sources=invocation.should_load_sources, - include_python=request.include_python, - select=request.select, - exclude=request.exclude, - cli_vars=request.cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - max_microbatches=request.max_microbatches, - ), - hooks=ConnectionHooks( - on_progress=invocation.planning_progress.on_progress, - on_connection_start=invocation.connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - invocation.connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - enforce_snapshot_full_refresh_policy( - plan=virtual_result.plan_output, - snapshots_config=invocation.discovered_inputs.project_config.snapshots, - allow_snapshot_full_refresh=False, - input_stream=StringIO(), - output_stream=StringIO(), - ) - return virtual_result if request.as_target is not None: invocation.progress_stream.write( f"Previewing plan as target '{request.as_target}' through the active target's " diff --git a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py index e206cab35c..e8d1d6f59c 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py @@ -46,20 +46,7 @@ def _suggested_commands(*, template: PlaygroundTemplate, display_path: str) -> t f"{display_path}/{PLAYGROUND_ORCHESTRATED_PROJECT_DIR}" if orchestrated else display_path ) commands: list[str] = [f"cd {project_path}"] - if template == PlaygroundTemplate.VIRTUAL: - commands.extend( - [ - "sqb state init", - "sqb build", - "sqb build --virtual-env pr", - "sqb test", - "sqb audit", - "sqb scenario test", - "sqb diff dev:pr --schema-only", - "sqb promote --from pr --to dev", - ] - ) - elif template == PlaygroundTemplate.PYTHON_NODES: + if template == PlaygroundTemplate.PYTHON_NODES: commands.extend( [ "sqb plan --select +fact_orders --select +orders_export", @@ -82,7 +69,6 @@ def _example_name(template: PlaygroundTemplate) -> str: labels: dict[PlaygroundTemplate, str] = { PlaygroundTemplate.DAGSTER: "waffle shop + Dagster", PlaygroundTemplate.RIVERS: "waffle shop + Rivers", - PlaygroundTemplate.VIRTUAL: "virtual environments waffle shop", PlaygroundTemplate.LOADER_WAFFLE_SHOP: "loader-focused waffle shop", PlaygroundTemplate.PYTHON_NODES: "Python nodes demo", } diff --git a/src/sqlbuild/cli/commands/_helpers/playground/copy.py b/src/sqlbuild/cli/commands/_helpers/playground/copy.py index 9951baf7a8..76e3b3ca2b 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/copy.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/copy.py @@ -185,236 +185,6 @@ def check_orders_export(ctx): ) ''' -_VIRTUAL_PROJECT_TOML: str = """name = "loader_waffle_shop_virtual" -adapter = "duckdb" -default_target = "dev" - -[connections.developer] -database = "loader_waffle_shop_virtual.duckdb" - -[settings] -virtual_environments = true -default_audit_severity = "warn" - -[defaults] -materialized = "table" - -[targets.dev] -connection = "developer" -schema = "dev" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" -unsuffixed_virtual_env = "dev" -allow_reset = true - -[targets.dev.state.connection] -database = "loader_waffle_shop_state.duckdb" -""" - -_VIRTUAL_FACT_WAFFLE_ORDERS_SQL: str = """MODEL ( - materialized table, - columns ( - order_id (nullable false, audits [not_null, unique]), - revenue_cents (nullable false, audits [not_null]), - ), -); - -SELECT - o.order_id, - o.customer_id, - c.plan_name, - o.waffle_type, - p.waffle_category, - o.quantity, - o.price_cents, - o.quantity * o.price_cents AS revenue_cents, - o.load_seq -FROM __source("raw_orders") o -LEFT JOIN __source("raw_customers") c ON o.customer_id = c.customer_id -LEFT JOIN __seed("waffle_price_tiers") p ON o.waffle_type = p.waffle_type -""" - -_VIRTUAL_TEST_FACT_WAFFLE_ORDERS_SQL: str = """TEST(); - -WITH -__source__raw_orders AS ( - SELECT - 1 AS order_id, - 10 AS customer_id, - 'classic' AS waffle_type, - 2 AS quantity, - 600 AS price_cents, - 1 AS load_seq -), -__source__raw_customers AS ( - SELECT - 10 AS customer_id, - 'plus' AS plan_name, - 1 AS load_seq -), -__seed__waffle_price_tiers AS ( - SELECT 'classic' AS waffle_type, 'standard' AS waffle_category -), -__expected__fact_waffle_orders AS ( - SELECT - 1 AS order_id, - 10 AS customer_id, - 'plus' AS plan_name, - 'classic' AS waffle_type, - 'standard' AS waffle_category, - 2 AS quantity, - 600 AS price_cents, - 1200 AS revenue_cents, - 1 AS load_seq -), -__assert__revenue_is_non_negative AS ( - SELECT * - FROM __ref("fact_waffle_orders") - WHERE revenue_cents < 0 -) -SELECT 1 -""" - -_VIRTUAL_SCENARIO_CUSTOMER_REVENUE_SQL: str = """SCENARIO ( - description "Customer revenue includes seed-backed waffle categories", - tags ["virtual", "example"] -); - -WITH -__source__raw_orders AS ( - SELECT - 1 AS order_id, - 10 AS customer_id, - 'classic' AS waffle_type, - 2 AS quantity, - 600 AS price_cents, - 1 AS load_seq - UNION ALL - SELECT - 2 AS order_id, - 10 AS customer_id, - 'blueberry' AS waffle_type, - 1 AS quantity, - 750 AS price_cents, - 1 AS load_seq -), -__source__raw_customers AS ( - SELECT - 10 AS customer_id, - 'plus' AS plan_name, - 1 AS load_seq -), -__seed__waffle_price_tiers AS ( - SELECT 'classic' AS waffle_type, 'standard' AS waffle_category - UNION ALL - SELECT 'blueberry' AS waffle_type, 'fruit' AS waffle_category -), -__expected__customer_revenue AS ( - SELECT - 10 AS customer_id, - 'plus' AS plan_name, - 1950 AS revenue_cents, - 2 AS order_count -), -__assert__no_negative_customer_revenue AS ( - SELECT * - FROM __ref("customer_revenue") - WHERE revenue_cents < 0 -) -SELECT 1 -""" - -_VIRTUAL_SEED_SCHEMA_YML: str = """seeds: - - name: waffle_price_tiers - description: Waffle category lookup used by the virtual playground. - columns: - - name: waffle_type - type: VARCHAR - - name: waffle_category - type: VARCHAR -""" - -_VIRTUAL_SEED_CSV: str = """waffle_type,waffle_category -classic,standard -blueberry,fruit -chocolate,dessert -""" - -_VIRTUAL_README: str = """# SQLBuild Virtual Environments Playground - -This project is a local DuckDB playground for SQLBuild virtual targets. It is -self-contained and does not require warehouse credentials. - -Virtual environments are an advanced, state-backed workflow. This playground uses a -local DuckDB state database so you can try the lifecycle without operating shared -infrastructure. - -## Try It - -Initialize the state store: - -```bash -sqb state init -``` - -Build the default `dev` virtual environment: - -```bash -sqb build -``` - -Create a branch-like virtual environment: - -```bash -sqb build --virtual-env pr -``` - -Modify a model, then rebuild only the branch: - -```bash -sqb build --virtual-env pr -``` - -Run the project checks: - -```bash -sqb test -sqb audit -sqb scenario test -``` - -Compare the branch with `dev`: - -```bash -sqb diff dev:pr --schema-only -``` - -Promote the branch into `dev`: - -```bash -sqb promote --from pr --to dev -``` - -Inspect and roll back checkpoints: - -```bash -sqb state checkpoints list --virtual-env dev -sqb rollback --virtual-env dev -``` - -## What This Shows - -- DuckDB-backed local virtual environment execution -- A local DuckDB state store -- Branch-like virtual environments with `--virtual-env` -- Low-copy promotion through versioned pointers and logical views -- Checkpoint-backed rollback -- Loader-backed waffle shop models without external services -- A seed lookup that is loaded before virtual model execution -- Unit test, audit, and scenario commands that exercise the generated project -""" _DAGSTER_DEFINITIONS: str = '''"""Dagster definitions for the SQLBuild waffle shop playground.""" @@ -586,7 +356,7 @@ def create_playground_project( template_path: str = ( _LOADER_WAFFLE_SHOP_TEMPLATE - if resolved_template in (PlaygroundTemplate.LOADER_WAFFLE_SHOP, PlaygroundTemplate.VIRTUAL) + if resolved_template == PlaygroundTemplate.LOADER_WAFFLE_SHOP else _WAFFLE_SHOP_TEMPLATE ) template_root: Traversable = files(_TEMPLATE_PACKAGE).joinpath(template_path) @@ -613,8 +383,6 @@ def create_playground_project( ) return _copy_tree(source=template_root, target=target_dir) - if resolved_template == PlaygroundTemplate.VIRTUAL: - _write_virtual_template_files(target_dir=target_dir) def _copy_tree(*, source: Traversable, target: Path) -> None: @@ -674,25 +442,3 @@ def _write_python_nodes_template_files(*, target_dir: Path) -> None: hook_dir: Path = target_dir / "hooks" / hook_language hook_dir.mkdir(parents=True) (hook_dir / ".gitkeep").touch() - - -def _write_virtual_template_files(*, target_dir: Path) -> None: - (target_dir / "sqlbuild_project.toml").write_text(_VIRTUAL_PROJECT_TOML, encoding="utf-8") - (target_dir / "README.md").write_text(_VIRTUAL_README, encoding="utf-8") - (target_dir / "models/fact_waffle_orders.sql").write_text( - _VIRTUAL_FACT_WAFFLE_ORDERS_SQL, encoding="utf-8" - ) - seeds_dir: Path = target_dir / "seeds" - seeds_dir.mkdir() - (seeds_dir / "lookups.yml").write_text(_VIRTUAL_SEED_SCHEMA_YML, encoding="utf-8") - (seeds_dir / "waffle_price_tiers.csv").write_text(_VIRTUAL_SEED_CSV, encoding="utf-8") - unit_tests_dir: Path = target_dir / "tests" / "unit" - unit_tests_dir.mkdir(parents=True) - (unit_tests_dir / "test_fact_waffle_orders.sql").write_text( - _VIRTUAL_TEST_FACT_WAFFLE_ORDERS_SQL, encoding="utf-8" - ) - scenarios_dir: Path = target_dir / "tests" / "scenarios" - scenarios_dir.mkdir(parents=True) - (scenarios_dir / "customer_revenue_minimal.sql").write_text( - _VIRTUAL_SCENARIO_CUSTOMER_REVENUE_SQL, encoding="utf-8" - ) diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/README.md b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/README.md index fe75b687c7..9b1583857d 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/README.md +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/README.md @@ -20,6 +20,12 @@ Repeated `sqb build` runs cause source-loader data to grow or change: ## What This Shows +The complete source contracts also support offline `sqb compile` column and type checks. +DuckDB rejects proven bind-time type errors; accepted conversions with data-dependent risks are +non-blocking W21x warnings. PostgreSQL, Snowflake, and BigQuery use their own coercion rules; +MotherDuck uses DuckDB's rules. Unknown types remain unchecked. The escape hatches are +`MODEL (sql_analysis false)`, the corresponding path default, and `--no-sql-analysis`. + - chained intermediate source loaders - different intermediate write strategies - terminal managed source loaders diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml index 22c02783f9..be3905ca72 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw_orders + contract: enforced managed: true write_strategy: table columns: @@ -16,6 +17,7 @@ sources: - name: load_seq type: INTEGER - name: raw_customers + contract: enforced managed: true write_strategy: table columns: diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/README.md b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/README.md index 9c679a8a59..a538e3f997 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/README.md +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/README.md @@ -36,6 +36,21 @@ sqb lineage fact_orders ## What This Shows +### Try semantic compilation + +`sqb compile` checks the enforced source schemas offline. Change `o.quantity` to `o.qty` in +`models/marts/fact_orders.sql` to see an unknown-column error (B002). Add +`WHERE ordered_at > 5` to `models/staging/stg_orders.sql` to see DuckDB's timestamp/integer +comparison error (B217). Restore each edit after trying it; downstream models and SQL tests may +also report the consequences of a changed output column. + +Proven bind-time failures are errors. Accepted conversions that may fail on data produce W21x +warnings and do not fail compilation. Unknown types remain unchecked. Type checks are enabled for +DuckDB/MotherDuck, PostgreSQL, Snowflake, and BigQuery using each dialect's coercion rules. +Use `MODEL (sql_analysis false)`, a path default, or `--no-sql-analysis` to opt out. + +### Project features + - DuckDB-backed local execution - SQL models across staging, intermediate, and mart layers - Seeds, expression-backed sources, and Python source loaders diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml index 5bdbae9c55..3d5b2d89a9 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml +++ b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/sources/raw.yml @@ -1,5 +1,6 @@ sources: - name: raw__customers + contract: enforced managed: true write_strategy: table columns: @@ -15,6 +16,7 @@ sources: type: TIMESTAMP - name: raw__orders + contract: enforced managed: true write_strategy: delete_insert cursor_column: ordered_at diff --git a/src/sqlbuild/cli/commands/_helpers/promote/output.py b/src/sqlbuild/cli/commands/_helpers/promote/output.py deleted file mode 100644 index 07f192e854..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/promote/output.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Promote command output helpers.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.types import CompletionState -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - -_MODEL_SET_CAP: int = 20 - - -def format_promote_output( - *, - from_virtual_environment: str, - to_virtual_environment: str, - status: str, - promoted_models: tuple[str, ...], - remaining_stale: tuple[str, ...], - verbose: bool = False, - use_color: bool = False, -) -> str: - """Format virtual promotion output.""" - - style: CliStyle = CliStyle(use_color=use_color) - from_label: str = style.object_name(from_virtual_environment) - to_label: str = style.object_name(to_virtual_environment) - status_label: str = ( - VirtualEnvironmentStatus.FINALIZED - if status == VirtualEnvironmentStatus.FINALIZED - else "working" - ) - status_value: str = ( - style.success_strong(status_label) - if status_label == VirtualEnvironmentStatus.FINALIZED - else style.warning_strong(status_label) - ) - promoted_count: str = style.value(f"{len(promoted_models):,}") - remaining_count: str = ( - style.warning_strong(f"{len(remaining_stale):,}") - if remaining_stale - else f"{len(remaining_stale):,}" - ) - doc: CliDocument = CliDocument(style) - doc.blank() - doc.line( - format_completion_line( - style=style, - state=( - CompletionState.OK - if status_label == VirtualEnvironmentStatus.FINALIZED - else CompletionState.WARN - ), - label="Virtual promotion complete", - summary=f"{from_label} -> {to_label}", - ) - ) - doc.line(f" {style.label('target status')} {status_value}") - doc.line(f" {style.label('promoted models')} {promoted_count}") - if promoted_models: - for line in _format_model_set_lines( - label="promoted model set", - model_names=promoted_models, - verbose=verbose, - style=style, - ): - doc.line(line) - doc.line(f" {style.label('remaining stale models')} {remaining_count}") - if remaining_stale: - for line in _format_model_set_lines( - label="remaining stale set", - model_names=remaining_stale, - verbose=verbose, - style=style, - ): - doc.line(line) - return doc.render(trailing_newline=False) - - -def _format_model_set_lines( - *, label: str, model_names: tuple[str, ...], verbose: bool, style: CliStyle -) -> list[str]: - visible: tuple[str, ...] = model_names if verbose else model_names[:_MODEL_SET_CAP] - label_text: str = style.muted(label) - lines: list[str] = [f" {label_text}: " + ", ".join(visible)] - remaining: int = len(model_names) - len(visible) - if remaining > 0: - help_text: str = f" ... {remaining:,} more; use --verbose to show all" - lines.append(style.muted(help_text)) - return lines diff --git a/src/sqlbuild/cli/commands/_helpers/reconcile/output.py b/src/sqlbuild/cli/commands/_helpers/reconcile/output.py deleted file mode 100644 index e383f7fbd1..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/reconcile/output.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Reconcile command output helpers.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle - -_NO_ISSUES_MESSAGE: str = "no issues" -_RECONCILE_ACTION_HEADERS: frozenset[str] = frozenset({"Repair", "Attach"}) -_OBJECT_LABELS: frozenset[str] = frozenset({"model", "VDE", "physical"}) -_RESULT_LABEL: str = "result" - - -def format_reconcile_output(*, message: str, use_color: bool) -> str: - """Format virtual reconcile output.""" - - style: CliStyle = CliStyle(use_color=use_color) - doc: CliDocument = CliDocument(style) - doc.blank() - doc.header(text="Virtual reconcile") - doc.blank() - doc.line(_format_reconcile_message(message=message, style=style)) - return doc.render() - - -def _format_reconcile_message(*, message: str, style: CliStyle) -> str: - if not style.use_color: - return message - if _NO_ISSUES_MESSAGE in message: - return style.value(message) - lines: list[str] = message.splitlines() - if not lines or lines[0] not in _RECONCILE_ACTION_HEADERS: - return message - formatted: list[str] = [style.success(lines[0])] - for line in lines[1:]: - stripped: str = line.strip() - label, _, value = stripped.partition(" ") - rendered_value: str = _format_reconcile_value(label=label, value=value.strip(), style=style) - formatted.append(f" {style.muted(f'{label:<8}')} {rendered_value}") - return "\n".join(formatted) - - -def _format_reconcile_value(*, label: str, value: str, style: CliStyle) -> str: - if label in _OBJECT_LABELS: - return style.object_name(value) - if label == _RESULT_LABEL: - return style.success(value) - return value diff --git a/src/sqlbuild/cli/commands/_helpers/rollback/output.py b/src/sqlbuild/cli/commands/_helpers/rollback/output.py deleted file mode 100644 index 4acea5034f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/rollback/output.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Rollback command output formatting.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.completion_line import format_completion_line -from sqlbuild.presentation.types import CompletionState -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - -_MODEL_SET_CAP: int = 20 - - -def format_rollback_output( - *, - virtual_environment: str, - checkpoint_id: str, - rolled_back_models: tuple[str, ...], - status: str, - verbose: bool = False, - use_color: bool = True, -) -> str: - """Format virtual rollback result output.""" - - style: CliStyle = CliStyle(use_color=use_color) - environment_label: str = style.object_name(virtual_environment) - checkpoint_label: str = style.object_name(checkpoint_id) - status_label: str = ( - style.success_strong(status) - if status == VirtualEnvironmentStatus.FINALIZED - else style.warning_strong(status) - ) - model_count: str = style.value(f"{len(rolled_back_models):,}") - doc: CliDocument = CliDocument(style) - doc.blank() - doc.line( - format_completion_line( - style=style, - state=( - CompletionState.OK - if status == VirtualEnvironmentStatus.FINALIZED - else CompletionState.WARN - ), - label="Virtual rollback complete", - ) - ) - doc.blank() - doc.line(f" {style.label('virtual environment')} {environment_label}") - doc.line(f" {style.label('checkpoint')} {checkpoint_label}") - doc.line(f" {style.label('status')} {status_label}") - doc.line(f" {style.label('rolled back models')} {model_count}") - if rolled_back_models: - for line in _format_model_set_lines( - label="rolled back model set", - model_names=rolled_back_models, - verbose=verbose, - style=style, - ): - doc.line(line) - doc.blank() - return doc.render(trailing_newline=False) - - -def _format_model_set_lines( - *, label: str, model_names: tuple[str, ...], verbose: bool, style: CliStyle -) -> list[str]: - visible: tuple[str, ...] = model_names if verbose else model_names[:_MODEL_SET_CAP] - label_text: str = style.muted(label) - lines: list[str] = [f" {label_text}: " + ", ".join(visible)] - remaining: int = len(model_names) - len(visible) - if remaining > 0: - help_text: str = f" ... {remaining:,} more; use --verbose to show all" - lines.append(style.muted(help_text)) - return lines diff --git a/src/sqlbuild/cli/commands/_helpers/rules/command.py b/src/sqlbuild/cli/commands/_helpers/rules/command.py index cf926e2834..da8aa5ff6b 100644 --- a/src/sqlbuild/cli/commands/_helpers/rules/command.py +++ b/src/sqlbuild/cli/commands/_helpers/rules/command.py @@ -162,6 +162,7 @@ def _run_rule_selection( result=RulesResult( findings=result.findings, evaluated_models=result.evaluated_models, + unevaluated_resources=result.unevaluated_resources, cache_hits=result.cache_hits, cache_misses=result.cache_misses, ), diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/mode_policy.py b/src/sqlbuild/cli/commands/_helpers/runtime/mode_policy.py deleted file mode 100644 index 5228349994..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/runtime/mode_policy.py +++ /dev/null @@ -1,50 +0,0 @@ -"""CLI live-mode policy checks.""" - -from __future__ import annotations - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs - - -def enforce_no_defer_to_in_virtual_mode( - *, discovered_inputs: DiscoveredProjectInputs, command_name: str, defer_to: str | None -) -> None: - """Block classic model-output deferral in virtual mode.""" - - if defer_to is None: - return - if discovered_inputs.project_config.settings.virtual_environments: - raise CliUserError( - f"{command_name} does not support --defer-to when virtual_environments = true", - code="C242", - ) - - -def enforce_virtual_only_flags_in_virtual_mode( - *, - discovered_inputs: DiscoveredProjectInputs, - command_name: str, - virtual_env: str | None, - include_stale_upstreams: bool, - changes_only: bool = False, -) -> None: - """Block virtual-environment-only flags on direct-mode projects.""" - - if discovered_inputs.project_config.settings.virtual_environments: - return - if virtual_env is not None: - raise CliUserError( - f"{command_name} does not support --virtual-env unless virtual_environments = true", - code="C263", - ) - if include_stale_upstreams: - raise CliUserError( - f"{command_name} does not support --include-stale-upstreams unless " - "virtual_environments = true", - code="C263", - ) - if changes_only: - raise CliUserError( - f"{command_name} does not support --changes-only unless virtual_environments = true", - code="C263", - ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py index d04d054ecc..44b4c2be3b 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture.py @@ -23,6 +23,7 @@ build_scenario_snapshot_capture_limits, scenario_snapshot_capture_warning, ) +from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios from sqlbuild.cli.commands.constants import ( SCENARIO_CLI_SQL_VALIDATION_REQUIRED, @@ -79,6 +80,9 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=effective_project_dir ) + discovered_inputs, namespace = resolve_scenario_namespace( + inputs=discovered_inputs, cli_value=request.scenario_namespace + ) _validate_capture_sql_analysis_enabled(discovered_inputs=discovered_inputs) adapter_name: str = resolve_effective_adapter_name( project_config=discovered_inputs.project_config, @@ -96,6 +100,9 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: ) use_color: bool = not no_color and supports_color() progress_stream: TextIO = sys.stdout + progress_stream.write( + f"Scenario namespace: {namespace.value or '(unset)'} (source: {namespace.source})\n" + ) target_label: str | None = " ".join(selectors) if selectors else None planning_progress: PlanningProgressReporter = PlanningProgressReporter( stream=progress_stream, @@ -154,6 +161,7 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: limits=capture_limits, ), output_context=ScenarioRunOutputContext( + namespace=namespace, progress_stream=progress_stream, use_color=use_color, ), diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py index 4157e4a682..d47847c33c 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_capture/capture_run.py @@ -8,6 +8,10 @@ from typing import TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( + scenario_activity_message, + write_namespace_completion, +) from sqlbuild.cli.commands._helpers.scenario_output.result_output import render_result_error from sqlbuild.cli.commands.constants import SUCCESS_STATUS from sqlbuild.cli.commands.models import ScenarioRunOutputContext @@ -67,8 +71,11 @@ def run_scenario_capture_run( use_color=use_color, ) status_is_tty: bool = hasattr(progress_stream, "isatty") and progress_stream.isatty() + activity: str = scenario_activity_message( + activity="Capturing scenarios...", context=output_context + ) if not status_is_tty: - progress_stream.write("Capturing scenarios...\n\n") + progress_stream.write(f"{activity}\n\n") progress_stream.flush() results: tuple[ScenarioSnapshotCaptureRunResult, ...] = run_scenario_capture_pipeline( project_dir=project_dir, @@ -92,7 +99,7 @@ def run_scenario_capture_run( ), ), on_scenario_start=lambda _scenario: ( - scenario_status.start("Capturing scenarios...") if status_is_tty else None + scenario_status.start(activity) if status_is_tty else None ), on_scenario_complete=lambda _scenario, scenario_plan, result: _complete_capture_run( scenario_status=scenario_status, @@ -122,6 +129,7 @@ def run_scenario_capture_run( + "\n" ) progress_stream.flush() + write_namespace_completion(context=output_context, succeeded=fail_count == 0) return (0 if fail_count == 0 else 1), capture_results_out diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py index 16f9853349..9ee8a048cc 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/local_run.py @@ -6,6 +6,10 @@ from typing import TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( + scenario_activity_message, + write_namespace_completion, +) from sqlbuild.cli.commands._helpers.scenario_output.result_output import complete_scenario_run from sqlbuild.cli.commands.models import ScenarioRunOutputContext from sqlbuild.cli.output.main._scenario_execution_json import ( @@ -49,8 +53,11 @@ def run_local_scenarios( use_color=use_color, ) status_is_tty: bool = hasattr(progress_stream, "isatty") and progress_stream.isatty() + activity: str = scenario_activity_message( + activity="Running scenarios...", context=output_context + ) if not status_is_tty: - progress_stream.write("Running scenarios...\n\n") + progress_stream.write(f"{activity}\n\n") progress_stream.flush() results: tuple[ScenarioRunResult, ...] = run_scenario_local_test_pipeline( project_dir=project_dir, @@ -62,7 +69,7 @@ def run_local_scenarios( capture_adapter=capture_adapter, capture_dialect=capture_dialect, on_scenario_start=lambda _scenario: ( - scenario_status.start("Running scenarios...") if status_is_tty else None + scenario_status.start(activity) if status_is_tty else None ), on_scenario_complete=lambda _scenario, scenario_plan, result: complete_scenario_run( scenario_status=scenario_status, @@ -79,8 +86,14 @@ def run_local_scenarios( exit_code: int = _write_local_summary( results=results, stream=progress_stream, use_color=use_color ) + write_namespace_completion(context=output_context, succeeded=exit_code == 0) write_execution_json_output( - payload=format_scenario_execution_json(results=results, local=True), + payload=format_scenario_execution_json( + results=results, + local=True, + run_namespace=output_context.namespace.value, + namespace_source=output_context.namespace.source, + ), json_output=json_output, json_output_path=json_output_path, ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/namespace.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/namespace.py new file mode 100644 index 0000000000..a200adfe9c --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/namespace.py @@ -0,0 +1,45 @@ +"""Resolve scenario namespace overrides before planning or warehouse work.""" + +from __future__ import annotations + +import os +from dataclasses import replace + +from sqlbuild.cli.commands.constants import SCENARIO_CLI_INVALID_NAMESPACE +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.cli.commands.models import ScenarioRunNamespace +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.spec.contracts.exceptions import SpecConfigError +from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import ( + parse_scenario_run_namespace, +) + + +def resolve_scenario_namespace( + *, inputs: DiscoveredProjectInputs, cli_value: str | None +) -> tuple[DiscoveredProjectInputs, ScenarioRunNamespace]: + """Apply CLI > environment > local config > project config precedence.""" + + candidates: tuple[tuple[str | None, str], ...] = ( + (cli_value, "cli"), + (os.environ.get("SQLBUILD_SCENARIO_NAMESPACE"), "env"), + (inputs.local_config.scenario.run_namespace, "local config"), + (inputs.project_config.scenario.run_namespace, "project config"), + ) + value: str | None + source: str + for value, source in candidates: + if value is None: + continue + try: + value = parse_scenario_run_namespace(value) + except SpecConfigError as exc: + raise CliUserError(str(exc), code=SCENARIO_CLI_INVALID_NAMESPACE) from exc + return replace( + inputs, + local_config=replace( + inputs.local_config, + scenario=replace(inputs.local_config.scenario, run_namespace=value), + ), + ), ScenarioRunNamespace(value=value, source=source) + return inputs, ScenarioRunNamespace() diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py index 6f19611a40..f32081a3f1 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/runner.py @@ -26,6 +26,7 @@ scenario_snapshot_capture_warning, ) from sqlbuild.cli.commands._helpers.scenario_execution.local_run import run_local_scenarios +from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace from sqlbuild.cli.commands._helpers.scenario_execution.selection import select_scenarios from sqlbuild.cli.commands._helpers.scenario_execution.warehouse_run import ( run_warehouse_scenarios, @@ -122,6 +123,9 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=effective_project_dir ) + discovered_inputs, namespace = resolve_scenario_namespace( + inputs=discovered_inputs, cli_value=request.scenario_namespace + ) if local: _validate_local_scenario_sql_analysis_enabled( discovered_inputs=discovered_inputs, @@ -154,6 +158,9 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: ) use_color: bool = not no_color and supports_color() progress_stream: TextIO = sys.stderr if json_output else sys.stdout + progress_stream.write( + f"Scenario namespace: {namespace.value or '(unset)'} (source: {namespace.source})\n" + ) target_label: str | None = " ".join(selectors) if selectors else None planning_progress: PlanningProgressReporter = PlanningProgressReporter( stream=progress_stream, @@ -214,6 +221,7 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: ), limit_inputs=limit_inputs, output_context=ScenarioRunOutputContext( + namespace=namespace, progress_stream=progress_stream, use_color=use_color, json_output=json_output, @@ -224,6 +232,8 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: if capture_exit_code != 0: write_execution_json_output( payload=format_scenario_snapshot_execution_json( + run_namespace=namespace.value, + namespace_source=namespace.source, results=tuple(capture_results), refresh=refresh, ), @@ -242,6 +252,7 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: target_dir=effective_project_dir / "target", retain=retain, output_context=ScenarioRunOutputContext( + namespace=namespace, progress_stream=progress_stream, use_color=use_color, json_output=json_output, @@ -259,6 +270,7 @@ def run_scenario(request: ScenarioTestCommandRequest) -> int: capture_dialect=local_capture_dialect, target_dir=effective_project_dir / "target", output_context=ScenarioRunOutputContext( + namespace=namespace, progress_stream=progress_stream, use_color=use_color, json_output=json_output, diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py b/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py index 5dac794b1d..4d1cc962fc 100644 --- a/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py +++ b/src/sqlbuild/cli/commands/_helpers/scenario_execution/warehouse_run.py @@ -6,6 +6,10 @@ from typing import TextIO from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.cli.commands._helpers.scenario_output.namespace import ( + scenario_activity_message, + write_namespace_completion, +) from sqlbuild.cli.commands._helpers.scenario_output.result_output import complete_scenario_run from sqlbuild.cli.commands.constants import SUCCESS_STATUS from sqlbuild.cli.commands.models import ScenarioRunOutputContext @@ -50,8 +54,11 @@ def run_warehouse_scenarios( use_color=use_color, ) status_is_tty: bool = hasattr(progress_stream, "isatty") and progress_stream.isatty() + activity: str = scenario_activity_message( + activity="Running scenarios...", context=output_context + ) if not status_is_tty: - progress_stream.write("Running scenarios...\n\n") + progress_stream.write(f"{activity}\n\n") progress_stream.flush() execution_connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( adapter_name=adapter_name, @@ -80,7 +87,7 @@ def run_warehouse_scenarios( ), ), on_scenario_start=lambda _scenario: ( - scenario_status.start("Running scenarios...") if status_is_tty else None + scenario_status.start(activity) if status_is_tty else None ), on_scenario_complete=lambda _scenario, scenario_plan, result: complete_scenario_run( scenario_status=scenario_status, @@ -97,8 +104,14 @@ def run_warehouse_scenarios( exit_code: int = _write_remote_summary( results=results, stream=progress_stream, use_color=use_color ) + write_namespace_completion(context=output_context, succeeded=exit_code == 0) write_execution_json_output( - payload=format_scenario_execution_json(results=results, local=False), + payload=format_scenario_execution_json( + results=results, + local=False, + run_namespace=output_context.namespace.value, + namespace_source=output_context.namespace.source, + ), json_output=json_output, json_output_path=json_output_path, ) diff --git a/src/sqlbuild/cli/commands/_helpers/scenario_output/namespace.py b/src/sqlbuild/cli/commands/_helpers/scenario_output/namespace.py new file mode 100644 index 0000000000..988c1b1b32 --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/scenario_output/namespace.py @@ -0,0 +1,23 @@ +"""Namespace attribution for scenario lifecycle progress.""" + +from __future__ import annotations + +from sqlbuild.cli.commands.models import ScenarioRunOutputContext + + +def scenario_activity_message(*, activity: str, context: ScenarioRunOutputContext) -> str: + """Label scenario work with the effective namespace and its source.""" + return ( + f"{activity} (namespace: {context.namespace.value or '(unset)'}; " + f"source: {context.namespace.source})" + ) + + +def write_namespace_completion(*, context: ScenarioRunOutputContext, succeeded: bool) -> None: + """Emit the explicit terminal state for the namespaced run.""" + message: str = scenario_activity_message( + activity="Scenario execution complete" if succeeded else "Scenario execution failed", + context=context, + ) + context.progress_stream.write(f"{message}\n") + context.progress_stream.flush() diff --git a/src/sqlbuild/cli/commands/_helpers/seed/virtual.py b/src/sqlbuild/cli/commands/_helpers/seed/virtual.py deleted file mode 100644 index d44814846f..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/seed/virtual.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Virtual mode delegation for the seed command.""" - -from __future__ import annotations - -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands.main.execution._virtual_build import run_virtual_build -from sqlbuild.cli.commands.models import SeedCommandRequest, SeedInvocation, VirtualBuildCliRequest -from sqlbuild.provider.classes.session import ProviderSession -from sqlbuild.provider.main.session import build_provider_session - - -def execute_virtual_seed(*, request: SeedCommandRequest, invocation: SeedInvocation) -> int: - """Delegate seed-only virtual mode execution to virtual build.""" - - provider_session: ProviderSession = build_provider_session( - discovered_providers=invocation.discovered_inputs.providers - ) - try: - return run_virtual_build( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - adapter_name=invocation.adapter_name, - connection_config=invocation.connection_config, - request=VirtualBuildCliRequest( - selected_target=request.selected_target, - include_python=False, - seed_only=True, - select=request.select, - exclude=request.exclude, - concurrency=request.concurrency, - cli_vars=request.cli_vars, - json_output=request.json_output, - json_output_path=request.json_output_path, - execution_command="seed", - use_color=invocation.use_color, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - ), - providers=provider_session.providers, - ), - ) - finally: - provider_session.close() diff --git a/src/sqlbuild/cli/commands/_helpers/state/checkpoints.py b/src/sqlbuild/cli/commands/_helpers/state/checkpoints.py deleted file mode 100644 index 7bdd1d4650..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/state/checkpoints.py +++ /dev/null @@ -1,380 +0,0 @@ -"""State checkpoint command helpers.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.classes.cli_document import CliDocument -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.state.main.checkpoints.list_checkpoints import ( - list_virtual_environment_checkpoints, -) -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) - -_LIST_CHECKPOINTS_COMMAND: str = "list" -_SHOW_CHECKPOINT_COMMAND: str = "show" -_DIFF_CHECKPOINT_COMMAND: str = "diff" - - -def run_state_checkpoints( - *, - project_dir: Path | None, - command: str | None, - checkpoint_id: str | None = None, - virtual_environment_name: str | None = None, - no_color: bool = False, -) -> int: - """Run state checkpoint inspection commands.""" - - if command is None: - raise CliUserError("state checkpoints requires a subcommand such as 'list'", code="C902") - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - resolved_target_name: str | None = virtual_environment_name or resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if resolved_target_name is None: - raise CliUserError( - "state checkpoints requires --virtual-env or a default target", - code="C903", - ) - style: CliStyle = CliStyle(use_color=not no_color and supports_color()) - if command == _LIST_CHECKPOINTS_COMMAND: - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - list_virtual_environment_checkpoints( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - virtual_environment_name=resolved_target_name, - ) - ) - print( - _format_checkpoint_list( - virtual_environment_name=resolved_target_name, - checkpoints=checkpoints, - style=style, - ) - ) - return 0 - if command == _SHOW_CHECKPOINT_COMMAND: - if checkpoint_id is None: - raise CliUserError("state checkpoints show requires checkpoint id", code="C904") - refs, seed_refs = _read_checkpoint_refs( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - checkpoint_id=checkpoint_id, - ) - if not refs and not seed_refs: - raise CliUserError(f"unknown checkpoint '{checkpoint_id}'", code="C905") - print( - _format_checkpoint_show( - checkpoint_id=checkpoint_id, - refs=refs, - seed_refs=seed_refs, - style=style, - ) - ) - return 0 - if command == _DIFF_CHECKPOINT_COMMAND: - if checkpoint_id is None: - raise CliUserError("state checkpoints diff requires checkpoint id", code="C907") - checkpoint_model_refs, checkpoint_seed_refs = _read_checkpoint_refs( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - checkpoint_id=checkpoint_id, - ) - if not checkpoint_model_refs and not checkpoint_seed_refs: - raise CliUserError(f"unknown checkpoint '{checkpoint_id}'", code="C905") - current_refs, current_seed_refs = _read_current_refs( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - virtual_environment_name=resolved_target_name, - ) - print( - _format_checkpoint_diff( - virtual_environment_name=resolved_target_name, - checkpoint_id=checkpoint_id, - current_refs=current_refs, - current_seed_refs=current_seed_refs, - checkpoint_model_refs=checkpoint_model_refs, - checkpoint_seed_refs=checkpoint_seed_refs, - style=style, - ) - ) - return 0 - raise CliUserError(f"unknown state checkpoints command '{command}'", code="C906") - - -def _format_checkpoint_list( - *, - virtual_environment_name: str, - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...], - style: CliStyle, -) -> str: - document: CliDocument = CliDocument(style) - document.blank() - document.header( - text="Virtual environment checkpoints", suffix=style.object_name(virtual_environment_name) - ) - document.blank() - if not checkpoints: - document.line(" no checkpoints") - document.blank() - return document.render(trailing_newline=False) - checkpoint: VirtualEnvironmentCheckpointRecord - for checkpoint in checkpoints: - created_at: str = ( - str(checkpoint.created_at) if checkpoint.created_at is not None else "unknown" - ) - document.line(f" {style.accent(checkpoint.checkpoint_id)} {style.muted(created_at)}") - document.blank() - return document.render(trailing_newline=False) - - -def _format_checkpoint_show( - *, - checkpoint_id: str, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...], - style: CliStyle, -) -> str: - document: CliDocument = CliDocument(style) - document.blank() - document.header(text="Virtual environment checkpoint") - document.blank() - document.line(f" checkpoint {style.accent(checkpoint_id)}") - document.blank() - document.line(style.success("Refs")) - ref: VirtualEnvironmentCheckpointModelRefRecord - for ref in refs: - document.line( - f" {style.object_name(f'{ref.model_name:<24}')} {style.muted(ref.version_hash)}" - ) - if seed_refs: - document.blank() - document.line(style.success("Seed refs")) - seed_ref: VirtualEnvironmentCheckpointSeedRefRecord - for seed_ref in seed_refs: - document.line( - f" {style.object_name(f'{seed_ref.seed_name:<24}')} " - f"{style.muted(seed_ref.version_hash)}" - ) - document.blank() - return document.render(trailing_newline=False) - - -def _format_checkpoint_diff( - *, - virtual_environment_name: str, - checkpoint_id: str, - current_refs: tuple[VirtualEnvironmentModelRefRecord, ...], - current_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - checkpoint_model_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...], - style: CliStyle, -) -> str: - current_ref_map: dict[str, str] = {ref.model_name: ref.version_hash for ref in current_refs} - checkpoint_ref_map: dict[str, str] = { - ref.model_name: ref.version_hash for ref in checkpoint_model_refs - } - current_seed_ref_map: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in current_seed_refs - } - checkpoint_seed_ref_map: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in checkpoint_seed_refs - } - changed: tuple[str, ...] = tuple( - sorted( - model_name - for model_name, version_hash in current_ref_map.items() - if model_name in checkpoint_ref_map and checkpoint_ref_map[model_name] != version_hash - ) - ) - current_only: tuple[str, ...] = tuple( - sorted(model_name for model_name in current_ref_map if model_name not in checkpoint_ref_map) - ) - checkpoint_only: tuple[str, ...] = tuple( - sorted(model_name for model_name in checkpoint_ref_map if model_name not in current_ref_map) - ) - changed_seed_refs: tuple[str, ...] = tuple( - sorted( - seed_name - for seed_name, version_hash in current_seed_ref_map.items() - if seed_name in checkpoint_seed_ref_map - and checkpoint_seed_ref_map[seed_name] != version_hash - ) - ) - current_only_seed_refs: tuple[str, ...] = tuple( - sorted( - seed_name - for seed_name in current_seed_ref_map - if seed_name not in checkpoint_seed_ref_map - ) - ) - checkpoint_only_seed_refs: tuple[str, ...] = tuple( - sorted( - seed_name - for seed_name in checkpoint_seed_ref_map - if seed_name not in current_seed_ref_map - ) - ) - document: CliDocument = CliDocument(style) - document.blank() - document.header( - text="Virtual environment checkpoint diff", - suffix=style.object_name(virtual_environment_name), - ) - document.blank() - document.line(f" {'checkpoint':<16} {style.accent(checkpoint_id)}") - document.line(f" {'changed refs':<16} {style.accent(f'{len(changed):,}')}") - document.line(f" {'current only':<16} {style.accent(f'{len(current_only):,}')}") - document.line(f" {'checkpoint only':<16} {style.accent(f'{len(checkpoint_only):,}')}") - document.line(f" {'changed seeds':<16} {style.accent(f'{len(changed_seed_refs):,}')}") - document.line(f" {'current seeds':<16} {style.accent(f'{len(current_only_seed_refs):,}')}") - document.line( - f" {'checkpoint seeds':<16} {style.accent(f'{len(checkpoint_only_seed_refs):,}')}" - ) - _append_ref_diff_lines( - document=document, - label="Changed refs", - model_names=changed, - current_ref_map=current_ref_map, - checkpoint_ref_map=checkpoint_ref_map, - style=style, - ) - _append_ref_diff_lines( - document=document, - label="Current only", - model_names=current_only, - current_ref_map=current_ref_map, - checkpoint_ref_map=checkpoint_ref_map, - style=style, - ) - _append_ref_diff_lines( - document=document, - label="Checkpoint only", - model_names=checkpoint_only, - current_ref_map=current_ref_map, - checkpoint_ref_map=checkpoint_ref_map, - style=style, - ) - _append_ref_diff_lines( - document=document, - label="Changed seed refs", - model_names=changed_seed_refs, - current_ref_map=current_seed_ref_map, - checkpoint_ref_map=checkpoint_seed_ref_map, - style=style, - ) - _append_ref_diff_lines( - document=document, - label="Current only seed refs", - model_names=current_only_seed_refs, - current_ref_map=current_seed_ref_map, - checkpoint_ref_map=checkpoint_seed_ref_map, - style=style, - ) - _append_ref_diff_lines( - document=document, - label="Checkpoint only seed refs", - model_names=checkpoint_only_seed_refs, - current_ref_map=current_seed_ref_map, - checkpoint_ref_map=checkpoint_seed_ref_map, - style=style, - ) - document.blank() - return document.render(trailing_newline=False) - - -def _append_ref_diff_lines( - *, - document: CliDocument, - label: str, - model_names: tuple[str, ...], - current_ref_map: dict[str, str], - checkpoint_ref_map: dict[str, str], - style: CliStyle, -) -> None: - if not model_names: - return - document.blank() - document.line(style.success(label)) - for model_name in model_names: - current_hash: str = current_ref_map.get(model_name, "") - checkpoint_hash: str = checkpoint_ref_map.get(model_name, "") - model_label: str = style.object_name(f"{model_name:<24}") - current_label: str = style.muted(current_hash) - checkpoint_label: str = style.muted(checkpoint_hash) - document.line(f" {model_label} {current_label} -> {checkpoint_label}") - - -def _read_checkpoint_refs( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - checkpoint_id: str, -) -> tuple[ - tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...], -]: - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, project_dir=project_dir - ) - connection: Any = backend.connect(config.connection) - try: - return ( - backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, schema=config.schema, checkpoint_id=checkpoint_id - ), - backend.get_virtual_environment_checkpoint_seed_refs( - connection=connection, schema=config.schema, checkpoint_id=checkpoint_id - ), - ) - finally: - backend.close(connection) - - -def _read_current_refs( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, -) -> tuple[ - tuple[VirtualEnvironmentModelRefRecord, ...], - tuple[VirtualEnvironmentSeedRefRecord, ...], -]: - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, project_dir=project_dir - ) - connection: Any = backend.connect(config.connection) - try: - return ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ), - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ), - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/cli/commands/_helpers/state/interactive_operation.py b/src/sqlbuild/cli/commands/_helpers/state/interactive_operation.py deleted file mode 100644 index 534b4cb303..0000000000 --- a/src/sqlbuild/cli/commands/_helpers/state/interactive_operation.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Shared CLI setup for interactive state operations.""" - -from __future__ import annotations - -import sys -import time -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.models import TargetConfig -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.types import StateCommand - - -def run_interactive_state_operation( - *, - project_dir: Path | None, - state_command: StateCommand, - operation_runner: Callable[..., str], - auto_approve: bool, - allow_copy: bool, - no_color: bool = False, -) -> int: - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - target_name: str | None = ( - discovered_inputs.local_config.target or discovered_inputs.project_config.default_target - ) - if target_name is None: - raise CliUserError( - f"state {state_command.value} requires an active target", - code="C255", - ) - effective_target: TargetConfig | None = discovered_inputs.project_config.targets.get( - target_name - ) - if effective_target is None or effective_target.state.unsuffixed_virtual_env != target_name: - raise CliUserError( - ( - f"state {state_command.value} requires " - f"[targets.{target_name}.state] unsuffixed_virtual_env = " - f"'{target_name}'" - ), - code="C256", - ) - if auto_approve: - raise CliUserError( - ( - f"state {state_command.value} is interactive-only and does not support " - "--auto-approve" - ), - code="C257", - ) - prompt: str = f"Type '{state_command.value} {target_name}' to confirm: " - if input(prompt).strip() != f"{state_command.value} {target_name}": - raise CliUserError(f"state {state_command.value} cancelled", code="C258") - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - ) - use_color: bool = not no_color and supports_color() - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - ) - state_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=f"{config.backend.value} state store", - stream=sys.stdout, - use_color=use_color, - ) - state_started_at: float = time.perf_counter() - state_progress.on_connection_start(1) - try: - state_connection: Any = backend.connect(config.connection) - except BaseException: - state_progress.on_connection_error( - connection_count=1, elapsed_seconds=time.perf_counter() - state_started_at - ) - raise - state_progress.on_connection_complete( - connection_count=1, elapsed_seconds=time.perf_counter() - state_started_at - ) - warehouse_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - stream=sys.stdout, - use_color=use_color, - ) - warehouse_started_at: float = time.perf_counter() - warehouse_progress.on_connection_start(1) - try: - warehouse_connection: Any = adapter.connect(connection_config) - except BaseException: - warehouse_progress.on_connection_error( - connection_count=1, elapsed_seconds=time.perf_counter() - warehouse_started_at - ) - backend.close(state_connection) - raise - warehouse_progress.on_connection_complete( - connection_count=1, elapsed_seconds=time.perf_counter() - warehouse_started_at - ) - status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stdout, - use_color=use_color, - ) - try: - status.start(f"Running state {state_command.value}...") - message: str = operation_runner( - discovered_inputs=discovered_inputs, - config=config, - backend=backend, - state_connection=state_connection, - adapter=adapter, - connection=warehouse_connection, - allow_copy=allow_copy, - ) - status.complete(message=f"State {state_command.value} complete.") - finally: - status.close() - adapter.close(warehouse_connection) - backend.close(state_connection) - print(message) - return 0 diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index 56cf87f724..44c1af0838 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -29,8 +29,6 @@ "target_range": None, "from_target": None, "to_target": None, - "from_virtual_environment": None, - "to_virtual_environment": None, "hard_copy": False, "json": False, "json_output": None, @@ -49,11 +47,7 @@ "no_color": False, "fail_fast": False, "full_refresh": False, - "virtual_env": None, "as_target": None, - "skip_locked": False, - "include_stale_upstreams": False, - "changes_only": False, "include_python": True, "load_sources": None, "reload": False, @@ -63,19 +57,12 @@ "allow_table_type_downgrade": False, "allow_retention_decrease": False, "allow_snapshot_schema_change": False, - "allow_partial_promotion": False, - "allow_partial_rollback": False, - "rollback_checkpoint_id": None, "concurrency": None, "verbose": False, "debug": False, "auto_approve": False, "retention_days": None, "direct_state_history_versions": None, - "state_command": None, - "state_checkpoint_command": None, - "state_checkpoint_id": None, - "state_backup_id": None, "bounded": None, "max_column_examples": None, "max_row_only_examples": None, @@ -108,7 +95,6 @@ "state": False, "full": False, "schema_only": False, - "allow_partial_diff": False, "left_query": None, "left_query_file": None, "right_query": None, @@ -122,14 +108,12 @@ "max_value_length": None, "no_example_values": False, "full_example_values": False, - "reconcile_command": None, - "reconcile_model": None, - "reconcile_physical_relation": None, "playground_path": "sqlbuild-playground", "playground_template": "waffle_shop", "scenario_command": None, "scenario_selector": [], "scenario_retain": False, + "scenario_namespace": None, "scenario_local": False, "scenario_strict": False, "scenario_sync_snapshots": False, @@ -202,8 +186,6 @@ class CliNamespace: target_range: str | None from_target: str | None to_target: str | None - from_virtual_environment: str | None - to_virtual_environment: str | None hard_copy: bool json: bool json_output: Path | None @@ -223,11 +205,7 @@ class CliNamespace: no_color: bool fail_fast: bool full_refresh: bool - virtual_env: str | None as_target: str | None - skip_locked: bool - include_stale_upstreams: bool - changes_only: bool selection_diagnostics: bool include_python: bool load_sources: bool | None @@ -238,19 +216,12 @@ class CliNamespace: allow_table_type_downgrade: bool allow_retention_decrease: bool allow_snapshot_schema_change: bool - allow_partial_promotion: bool - allow_partial_rollback: bool - rollback_checkpoint_id: str | None concurrency: int | None verbose: bool debug: bool auto_approve: bool retention_days: int | None direct_state_history_versions: int | None - state_command: str | None - state_checkpoint_command: str | None - state_checkpoint_id: str | None - state_backup_id: str | None bounded: str | None max_column_examples: int | None max_row_only_examples: int | None @@ -283,7 +254,6 @@ class CliNamespace: state: bool full: bool schema_only: bool - allow_partial_diff: bool left_query: str | None left_query_file: Path | None left_label: str | None @@ -297,14 +267,12 @@ class CliNamespace: max_value_length: int | None no_example_values: bool full_example_values: bool - reconcile_command: str | None - reconcile_model: str | None - reconcile_physical_relation: str | None playground_path: str playground_template: str scenario_command: str | None scenario_selector: list[str] scenario_retain: bool + scenario_namespace: str | None scenario_local: bool scenario_strict: bool scenario_sync_snapshots: bool diff --git a/src/sqlbuild/cli/commands/classes/virtual_build_plan_hook.py b/src/sqlbuild/cli/commands/classes/virtual_build_plan_hook.py deleted file mode 100644 index dc003570f2..0000000000 --- a/src/sqlbuild/cli/commands/classes/virtual_build_plan_hook.py +++ /dev/null @@ -1,198 +0,0 @@ -"""Plan-ready hook for virtual build CLI progress and safety checks.""" - -from __future__ import annotations - -import sys -from pathlib import Path -from typing import TextIO - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.build_execution.run_context import ( - write_build_run_context, -) -from sqlbuild.cli.commands._helpers.build_planning.execution_limits import ( - enforce_model_execution_limit, - executable_model_count, -) -from sqlbuild.cli.commands._helpers.build_planning.full_refresh import ( - enforce_snapshot_full_refresh_policy, -) -from sqlbuild.cli.commands._helpers.build_planning.retention_decrease import ( - enforce_retention_decrease_policy, -) -from sqlbuild.cli.commands._helpers.build_planning.table_type import ( - enforce_table_type_downgrade_policy, -) -from sqlbuild.cli.commands._helpers.compile.target_writer import write_compile_target -from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks -from sqlbuild.cli.commands.models import BuildRunContext, VirtualBuildPlanHookConfig -from sqlbuild.cli.output.main.plan import format_plan -from sqlbuild.cli.progress.main._write_execution_header import write_execution_header -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.presentation.models import DisplayOptions -from sqlbuild.virtual.executor.models import VirtualBuildExecutionHooks - - -class VirtualBuildPlanHook: - """Render the plan, enforce safety, and expose progress callbacks on plan ready.""" - - def __init__( - self, - *, - stream: TextIO, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - config: VirtualBuildPlanHookConfig, - ) -> None: - self._stream = stream - self._project_dir = project_dir - self._discovered_inputs = discovered_inputs - self._adapter = adapter - self._full_refresh = config.full_refresh - self._allow_snapshot_full_refresh = config.allow_snapshot_full_refresh - self._allow_table_type_downgrade = config.allow_table_type_downgrade - self._allow_retention_decrease = config.allow_retention_decrease - self._use_color = config.use_color - self._verbose = config.verbose - self._debug = config.debug - self._json_output = config.json_output - self._execution_command = config.execution_command - self._concurrency = config.concurrency - self._connection_config = config.connection_config - self._selector_files = config.selector_files - self._virtual_environment_name = config.virtual_environment_name or "default" - self._unsuffixed_virtual_environment_name = config.unsuffixed_virtual_environment_name - self._effective_target_name = config.effective_target_name - self._execution_limits = config.execution_limits - self.callbacks: BuildProgressCallbacks | None = None - self._closed: bool = False - - @property - def elapsed(self) -> float: - """Elapsed execution seconds since the plan was rendered.""" - - return self.callbacks.elapsed if self.callbacks is not None else 0 - - def on_plan_ready( - self, - *, - project: CompiledProject, - plan_output: PlanOutput, - python_plan_entries: tuple[PythonPlanEntry, ...], - ) -> VirtualBuildExecutionHooks: - """Render the plan and return node progress hooks for execution.""" - - plan_text: str = format_plan( - plan=plan_output, - full_refresh=self._full_refresh, - use_color=self._use_color, - display_options=DisplayOptions(max_entries_per_section=None if self._verbose else 50), - python_plan_entries=python_plan_entries, - ) - self._stream.write("\n" + plan_text + "\n\n") - self._stream.flush() - enforce_model_execution_limit( - model_count=executable_model_count(plan=plan_output), - target_name=self._effective_target_name, - limits=self._execution_limits, - ) - enforce_snapshot_full_refresh_policy( - plan=plan_output, - snapshots_config=self._discovered_inputs.project_config.snapshots, - allow_snapshot_full_refresh=self._allow_snapshot_full_refresh, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - enforce_table_type_downgrade_policy( - plan=plan_output, - allow_table_type_downgrade=self._allow_table_type_downgrade, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - enforce_retention_decrease_policy( - plan=plan_output, - allow_retention_decrease=self._allow_retention_decrease, - input_stream=sys.stdin, - output_stream=sys.stdout, - ) - write_compile_target( - target_dir=self._project_dir / "target", - adapter=self._adapter, - plan_output=plan_output, - ) - callbacks: BuildProgressCallbacks = BuildProgressCallbacks( - plan=plan_output, - use_color=self._use_color, - verbose=self._verbose, - debug=self._debug or self._json_output, - ) - self.callbacks = callbacks - effective_concurrency: int = ( - self._concurrency if self._concurrency is not None else project.settings.concurrency - ) - if self._verbose or self._debug: - base_schema: str | None = project.effective_target_schema - logical_schema: str | None = base_schema - if ( - base_schema is not None - and self._unsuffixed_virtual_environment_name != self._virtual_environment_name - ): - logical_schema = f"{base_schema}__{self._virtual_environment_name}" - physical_schema: str | None = ( - f"{base_schema}__sqb_physical" if base_schema is not None else None - ) - write_build_run_context( - stream=self._stream, - context=BuildRunContext( - command=f"sqb {self._execution_command}", - project=project, - plan=plan_output, - discovered_inputs=self._discovered_inputs, - python_plan_entries=python_plan_entries, - connection_config=self._connection_config, - concurrency=effective_concurrency, - full_refresh=self._full_refresh, - selector_files=self._selector_files, - virtual_logical_schema=logical_schema, - virtual_physical_schema=physical_schema, - ), - use_color=self._use_color, - ) - else: - write_execution_header( - stream=self._stream, - command=f"sqb {self._execution_command}", - target=None, - concurrency=effective_concurrency, - use_color=self._use_color, - ) - return VirtualBuildExecutionHooks( - on_node_start=lambda name, resource_kind: callbacks.on_node_start( - name=name, resource_kind=resource_kind - ), - on_node_complete=callbacks.on_node_complete, - on_sub_progress=callbacks.on_sub_progress, - on_scheduler_state=( - callbacks.on_scheduler_state if self._verbose or self._debug else None - ), - on_statement_complete=( - callbacks.on_statement_complete if self._verbose or self._debug else None - ), - ) - - def close(self) -> None: - """Close plan callbacks once without changing the build outcome.""" - - if self._closed: - return - self._closed = True - if self.callbacks is None: - return - try: - self.callbacks.close() - except BaseException: - return diff --git a/src/sqlbuild/cli/commands/classes/virtual_build_project_capture.py b/src/sqlbuild/cli/commands/classes/virtual_build_project_capture.py deleted file mode 100644 index fa7fe49446..0000000000 --- a/src/sqlbuild/cli/commands/classes/virtual_build_project_capture.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Mutable virtual-build project capture for exception finalization.""" - -from sqlbuild.compiler.compile.models import CompiledProject - - -class VirtualBuildProjectCapture: - """Capture the compiled project once the virtual plan establishes a run ID.""" - - def __init__(self) -> None: - self.project: CompiledProject | None = None - - def capture(self, project: CompiledProject) -> None: - self.project = project diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index c4c976c281..c2d025fcca 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -68,6 +68,7 @@ SCENARIO_CLI_SQL_VALIDATION_REQUIRED: str = "C455" SCENARIO_CLI_CAPTURE_DIALECT_REQUIRED: str = "C456" SCENARIO_CLI_UNSUPPORTED_GRAPH_SELECTOR: str = "C457" +SCENARIO_CLI_INVALID_NAMESPACE: str = "C458" SQL_ANALYSIS_CONFIG_KEY: str = "sql_analysis" GRAPH_SELECTOR_EXPANSION_MARKER: str = "+" GRAPH_SELECTOR_PATH_MARKER: str = "~" @@ -76,8 +77,6 @@ DBT_VERBOSE_OPTIONS: frozenset[str] = frozenset({"--verbose", "-v"}) DBT_CLI_OUTPUT_OPTIONS: frozenset[str] = frozenset({DBT_JSON_OUTPUT_OPTION, *DBT_VERBOSE_OPTIONS}) DBT_NO_CONNECTION_OPTION: str = "--no-connection" -RECONCILE_ATTACH_COMMAND: str = "attach" -STATE_CHECKPOINTS_COMMAND: str = "checkpoints" COST_HISTORY_SELECTOR: str = "history" COST_LATEST_SELECTOR: str = "latest" COST_DESCENDING_ORDER: str = "desc" diff --git a/src/sqlbuild/cli/commands/main/execution/_build.py b/src/sqlbuild/cli/commands/main/execution/_build.py index 1f2458c546..3a9f7b505a 100644 --- a/src/sqlbuild/cli/commands/main/execution/_build.py +++ b/src/sqlbuild/cli/commands/main/execution/_build.py @@ -33,14 +33,12 @@ from sqlbuild.cli.commands._helpers.cost.collection import ( finalize_build_cost, ) -from sqlbuild.cli.commands.main.execution._virtual_build import run_virtual_build from sqlbuild.cli.commands.models import ( BuildCommandRequest, BuildCostFinalization, BuildExecutionPreparation, BuildInvocation, BuildRunOutcome, - VirtualBuildCliRequest, ) from sqlbuild.compiler.pipeline.models import CompilePipelineResult from sqlbuild.cost.classes.cost_context import CostContext @@ -95,49 +93,6 @@ def _run_build( discovered_providers=invocation.discovered_inputs.providers ) try: - if invocation.virtual_mode: - return run_virtual_build( - project_dir=invocation.effective_project_dir, - discovered_inputs=invocation.discovered_inputs, - adapter=invocation.adapter, - adapter_name=invocation.adapter_name, - connection_config=invocation.connection_config, - progress_stream=invocation.progress_stream, - request=VirtualBuildCliRequest( - selected_target=request.selected_target, - no_sql_validation=request.no_sql_validation, - no_cache=request.no_cache, - defer_sources_to=request.defer_sources_to, - cursor_overrides=request.cursor_overrides, - full_refresh=request.full_refresh, - virtual_environment_name=request.virtual_env, - include_stale_upstreams=request.include_stale_upstreams, - changes_only=invocation.effective_changes_only, - auto_load_sources=invocation.should_load_sources, - reload_sources=request.reload_sources, - include_python=request.include_python, - select=request.select, - selector_files=request.selector_files, - exclude=request.exclude, - fail_fast=request.fail_fast, - allow_snapshot_full_refresh=request.allow_snapshot_full_refresh, - allow_table_type_downgrade=request.allow_table_type_downgrade, - allow_retention_decrease=request.allow_retention_decrease, - allow_snapshot_schema_change=request.allow_snapshot_schema_change, - concurrency=request.concurrency, - verbose=request.verbose, - debug=request.debug, - cli_vars=request.cli_vars, - run_tests=request.run_tests, - run_audits=request.run_audits, - json_output=request.json_output, - json_output_path=request.json_output_path, - use_color=invocation.use_color, - providers=provider_session.providers, - command_started_at=command_started_at, - max_microbatches=request.max_microbatches, - ), - ) if invocation.effective_defer_clone_from is not None: _ = run_defer_clone_boundary_prephase( request=request, diff --git a/src/sqlbuild/cli/commands/main/execution/_freshness.py b/src/sqlbuild/cli/commands/main/execution/_freshness.py index 09c69a9b89..ea97f7cf07 100644 --- a/src/sqlbuild/cli/commands/main/execution/_freshness.py +++ b/src/sqlbuild/cli/commands/main/execution/_freshness.py @@ -18,7 +18,6 @@ from sqlbuild.cli.commands._helpers.freshness.selection import resolve_freshness_source_names from sqlbuild.cli.commands._helpers.freshness.state import ( read_direct_freshness_state_for_command, - read_virtual_freshness_state_for_command, ) from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter from sqlbuild.cli.commands._helpers.runtime.connection import ( @@ -49,7 +48,6 @@ def run_freshness(request: FreshnessCommandRequest) -> int: select: tuple[str, ...] = request.select cli_vars: dict[str, object] | None = request.cli_vars compare_state: bool = request.compare_state - virtual_environment_name: str | None = request.virtual_environment_name if request.fail_on_stale and not compare_state: raise CliUserError("freshness --fail-on-stale requires --state", code="C238") effective_project_dir: Path = ( @@ -93,13 +91,7 @@ def run_freshness(request: FreshnessCommandRequest) -> int: try: previous_records_by_source_name: dict[str, SourceFreshnessRecord] | None = None previous_records: dict[SourceFreshnessIdentity, SourceFreshnessRecord] | None = None - if compare_state and virtual_environment_name is not None: - previous_records_by_source_name = read_virtual_freshness_state_for_command( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - virtual_environment_name=virtual_environment_name, - ) - elif compare_state: + if compare_state: previous_records = read_direct_freshness_state_for_command( adapter=adapter, connection=connection, diff --git a/src/sqlbuild/cli/commands/main/execution/_seed.py b/src/sqlbuild/cli/commands/main/execution/_seed.py index 0c253ea443..c7261e2a56 100644 --- a/src/sqlbuild/cli/commands/main/execution/_seed.py +++ b/src/sqlbuild/cli/commands/main/execution/_seed.py @@ -10,7 +10,6 @@ write_seed_execution_header, ) from sqlbuild.cli.commands._helpers.seed.planning import prepare_seed_execution -from sqlbuild.cli.commands._helpers.seed.virtual import execute_virtual_seed from sqlbuild.cli.commands.models import ( SeedCommandRequest, SeedExecutionPreparation, @@ -23,8 +22,6 @@ def run_seed(request: SeedCommandRequest) -> int: """Execute the seed command.""" invocation: SeedInvocation = resolve_seed_invocation(request=request) - if invocation.discovered_inputs.project_config.settings.virtual_environments: - return execute_virtual_seed(request=request, invocation=invocation) preparation: SeedExecutionPreparation = prepare_seed_execution( request=request, invocation=invocation, diff --git a/src/sqlbuild/cli/commands/main/execution/_virtual_build.py b/src/sqlbuild/cli/commands/main/execution/_virtual_build.py deleted file mode 100644 index 5d71f4bc1d..0000000000 --- a/src/sqlbuild/cli/commands/main/execution/_virtual_build.py +++ /dev/null @@ -1,301 +0,0 @@ -"""CLI orchestration for virtual-mode build.""" - -from __future__ import annotations - -import sys -import time -from datetime import UTC, datetime -from pathlib import Path -from typing import TextIO - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.build_execution.outputs import ( - execution_json_payload_with_degradation, -) -from sqlbuild.cli.commands._helpers.build_execution.phase_timings import ( - record_and_write_virtual_build_phase_timings, -) -from sqlbuild.cli.commands._helpers.build_python_nodes.python_node_output import ( - write_python_node_results, -) -from sqlbuild.cli.commands._helpers.build_virtual.virtual_checks import ( - run_post_virtual_build_checks, -) -from sqlbuild.cli.commands._helpers.build_virtual.virtual_execution import execute_virtual_build -from sqlbuild.cli.commands._helpers.check.core import check_results_failed -from sqlbuild.cli.commands._helpers.cost.collection import ( - finalize_build_cost, - render_build_cost, -) -from sqlbuild.cli.commands.classes.build_progress_callbacks import format_build_footer -from sqlbuild.cli.commands.classes.virtual_build_project_capture import ( - VirtualBuildProjectCapture, -) -from sqlbuild.cli.commands.models import ( - BuildCostFinalization, - VirtualBuildCliRequest, - VirtualBuildExecution, -) -from sqlbuild.cli.output.main._write_execution_json_output import write_execution_json_output -from sqlbuild.cli.target_artifacts.main._write_python_check_runtime_target import ( - write_python_check_runtime_target, -) -from sqlbuild.cli.target_artifacts.main._write_runtime_target import write_runtime_target -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.plan_work import plan_has_executable_work -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.models import CostRunRecord -from sqlbuild.cost.types import CostStatus -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.executor.python_nodes.models import PythonCheckExecutionResult -from sqlbuild.virtual.executor.models import VirtualBuildPipelineResult - - -def run_virtual_build( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - adapter_name: str, - connection_config: dict[str, object], - request: VirtualBuildCliRequest, - progress_stream: TextIO | None = None, -) -> int: - """Execute a virtual build and render CLI output.""" - - build_started_at: datetime = datetime.now(UTC) - stream: TextIO = progress_stream or ( - sys.stderr if request.debug or request.json_output else sys.stdout - ) - project_capture: VirtualBuildProjectCapture = VirtualBuildProjectCapture() - - def persist_started_cost(project: CompiledProject) -> None: - project_capture.capture(project) - try: - _ = _persist_virtual_started_cost( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - adapter_name=adapter_name, - connection_config=connection_config, - project=project, - stream=stream, - started_at=build_started_at, - use_color=request.use_color, - ) - except BaseException: - return - - try: - execution: VirtualBuildExecution = execute_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - adapter_name=adapter_name, - connection_config=connection_config, - request=request, - progress_stream=stream, - on_project_ready=persist_started_cost, - ) - result: VirtualBuildPipelineResult = execution.result - return _complete_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - adapter_name=adapter_name, - connection_config=connection_config, - request=request, - execution=execution, - result=result, - stream=stream, - build_started_at=build_started_at, - ) - except BaseException as error: - compiled_project: CompiledProject | None = project_capture.project - if compiled_project is None: - raise - interrupted: bool = isinstance(error, KeyboardInterrupt) - exceptional_cost_started_at: float = time.monotonic() - try: - _ = finalize_build_cost( - BuildCostFinalization( - project_dir=project_dir, - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - target_name=compiled_project.effective_target_name, - target_database=compiled_project.effective_target_database, - run_id=compiled_project.run_id, - build_status="interrupted" if interrupted else "failed", - started_at=build_started_at, - completed_at=datetime.now(UTC), - config=discovered_inputs.project_config.cost, - output_stream=stream, - use_color=request.use_color, - collect=not interrupted, - render=False, - cost_status=CostStatus.PARTIAL, - cost_message="Build was interrupted before cost collection completed.", - ) - ) - except BaseException: - pass - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - if timing_tracker is not None: - timing_tracker.cost_collection_seconds = time.monotonic() - exceptional_cost_started_at - raise - - -def _complete_virtual_build( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - adapter_name: str, - connection_config: dict[str, object], - request: VirtualBuildCliRequest, - execution: VirtualBuildExecution, - result: VirtualBuildPipelineResult, - stream: TextIO, - build_started_at: datetime, -) -> int: - plan_output: PlanOutput = result.display_plan_output - write_python_node_results( - stream=stream, - results=result.python_node_results, - use_color=request.use_color, - ) - with CostContext.scope( - run_id=result.project.run_id, - resource_type="run", - resource_name=result.project.effective_target_name or adapter_name, - ledger_path=( - project_dir / "target" / "executions" / result.project.run_id / "statements.jsonl" - ), - phase="post_build_checks", - ): - check_results: tuple[PythonCheckExecutionResult, ...] = run_post_virtual_build_checks( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - result=result, - exclude=request.exclude, - reload_sources=request.reload_sources, - providers=request.providers, - stream=stream, - use_color=request.use_color, - ) - build_completed_at: datetime = datetime.now(UTC) - footer: str = format_build_footer( - result=result.execution_result, - elapsed=execution.elapsed, - use_color=request.use_color, - python_node_results=result.python_node_results, - ) - write_runtime_target( - target_dir=project_dir / "target", - plan_output=plan_output, - result=result.execution_result, - ) - write_python_check_runtime_target(target_dir=project_dir / "target", results=check_results) - exit_code: int = ( - 0 - if result.execution_result.status == BuildStatus.SUCCESS - and not check_results_failed(check_results) - else 1 - ) - had_executable_work: bool = bool(result.python_node_results) or plan_has_executable_work( - plan=result.execution_plan, - python_plan_entries=(), - ) - cost_started_at: float = time.monotonic() - cost_record: CostRunRecord | None = finalize_build_cost( - BuildCostFinalization( - project_dir=project_dir, - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - target_name=result.project.effective_target_name, - target_database=result.project.effective_target_database, - run_id=result.project.run_id, - build_status="success" if exit_code == 0 else "failed", - started_at=build_started_at, - completed_at=build_completed_at, - config=discovered_inputs.project_config.cost, - output_stream=stream, - use_color=request.use_color, - collect=had_executable_work, - render=False, - had_executable_work=had_executable_work, - ) - ) - cost_collection_seconds: float = time.monotonic() - cost_started_at - stream.write("\n" + footer + "\n") - stream.flush() - payload: str = execution_json_payload_with_degradation( - result=result.execution_result, - plan=plan_output, - python_node_results=result.python_node_results, - python_check_results=check_results, - command=request.execution_command, - run_id=None if cost_record is None else result.project.run_id, - cost_record=cost_record, - execution_succeeded=exit_code == 0, - ) - write_execution_json_output( - payload=payload, - json_output=request.json_output, - json_output_path=request.json_output_path, - ) - _ = render_build_cost( - record=cost_record, - output_stream=stream, - use_color=request.use_color, - ) - _ = record_and_write_virtual_build_phase_timings( - stream=stream, - request=request, - execution=execution, - result=result, - cost_collection_seconds=cost_collection_seconds, - ) - return exit_code - - -def _persist_virtual_started_cost( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - adapter_name: str, - connection_config: dict[str, object], - project: CompiledProject, - stream: TextIO, - started_at: datetime, - use_color: bool, -) -> None: - _ = finalize_build_cost( - BuildCostFinalization( - project_dir=project_dir, - adapter_name=adapter_name, - adapter=adapter, - connection_config=connection_config, - target_name=project.effective_target_name, - target_database=project.effective_target_database, - run_id=project.run_id, - build_status="running", - started_at=started_at, - completed_at=started_at, - config=discovered_inputs.project_config.cost, - output_stream=stream, - use_color=use_color, - collect=False, - render=False, - cost_status=CostStatus.PENDING, - cost_message="Build cost collection has not completed.", - ) - ) diff --git a/src/sqlbuild/cli/commands/main/inspection/_clone.py b/src/sqlbuild/cli/commands/main/inspection/_clone.py index 2ee1f8b9bf..996a76626b 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_clone.py +++ b/src/sqlbuild/cli/commands/main/inspection/_clone.py @@ -15,7 +15,6 @@ write_clone_execution_json_output, ) from sqlbuild.cli.commands._helpers.clone.planning import prepare_clone_execution -from sqlbuild.cli.commands._helpers.clone.virtual import execute_virtual_clone from sqlbuild.cli.commands.models import ( CloneCommandRequest, CloneConnectionContext, @@ -29,8 +28,6 @@ def run_clone(request: CloneCommandRequest) -> int: """Execute the clone command.""" invocation: CloneInvocation = resolve_clone_invocation(request=request) - if invocation.discovered_inputs.project_config.settings.virtual_environments: - return execute_virtual_clone(request=request, invocation=invocation) connection_context: CloneConnectionContext = connect_clone_targets( request=request, invocation=invocation, diff --git a/src/sqlbuild/cli/commands/main/inspection/_diff.py b/src/sqlbuild/cli/commands/main/inspection/_diff.py index d8608b0155..e48ffabe89 100644 --- a/src/sqlbuild/cli/commands/main/inspection/_diff.py +++ b/src/sqlbuild/cli/commands/main/inspection/_diff.py @@ -6,10 +6,8 @@ from sqlbuild.cli.commands._helpers.diff.execution import ( execute_direct_diff, execute_query_diff, - execute_virtual_diff, prepare_direct_diff, prepare_query_diff, - prepare_virtual_diff, ) from sqlbuild.cli.commands._helpers.diff.invocation import ( is_query_diff_request, @@ -18,7 +16,6 @@ from sqlbuild.cli.commands._helpers.diff.outputs import ( resolve_diff_exit_code, write_direct_diff_output, - write_virtual_diff_output, ) from sqlbuild.cli.commands.constants import ( QUERY_DIFF_INCOMPLETE_EXECUTION_CODES, @@ -36,8 +33,6 @@ DirectDiffPreparation, QueryDiffPreparation, QueryDiffRunOutcome, - VirtualDiffPreparation, - VirtualDiffRunOutcome, ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.diff.constants import ( @@ -130,22 +125,6 @@ def run_diff(request: DiffCommandRequest) -> int: f"failed to publish query diff output: {error}", code="C257" ) from error return query_outcome.exit_code - if invocation.is_virtual_mode: - virtual_preparation: VirtualDiffPreparation = prepare_virtual_diff( - request=request, - invocation=invocation, - ) - virtual_outcome: VirtualDiffRunOutcome = execute_virtual_diff( - request=request, - invocation=invocation, - preparation=virtual_preparation, - ) - write_virtual_diff_output( - request=request, - preparation=virtual_preparation, - outcome=virtual_outcome, - ) - return resolve_diff_exit_code(virtual_outcome.result) direct_preparation: DirectDiffPreparation = prepare_direct_diff( request=request, diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 7818c3dd77..404cc1c35a 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -18,6 +18,7 @@ write_compile_artifacts, write_compile_dag_artifact, ) +from sqlbuild.cli.commands._helpers.compile.semantic_notice import semantic_coverage_notice from sqlbuild.cli.commands._helpers.compile.status import elapsed_ms, start_compile_status from sqlbuild.cli.commands.classes.prepared_compile_artifacts import PreparedCompileArtifacts from sqlbuild.cli.commands.types import CompileLineageMode @@ -152,6 +153,13 @@ def _run_compile_with_status( print(f"note: {skipped_rules_note}", file=sys.stderr) if json_output: + notice: str | None = semantic_coverage_notice( + project=analysis.graph.project, + selected_keys=analysis.selected_keys, + sql_validation_enabled=not request.no_sql_validation, + ) + if notice: + print(notice, file=sys.stderr) print( format_compile_json( graph=analysis.graph, @@ -162,6 +170,7 @@ def _run_compile_with_status( lineage_mode=lineage_mode, diagnostics=diagnostics, selected_keys=analysis.selected_keys, + sql_validation_enabled=not request.no_sql_validation, ) ) return exit_code @@ -176,6 +185,7 @@ def _run_compile_with_status( diagnostics=diagnostics, selected_keys=analysis.selected_keys, use_color=(not no_color) and supports_color(), + sql_validation_enabled=not request.no_sql_validation, ) ) return exit_code diff --git a/src/sqlbuild/cli/commands/main/state/_janitor.py b/src/sqlbuild/cli/commands/main/state/_janitor.py index c64d644d9e..c5cf44e41a 100644 --- a/src/sqlbuild/cli/commands/main/state/_janitor.py +++ b/src/sqlbuild/cli/commands/main/state/_janitor.py @@ -22,7 +22,6 @@ ) from sqlbuild.cli.commands._helpers.janitor_runtime.planning import ( build_janitor_execution_plan, - inspect_janitor_retention, ) from sqlbuild.cli.commands.models import ( JanitorCommandRequest, @@ -30,7 +29,6 @@ JanitorConnectionContext, JanitorInvocation, JanitorPlanningResult, - JanitorRetentionInspection, JanitorSettings, ) from sqlbuild.executor.janitor.models import JanitorExecutionResult @@ -50,17 +48,11 @@ def run_janitor(request: JanitorCommandRequest) -> int: compile_context=compile_context, ) try: - inspection: JanitorRetentionInspection = inspect_janitor_retention( - invocation=invocation, - settings=settings, - compile_context=compile_context, - ) planning_result: JanitorPlanningResult = build_janitor_execution_plan( invocation=invocation, settings=settings, compile_context=compile_context, connection_context=connection_context, - inspection=inspection, ) write_janitor_plan(invocation=invocation, planning_result=planning_result) if planning_result.plan.blocked_schemas: @@ -71,10 +63,8 @@ def run_janitor(request: JanitorCommandRequest) -> int: write_janitor_cancelled() return 1 result: JanitorExecutionResult = execute_janitor_cleanup( - invocation=invocation, compile_context=compile_context, connection_context=connection_context, - inspection=inspection, planning_result=planning_result, ) write_janitor_completion(invocation=invocation, result=result) diff --git a/src/sqlbuild/cli/commands/main/state/_promote.py b/src/sqlbuild/cli/commands/main/state/_promote.py deleted file mode 100644 index 913587e2db..0000000000 --- a/src/sqlbuild/cli/commands/main/state/_promote.py +++ /dev/null @@ -1,121 +0,0 @@ -"""CLI promote command entry point.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.promote.output import format_promote_output -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import PromoteCommandRequest -from sqlbuild.cli.progress.classes.connection_progress_reporter import ( - ConnectionProgressReporter, -) -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.virtual.executor.main.promote import run_virtual_promote -from sqlbuild.virtual.executor.models import PromoteOptions - - -def run_promote(request: PromoteCommandRequest) -> int: - """Execute the promote command.""" - - project_dir: Path | None = request.project_dir - no_color: bool = request.no_color - no_sql_validation: bool = request.no_sql_validation - from_virtual_environment: str = request.from_virtual_environment - to_virtual_environment: str = request.to_virtual_environment - select: tuple[str, ...] = request.select - exclude: tuple[str, ...] = request.exclude - allow_partial_promotion: bool = request.allow_partial_promotion - include_stale_upstreams: bool = request.include_stale_upstreams - verbose: bool = request.verbose - cli_vars: dict[str, object] | None = request.cli_vars - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - if not discovered_inputs.project_config.settings.virtual_environments: - raise CliUserError("promote requires virtual_environments = true", code="C243") - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - cli_vars=cli_vars, - ) - use_color: bool = not no_color and supports_color() - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=sys.stdout, - use_color=use_color, - ) - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - stream=sys.stdout, - use_color=use_color, - ) - status, promoted_models, remaining_stale = run_virtual_promote( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - from_virtual_environment_name=from_virtual_environment, - to_virtual_environment_name=to_virtual_environment, - options=PromoteOptions( - select=select, - exclude=exclude, - allow_partial_promotion=allow_partial_promotion, - include_stale_upstreams=include_stale_upstreams, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ), - ), - hooks=ConnectionHooks( - on_progress=planning_progress.on_progress, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - print( - format_promote_output( - from_virtual_environment=from_virtual_environment, - to_virtual_environment=to_virtual_environment, - status=status, - promoted_models=promoted_models, - remaining_stale=remaining_stale, - verbose=verbose, - use_color=use_color, - ) - ) - return 0 diff --git a/src/sqlbuild/cli/commands/main/state/_reconcile.py b/src/sqlbuild/cli/commands/main/state/_reconcile.py deleted file mode 100644 index 828a157daa..0000000000 --- a/src/sqlbuild/cli/commands/main/state/_reconcile.py +++ /dev/null @@ -1,87 +0,0 @@ -"""CLI reconcile command entry point.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.reconcile.output import format_reconcile_output -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) -from sqlbuild.cli.commands.constants import RECONCILE_ATTACH_COMMAND -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.virtual.reconcile.main.reconcile import run_virtual_reconcile - - -def run_reconcile( - *, - project_dir: Path | None, - no_color: bool, - virtual_environment: str | None, - reconcile_command: str | None, - model_name: str | None, - seed_name: str | None, - physical_relation_name: str | None, - auto_approve: bool = False, - cli_vars: dict[str, object] | None = None, -) -> int: - """Execute the reconcile command.""" - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - if not discovered_inputs.project_config.settings.virtual_environments: - raise CliUserError("reconcile requires virtual_environments = true", code="C254") - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - cli_vars=cli_vars, - ) - if reconcile_command == RECONCILE_ATTACH_COMMAND and not auto_approve: - if model_name is None: - raise CliUserError("reconcile attach requires --model", code="C249") - prompt: str = f"Type 'attach {model_name}' to confirm: " - if input(prompt).strip() != f"attach {model_name}": - raise CliUserError("reconcile attach cancelled", code="C262") - use_color: bool = not no_color and supports_color() - status: TransientStatusReporter = TransientStatusReporter( - stream=sys.stdout, - use_color=use_color, - ) - status.start("Reconciling virtual environment...") - try: - message: str = run_virtual_reconcile( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment, - command=reconcile_command, - model_name=model_name, - seed_name=seed_name, - physical_relation_name=physical_relation_name, - ) - status.complete(message="Reconciled virtual environment.") - except BaseException: - status.error("Reconcile failed.") - raise - print(format_reconcile_output(message=message, use_color=use_color)) - return 0 diff --git a/src/sqlbuild/cli/commands/main/state/_rollback.py b/src/sqlbuild/cli/commands/main/state/_rollback.py deleted file mode 100644 index e312cd895f..0000000000 --- a/src/sqlbuild/cli/commands/main/state/_rollback.py +++ /dev/null @@ -1,119 +0,0 @@ -"""CLI rollback command entry point.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands._helpers.planning.external_refs import ( - resolve_external_sql_reference_resolver, -) -from sqlbuild.cli.commands._helpers.rollback.output import format_rollback_output -from sqlbuild.cli.commands._helpers.runtime.adapters import resolve_adapter -from sqlbuild.cli.commands._helpers.runtime.connection import ( - resolve_project_connection_config, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.models import RollbackCommandRequest -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.cli.progress.classes.planning_progress_reporter import PlanningProgressReporter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( - resolve_effective_adapter_name, -) -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.executor.main.rollback import run_virtual_rollback -from sqlbuild.virtual.executor.models import RollbackOptions - - -def run_rollback(request: RollbackCommandRequest) -> int: - """Execute the rollback command.""" - - cli_vars: dict[str, object] | None = request.cli_vars - effective_project_dir: Path = ( - request.project_dir if request.project_dir is not None else Path.cwd() - ) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - if not discovered_inputs.project_config.settings.virtual_environments: - raise CliUserError("rollback requires virtual_environments = true", code="C245") - adapter_name: str = resolve_effective_adapter_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - ) - adapter: BaseAdapter = resolve_adapter( - adapter_name=adapter_name, project_dir=effective_project_dir - ) - connection_config: dict[str, object] = resolve_project_connection_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - cli_vars=cli_vars, - ) - resolved_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - virtual_environment_name: str | None = request.virtual_environment or resolved_target_name - if virtual_environment_name is None: - raise CliUserError("rollback requires --virtual-env or a default environment", code="C246") - use_color: bool = not request.no_color and supports_color() - planning_progress: PlanningProgressReporter = PlanningProgressReporter( - stream=sys.stdout, - use_color=use_color, - ) - connection_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name=adapter_name, - stream=sys.stdout, - use_color=use_color, - ) - restored_checkpoint_id, rolled_back_models, status = run_virtual_rollback( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - options=RollbackOptions( - checkpoint_id=request.checkpoint_id, - select=request.select, - exclude=request.exclude, - allow_partial_rollback=request.allow_partial_rollback, - include_stale_upstreams=request.include_stale_upstreams, - no_sql_validation=request.no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=resolve_external_sql_reference_resolver( - project_dir=effective_project_dir, - discovered_inputs=discovered_inputs, - ), - ), - hooks=ConnectionHooks( - on_progress=planning_progress.on_progress, - on_connection_start=connection_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - connection_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ), - ) - print( - format_rollback_output( - virtual_environment=virtual_environment_name, - checkpoint_id=restored_checkpoint_id, - rolled_back_models=rolled_back_models, - status=status.value, - verbose=request.verbose, - use_color=use_color, - ) - ) - return 0 diff --git a/src/sqlbuild/cli/commands/main/state/_state.py b/src/sqlbuild/cli/commands/main/state/_state.py deleted file mode 100644 index 1202c915c3..0000000000 --- a/src/sqlbuild/cli/commands/main/state/_state.py +++ /dev/null @@ -1,114 +0,0 @@ -"""CLI state command entry point.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from sqlbuild.cli.commands._helpers.state.checkpoints import run_state_checkpoints -from sqlbuild.cli.commands._helpers.state.interactive_operation import ( - run_interactive_state_operation, -) -from sqlbuild.cli.commands.constants import STATE_CHECKPOINTS_COMMAND -from sqlbuild.cli.progress.classes.connection_progress_reporter import ConnectionProgressReporter -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.virtual.state.main.environments.adopt import run_state_adopt -from sqlbuild.virtual.state.main.environments.detach import run_state_detach -from sqlbuild.virtual.state.main.environments.lifecycle import run_state_lifecycle -from sqlbuild.virtual.state.types import StateCommand - - -def run_state( - *, - project_dir: Path | None, - state_command: str, - backup_id: str | None = None, - auto_approve: bool = False, - no_color: bool = False, - checkpoint_command: str | None = None, - checkpoint_id: str | None = None, - virtual_environment: str | None = None, - allow_copy: bool = False, -) -> int: - """Execute a virtual state lifecycle command.""" - - if state_command == StateCommand.ADOPT.value: - return _run_state_adopt_command( - project_dir=project_dir, - auto_approve=auto_approve, - allow_copy=allow_copy, - no_color=no_color, - ) - if state_command == StateCommand.DETACH.value: - return _run_state_detach_command( - project_dir=project_dir, - auto_approve=auto_approve, - allow_copy=allow_copy, - no_color=no_color, - ) - if state_command == STATE_CHECKPOINTS_COMMAND: - return run_state_checkpoints( - project_dir=project_dir, - command=checkpoint_command, - checkpoint_id=checkpoint_id, - virtual_environment_name=virtual_environment, - no_color=no_color, - ) - use_color: bool = not no_color and supports_color() - state_progress: ConnectionProgressReporter = ConnectionProgressReporter( - adapter_name="state store", - stream=sys.stdout, - use_color=use_color, - ) - return run_state_lifecycle( - project_dir=project_dir, - command=StateCommand(state_command), - backup_id=backup_id, - auto_approve=auto_approve, - no_color=no_color, - on_connection_start=state_progress.on_connection_start, - on_connection_complete=lambda connection_count, elapsed_seconds: ( - state_progress.on_connection_complete( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - on_connection_error=lambda connection_count, elapsed_seconds: ( - state_progress.on_connection_error( - connection_count=connection_count, elapsed_seconds=elapsed_seconds - ) - ), - ) - - -def _run_state_adopt_command( - *, - project_dir: Path | None, - auto_approve: bool, - allow_copy: bool, - no_color: bool, -) -> int: - return run_interactive_state_operation( - project_dir=project_dir, - state_command=StateCommand.ADOPT, - operation_runner=run_state_adopt, - auto_approve=auto_approve, - allow_copy=allow_copy, - no_color=no_color, - ) - - -def _run_state_detach_command( - *, - project_dir: Path | None, - auto_approve: bool, - allow_copy: bool, - no_color: bool, -) -> int: - return run_interactive_state_operation( - project_dir=project_dir, - state_command=StateCommand.DETACH, - operation_runner=run_state_detach, - auto_approve=auto_approve, - allow_copy=allow_copy, - no_color=no_color, - ) diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 626d53e259..945f2fa93a 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -69,7 +69,6 @@ ) from sqlbuild.compiler.planner.models import CursorOverrides, PlanOutput from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus from sqlbuild.cost.types import CostStatus from sqlbuild.executor.build.models import BuildExecutionResult, SeedExecutionResult @@ -80,18 +79,9 @@ from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult from sqlbuild.integrations.dbt.models import DbtInitRequest from sqlbuild.presentation.classes.transient_status_reporter import TransientStatusReporter -from sqlbuild.provider.main.runtime import ProviderContainer from sqlbuild.python_nodes.models import SqlResourceRef from sqlbuild.runtime.contracts.types import NodeStartCallback from sqlbuild.spec.contracts.models import CostConfig, ExecutionLimitsConfig, SourceEntry -from sqlbuild.virtual.executor.models import VirtualBuildPipelineResult -from sqlbuild.virtual.state.models import ( - CheckpointRetentionInspection, - DetachedVirtualEnvironmentInspection, - ExpiredVirtualEnvironmentInspection, - PhysicalRelationRecord, - StateJanitorInspection, -) @dataclass(frozen=True) @@ -149,8 +139,6 @@ class BuildRunContext: concurrency: int full_refresh: bool selector_files: tuple[SelectorFileSummary, ...] - virtual_logical_schema: str | None = None - virtual_physical_schema: str | None = None @dataclass(frozen=True) @@ -167,7 +155,6 @@ class BuildCommandRequest: no_color: bool = False fail_fast: bool = False full_refresh: bool = False - virtual_env: str | None = None load_sources: bool | None = None reload_sources: bool = False include_python: bool = True @@ -181,8 +168,6 @@ class BuildCommandRequest: verbose: bool = False debug: bool = False cli_vars: dict[str, object] | None = None - include_stale_upstreams: bool = False - changes_only: bool = False run_tests: bool = True run_audits: bool = True manifest: bool = False @@ -212,7 +197,6 @@ class BuildInvocation: effective_project_dir: Path discovered_inputs: DiscoveredProjectInputs effective_defer_clone_from: str | None - effective_changes_only: bool adapter_name: str adapter: BaseAdapter connection_config: dict[str, object] @@ -221,7 +205,6 @@ class BuildInvocation: connection_progress: ConnectionProgressReporter planning_progress: PlanningProgressReporter should_load_sources: bool - virtual_mode: bool effective_target_name: str | None = None execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) @@ -296,78 +279,6 @@ class BuildPhaseTimings: total_seconds: float | None = None -@dataclass(frozen=True) -class VirtualBuildPlanHookConfig: - """Rendering, safety, and header options for the virtual build plan hook.""" - - full_refresh: bool - allow_snapshot_full_refresh: bool - allow_table_type_downgrade: bool - allow_retention_decrease: bool - use_color: bool - verbose: bool - debug: bool - json_output: bool - execution_command: str - concurrency: int | None - connection_config: dict[str, object] = field(default_factory=dict) - selector_files: tuple[SelectorFileSummary, ...] = () - virtual_environment_name: str | None = None - unsuffixed_virtual_environment_name: str | None = None - effective_target_name: str | None = None - execution_limits: ExecutionLimitsConfig = field(default_factory=ExecutionLimitsConfig) - - -@dataclass(frozen=True) -class VirtualBuildCliRequest: - """Flag and option inputs for the virtual-build CLI entrypoint.""" - - selected_target: str | None = None - no_sql_validation: bool = False - defer_sources_to: str | None = None - cursor_overrides: CursorOverrides | None = None - full_refresh: bool = False - virtual_environment_name: str | None = None - include_stale_upstreams: bool = False - changes_only: bool = False - auto_load_sources: bool = False - reload_sources: bool = False - include_python: bool = True - seed_only: bool = False - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - fail_fast: bool = False - allow_snapshot_full_refresh: bool = False - allow_table_type_downgrade: bool = False - allow_retention_decrease: bool = False - allow_snapshot_schema_change: bool = False - concurrency: int | None = None - verbose: bool = False - debug: bool = False - cli_vars: dict[str, object] | None = None - run_tests: bool = True - run_audits: bool = True - json_output: bool = False - json_output_path: Path | None = None - execution_command: str = "build" - use_color: bool = False - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - providers: ProviderContainer | None = None - no_cache: bool = False - selector_files: tuple[SelectorFileSummary, ...] = () - max_microbatches: int | None = None - command_started_at: float | None = None - - -@dataclass(frozen=True) -class VirtualBuildExecution: - """Pipeline result and output context produced by virtual build execution.""" - - result: VirtualBuildPipelineResult - stream: TextIO - elapsed: float - - @dataclass(frozen=True) class CheckCommandRequest: """CLI inputs for one check command invocation.""" @@ -421,8 +332,6 @@ class CloneCommandRequest: origin_target_name: str destination_target_name: str | None hard_copy: bool - virtual_env: str | None = None - skip_locked: bool = False select: tuple[str, ...] = () exclude: tuple[str, ...] = () verbose: bool = False @@ -551,7 +460,6 @@ class DiffCommandRequest: exclude: tuple[str, ...] = () verbose: bool = False cli_vars: dict[str, object] | None = None - allow_partial_diff: bool = False selected_target: str | None = None left_query: str | None = None left_query_file: Path | None = None @@ -574,7 +482,6 @@ class DiffInvocation: effective_project_dir: Path discovered_inputs: DiscoveredProjectInputs - is_virtual_mode: bool @dataclass(frozen=True) @@ -640,32 +547,6 @@ class QueryDiffRunOutcome: exit_code: int -@dataclass(frozen=True) -class VirtualDiffPreparation: - """Resolved virtual diff adapter, connection, reporters, and sample limits.""" - - from_virtual_environment: str - to_virtual_environment: str - adapter: BaseAdapter - connection_config: dict[str, object] - effective_max_column_examples: int - effective_max_row_only_examples: int - use_color: bool - - -@dataclass(frozen=True) -class VirtualDiffRunOutcome: - """Virtual diff result plus virtual environment freshness metadata.""" - - result: DiffExecutionResult - selected_names: tuple[str, ...] - skipped_names: tuple[str, ...] - from_stale: tuple[str, ...] - to_stale: tuple[str, ...] - from_working: bool - to_working: bool - - @dataclass(frozen=True) class FreshnessCommandRequest: """CLI inputs for one `sqb freshness` invocation.""" @@ -682,7 +563,6 @@ class FreshnessCommandRequest: fail_on_error: bool = False compare_state: bool = False fail_on_stale: bool = False - virtual_environment_name: str | None = None @dataclass(frozen=True) @@ -804,18 +684,6 @@ class JanitorConnectionContext: connection: object -@dataclass(frozen=True) -class JanitorRetentionInspection: - """Retention inspection results used to build a janitor plan.""" - - checkpoint: CheckpointRetentionInspection | None - detached_environment: DetachedVirtualEnvironmentInspection | None - expired_environment: ExpiredVirtualEnvironmentInspection | None - state: StateJanitorInspection | None - unsuffixed_virtual_environment_name: str | None - active_microbatch_replay_relations: tuple[PhysicalRelationRecord, ...] = () - - @dataclass(frozen=True) class JanitorPlanningResult: """Built janitor plan.""" @@ -981,7 +849,6 @@ class PlanCommandRequest: cursor_overrides: CursorOverrides | None = None json_output: bool = False full_refresh: bool = False - virtual_env: str | None = None load_sources: bool | None = None include_python: bool = True no_color: bool = False @@ -989,8 +856,6 @@ class PlanCommandRequest: exclude: tuple[str, ...] = () verbose: bool = False cli_vars: dict[str, object] | None = None - include_stale_upstreams: bool = False - changes_only: bool = False no_cache: bool = False max_microbatches: int | None = None selection_diagnostics: bool = False @@ -1002,7 +867,6 @@ class PlanInvocation: effective_project_dir: Path discovered_inputs: DiscoveredProjectInputs - effective_changes_only: bool adapter: BaseAdapter connection_config: dict[str, object] use_color: bool @@ -1010,7 +874,6 @@ class PlanInvocation: connection_progress: ConnectionProgressReporter planning_progress: PlanningProgressReporter should_load_sources: bool - virtual_mode: bool @dataclass(frozen=True) @@ -1030,40 +893,6 @@ class PlaygroundTarget: template: PlaygroundTemplate -@dataclass(frozen=True) -class PromoteCommandRequest: - """CLI inputs for one `sqb promote` invocation.""" - - project_dir: Path | None - no_color: bool - no_sql_validation: bool - from_virtual_environment: str - to_virtual_environment: str - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - allow_partial_promotion: bool = False - include_stale_upstreams: bool = False - verbose: bool = False - cli_vars: dict[str, object] | None = None - - -@dataclass(frozen=True) -class RollbackCommandRequest: - """CLI inputs for one `sqb rollback` invocation.""" - - project_dir: Path | None - no_color: bool - no_sql_validation: bool - virtual_environment: str | None - verbose: bool = False - checkpoint_id: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - allow_partial_rollback: bool = False - include_stale_upstreams: bool = False - cli_vars: dict[str, object] | None = None - - @dataclass(frozen=True) class AdapterConnectionContext: """Resolved adapter and connection configuration for one CLI command.""" @@ -1073,12 +902,21 @@ class AdapterConnectionContext: connection_config: dict[str, object] +@dataclass(frozen=True) +class ScenarioRunNamespace: + """Effective artifact namespace and its configuration source.""" + + value: str | None = None + source: str = "unset" + + @dataclass(frozen=True) class ScenarioRunOutputContext: """Progress stream and JSON output settings for one scenario CLI run.""" progress_stream: TextIO use_color: bool + namespace: ScenarioRunNamespace = ScenarioRunNamespace() json_output: bool = False json_output_path: Path | None = None @@ -1099,6 +937,7 @@ class ScenarioTestCommandRequest: """CLI inputs for one `sqb scenario test` invocation.""" project_dir: Path | None = None + scenario_namespace: str | None = None no_sql_validation: bool = False no_color: bool = False selectors: tuple[str, ...] = () @@ -1118,6 +957,7 @@ class ScenarioCaptureCommandRequest: """CLI inputs for one `sqb scenario capture` invocation.""" project_dir: Path | None = None + scenario_namespace: str | None = None no_sql_validation: bool = False no_color: bool = False selectors: tuple[str, ...] = () diff --git a/src/sqlbuild/cli/commands/types.py b/src/sqlbuild/cli/commands/types.py index 0f2d1036e4..57203149cf 100644 --- a/src/sqlbuild/cli/commands/types.py +++ b/src/sqlbuild/cli/commands/types.py @@ -38,16 +38,12 @@ class CliCommand(StrEnum): SEED = "seed" CLONE = "clone" DIFF = "diff" - RECONCILE = "reconcile" - PROMOTE = "promote" - ROLLBACK = "rollback" DEBUG = "debug" LINEAGE = "lineage" QUERY = "query" COST = "cost" CLEAN = "clean" JANITOR = "janitor" - STATE = "state" INIT = "init" PLAYGROUND = "playground" SCENARIO = "scenario" @@ -78,22 +74,6 @@ def __call__( ) -> int: ... -class ReconcileCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - no_color: bool, - virtual_environment: str | None, - reconcile_command: str | None, - model_name: str | None, - seed_name: str | None, - physical_relation_name: str | None, - auto_approve: bool, - cli_vars: dict[str, object] | None, - ) -> int: ... - - class QueryCommandHandler(Protocol): def __call__( self, @@ -123,22 +103,6 @@ class LineageCommandHandler(Protocol): def __call__(self, request: LineageCommandRequest) -> int: ... -class StateCommandHandler(Protocol): - def __call__( - self, - project_dir: Path | None, - *, - state_command: str, - backup_id: str | None, - auto_approve: bool, - no_color: bool, - checkpoint_command: str | None, - checkpoint_id: str | None, - virtual_environment: str | None, - allow_copy: bool, - ) -> int: ... - - class SkillsUpdateCommandHandler(Protocol): def __call__( self, @@ -183,5 +147,4 @@ class PlaygroundTemplate(StrEnum): LOADER_WAFFLE_SHOP = "loader_waffle_shop" DAGSTER = "dagster" RIVERS = "rivers" - VIRTUAL = "virtual" PYTHON_NODES = "python_nodes" diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py index 6a5faa30f9..c0b4ec4faa 100644 --- a/src/sqlbuild/cli/entry/models.py +++ b/src/sqlbuild/cli/entry/models.py @@ -13,10 +13,8 @@ FormatCommandHandler, LineageCommandHandler, QueryCommandHandler, - ReconcileCommandHandler, ScopeCommandHandler, SkillsUpdateCommandHandler, - StateCommandHandler, ) from sqlbuild.cli.compile.models import CompileCommandRequest @@ -67,14 +65,10 @@ class CliEntrypointHandlers: run_load: Callable[..., int] run_clone: Callable[..., int] run_diff: Callable[..., int] - run_reconcile: ReconcileCommandHandler - run_promote: Callable[..., int] - run_rollback: Callable[..., int] run_query: QueryCommandHandler run_debug: DebugCommandHandler run_lineage: LineageCommandHandler run_janitor: Callable[..., int] - run_state: StateCommandHandler run_init: Callable[[Path | None], int] run_playground: Callable[..., int] run_skills_update: SkillsUpdateCommandHandler diff --git a/src/sqlbuild/cli/output/_helpers/execution_result_document.py b/src/sqlbuild/cli/output/_helpers/execution_result_document.py index b2421d1c4c..e7031c96ac 100644 --- a/src/sqlbuild/cli/output/_helpers/execution_result_document.py +++ b/src/sqlbuild/cli/output/_helpers/execution_result_document.py @@ -59,12 +59,6 @@ from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.sql_values.models import SqlValue from sqlbuild.sql_values.types import SqlValueKind -from sqlbuild.virtual.executor.constants import ( - VIRTUAL_CLONE_FOUND_ACTIONS, - VIRTUAL_CLONE_MISSING_ACTION, - VIRTUAL_CLONE_SKIPPED_LOCKED_ACTION, -) -from sqlbuild.virtual.executor.models import VirtualCloneResult _JSON_VERSION: int = 1 _SCENARIO_RESOURCE_NAMESPACE: ContextVar[str | None] = ContextVar( @@ -417,47 +411,6 @@ def _format_clone_asset(*, item: CloneItemResult, resource_type: str) -> dict[st ) -def format_virtual_clone_execution_json(*, result: VirtualCloneResult) -> str: - """Format virtual clone command execution results as JSON.""" - - return _format_execution_json( - command="clone", - status=( - BuildStatus.SUCCESS.value if result.missing_count == 0 else BuildStatus.FAILED.value - ), - assets=tuple( - _drop_none( - { - "kind": item.artifact_type.value, - "name": item.artifact_name, - "status": _virtual_clone_item_status(action=item.action), - "action": item.action, - "version_hash": item.version_hash, - "message": item.message, - } - ) - for item in result.item_results - ), - checks=(), - summary={ - "success_count": result.found_count, - "failure_count": result.missing_count, - "skipped_count": result.skipped_locked_count, - "total_count": result.selected_count, - }, - ) - - -def _virtual_clone_item_status(*, action: str) -> str: - if action in VIRTUAL_CLONE_FOUND_ACTIONS: - return "success" - if action == VIRTUAL_CLONE_SKIPPED_LOCKED_ACTION: - return "skipped" - if action == VIRTUAL_CLONE_MISSING_ACTION: - return "warning" - return "failed" - - def format_test_execution_json(*, results: tuple[SqlTestExecutionResult, ...]) -> str: """Format test command execution results as JSON.""" @@ -507,7 +460,11 @@ def format_audit_execution_json( def format_scenario_execution_json( - *, results: tuple[ScenarioRunResult, ...], local: bool = False + *, + results: tuple[ScenarioRunResult, ...], + local: bool = False, + run_namespace: str | None = None, + namespace_source: str = "unset", ) -> str: """Format scenario test command execution results as JSON.""" @@ -523,6 +480,10 @@ def format_scenario_execution_json( checks.extend(_format_scenario_checks(result)) return _format_execution_json( command="scenario test", + execution={ + "scenario_namespace": run_namespace, + "scenario_namespace_source": namespace_source, + }, status=BuildStatus.SUCCESS.value if fail_count == 0 else BuildStatus.FAILED.value, assets=tuple(assets), checks=tuple(checks), @@ -536,13 +497,21 @@ def format_scenario_execution_json( def format_scenario_snapshot_execution_json( - *, results: tuple[ScenarioSnapshotCaptureRunResult, ...], refresh: bool = False + *, + results: tuple[ScenarioSnapshotCaptureRunResult, ...], + refresh: bool = False, + run_namespace: str | None = None, + namespace_source: str = "unset", ) -> str: """Format scenario snapshot sync/refresh execution results as JSON.""" fail_count: int = sum(1 for result in results if result.status == ExecutionStatus.FAILED) return _format_execution_json( command="scenario snapshot refresh" if refresh else "scenario snapshot sync", + execution={ + "scenario_namespace": run_namespace, + "scenario_namespace_source": namespace_source, + }, status=BuildStatus.SUCCESS.value if fail_count == 0 else BuildStatus.FAILED.value, assets=(), checks=(), diff --git a/src/sqlbuild/cli/output/_helpers/plan_json.py b/src/sqlbuild/cli/output/_helpers/plan_json.py index 9473f13c62..535d274922 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_json.py +++ b/src/sqlbuild/cli/output/_helpers/plan_json.py @@ -20,7 +20,6 @@ PlanOutput, PlanProviderUsage, PlanWarning, - RunDespiteUnchangedDecision, SeedPlanEntry, SourceLoadPlanEntry, ) @@ -192,14 +191,6 @@ def _serialize_model_entry(entry: ModelPlanEntry) -> dict[str, object]: if entry.destination.qualified_name is not None: model["qualified_name"] = entry.destination.qualified_name - if entry.run_despite_unchanged is not None: - decision: RunDespiteUnchangedDecision = entry.run_despite_unchanged - model["run_despite_unchanged"] = { - "mode": decision.mode.value, - "duration": decision.duration, - "newest_source_name": decision.newest_source_name, - "newest_source_data_age_seconds": decision.newest_source_data_age_seconds, - } return model diff --git a/src/sqlbuild/cli/output/_helpers/plan_text.py b/src/sqlbuild/cli/output/_helpers/plan_text.py index 510990d0e9..9b27986b3a 100644 --- a/src/sqlbuild/cli/output/_helpers/plan_text.py +++ b/src/sqlbuild/cli/output/_helpers/plan_text.py @@ -33,7 +33,6 @@ PlanOutput, PlanProviderUsage, PlanWarning, - RunDespiteUnchangedDecision, SchemaFinding, SourceLoadPlanEntry, ) @@ -57,7 +56,6 @@ from sqlbuild.presentation.main.visible_entries import visible_entries from sqlbuild.presentation.models import DisplayOptions from sqlbuild.runtime.contracts.types import ExecutionResourceKind -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus _DIFF_HEADER_MARKER: str = "# " _ENTRY_ROW_PREFIX: str = " " @@ -73,7 +71,6 @@ PlanReason.QUERY_CHANGED, PlanReason.CONFIG_CHANGED, PlanReason.SCHEMA_CHANGED, - PlanReason.RUN_DESPITE_UNCHANGED, PlanReason.FIRST_RUN, PlanReason.RENAMED, ) @@ -83,7 +80,6 @@ PlanReason.QUERY_CHANGED: "Query changed", PlanReason.CONFIG_CHANGED: "Config changed", PlanReason.SCHEMA_CHANGED: "Schema changed", - PlanReason.RUN_DESPITE_UNCHANGED: "Runs despite unchanged", PlanReason.FIRST_RUN: "First run", PlanReason.RENAMED: "Renamed", } @@ -159,12 +155,6 @@ def format_plan( ) lines.append(format_surface_header(style=style, title="Plan ready", context=header)) - lines = _format_virtual_metadata( - lines=lines, - plan=plan, - section_header_style=resolved_section_header_style, - display_options=resolved_display_options, - ) if include_direct_freshness_diagnostics: lines = _format_direct_source_freshness_metadata( lines=lines, @@ -934,7 +924,6 @@ def _format_detail_entry( show_range=reason != PlanReason.FULL_REFRESH, ) lines = _append_policy_line(lines=lines, entry=entry) - lines = _append_run_despite_unchanged_detail(lines=lines, entry=entry) lines = _append_schema_diff(lines=lines, entry=entry) lines = _append_config_diff(lines=lines, entry=entry) lines = _append_query_diff(lines=lines, entry=entry) @@ -1121,8 +1110,6 @@ def _cascade_cause_description(cascade: CascadeResult) -> str: """Format the cause line content for an upstream-changed entry.""" root: str = cascade.root_cause or "unknown" - if cascade.root_reason == PlanReason.RUN_DESPITE_UNCHANGED: - return f"{root} ran despite unchanged inputs" if cascade.root_reason is not None: reason_text: str = _plan_reason_text(cascade.root_reason) if reason_text: @@ -1154,35 +1141,9 @@ def _plan_reason_text(reason: PlanReason) -> str: return "renamed" if reason == PlanReason.FULL_REFRESH: return "full refresh" - if reason == PlanReason.RUN_DESPITE_UNCHANGED: - return "ran despite unchanged inputs" return "" -def _append_run_despite_unchanged_detail(*, lines: list[str], entry: ModelPlanEntry) -> list[str]: - decision: RunDespiteUnchangedDecision | None = entry.run_despite_unchanged - if decision is None: - return lines - detail: str = decision.duration or decision.mode.value - lines.append(f" run_despite_unchanged: {detail}") - if decision.newest_source_data_age_seconds is not None: - lines.append( - " newest source data age: " - f"{_format_age_seconds(decision.newest_source_data_age_seconds)}" - ) - return lines - - -def _format_age_seconds(age_seconds: int) -> str: - if age_seconds % 86400 == 0: - return f"{age_seconds // 86400}d" - if age_seconds % 3600 == 0: - return f"{age_seconds // 3600}h" - if age_seconds % 60 == 0: - return f"{age_seconds // 60}m" - return f"{age_seconds}s" - - def _schema_change_suffix(entry: ModelPlanEntry) -> str: """Short suffix describing schema changes for inline display.""" @@ -1486,151 +1447,6 @@ def _format_diagnostic_section( ) -def _format_virtual_metadata( - *, - lines: list[str], - plan: PlanOutput, - section_header_style: Callable[[str], str], - display_options: DisplayOptions, -) -> list[str]: - """Append a virtual planner metadata section when present.""" - - virtual_environment_name: object | None = plan.metadata.get("virtual_environment_name") - if not isinstance(virtual_environment_name, str): - return lines - virtual_environment_status: str = str( - plan.metadata.get("virtual_environment_status", "unknown") - ) - raw_stale_model_names: object | None = plan.metadata.get("virtual_stale_model_names") - stale_model_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_stale_model_names) - if isinstance(raw_stale_model_names, (tuple, list)) - else () - ) - raw_stale_root_names: object | None = plan.metadata.get("virtual_stale_root_names") - stale_root_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_stale_root_names) - if isinstance(raw_stale_root_names, (tuple, list)) - else () - ) - raw_remaining_stale_model_names: object | None = plan.metadata.get( - "virtual_remaining_stale_model_names" - ) - remaining_stale_model_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_remaining_stale_model_names) - if isinstance(raw_remaining_stale_model_names, (tuple, list)) - else () - ) - raw_observed_source_names: object | None = plan.metadata.get( - "virtual_source_freshness_observed_source_names" - ) - observed_source_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_observed_source_names) - if isinstance(raw_observed_source_names, (tuple, list)) - else () - ) - raw_incomplete_source_names: object | None = plan.metadata.get( - "virtual_source_freshness_incomplete_source_names" - ) - raw_unchanged_source_names: object | None = plan.metadata.get( - "virtual_source_freshness_unchanged_source_names" - ) - unchanged_source_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_unchanged_source_names) - if isinstance(raw_unchanged_source_names, (tuple, list)) - else () - ) - incomplete_source_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_incomplete_source_names) - if isinstance(raw_incomplete_source_names, (tuple, list)) - else () - ) - raw_incomplete_model_names: object | None = plan.metadata.get( - "virtual_source_freshness_incomplete_model_names" - ) - incomplete_model_names: tuple[str, ...] = ( - tuple(str(item) for item in raw_incomplete_model_names) - if isinstance(raw_incomplete_model_names, (tuple, list)) - else () - ) - unchanged_source_name_set: frozenset[str] = frozenset(unchanged_source_names) - new_or_changed_source_names: tuple[str, ...] = tuple( - name for name in observed_source_names if name not in unchanged_source_name_set - ) - stale_root_name_set: frozenset[str] = frozenset(stale_root_names) - downstream_stale_model_names: tuple[str, ...] = tuple( - name for name in stale_model_names if name not in stale_root_name_set - ) - status_summary: str = virtual_environment_status - if virtual_environment_status == VirtualEnvironmentStatus.FINALIZED: - status_summary = "finalized, up to date" - elif stale_model_names: - status_summary = f"{virtual_environment_status}, build required" - lines.append("") - lines.append( - section_header_style(f"Virtual environment {virtual_environment_name} ({status_summary})") - ) - if observed_source_names or incomplete_source_names: - source_count: int = len(observed_source_names) + len(incomplete_source_names) - lines.append(f" Source freshness ({len(observed_source_names)} of {source_count} checked)") - if new_or_changed_source_names: - new_or_changed_source_set: str = _format_capped_name_list( - names=new_or_changed_source_names, - display_options=display_options, - ) - lines.append( - f" new or changed ({len(new_or_changed_source_names)}): " - f"{new_or_changed_source_set}" - ) - if unchanged_source_names: - unchanged_source_set: str = _format_capped_name_list( - names=unchanged_source_names, - display_options=display_options, - ) - lines.append(f" unchanged ({len(unchanged_source_names)}): {unchanged_source_set}") - if incomplete_source_names: - incomplete_source_set: str = _format_capped_name_list( - names=incomplete_source_names, - display_options=display_options, - ) - lines.append( - f" not verifiable ({len(incomplete_source_names)}): {incomplete_source_set}" - ) - if incomplete_model_names: - incomplete_model_set: str = _format_capped_name_list( - names=incomplete_model_names, - display_options=display_options, - ) - lines.append( - f" affected models ({len(incomplete_model_names)}): {incomplete_model_set}" - ) - lines.append(f" Models needing build ({len(stale_model_names)})") - if stale_root_names: - stale_root_set: str = _format_capped_name_list( - names=stale_root_names, - display_options=display_options, - ) - lines.append(f" directly affected ({len(stale_root_names)}): {stale_root_set}") - if downstream_stale_model_names: - downstream_stale_model_set: str = _format_capped_name_list( - names=downstream_stale_model_names, - display_options=display_options, - ) - lines.append( - f" downstream affected ({len(downstream_stale_model_names)}): " - f"{downstream_stale_model_set}" - ) - if remaining_stale_model_names: - remaining_stale_set: str = _format_capped_name_list( - names=remaining_stale_model_names, - display_options=display_options, - ) - lines.append( - f" outside this plan ({len(remaining_stale_model_names)}): {remaining_stale_set}" - ) - return lines - - def _format_direct_source_freshness_metadata( *, lines: list[str], diff --git a/src/sqlbuild/cli/output/main/_scenario_execution_json.py b/src/sqlbuild/cli/output/main/_scenario_execution_json.py index 834b18f59d..dc2278c6f9 100644 --- a/src/sqlbuild/cli/output/main/_scenario_execution_json.py +++ b/src/sqlbuild/cli/output/main/_scenario_execution_json.py @@ -9,8 +9,14 @@ def format_scenario_execution_json( - *, results: tuple[ScenarioRunResult, ...], local: bool = False + *, + results: tuple[ScenarioRunResult, ...], + local: bool = False, + run_namespace: str | None = None, + namespace_source: str = "unset", ) -> str: """Format scenario test command execution results as JSON.""" - return _format_scenario_execution_json(results=results, local=local) + return _format_scenario_execution_json( + results=results, local=local, run_namespace=run_namespace, namespace_source=namespace_source + ) diff --git a/src/sqlbuild/cli/output/main/_scenario_snapshot_execution_json.py b/src/sqlbuild/cli/output/main/_scenario_snapshot_execution_json.py index f0fd72d2a8..e6c933ac9b 100644 --- a/src/sqlbuild/cli/output/main/_scenario_snapshot_execution_json.py +++ b/src/sqlbuild/cli/output/main/_scenario_snapshot_execution_json.py @@ -9,8 +9,17 @@ def format_scenario_snapshot_execution_json( - *, results: tuple[ScenarioSnapshotCaptureRunResult, ...], refresh: bool = False + *, + results: tuple[ScenarioSnapshotCaptureRunResult, ...], + refresh: bool = False, + run_namespace: str | None = None, + namespace_source: str = "unset", ) -> str: """Format scenario snapshot sync/refresh execution results as JSON.""" - return _format_scenario_snapshot_execution_json(results=results, refresh=refresh) + return _format_scenario_snapshot_execution_json( + results=results, + refresh=refresh, + run_namespace=run_namespace, + namespace_source=namespace_source, + ) diff --git a/src/sqlbuild/cli/output/main/_virtual_clone_execution_json.py b/src/sqlbuild/cli/output/main/_virtual_clone_execution_json.py deleted file mode 100644 index c14ac349ba..0000000000 --- a/src/sqlbuild/cli/output/main/_virtual_clone_execution_json.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public virtual clone execution JSON formatting entrypoint.""" - -from __future__ import annotations - -from sqlbuild.cli.output._helpers.execution_result_document import ( - format_virtual_clone_execution_json as _format_virtual_clone_execution_json, -) -from sqlbuild.virtual.executor.models import VirtualCloneResult - - -def format_virtual_clone_execution_json(*, result: VirtualCloneResult) -> str: - """Format virtual clone command execution results as JSON.""" - - return _format_virtual_clone_execution_json(result=result) diff --git a/src/sqlbuild/compiler/compile/main/_project_var_values.py b/src/sqlbuild/compiler/authored_values/main/_project_var_values.py similarity index 100% rename from src/sqlbuild/compiler/compile/main/_project_var_values.py rename to src/sqlbuild/compiler/authored_values/main/_project_var_values.py diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py b/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py index ddfaf5e0f5..0e76848f80 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py @@ -43,10 +43,11 @@ from sqlbuild.compiler.profiling.main._metric import record_compile_metric from sqlbuild.compiler.profiling.main.record import record_compile_timing from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import BINDING_SEVERITIES, TYPE_CHECKED_DIALECTS from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic -_ANALYSIS_CACHE_VERSION: int = 10 -_ANALYSIS_ALGORITHM_FINGERPRINT: str = "model-sql-analysis-v13-bound-lineage-type-recovery" +_ANALYSIS_CACHE_VERSION: int = 12 +_ANALYSIS_ALGORITHM_FINGERPRINT: str = "model-sql-analysis-v21-completed-star-shapes" _LINEAGE_COLUMN_VALUE_COUNT: int = 4 _LINEAGE_SOURCE_VALUE_COUNT: int = 3 _COMPACT_TRANSFORM_CODES: dict[str, int] = { @@ -126,6 +127,7 @@ def build_analysis_cache_context( "python_version": platform.python_version_tuple()[:2], "allow_compact_analysis": allow_compact_analysis, "rich_type_inference": rich_type_inference, + "type_checked_dialects": sorted(TYPE_CHECKED_DIALECTS), "inference_profile": profile_payload, } try: @@ -675,6 +677,7 @@ def model_analysis_output_signature(analysis: PolyglotAnalysisResult) -> str: ] ), "has_star": analysis.has_star, + "star_resolved": analysis.star_resolved, } ) @@ -738,6 +741,9 @@ def _inference_profile_payload( ) return { "sql_analysis_dialect": profile.sql_analysis_dialect, + "quoted_identifiers_ignore_case": profile.quoted_identifiers_ignore_case, + "semantic_known_functions": profile.semantic_known_functions, + "semantic_known_types": profile.semantic_known_types, "function_nullability_rules": rules, "function_return_types": dict(sorted(profile.function_return_types.items())), } @@ -802,6 +808,7 @@ def _analysis_payload(*, cache_key: str, analysis: PolyglotAnalysisResult) -> di ), "l": [_lineage_column_payload(column) for column in analysis.lineage_columns], "h": analysis.has_star, + "sr": analysis.star_resolved, "b": [ [ diagnostic.code, @@ -810,6 +817,7 @@ def _analysis_payload(*, cache_key: str, analysis: PolyglotAnalysisResult) -> di diagnostic.column, diagnostic.start, diagnostic.end, + diagnostic.severity, ] for diagnostic in analysis.binding_diagnostics ], @@ -888,6 +896,7 @@ def _analysis_from_payload( columns=columns, lineage_columns=lineage_columns, has_star=has_star, + star_resolved=values.get("sr") is True, binding_diagnostics=binding_diagnostics, binding_validated=binding_validated, ), @@ -896,10 +905,12 @@ def _analysis_from_payload( def _binding_diagnostic_from_payload(payload: list[object]) -> SqlBindingDiagnostic: - diagnostic_value_count: int = 6 + diagnostic_value_count: int = 7 if len(payload) != diagnostic_value_count: - raise AnalysisCacheEntryError("analysis cache binding diagnostic must contain six values") - code, message, line, column, start, end = payload + raise AnalysisCacheEntryError("analysis cache binding diagnostic must contain seven values") + code, message, line, column, start, end, severity = payload + if severity not in BINDING_SEVERITIES: + raise AnalysisCacheEntryError("analysis cache binding diagnostic severity is invalid") if not isinstance(code, str) or not isinstance(message, str): raise AnalysisCacheEntryError( "analysis cache binding diagnostic code/message must be strings" @@ -914,6 +925,7 @@ def _binding_diagnostic_from_payload(payload: list[object]) -> SqlBindingDiagnos column=cast(int | None, column), start=cast(int | None, start), end=cast(int | None, end), + severity=cast(str, severity), ) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py b/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py index 0cb65058e7..6ec89bc911 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/columns.py @@ -36,12 +36,6 @@ ColumnTransformKind, InferredNullability, ) -from sqlbuild.compiler.references.main._quoted_reference_call_pattern import ( - quoted_reference_call_pattern, -) -from sqlbuild.compiler.references.main.reference_call_prefix_pattern_text import ( - reference_call_prefix_pattern_text, -) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.sql_analysis.constants import ( POLYGLOT_AGGREGATE_KINDS as _POLYGLOT_AGGREGATE_KINDS, @@ -151,26 +145,11 @@ from sqlbuild.compiler.sql_analysis.constants import ( TIMESTAMP_WITH_TIME_ZONE_SQL_TYPE_NAME as _TIMESTAMP_WITH_TIME_ZONE_SQL_TYPE_NAME, ) -from sqlbuild.compiler.sql_analysis.main._find_matching_paren import find_matching_paren -from sqlbuild.compiler.sql_analysis.main._normalize_for_polyglot import ( - normalize_sql_for_polyglot, -) +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.diagnostics.main.log_debug_event import log_debug_event _DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.compile") -_REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF) -_SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED) -_SOURCE_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SOURCE) -_DBT_REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.DBT_REF) -_UDF_PATTERN: re.Pattern[str] = re.compile( - rf"{reference_call_prefix_pattern_text(SqlReferenceKind.UDF)}" - r'"([A-Za-z_][A-Za-z0-9_]*)"\)\s*(?=\()' -) -_TABLE_FUNCTION_PATTERN: re.Pattern[str] = re.compile( - rf"{reference_call_prefix_pattern_text(SqlReferenceKind.TABLE_FUNCTION)}" - r'"([A-Za-z_][A-Za-z0-9_]*)"\)\s*(?=\()' -) _PLACEHOLDER_PATTERN: re.Pattern[str] = re.compile(r"@@@(\w+)") _QUALIFIED_IDENTIFIER_PATTERN: re.Pattern[str] = re.compile( r'(?[^"]+)"|`(?P[^`]+)`|' @@ -961,19 +940,7 @@ def _replace_refs_with_stubs( ) -> str: """Replace SQLBuild marker calls with parseable SQL stubs.""" - stubs: dict[str, str] = relation_stubs or {} - - def relation_stub(match: re.Match[str]) -> str: - name: str = match.group(1) - return stubs.get(name, name) - - result: str = _REF_PATTERN.sub(relation_stub, query_sql) - result = _SEED_PATTERN.sub(relation_stub, result) - result = _SOURCE_PATTERN.sub(relation_stub, result) - result = _DBT_REF_PATTERN.sub(r"\1", result) - result = _UDF_PATTERN.sub(r"__sqlbuild_udf_\1", result) - result = _replace_table_function_calls_with_stubs(query_sql=result, relation_stubs=stubs) - return normalize_sql_for_polyglot(sql=result, dialect=dialect) + return normalize_analysis_sql(sql=query_sql, dialect=dialect, stubs=relation_stubs) def _qualified_reference_names(*, query_sql: str, reference_names: Iterable[str]) -> frozenset[str]: @@ -993,26 +960,6 @@ def _qualified_reference_names(*, query_sql: str, reference_names: Iterable[str] return frozenset(qualified) -def _replace_table_function_calls_with_stubs( - *, query_sql: str, relation_stubs: dict[str, str] | None = None -) -> str: - parts: list[str] = [] - last_index: int = 0 - match: re.Match[str] - for match in _TABLE_FUNCTION_PATTERN.finditer(query_sql): - parts.append(query_sql[last_index : match.start()]) - call_end: int = find_matching_paren( - sql=query_sql, - open_paren_index=match.end(), - context="SQL table function analysis", - ) - table_name: str = table_function_analysis_name(match.group(1)) - parts.append((relation_stubs or {}).get(table_name, table_name)) - last_index = call_end + 1 - parts.append(query_sql[last_index:]) - return "".join(parts) - - def _infer_expression_nullability( *, expression: Any, diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py index 42e9da6806..b9755c446a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py @@ -17,7 +17,6 @@ _infer_columns_with_polyglot, _infer_polyglot_nullability, _infer_polyglot_shallow_nullability, - _lineage_resource_type, _polyglot_alias_nullability_from_select, _polyglot_expression_type, _qualified_reference_names, @@ -28,18 +27,30 @@ _polyglot_cte_passthrough_facts, _polyglot_filtered_non_null_outputs, ) +from sqlbuild.compiler.compile._helpers.sharing.binding import ( + binding_query_fields, + lineage_reference_map, + prepare_binding_queries, + remembered_shared_results, + reused_shared_results, + shared_result_is_exact, + shared_result_keys, +) from sqlbuild.compiler.compile.constants import ( COMPACT_ANALYSIS_FACT_LENGTH, COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH, COMPACT_ANALYSIS_RESPONSE_LENGTH, COMPACT_ANALYSIS_SOURCE_LENGTH, + COMPACT_RELATION_STUB_PREFIX, RESOLVED_SOURCE_CONFIDENCE, SQL_WILDCARD_TOKEN, UNKNOWN_SQL_TYPE_NAME, ) from sqlbuild.compiler.compile.exceptions import CompactAnalysisInputError from sqlbuild.compiler.compile.models import ( + CompactBatchContext, CompactBatchExecutionOptions, + CompactBatchInputs, CompactBatchPreparation, CompactLineageFacts, CompactProjectedFacts, @@ -51,9 +62,11 @@ InferredColumn, NativeCompactAnalysis, PolyglotAnalysisResult, + PreparedBindingQuery, ProjectedAnalysisRequest, + SharedBindingQuery, ) -from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.compile.types import CompactBatchResponseCallback, CompiledResourceType from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -151,8 +164,10 @@ POLYGLOT_PAYLOAD_COLUMN as _POLYGLOT_PAYLOAD_COLUMN, ) from sqlbuild.compiler.sql_analysis.exceptions import SqlAnalysisBoundaryError +from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog from sqlbuild.compiler.sql_analysis.main._decode_schema_validation import decode_schema_validation -from sqlbuild.compiler.sql_analysis.main._prepare_schema_validation import prepare_schema_validation +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql +from sqlbuild.compiler.sql_analysis.main._normalize_analysis_batch import normalize_analysis_sqls from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.compiler.sql_analysis.models import ( @@ -245,6 +260,7 @@ def analyze_columns_and_lineage_with_polyglot( ), template_index=precomputed.compact_template_index, resource_name_indexes=precomputed.resource_name_indexes, + resource_names=precomputed.resource_names, binding_diagnostics=( precomputed.binding_diagnostics if precomputed.binding_diagnostics is not None @@ -331,6 +347,7 @@ def analyze_columns_and_lineage_with_polyglot( ), template_index=precomputed.compact_template_index, resource_name_indexes=precomputed.resource_name_indexes, + resource_names=precomputed.resource_names, binding_diagnostics=(), binding_validated=False, ) @@ -378,61 +395,290 @@ def analyze_queries_with_compact_polyglot_batch( if binding_schemas is not None and len(binding_schemas) != len(query_sqls): raise CompactAnalysisInputError("binding schemas must match the query batch length") if cached_batch is not None: - preparation: CompactBatchPreparation = cached_batch[0] - else: - with record_compile_timing("analysis_preparation_ms"): - preparation = _prepare_compact_analysis_batch( - query_sqls=query_sqls, - references=references, - placeholders=placeholders, - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - inference_profile=inference_profile, - recover_cte_facts=recover_cte_facts, - rich_type_inference=rich_type_inference, - binding_schemas=binding_schemas, + if len(cached_batch[0].cleaned_sql) != len(query_sqls): + raise CompactAnalysisInputError( + "cached compact query-analysis preparation has an invalid length" ) - if len(preparation.cleaned_sql) != len(query_sqls): - raise CompactAnalysisInputError( - "cached compact query-analysis preparation has an invalid length" + with record_compile_timing("analysis_projection_ms"): + return _project_compact_analysis_batch( + preparation=cached_batch[0], response_payload=cached_batch[1] + ) + inputs: CompactBatchInputs = CompactBatchInputs( + query_sqls=query_sqls, + references=references, + placeholders=placeholders, + column_nullability_by_table=column_nullability_by_table, + column_types_by_table=column_types_by_table, + inference_profile=inference_profile, + recover_cte_facts=recover_cte_facts, + rich_type_inference=rich_type_inference, + binding_schemas=binding_schemas, + ) + reuse_results: bool = options.on_response is None and binding_schemas is not None + with record_compile_timing("analysis_preparation_ms"): + context: CompactBatchContext = _compact_batch_context( + inputs=inputs, + share=True, + shareable_prekeys=options.shareable_prekeys if reuse_results else None, ) - if cached_batch is not None: - response_payload: object = cached_batch[1] - else: - with record_compile_timing("analysis_native_ms"): - response_payload = _run_compact_analysis_batch(preparation=preparation) + memo: dict[object, object] | None = ( + cast(dict[object, object], context.binding_catalog.shared_analyses) + if reuse_results and context.binding_catalog is not None + else None + ) + memo_keys: tuple[tuple[object, ...] | None, ...] = shared_result_keys( + inputs=inputs, context=context + ) + reused: dict[int, NativeCompactAnalysis] = ( + reused_shared_results(inputs=inputs, context=context, keys=memo_keys, memo=memo) + if memo is not None + else {} + ) + analyzed: tuple[int, ...] = tuple( + index for index in range(len(query_sqls)) if index not in reused + ) + analyzed_results: tuple[NativeCompactAnalysis, ...] = () + if analyzed: + analyzed_inputs: CompactBatchInputs = _batch_input_subset(inputs=inputs, indexes=analyzed) + recomputed: tuple[int, ...] + analyzed_results, recomputed = _analyze_compact_inputs( + inputs=analyzed_inputs, + context=_batch_context_subset(context=context, indexes=analyzed), + on_response=options.on_response if not reused else None, + ) + if memo is not None: + for key, result in remembered_shared_results( + keys=tuple( + None if position in recomputed else memo_keys[index] + for position, index in enumerate(analyzed) + ), + results=analyzed_results, + ).items(): + memo.setdefault(key, result) + results: list[NativeCompactAnalysis | None] = [None] * len(query_sqls) + for index, result in zip(analyzed, analyzed_results, strict=True): + results[index] = result + for index, result in reused.items(): + results[index] = result + return tuple(cast(NativeCompactAnalysis, result) for result in results) + + +def _analyze_compact_inputs( + *, + inputs: CompactBatchInputs, + context: CompactBatchContext, + on_response: CompactBatchResponseCallback | None, +) -> tuple[tuple[NativeCompactAnalysis, ...], tuple[int, ...]]: + """Run one native batch, re-analysing inexact shared members with their own names.""" + + with record_compile_timing("analysis_preparation_ms"): + preparation: CompactBatchPreparation = _prepare_compact_analysis_batch( + inputs=inputs, context=context + ) + with record_compile_timing("analysis_native_ms"): + response_payload: object = _run_compact_analysis_batch(preparation=preparation) with record_compile_timing("analysis_projection_ms"): - projected: tuple[NativeCompactAnalysis, ...] = _project_compact_analysis_batch( + projected: tuple[NativeCompactAnalysis, ...] = _attach_compiled_bindings( preparation=preparation, - response_payload=response_payload, - ) - if cached_batch is None: - projected = _attach_compiled_bindings( + response=response_payload, + analyses=_project_compact_analysis_batch( preparation=preparation, - response=response_payload, - analyses=projected, + response_payload=response_payload, + ), + ) + member_indexes: tuple[int, ...] = _inexact_shared_members( + preparation=preparation, response=response_payload + ) + if member_indexes: + projected = _reanalyze_shared_members( + member_indexes=member_indexes, projected=projected, request=inputs + ) + elif on_response is not None: + on_response(preparation=preparation, response=response_payload) + return projected, member_indexes + + +def _batch_input_subset( + *, inputs: CompactBatchInputs, indexes: tuple[int, ...] +) -> CompactBatchInputs: + if len(indexes) == len(inputs.query_sqls): + return inputs + return replace( + inputs, + query_sqls=tuple(inputs.query_sqls[index] for index in indexes), + references=tuple(inputs.references[index] for index in indexes), + placeholders=tuple(inputs.placeholders[index] for index in indexes), + recover_cte_facts=tuple(inputs.recover_cte_facts[index] for index in indexes), + binding_schemas=( + tuple(inputs.binding_schemas[index] for index in indexes) + if inputs.binding_schemas is not None + else None + ), + ) + + +def _batch_context_subset( + *, context: CompactBatchContext, indexes: tuple[int, ...] +) -> CompactBatchContext: + if len(indexes) == len(context.normalized_sqls): + return context + return replace( + context, + normalized_sqls=[context.normalized_sqls[index] for index in indexes], + binding_queries=tuple(context.binding_queries[index] for index in indexes), + ) + + +def _inexact_shared_members( + *, preparation: CompactBatchPreparation, response: object +) -> tuple[int, ...]: + if not preparation.shared_query_indexes or not isinstance(response, dict): + return () + payload: dict[str, object] = cast(dict[str, object], response) + validations: object = payload.get("validations") + analyses: object = payload.get("analyses") + templates: object = payload.get("templates") + if not isinstance(analyses, list) or not isinstance(templates, list): + return () + members: list[int] = [] + for index, (projection, analysis) in enumerate( + zip(preparation.projections, analyses, strict=True) + ): + template_index: int = cast(int, projection["templateIndex"]) + query_index: int = cast(int, preparation.templates[template_index]["queryIndex"]) + if query_index not in preparation.shared_query_indexes: + continue + template: object = ( + templates[analysis[0]] + if isinstance(analysis, list) + and analysis + and isinstance(analysis[0], int) + and 0 <= analysis[0] < len(templates) + else str(analysis) + ) + validation: object = ( + cast(list[object], validations)[query_index] if isinstance(validations, list) else None + ) + if not shared_result_is_exact(validation=validation, template=template): + members.append(index) + return tuple(members) + + +def _reanalyze_shared_members( + *, + member_indexes: tuple[int, ...], + projected: tuple[NativeCompactAnalysis, ...], + request: CompactBatchInputs, +) -> tuple[NativeCompactAnalysis, ...]: + """Recompute members of inexact shared queries with their own relation names.""" + + with record_compile_timing("analysis_preparation_ms"): + preparation: CompactBatchPreparation = _prepare_compact_analysis_batch( + inputs=_batch_input_subset(inputs=request, indexes=member_indexes), + share_binding_queries=False, + ) + with record_compile_timing("analysis_native_ms"): + response: object = _run_compact_analysis_batch(preparation=preparation) + with record_compile_timing("analysis_projection_ms"): + replacements: tuple[NativeCompactAnalysis, ...] = _attach_compiled_bindings( + preparation=preparation, + response=response, + analyses=_project_compact_analysis_batch( + preparation=preparation, response_payload=response + ), + ) + results: list[NativeCompactAnalysis] = list(projected) + for index, replacement in zip(member_indexes, replacements, strict=True): + results[index] = replacement + return tuple(results) + + +def _compact_batch_context( + *, + inputs: CompactBatchInputs, + share: bool, + shareable_prekeys: frozenset[str] | None, +) -> CompactBatchContext: + """Prepare the catalog, cleaned SQL, and binding references for one batch.""" + + profile: ExpressionInferenceProfile = inputs.inference_profile + dialect: str | None = profile.sql_analysis_dialect + binding_catalog: Any | None = profile.binding_catalog + if binding_catalog is None and inputs.binding_schemas is not None: + binding_catalog = create_binding_catalog( + dialect=dialect or "generic", + quoted_ignore_case=profile.quoted_identifiers_ignore_case, + known_functions=profile.semantic_known_functions, + known_types=profile.semantic_known_types, + relations=inputs.column_types_by_table, + ) + if binding_catalog is not None: + required_tables: set[str] = set() + for query_references in inputs.references: + required_tables.update( + _analysis_reference_name(reference) for reference in query_references ) - if cached_batch is None and options.on_response is not None: - options.on_response(preparation=preparation, response=response_payload) - return projected + binding_catalog.prepare_analysis( + types={ + name: inputs.column_types_by_table[name] + for name in required_tables + if name in inputs.column_types_by_table + }, + nullability={ + name: inputs.column_nullability_by_table[name] + for name in required_tables + if name in inputs.column_nullability_by_table + }, + ) + normalized_sqls: list[str] = normalize_analysis_sqls( + sqls=inputs.query_sqls, + dialect=dialect, + placeholders=inputs.placeholders, + ) + binding_queries: tuple[PreparedBindingQuery | None, ...] = ( + prepare_binding_queries( + binding_catalog=binding_catalog, + inputs=inputs, + cleaned_sqls=normalized_sqls, + share=share, + shareable_prekeys=shareable_prekeys, + ) + if inputs.binding_schemas is not None and binding_catalog is not None + else (None,) * len(inputs.query_sqls) + ) + return CompactBatchContext( + binding_catalog=binding_catalog, + normalized_sqls=normalized_sqls, + binding_queries=binding_queries, + ) def _prepare_compact_analysis_batch( *, - query_sqls: tuple[str, ...], - references: tuple[tuple[CompileSqlReference, ...], ...], - placeholders: tuple[dict[str, str] | None, ...], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - column_types_by_table: dict[str, dict[str, str]], - inference_profile: ExpressionInferenceProfile, - recover_cte_facts: tuple[bool, ...], - rich_type_inference: bool, - binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] | None = None, + inputs: CompactBatchInputs, + share_binding_queries: bool = True, + context: CompactBatchContext | None = None, ) -> CompactBatchPreparation: + query_sqls: tuple[str, ...] = inputs.query_sqls + references: tuple[tuple[CompileSqlReference, ...], ...] = inputs.references + placeholders: tuple[dict[str, str] | None, ...] = inputs.placeholders + column_nullability_by_table: dict[str, dict[str, InferredNullability]] = ( + inputs.column_nullability_by_table + ) + column_types_by_table: dict[str, dict[str, str]] = inputs.column_types_by_table + inference_profile: ExpressionInferenceProfile = inputs.inference_profile + recover_cte_facts: tuple[bool, ...] = inputs.recover_cte_facts + rich_type_inference: bool = inputs.rich_type_inference + binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] | None = inputs.binding_schemas dialect: str | None = inference_profile.sql_analysis_dialect + if context is None: + context = _compact_batch_context( + inputs=inputs, share=share_binding_queries, shareable_prekeys=None + ) + binding_catalog: Any | None = context.binding_catalog prepared: list[str] = [] queries: list[dict[str, object]] = [] - query_indexes: dict[tuple[str, str, bytes | None, bytes | None], int] = {} + query_indexes: dict[tuple[object, ...], int] = {} templates: list[dict[str, object]] = [] template_indexes: dict[ tuple[ @@ -447,28 +693,36 @@ def _prepare_compact_analysis_batch( ] = {} projections: list[dict[str, object]] = [] function_return_types: dict[str, str] = dict(inference_profile.function_return_types) + normalized_sqls: list[str] = context.normalized_sqls + member_binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] = ( + binding_schemas if binding_schemas is not None else (None,) * len(query_sqls) + ) + binding_queries: tuple[PreparedBindingQuery | None, ...] = context.binding_queries + shared_query_indexes: set[int] = set() for ( query_sql, + cleaned_sql, query_references, query_placeholders, query_recover_cte_facts, binding_schema, + binding_query, ) in zip( query_sqls, + normalized_sqls, references, placeholders, recover_cte_facts, - binding_schemas if binding_schemas is not None else (None,) * len(query_sqls), + member_binding_schemas, + binding_queries, strict=True, ): - cleaned_sql: str = _cleaned_analysis_sql( - query_sql=query_sql, - placeholders=query_placeholders, - dialect=dialect, - ) - lineage_references: dict[str, tuple[CompiledResourceType, str]] = _lineage_reference_map( + lineage_references: dict[str, tuple[CompiledResourceType, str]] = lineage_reference_map( query_references ) + shared: SharedBindingQuery | None = ( + binding_query.shared if binding_query is not None else None + ) qualified_reference_names: frozenset[str] = ( _qualified_reference_names( query_sql=cleaned_sql, @@ -477,16 +731,22 @@ def _prepare_compact_analysis_batch( if binding_schema is None else frozenset() ) - canonical_stubs: dict[str, str] = { - name: ( - name - if binding_schema is not None or name in qualified_reference_names - else f"__sqlbuild_project_input_{index}" - ) - for index, name in enumerate(lineage_references) - } + canonical_stubs: dict[str, str] = ( + shared.stubs + if shared is not None + else { + name: ( + name + if binding_schema is not None or name in qualified_reference_names + else f"{COMPACT_RELATION_STUB_PREFIX}{index}" + ) + for index, name in enumerate(lineage_references) + } + ) analysis_sql: str = ( - cleaned_sql + shared.sql + if shared is not None + else cleaned_sql if binding_schema is not None else _cleaned_analysis_sql( query_sql=query_sql, @@ -499,41 +759,63 @@ def _prepare_compact_analysis_batch( "sql": analysis_sql, "dialect": dialect or "generic", } - schema: dict[str, object] | None = _compact_analysis_schema( - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - table_names=frozenset( - _analysis_reference_name(reference) for reference in query_references - ), - table_name_aliases=canonical_stubs, - ) + schema: dict[str, object] | None = None + if binding_catalog is not None: + query["analysis_references"] = [ + (name, canonical_stubs.get(name, name)) + for name in sorted( + lineage_references, key=lambda name: canonical_stubs.get(name, name) + ) + ] + else: + schema = _compact_analysis_schema( + column_nullability_by_table=column_nullability_by_table, + column_types_by_table=column_types_by_table, + table_names=frozenset( + _analysis_reference_name(reference) for reference in query_references + ), + table_name_aliases=canonical_stubs, + ) if schema is not None: query["schema"] = schema binding_payload: dict[str, object] | None = None - if binding_schema is not None: - binding_payload = prepare_schema_validation( - request=SqlSchemaValidationRequest( - sql=cleaned_sql, - dialect=dialect, - schema=binding_schema, - ) - ) - query["binding_schema"] = binding_payload["schema"] - query_key: tuple[str, str, bytes | None, bytes | None] = ( - analysis_sql, - dialect or "generic", - (orjson.dumps(schema, option=orjson.OPT_SORT_KEYS) if schema is not None else None), - ( - orjson.dumps(binding_payload["schema"], option=orjson.OPT_SORT_KEYS) - if binding_payload is not None - else None - ), + shared_key: tuple[object, ...] | None = ( + (shared.key, query_recover_cte_facts) if shared is not None else None ) - query_index: int | None = query_indexes.get(query_key) + query_index: int | None = query_indexes.get(shared_key) if shared_key is not None else None + if query_index is None and binding_schema is not None: + if binding_query is None or binding_catalog is None: + raise SqlAnalysisBoundaryError("binding schemas require a native project catalog") + binding_fields: dict[str, object] + binding_fields, binding_payload = binding_query_fields( + binding_query=binding_query, binding_catalog=binding_catalog + ) + query.update(binding_fields) + query_key: tuple[object, ...] + if shared_key is not None: + query_key = shared_key + else: + query_key = ( + analysis_sql, + dialect or "generic", + ( + orjson.dumps( + schema or query.get("analysis_references"), option=orjson.OPT_SORT_KEYS + ) + ), + ( + orjson.dumps(binding_payload, option=orjson.OPT_SORT_KEYS) + if binding_payload is not None + else None + ), + ) + query_index = query_indexes.get(query_key) if query_index is None: query_index = len(queries) query_indexes[query_key] = query_index queries.append(query) + if shared is not None: + shared_query_indexes.add(query_index) declared_column_order: tuple[str, ...] = ( tuple( column_nullability_by_table.get(_analysis_reference_name(query_references[0]), {}) @@ -595,6 +877,8 @@ def _prepare_compact_analysis_batch( queries=tuple(queries), templates=tuple(templates), projections=tuple(projections), + binding_catalog=binding_catalog, + shared_query_indexes=frozenset(shared_query_indexes), ) @@ -607,6 +891,19 @@ def _run_compact_analysis_batch(*, preparation: CompactBatchPreparation) -> obje >= _LARGE_COMPACT_SQL_BYTES ) workers: int = 1 if large_project else 4 + if preparation.binding_catalog is not None: + return orjson.loads( + preparation.binding_catalog.native.analyze_compact( + orjson.dumps( + { + "queries": preparation.queries, + "templates": preparation.templates, + "projections": preparation.projections, + "workers": workers, + } + ) + ) + ) return orjson.loads( cast(NativeQueryAnalysisModule, _native).analyze_project_queries_compact_json( orjson.dumps( @@ -723,7 +1020,14 @@ def _project_compact_analysis_batch( ) if not ( isinstance(template, list) - and (len(template) == COMPACT_ANALYSIS_RESPONSE_LENGTH or legacy_type_recovery) + and ( + len(template) == COMPACT_ANALYSIS_RESPONSE_LENGTH + or legacy_type_recovery + or ( + len(template) == COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH + and isinstance(template[2], bool) + ) + ) and isinstance(template[0], list) and isinstance(template[1], bool) ): @@ -755,7 +1059,13 @@ def _project_compact_analysis_batch( results.append( NativeCompactAnalysis( cleaned_sql=cleaned_sql, - analysis=None if legacy_type_recovery else {"hasStar": template[1]}, + analysis=None + if legacy_type_recovery + else { + "hasStar": template[1], + "starResolved": len(template) == COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH + and template[2] is True, + }, projected=not legacy_type_recovery, compact_rows=cast(list[object], template[0]), compact_fact_rows=cast(list[object], raw_facts), @@ -840,6 +1150,7 @@ def _projected_analysis_result(*, request: ProjectedAnalysisRequest) -> Polyglot columns=tuple(columns), lineage_columns=tuple(lineage_columns), has_star=bool(analysis.get("hasStar")), + star_resolved=bool(analysis.get("starResolved")), binding_diagnostics=request.binding_diagnostics, binding_validated=request.binding_validated, ) @@ -862,6 +1173,7 @@ def _compact_projected_analysis_result( resource_name_indexes=request.resource_name_indexes, ), has_star=bool(request.analysis and request.analysis.get("hasStar")), + star_resolved=bool(request.analysis and request.analysis.get("starResolved")), binding_diagnostics=request.binding_diagnostics, binding_validated=request.binding_validated, ) @@ -969,8 +1281,10 @@ def pooled_string(value: object) -> str: raise SqlAnalysisBoundaryError( "native compact analysis returned invalid upstream lineage" ) - if not all( - isinstance(value, int) and not isinstance(value, bool) for value in raw_source + if ( + type(raw_source[0]) is not int + or type(raw_source[1]) is not int + or type(raw_source[2]) is not int ): raise SqlAnalysisBoundaryError( "native compact analysis returned invalid upstream indexes" @@ -1000,8 +1314,10 @@ def pooled_string(value: object) -> str: string_pool=string_pool, rows=compact_facts.lineage_rows, resource_name_indexes=request.resource_name_indexes, + resource_names=request.resource_names, ), has_star=bool(request.analysis and request.analysis.get("hasStar")), + star_resolved=bool(request.analysis and request.analysis.get("starResolved")), binding_diagnostics=request.binding_diagnostics, binding_validated=request.binding_validated, ) @@ -1014,17 +1330,9 @@ def _cleaned_analysis_sql( dialect: str | None, relation_stubs: dict[str, str] | None = None, ) -> str: - cleaned_sql: str = _replace_refs_with_stubs( - query_sql=query_sql, - dialect=dialect, - relation_stubs=relation_stubs, + return normalize_analysis_sql( + sql=query_sql, dialect=dialect, stubs=relation_stubs, placeholders=placeholders ) - if placeholders: - cleaned_sql = substitute_placeholder_defaults( - query_sql=cleaned_sql, - placeholders=placeholders, - ) - return cleaned_sql def get_complete_schema_binding_request( @@ -1033,19 +1341,22 @@ def get_complete_schema_binding_request( placeholders: dict[str, str] | None, dialect: str | None, binding_schema: dict[str, dict[str, str]], + known_functions: tuple[str, ...] = (), + known_types: tuple[str, ...] = (), + cleaned_sql: str | None = None, ) -> SqlSchemaValidationRequest: """Build one stable native schema-validation request.""" - cleaned_sql: str = _replace_refs_with_stubs(query_sql=query_sql, dialect=dialect) - if placeholders: - cleaned_sql = substitute_placeholder_defaults( - query_sql=cleaned_sql, - placeholders=placeholders, + if cleaned_sql is None: + cleaned_sql = normalize_analysis_sql( + sql=query_sql, dialect=dialect, placeholders=placeholders ) return SqlSchemaValidationRequest( sql=cleaned_sql, dialect=dialect, schema=binding_schema, + known_functions=known_functions, + known_types=known_types, ) @@ -1108,7 +1419,7 @@ def _analyze_columns_and_lineage_with_compact_polyglot( return None if not _compact_analysis_is_eligible(analysis=analysis, projections=projections): return None - reference_map: dict[str, tuple[CompiledResourceType, str]] = _lineage_reference_map(references) + reference_map: dict[str, tuple[CompiledResourceType, str]] = lineage_reference_map(references) relation_alias_by_name: dict[str, str | None] = _compact_relation_alias_by_name(analysis) cte_passthrough_types, cte_passthrough_nullability, direct_cte_outputs, parsed = ( _polyglot_cte_passthrough_facts( @@ -1288,22 +1599,6 @@ def _compact_analysis_schema_column( return column -def _lineage_reference_map( - references: tuple[CompileSqlReference, ...], -) -> dict[str, tuple[CompiledResourceType, str]]: - reference_map: dict[str, tuple[CompiledResourceType, str]] = {} - reference: CompileSqlReference - for reference in references: - resource_type: CompiledResourceType | None = _lineage_resource_type(reference) - if resource_type is None: - continue - reference_map[_analysis_reference_name(reference)] = ( - resource_type, - reference.ref_name, - ) - return reference_map - - def _compact_relation_alias_by_name(analysis: dict[str, Any]) -> dict[str, str | None]: alias_by_name: dict[str, str | None] = {} relation_key: str @@ -1334,14 +1629,16 @@ def _compact_projection_type( cast_type: object = projection.get(_POLYGLOT_ANALYSIS_CAST_TYPE) if isinstance(cast_type, str) and cast_type and cast_type != UNKNOWN_SQL_TYPE_NAME: return cast_type - type_hint: object = projection.get(_POLYGLOT_ANALYSIS_TYPE_HINT) - if isinstance(type_hint, str) and type_hint and type_hint != UNKNOWN_SQL_TYPE_NAME: - return type_hint transform_function: object = projection.get(_POLYGLOT_ANALYSIS_TRANSFORM_FUNCTION) if isinstance(transform_function, dict): function_name: object = transform_function.get(_POLYGLOT_ANALYSIS_FUNCTION_NAME) if isinstance(function_name, str): - return inference_profile.function_return_type(function_name) + declared_type: str | None = inference_profile.function_return_type(function_name) + if declared_type is not None: + return declared_type + type_hint: object = projection.get(_POLYGLOT_ANALYSIS_TYPE_HINT) + if isinstance(type_hint, str) and type_hint and type_hint != UNKNOWN_SQL_TYPE_NAME: + return type_hint return None diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py new file mode 100644 index 0000000000..4e265b80ea --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_positions.py @@ -0,0 +1,105 @@ +"""Construct authored locations from native normalization and expansion maps.""" + +from functools import lru_cache +from pathlib import Path +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.models import CompiledSqlExpansion, CompileModelInput, ExpansionSpan +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic +from sqlbuild.compiler.sql_analysis.types import NativeBindingPositions, NativePositionsModule +from sqlbuild.spec.contracts.models import SourceLocation + + +def query_line_offset(model_input: CompileModelInput) -> int | None: + query_sql: str = model_input.model_file.query_sql + query_start: int = model_input.model_file.contents.find(query_sql) + if query_start < 0: + return None + return model_input.model_file.contents[:query_start].count("\n") + + +@lru_cache(maxsize=32) +def _mapper( + *, + authored_sql: str, + authored_query_sql: str, + expanded_sql: str, + cleaned_sql: str, + passes: tuple[tuple[ExpansionSpan, ...], ...], +) -> NativeBindingPositions: + native_passes: list[tuple[tuple[int, int, int, int], ...]] = [] + for spans in passes: + native_passes.append( + tuple( + (span.source_start, span.source_end, span.output_start, span.output_end) + for span in spans + ) + ) + return cast(NativePositionsModule, _native).BindingPositions( + { + "authored": authored_sql, + "query": authored_query_sql, + "expanded": expanded_sql, + "cleaned": cleaned_sql, + "passes": native_passes, + } + ) + + +def get_authored_binding_location( + *, + path: Path, + authored_sql: str, + authored_query_sql: str, + cleaned_sql: str, + diagnostic: SqlBindingDiagnostic, + expansion: CompiledSqlExpansion | None = None, +) -> SourceLocation | None: + mapper: NativeBindingPositions = _mapper( + authored_sql=authored_sql, + authored_query_sql=authored_query_sql, + expanded_sql=expansion.expanded_sql if expansion is not None else authored_query_sql, + cleaned_sql=cleaned_sql, + passes=expansion.passes if expansion is not None else (), + ) + line, column = mapper.position( + start=diagnostic.start, + line=diagnostic.line, + column=diagnostic.column, + message=diagnostic.message, + ) + if line is None or column is None: + return None + end_line: int | None = None + end_column: int | None = None + if diagnostic.end is not None: + end_line, end_column = mapper.position( + start=diagnostic.end, line=None, column=None, message="" + ) + return SourceLocation( + path=path, line=line, column=column, end_line=end_line, end_column=end_column + ) + + +def get_authored_binding_position( + *, + authored_sql: str, + authored_query_sql: str, + cleaned_sql: str, + diagnostic: SqlBindingDiagnostic, + expansion: CompiledSqlExpansion | None = None, +) -> tuple[int | None, int | None]: + mapper: NativeBindingPositions = _mapper( + authored_sql=authored_sql, + authored_query_sql=authored_query_sql, + expanded_sql=expansion.expanded_sql if expansion is not None else authored_query_sql, + cleaned_sql=cleaned_sql, + passes=expansion.passes if expansion is not None else (), + ) + return mapper.position( + start=diagnostic.start, + line=diagnostic.line, + column=diagnostic.column, + message=diagnostic.message, + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py new file mode 100644 index 0000000000..8922440b40 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py @@ -0,0 +1,164 @@ +"""Analyze open model inputs after their producers have supplied closed shapes.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from graphlib import CycleError, TopologicalSorter + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.analysis.cache import model_analysis_output_signature +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( + binding_relation_names, + published_model_shape, +) +from sqlbuild.compiler.compile.models import CompileModelInput, PolyglotAnalysisResult +from sqlbuild.compiler.compile.models import ( + ModelSqlAnalysis as _ModelSqlAnalysis, +) +from sqlbuild.compiler.compile.models import ( + ModelSqlAnalysisRequest as _ModelSqlAnalysisRequest, +) +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.references.types import SqlReferenceKind + + +def analyze_binding_waves( + *, + requests: tuple[_ModelSqlAnalysisRequest, ...], + names: tuple[str, ...], + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], + shapes: dict[str, dict[str, str]], + types: dict[str, dict[str, str]], + nullability: dict[str, dict[str, InferredNullability]], + profile: ExpressionInferenceProfile, + analyze: Callable[..., tuple[_ModelSqlAnalysis, ...]], + complete: Callable[..., tuple[_ModelSqlAnalysis, ...]], +) -> tuple[tuple[_ModelSqlAnalysis, ...], dict[str, PolyglotAnalysisResult]]: + by_name: dict[str, _ModelSqlAnalysisRequest] = dict(zip(names, requests, strict=True)) + available_names: frozenset[str] = frozenset(by_name) + dependencies: dict[str, tuple[str, ...]] = {} + for name, request in by_name.items(): + dependencies[name] = referenced_model_names( + model_input=request.model_input, available_names=available_names + ) + sorter: TopologicalSorter[str] = TopologicalSorter(dependencies) + try: + sorter.prepare() + except CycleError: + return complete( + requests=requests, + analyses=analyze( + requests=requests, + cached_analyses=cached, + column_types_by_table=types, + column_nullability_by_table=nullability, + ), + complete_binding_schemas=shapes, + ), cached + complete_shapes: dict[str, dict[str, str]] = dict(shapes) + available_types: dict[str, dict[str, str]] = dict(types) + available_nullability: dict[str, dict[str, InferredNullability]] = dict(nullability) + reusable: dict[str, PolyglotAnalysisResult] = dict(cached) + changed: set[str] = set() + results: dict[str, _ModelSqlAnalysis] = {} + while sorter.is_active(): + ready: tuple[str, ...] = tuple(sorter.get_ready()) + wave: list[_ModelSqlAnalysisRequest] = [] + for name in ready: + request: _ModelSqlAnalysisRequest = by_name[name] + if any(parent in changed for parent in dependencies[name]): + changed.add(name) + if request.cache_key is not None: + reusable.pop(request.cache_key, None) + bindings: dict[str, dict[str, str]] | None = ( + None + if request.binding_schema is None + else { + relation: complete_shapes.get(relation, {}) + for relation in binding_relation_names(request.model_input.references) + } + ) + wave.append(replace(request, binding_schema=bindings)) + wave_requests: tuple[_ModelSqlAnalysisRequest, ...] = tuple(wave) + wave_names: set[str] = set(ready) + for request in wave_requests: + wave_names.update(binding_relation_names(request.model_input.references)) + analyses: tuple[_ModelSqlAnalysis, ...] = complete( + requests=wave_requests, + analyses=analyze( + requests=wave_requests, + cached_analyses=reusable, + column_types_by_table=available_types, + column_nullability_by_table=available_nullability, + ), + complete_binding_schemas={ + name: complete_shapes[name] for name in wave_names if name in complete_shapes + }, + supplied_relations=frozenset( + name + for name in wave_names + if name in complete_shapes + and available_types.get(name, {}).keys() == complete_shapes[name].keys() + ), + ) + for name, request, result in zip(ready, wave_requests, analyses, strict=True): + results[name] = result + analysis: PolyglotAnalysisResult = result.polyglot_analysis + if model_analysis_output_signature(analysis) != previous_signatures.get(name): + changed.add(name) + required: frozenset[str] = binding_relation_names(request.model_input.references) + star_known: bool = not analysis.has_star or ( + analysis.star_resolved and required <= complete_shapes.keys() + ) + if analysis.columns and star_known: + shape: dict[str, str] = published_model_shape( + sql=result.cleaned_sql or request.query_sql, + columns={column.name: column.type or "UNKNOWN" for column in analysis.columns}, + inputs={table: complete_shapes.get(table, {}) for table in required}, + profile=profile, + config_values=request.model_input.config.values, + ) + complete_shapes.setdefault(name, shape) + available_types.setdefault(name, shape) + available_nullability.setdefault( + name, dict.fromkeys(shape, InferredNullability.UNKNOWN) + ) + sorter.done(*ready) + return tuple(results[name] for name in names), reusable + + +def downstream_model_names( + *, model_inputs: tuple[CompileModelInput, ...], changed_names: set[str] +) -> set[str]: + downstream_by_name: dict[str, set[str]] = {} + for model_input in model_inputs: + for reference in model_input.references: + if reference.ref_kind == SqlReferenceKind.REF: + downstream_by_name.setdefault(reference.ref_name, set()).add( + model_input.model_file.file_path.stem + ) + downstream_names: set[str] = set() + pending: list[str] = list(changed_names) + while pending: + for downstream_name in downstream_by_name.get(pending.pop(), set()): + if downstream_name not in downstream_names and downstream_name not in changed_names: + downstream_names.add(downstream_name) + pending.append(downstream_name) + return downstream_names + + +def referenced_model_names( + *, model_input: CompileModelInput, available_names: frozenset[str] | None = None +) -> tuple[str, ...]: + return tuple( + sorted( + { + reference.ref_name + for reference in model_input.references + if reference.ref_kind == SqlReferenceKind.REF + and (available_names is None or reference.ref_name in available_names) + } + ) + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py new file mode 100644 index 0000000000..cc0569c749 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/metadata_validation.py @@ -0,0 +1,579 @@ +"""Check column-bearing model metadata against authoritative output shapes.""" + +from __future__ import annotations + +import re +from collections.abc import Iterator +from functools import lru_cache +from typing import Any, cast + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.adapter.contract.types import TypeFamily +from sqlbuild.adapter.type_system.main.normalize_type import normalize_type +from sqlbuild.compiler.compile._helpers.analysis.columns import _polyglot_expression_type +from sqlbuild.compiler.compile._helpers.analysis.compact import ( + analyze_columns_and_lineage_with_polyglot, + analyze_queries_with_compact_polyglot_batch, + get_complete_schema_binding_request, +) +from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( + known_declared_types, + known_function_names, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile.models import ( + CompiledFunction, + CompiledModel, + CompiledModelSqlTestPayload, + CompiledProject, + CompilerDiagnostic, + NativeCompactAnalysis, + PolyglotAnalysisResult, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import BINDING_UNKNOWN_TABLE_INTERNAL_CODE +from sqlbuild.compiler.sql_analysis.main._resolve_binding_column import resolve_binding_column +from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest +from sqlbuild.spec.contracts.models import SchemaAuditInstance, SourceLocation + +_UNKNOWN_TYPE: str = "UNKNOWN" +_EXPRESSION_AUDIT: str = "expression_is_true" +_RELATIONSHIPS_AUDIT: str = "relationships" +_ACCEPTED_VALUES_AUDIT: str = "accepted_values" +_SELECT_KIND: str = "select" +_FUNCTION_KIND: str = "function" +_CURSOR_FAMILIES: dict[str, frozenset[TypeFamily]] = { + "integer": frozenset({TypeFamily.INTEGER, TypeFamily.DECIMAL}), + "timestamp": frozenset({TypeFamily.TIMESTAMP, TypeFamily.DATETIME, TypeFamily.DATE}), + "date": frozenset({TypeFamily.TIMESTAMP, TypeFamily.DATETIME, TypeFamily.DATE}), +} + + +def get_semantic_metadata_diagnostics( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, +) -> tuple[CompilerDiagnostic, ...]: + """Reject metadata names only where absence can be proven.""" + + if not project.settings.sql_analysis: + return () + shapes: dict[str, dict[str, str]] = semantic_shapes(project=project, profile=profile) + diagnostics: list[CompilerDiagnostic] = [] + for model in project.models: + if model.config.values.get("sql_analysis") is False or not model.binding_validated: + continue + diagnostics.extend( + _function_errors(model=model, project=project, shapes=shapes, profile=profile) + ) + shape: dict[str, str] | None = shapes.get(model.name) + if shape is None: + continue + diagnostics.extend( + _audit_errors(model=model, project=project, shape=shape, shapes=shapes, profile=profile) + ) + values: dict[str, object] = model.config.values + references: list[tuple[str, str, dict[str, str]]] = [] + for key in ("unique_key", "cursor", "partition_column", "row_diff_exclude_columns"): + references.extend((key, name, shape) for name in _names(values.get(key))) + custom: object = values.get("config") + if isinstance(custom, dict): + references.extend( + ("partition_column", name, shape) + for name in _names(cast(dict[str, object], custom).get("partition_column")) + ) + tolerances: object = values.get("row_diff_tolerances") + by_column: object = ( + cast(dict[str, object], tolerances).get("by_column") + if isinstance(tolerances, dict) + else None + ) + if isinstance(by_column, dict): + references.extend(("row_diff_tolerances", str(name), shape) for name in by_column) + cursor_inputs: object = values.get("cursor_inputs") + if isinstance(cursor_inputs, dict): + for upstream, columns in cursor_inputs.items(): + upstream_shape: dict[str, str] | None = shapes.get(str(upstream)) + if upstream_shape is not None: + references.extend( + (f"cursor_inputs {upstream}", name, upstream_shape) + for name in _names(columns) + ) + for key, name, reference_shape in references: + if name.casefold() not in {column.casefold() for column in reference_shape}: + diagnostics.append( + _model_error( + model=model, + code="B300", + name=name, + message=f"{key} references unknown column '{name}'", + ) + ) + cursor: object = values.get("cursor") + cursor_type: object = values.get("cursor_type") + if isinstance(cursor, str) and isinstance(cursor_type, str): + actual: str = next( + (value for key, value in shape.items() if key.casefold() == cursor.casefold()), + "UNKNOWN", + ) + if cursor_type.lower() in _CURSOR_FAMILIES: + family: TypeFamily = normalize_type( + type_sql=actual, dialect=profile.sql_analysis_dialect + ).family + if ( + family != TypeFamily.OTHER + and family not in _CURSOR_FAMILIES[cursor_type.lower()] + ): + diagnostics.append( + _model_error( + model=model, + code="B301", + name=cursor, + message=( + f"cursor_type {cursor_type} does not match " + f"column '{cursor}' type {actual}" + ), + ) + ) + for source in project.sources: + cursor = source.source_entry.cursor_column + shape = shapes.get(source.name) + if ( + cursor + and shape is not None + and cursor.casefold() not in {name.casefold() for name in shape} + ): + line: int + column: int + line, column = _text_position(text=source.source_file.contents, name=cursor) + diagnostics.append( + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code="B300", + message=f"cursor_column references unknown column '{cursor}'", + resource_type=CompiledResourceType.SOURCE, + resource_name=source.name, + path=source.source_file.relative_path, + line=line, + column=column, + ) + ) + diagnostics.extend(_sql_test_errors(project=project, shapes=shapes, profile=profile)) + return tuple(diagnostics) + + +def _audit_errors( + *, + model: CompiledModel, + project: CompiledProject, + shape: dict[str, str], + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, +) -> tuple[CompilerDiagnostic, ...]: + if model.schema_entry is None: + return () + diagnostics: list[CompilerDiagnostic] = [] + attached: list[tuple[str | None, SchemaAuditInstance]] = [ + (None, audit) for audit in model.schema_entry.audits + ] + for schema_column in model.schema_entry.columns: + attached.extend((schema_column.name, audit) for audit in schema_column.audits) + for column, audit in attached: + arguments: dict[str, object] = audit.arguments + if audit.definition_name == _EXPRESSION_AUDIT: + expression: object = arguments.get("expression") + if isinstance(expression, str): + result: SqlBindingResult = get_schema_validations( + requests=( + SqlSchemaValidationRequest( + sql=f"SELECT * FROM output WHERE {expression}", + dialect=profile.sql_analysis_dialect, + schema={"output": shape}, + known_functions=known_function_names(project.functions), + catalog=project.binding_catalog, + quoted_identifiers_ignore_case=profile.quoted_identifiers_ignore_case, + known_types=known_declared_types( + functions=project.functions, column_types=shapes + ), + ), + ) + )[0] + diagnostics.extend( + _model_error( + model=model, + code=diagnostic.code, + name=expression, + message=f"expression_is_true: {diagnostic.message}", + severity=DiagnosticSeverity(diagnostic.severity), + ) + for diagnostic in result.diagnostics + if diagnostic.code != BINDING_UNKNOWN_TABLE_INTERNAL_CODE + ) + elif audit.definition_name == _RELATIONSHIPS_AUDIT and column: + target: str = str(arguments.get("to", "")) + match: re.Match[str] | None = re.search( + r"__(?:ref|source|seed)\([\'\"]([^\'\"]+)[\'\"]\)", target + ) + target_shape: dict[str, str] | None = shapes.get(match.group(1) if match else target) + field: str = str(arguments.get("field", "")) + resolved_field: str | None = resolve_binding_column( + name=field, + columns=target_shape or {}, + dialect=profile.sql_analysis_dialect, + ignore_quoted_case=profile.quoted_identifiers_ignore_case, + ) + if target_shape and resolved_field is None: + diagnostics.append( + _model_error( + model=model, + code="B300", + name=field, + message=f"relationships target has no column '{field}'", + ) + ) + elif target_shape is not None and resolved_field is not None and column in shape: + diagnostics.extend( + _model_error( + model=model, + code="B301" + if diagnostic.severity == DiagnosticSeverity.ERROR + else diagnostic.code, + name=column, + message=f"relationships: {diagnostic.message}", + severity=DiagnosticSeverity(diagnostic.severity), + ) + for diagnostic in _comparison_result( + left=shape[column], + right=target_shape[resolved_field], + dialect=profile.sql_analysis_dialect, + ).diagnostics + ) + elif audit.definition_name == _ACCEPTED_VALUES_AUDIT and column in shape: + values: object = arguments.get("values") + if isinstance(values, (list, tuple)): + literals: list[str | None] = [_audit_literal(value) for value in values] + if literals and all(literal is not None for literal in literals): + result = _accepted_values_result( + literals=tuple(str(value) for value in literals), + dialect=profile.sql_analysis_dialect, + column_type=shape[column], + ) + diagnostics.extend( + _model_error( + model=model, + code="B301" if item.severity == DiagnosticSeverity.ERROR else item.code, + name=column, + message=f"accepted_values: {item.message}", + severity=DiagnosticSeverity(item.severity), + ) + for item in result.diagnostics + ) + return tuple(diagnostics) + + +def _different_argument_families( + *, left: str, right: str, profile: ExpressionInferenceProfile +) -> bool: + families: set[TypeFamily] = { + normalize_type(type_sql=value, dialect=profile.sql_analysis_dialect).family + for value in (left, right) + } + different_families: bool = ( + len(families) > 1 + and TypeFamily.OTHER not in families + and not families <= {TypeFamily.INTEGER, TypeFamily.DECIMAL, TypeFamily.FLOAT} + and not families <= {TypeFamily.TIMESTAMP, TypeFamily.DATE, TypeFamily.DATETIME} + ) + return different_families + + +@lru_cache(maxsize=256) +def _comparison_result(*, left: str, right: str, dialect: str | None) -> SqlBindingResult: + return get_schema_validations( + requests=( + SqlSchemaValidationRequest( + sql="SELECT lhs.value = rhs.value FROM lhs CROSS JOIN rhs", + dialect=dialect, + schema={"lhs": {"value": left}, "rhs": {"value": right}}, + ), + ) + )[0] + + +def _audit_literal(value: object) -> str | None: + if value is None: + return "NULL" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, int | float): + return str(value) + if isinstance(value, str): + return "'" + value.replace("'", "''") + "'" + return None + + +@lru_cache(maxsize=256) +def _accepted_values_result( + *, literals: tuple[str, ...], dialect: str | None, column_type: str +) -> SqlBindingResult: + return get_schema_validations( + requests=( + SqlSchemaValidationRequest( + sql=f"SELECT value NOT IN ({', '.join(literals)}) FROM output", + dialect=dialect, + schema={"output": {"value": column_type}}, + ), + ) + )[0] + + +def _sql_test_errors( + *, + project: CompiledProject, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, +) -> tuple[CompilerDiagnostic, ...]: + diagnostics: list[CompilerDiagnostic] = [] + columns_by_sql: dict[str, tuple[str, ...]] = _sql_test_columns(project=project, profile=profile) + for test in project.sql_tests: + if not isinstance(test.payload, CompiledModelSqlTestPayload): + continue + for cte in (*test.payload.authored_ctes, *test.payload.expected_ctes): + match: re.Match[str] | None = re.match( + r"__(?:expected|ref|source|seed)__(.+)$", cte.name + ) + shape: dict[str, str] | None = shapes.get(match.group(1)) if match else None + if shape is None: + continue + available: set[str] = {name.casefold() for name in shape} + for column_name in columns_by_sql[cte.sql_body]: + if column_name.casefold() not in available: + line: int + column_position: int + line, column_position = _text_position( + text=test.test_file.contents, + name=column_name, + offset=max(test.test_file.contents.find(cte.name), 0), + ) + diagnostics.append( + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.ERROR, + code="B302", + message=f"SQL test '{cte.name}' names unknown column '{column_name}'", + resource_type=CompiledResourceType.SQL_TEST, + resource_name=test.name, + path=test.test_file.relative_path, + line=line, + column=column_position, + location=SourceLocation( + path=test.test_file.relative_path, + line=line, + column=column_position, + end_line=line, + end_column=column_position + len(column_name), + ), + ) + ) + return tuple(diagnostics) + + +def _sql_test_columns( + *, project: CompiledProject, profile: ExpressionInferenceProfile +) -> dict[str, tuple[str, ...]]: + """Infer fixture interfaces in one batch instead of repeating Python AST walks.""" + bodies: dict[str, None] = {} + for test in project.sql_tests: + if isinstance(test.payload, CompiledModelSqlTestPayload): + for cte in (*test.payload.authored_ctes, *test.payload.expected_ctes): + bodies[cte.sql_body] = None + queries: tuple[str, ...] = tuple(bodies) + if not queries: + return {} + prepared: tuple[NativeCompactAnalysis, ...] = analyze_queries_with_compact_polyglot_batch( + query_sqls=queries, + references=((),) * len(queries), + placeholders=(None,) * len(queries), + column_nullability_by_table={}, + column_types_by_table={}, + inference_profile=profile, + recover_cte_facts=(False,) * len(queries), + ) + result: dict[str, tuple[str, ...]] = {} + for sql, native in zip(queries, prepared, strict=True): + analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + query_sql=sql, + references=(), + inference_profile=profile, + allow_compact_analysis=True, + precomputed=native, + ) + result[sql] = tuple(column.name for column in analysis.columns or ()) + return result + + +def _names(value: object) -> tuple[str, ...]: + if isinstance(value, str): + return (value,) + if isinstance(value, (tuple, list)): + return tuple(item for item in value if isinstance(item, str)) + return () + + +def _function_errors( + *, + model: CompiledModel, + project: CompiledProject, + shapes: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, +) -> tuple[CompilerDiagnostic, ...]: + functions: dict[str, CompiledFunction] = { + function.name.casefold(): function for function in project.functions + } + if not functions or not any( + reference.ref_kind in {SqlReferenceKind.UDF, SqlReferenceKind.TABLE_FUNCTION} + for reference in model.references + ): + return () + module: Any = import_polyglot_sql() + request: SqlSchemaValidationRequest = get_complete_schema_binding_request( + query_sql=model.query_sql, + placeholders=None, + dialect=profile.sql_analysis_dialect, + binding_schema=shapes, + ) + try: + parsed: Any = module.parse_one(request.sql, dialect=profile.sql_analysis_dialect) + except module.PolyglotError: + return () + diagnostics: list[CompilerDiagnostic] = [] + for call, select in _function_scopes(parsed): + payload: dict[str, Any] = call.to_dict().get("function", {}) + name: str = str(payload.get("name", "")).removeprefix("__sqlbuild_udf_") + function: CompiledFunction | None = functions.get(name.casefold()) + if function is None: + continue + arguments: list[dict[str, Any]] = payload.get("args", []) + if len(arguments) != len(function.arguments): + diagnostics.append( + _model_error( + model=model, + code="B102", + name=name, + message=( + f"Function '{name}' expects {len(function.arguments)} arguments " + f"but received {len(arguments)}" + ), + ) + ) + continue + argument_expressions: list[Any] = call.expressions + for argument, expression, declaration in zip( + arguments, argument_expressions, function.arguments, strict=True + ): + actual: str = ( + _polyglot_expression_type(expression=expression, inference_profile=profile) + or _UNKNOWN_TYPE + ) + literal: Any = argument.get("literal") + column: Any = argument.get("column") + if literal: + actual = {"string": "VARCHAR", "number": "DOUBLE", "boolean": "BOOLEAN"}.get( + literal.get("literal_type"), "UNKNOWN" + ) + elif column: + actual = _argument_column_type(column=column, select=select, shapes=shapes) + if actual != _UNKNOWN_TYPE and _different_argument_families( + left=actual, right=declaration.type, profile=profile + ): + diagnostics.append( + _model_error( + model=model, + code="B301", + severity=DiagnosticSeverity.ERROR, + name=name, + message=( + f"Function '{name}' argument '{declaration.name}' " + f"expects {declaration.type}, received {actual}; " + "convert the argument explicitly" + ), + ) + ) + return tuple(diagnostics) + + +def _function_scopes(root: Any) -> Iterator[tuple[Any, Any]]: + pending: list[tuple[Any, Any]] = [(root, None)] + node: Any + select: Any + while pending: + node, select = pending.pop() + if node.kind == _SELECT_KIND: + select = node + if node.kind == _FUNCTION_KIND: + yield node, select + pending.extend((child, select) for child in reversed(node.children())) + + +def _argument_column_type( + *, column: dict[str, Any], select: Any, shapes: dict[str, dict[str, str]] +) -> str: + if select is None: + return _UNKNOWN_TYPE + payload: dict[str, Any] = select.to_dict().get("select", {}) + relations: list[dict[str, Any]] = list((payload.get("from") or {}).get("expressions", [])) + relations.extend(join.get("this", {}) for join in payload.get("joins", [])) + qualifier: str | None = (column.get("table") or {}).get("name") + name: str = str(column.get("name", {}).get("name", "")) + candidates: list[str] = [] + for relation in relations: + table: dict[str, Any] = relation.get("table", {}) + table_name: str = str(table.get("name", {}).get("name", "")) + alias: str = str((table.get("alias") or {}).get("name", table_name)) + if qualifier is not None and qualifier.casefold() != alias.casefold(): + continue + shape: dict[str, str] | None = shapes.get(table_name) + if shape is None: + return _UNKNOWN_TYPE + candidates.extend( + value for key, value in shape.items() if key.casefold() == name.casefold() + ) + return candidates[0] if len(candidates) == 1 else _UNKNOWN_TYPE + + +def _model_error( + *, + model: CompiledModel, + code: str, + name: str, + message: str, + severity: DiagnosticSeverity = DiagnosticSeverity.ERROR, +) -> CompilerDiagnostic: + line: int + column: int + line, column = _text_position(text=model.authored_sql, name=name) + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=severity, + code=code, + message=message, + resource_type=CompiledResourceType.MODEL, + resource_name=model.name, + path=model.relative_path, + line=line, + column=column, + ) + + +def _text_position(*, text: str, name: str, offset: int = 0) -> tuple[int, int]: + match: re.Match[str] | None = re.search(rf"(? tuple[str, ...]: + """Register authored names and the analysis-only scalar/table function names.""" + names: set[str] = set() + for function in functions: + names.update( + ( + function.name, + f"__sqlbuild_udf_{function.name}", + f"__sqlbuild_table_function_{function.name}", + ) + ) + return tuple(sorted(names)) + + +def known_declared_types( + *, + functions: tuple[CompiledFunction, ...] | tuple[CompileSqlFunctionInput, ...], + column_types: dict[str, dict[str, str]], +) -> tuple[str, ...]: + """Register types named by relation contracts and scalar/table function signatures.""" + types: set[str] = set() + for columns in column_types.values(): + types.update(columns.values()) + for function in functions: + types.update(argument.type for argument in function.arguments) + types.update(column.type for column in function.return_columns) + if not function.return_columns: + types.add(function.returns) + types.discard("UNKNOWN") + return tuple(sorted(types)) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index 9998c664f2..a881ea5a9a 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -3,10 +3,12 @@ from __future__ import annotations import re -from collections import defaultdict, deque from collections.abc import Sequence from dataclasses import dataclass, replace +from functools import partial +from graphlib import CycleError, TopologicalSorter from pathlib import Path +from typing import Any, cast from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.adapter.contract.types import BuiltinAdapter @@ -39,6 +41,31 @@ validate_hook_sql_syntax, validate_sql_syntax, ) +from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( + get_authored_binding_location, + query_line_offset, +) +from sqlbuild.compiler.compile._helpers.assembly.binding_waves import analyze_binding_waves +from sqlbuild.compiler.compile._helpers.assembly.binding_waves import ( + downstream_model_names as _downstream_model_names, +) +from sqlbuild.compiler.compile._helpers.assembly.binding_waves import ( + referenced_model_names as _referenced_model_names, +) +from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( + known_declared_types, + known_function_names, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( + binding_required_names, + binding_schema_for_model, + build_complete_binding_schemas, + get_expression_source_shapes, + published_model_shape, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( + build_declared_column_types as _build_column_types_by_table, +) from sqlbuild.compiler.compile._helpers.assembly.targets import ( build_model_relation_target, build_seed_relation_target, @@ -52,6 +79,7 @@ model_build_deps, sql_test_scope_deps, ) +from sqlbuild.compiler.compile._helpers.diagnostics.recovery import complete_semantic_diagnostics from sqlbuild.compiler.compile._helpers.render.context_templates import ( resolve_early_model_templates, ) @@ -67,6 +95,7 @@ find_macro_call_names, ) from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data +from sqlbuild.compiler.compile._helpers.sharing.binding import shareable_binding_prekeys from sqlbuild.compiler.compile._helpers.sql_tests.core import extract_assertion_target_model_names from sqlbuild.compiler.compile._helpers.sql_tests.identity import ( build_sql_test_case_fingerprint, @@ -76,7 +105,6 @@ from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) -from sqlbuild.compiler.compile.main.function_node_type import function_node_type from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, CompactAnalysisCacheCandidate, @@ -114,6 +142,12 @@ MacroContext, PolyglotAnalysisResult, ) +from sqlbuild.compiler.compile.models import ( + ModelSqlAnalysis as _ModelSqlAnalysis, +) +from sqlbuild.compiler.compile.models import ( + ModelSqlAnalysisRequest as _ModelSqlAnalysisRequest, +) from sqlbuild.compiler.compile.types import ( AttachedAuditTargetKind, CompactBatchResponseCallback, @@ -126,11 +160,17 @@ from sqlbuild.compiler.planner.types import ContractPolicy from sqlbuild.compiler.profiling.main.record import record_compile_timing from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.compiler.scopes.exceptions import ScopeValidationError from sqlbuild.compiler.scopes.main._validate_scope_index import validate_scope_index from sqlbuild.compiler.scopes.models import ScopeIndex -from sqlbuild.compiler.sql_analysis.constants import BINDING_UNKNOWN_TABLE_INTERNAL_CODE +from sqlbuild.compiler.sql_analysis.constants import ( + BINDING_UNKNOWN_TABLE_INTERNAL_CODE, + NATIVE_DIALECT_ALIASES, +) from sqlbuild.compiler.sql_analysis.exceptions import SqlAnalysisBoundaryError +from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog +from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations from sqlbuild.compiler.sql_analysis.models import ( SqlBindingDiagnostic, @@ -150,25 +190,11 @@ SchemaDynamicColumnFamily, SourceColumnEntry, SourceEntry, + SourceLocation, TargetConfig, ) -@dataclass(frozen=True) -class _ModelSqlAnalysis: - polyglot_analysis: PolyglotAnalysisResult - placeholders: dict[str, str] | None - - -@dataclass(frozen=True) -class _ModelSqlAnalysisRequest: - model_input: CompileModelInput - query_sql: str - placeholders: dict[str, str] | None - cache_key: str | None - binding_schema: dict[str, dict[str, str]] | None - - @dataclass(frozen=True) class _DynamicContractAnalysisInputs: families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] @@ -203,14 +229,75 @@ def assemble_compiled_project( dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] = ( _build_dynamic_families_by_table(inputs) ) - complete_binding_schemas: dict[str, dict[str, str]] = _build_complete_binding_schemas(inputs) + profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() + profile = replace( + profile, + semantic_known_functions=known_function_names(inputs.sql_function_inputs), + semantic_known_types=known_declared_types( + functions=inputs.sql_function_inputs, column_types=column_types_by_table + ), + quoted_identifiers_ignore_case=ignores_quoted_case( + connection=inputs.effective_connection, dialect=profile.sql_analysis_dialect + ), + sql_analysis_dialect=NATIVE_DIALECT_ALIASES.get( + profile.sql_analysis_dialect or "", profile.sql_analysis_dialect + ), + function_return_types={ + **profile.function_return_types, + **{ + f"__sqlbuild_udf_{function.name}".upper(): function.returns + for function in inputs.sql_function_inputs + if not function.return_columns + }, + }, + ) + complete_binding_schemas: dict[str, dict[str, str]] = build_complete_binding_schemas(inputs) + binding_catalog: Any = create_binding_catalog( + dialect=profile.sql_analysis_dialect or "generic", + quoted_ignore_case=profile.quoted_identifiers_ignore_case, + known_functions=profile.semantic_known_functions, + known_types=profile.semantic_known_types, + relations={}, + ) + profile = replace(profile, binding_catalog=binding_catalog) + if sql_analysis_enabled: + expression_sources: tuple[CompileSourceInput, ...] = tuple( + source_input + for source_input in inputs.source_inputs + if source_input.source_entry.expression + ) + expression_shapes: tuple[dict[str, str] | None, ...] = get_expression_source_shapes( + expressions=tuple( + cast(str, source_input.source_entry.expression) + for source_input in expression_sources + ), + profile=profile, + ) + for source_input, shape in zip(expression_sources, expression_shapes, strict=True): + if shape is not None: + declared_types: dict[str, str] = column_types_by_table.get( + source_input.source_entry.name, {} + ) + declared_shape: dict[str, str] = { + name: declared_types.get(name, column_type) + for name, column_type in shape.items() + } + complete_binding_schemas[source_input.source_entry.name] = declared_shape + column_types_by_table[source_input.source_entry.name] = declared_shape + column_nullability_by_table[source_input.source_entry.name] = { + name: column_nullability_by_table.get(source_input.source_entry.name, {}).get( + name, InferredNullability.UNKNOWN + ) + for name in declared_shape + } + binding_catalog.native.update_relations(complete_binding_schemas) + binding_catalog.schemas.update(complete_binding_schemas) dynamic_contract_analysis_inputs: _DynamicContractAnalysisInputs = ( _DynamicContractAnalysisInputs( families_by_table=dynamic_families_by_table, authoritative_column_types_by_table=complete_binding_schemas, ) ) - profile: ExpressionInferenceProfile = inference_profile or ExpressionInferenceProfile() allow_compact_analysis: bool = column_lineage_mode in { ColumnLineageMode.FAST, ColumnLineageMode.RICH, @@ -223,6 +310,10 @@ def assemble_compiled_project( allow_compact_analysis=allow_compact_analysis, rich_type_inference=rich_type_inference, signature_namespace={ + "known_functions": known_function_names(inputs.sql_function_inputs), + "known_types": known_declared_types( + functions=inputs.sql_function_inputs, column_types=column_types_by_table + ), "target": inputs.effective_target_name, "vars": inputs.effective_vars, }, @@ -234,6 +325,10 @@ def assemble_compiled_project( if sql_analysis_enabled: with record_compile_timing("model_analysis_ms"): model_sql_analysis_by_name = _analyze_model_sql_in_parallel( + known_functions=known_function_names(inputs.sql_function_inputs), + known_types=known_declared_types( + functions=inputs.sql_function_inputs, column_types=column_types_by_table + ), model_inputs=tuple( model_input for model_input in inputs.model_inputs @@ -271,7 +366,8 @@ def assemble_compiled_project( effective_target_name=inputs.effective_target_name, run_id=inputs.run_id, ) - return CompiledProject( + project: CompiledProject = CompiledProject( + binding_catalog=profile.binding_catalog, sql_expansions={ model_input.model_file.file_path: model_input.sql_expansion for model_input in inputs.model_inputs @@ -283,6 +379,7 @@ def assemble_compiled_project( effective_vars=inputs.effective_vars, effective_target_database=_str_or_none(effective_target_values.get("database")), effective_target_schema=_str_or_none(effective_target_values.get("schema")), + sql_analysis_dialect=profile.sql_analysis_dialect, compile_cache_dir=inputs.compile_cache_dir, settings=inputs.effective_settings, scenario=resolve_effective_scenario_config( @@ -357,11 +454,20 @@ def assemble_compiled_project( *_project_binding_diagnostics( model_inputs=inputs.model_inputs, analyses_by_name=model_sql_analysis_by_name, + dialect=profile.sql_analysis_dialect, ), ), external_sql_reference_resolver=inputs.external_sql_reference_resolver, scope_index=scope_index, ) + return complete_semantic_diagnostics( + project=project, + profile=profile, + binding_results={ + name: analysis.polyglot_analysis.binding_diagnostics + for name, analysis in model_sql_analysis_by_name.items() + }, + ) def _str_or_none(value: object) -> str | None: @@ -400,6 +506,7 @@ def _assemble_compiled_model( inferred_columns: tuple[InferredColumn, ...] | None = None fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = None fast_lineage_has_star: bool = False + fast_lineage_star_resolved: bool = False placeholders: dict[str, str] | None = ( sql_analysis.placeholders if sql_analysis is not None else _model_placeholders(model_input) ) @@ -431,6 +538,7 @@ def _assemble_compiled_model( inferred_columns = polyglot_analysis.columns fast_lineage_columns = polyglot_analysis.lineage_columns fast_lineage_has_star = polyglot_analysis.has_star + fast_lineage_star_resolved = polyglot_analysis.star_resolved else: if model_input.sql_validation_enabled: validate_sql_syntax( @@ -496,6 +604,7 @@ def _assemble_compiled_model( inferred_columns=inferred_columns, fast_lineage_columns=fast_lineage_columns, fast_lineage_has_star=fast_lineage_has_star, + fast_lineage_star_resolved=fast_lineage_star_resolved, authored_sql=model_input.model_file.contents, authored_query_sql=model_input.model_file.query_sql, output_column_locations=model_input.model_file.output_column_locations, @@ -506,6 +615,8 @@ def _assemble_compiled_model( enum_columns=model_input.enum_columns, binding_diagnostics=_binding_compiler_diagnostics( model_input=model_input, + cleaned_sql=sql_analysis.cleaned_sql if sql_analysis is not None else None, + dialect=profile.sql_analysis_dialect, diagnostics=(polyglot_analysis.binding_diagnostics if sql_analysis_enabled else ()), ), binding_validated=(polyglot_analysis.binding_validated if sql_analysis_enabled else False), @@ -515,6 +626,8 @@ def _assemble_compiled_model( def _analyze_model_sql_in_parallel( *, + known_functions: tuple[str, ...], + known_types: tuple[str, ...], model_inputs: tuple[CompileModelInput, ...], column_nullability_by_table: dict[str, dict[str, InferredNullability]], column_types_by_table: dict[str, dict[str, str]], @@ -542,6 +655,10 @@ def _analyze_model_sql_in_parallel( request_cache_keys: tuple[str, ...] = tuple( request.cache_key for request in requests if request.cache_key is not None ) + dependency_ordered: bool = allow_compact_analysis and any( + request.binding_schema is not None and not all(request.binding_schema.values()) + for request in requests + ) compact_batch_plan: CompactAnalysisCachePlan | None = None if analysis_cache is not None: compact_batch_plan = build_compact_analysis_cache_plan( @@ -567,12 +684,12 @@ def _analyze_model_sql_in_parallel( expected_count=len(requests), min_model_count=_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT, ) - if compact_batch_plan is not None + if compact_batch_plan is not None and not dependency_ordered else None ) if compact_candidate is not None: completed_analyses: dict[str, PolyglotAnalysisResult] = {} - if analysis_cache is not None and len(compact_candidate.matching_indexes) == len(requests): + if analysis_cache is not None: completed_analyses, _, _ = read_model_analyses( context=analysis_cache, cache_keys=request_cache_keys, @@ -627,6 +744,8 @@ def _analyze_model_sql_in_parallel( bypasses=0, ) compact_analyses = _complete_inferred_bindings( + known_functions=known_functions, + known_types=known_types, requests=requests, analyses=compact_analyses, complete_binding_schemas=complete_binding_schemas, @@ -680,42 +799,71 @@ def _analyze_model_sql_in_parallel( for cache_key, analysis in compact_cached_analyses.items() } ) + analyses: tuple[_ModelSqlAnalysis, ...] + if dependency_ordered: + analyses, cached_analyses = analyze_binding_waves( + requests=requests, + names=tuple(_model_name(request.model_input) for request in requests), + cached=cached_analyses, + previous_signatures=previous_signatures, + shapes=complete_binding_schemas, + types=column_types_by_table, + nullability=column_nullability_by_table, + profile=inference_profile, + analyze=partial( + _analyze_model_sql_requests, + inference_profile=inference_profile, + allow_compact_analysis=allow_compact_analysis, + rich_type_inference=rich_type_inference, + shareable_prekeys=shareable_binding_prekeys(requests=requests), + ), + complete=partial( + _complete_inferred_bindings, + known_functions=known_functions, + known_types=known_types, + inference_profile=inference_profile, + ), + ) + if compact_batch_plan is not None: + compact_batch_hit_count = sum(analysis.cached for analysis in analyses) + entry_cache_hit_count -= compact_batch_hit_count + else: + analyses = _analyze_model_sql_requests( + requests=requests, + cached_analyses=cached_analyses, + column_nullability_by_table=column_nullability_by_table, + column_types_by_table=column_types_by_table, + inference_profile=inference_profile, + allow_compact_analysis=allow_compact_analysis, + rich_type_inference=rich_type_inference, + on_compact_response=( + compact_analysis_batch_response_writer( + plan=compact_batch_plan, + count=len(requests), + ) + if compact_batch_plan is not None and not cached_analyses + else None + ), + shareable_prekeys=shareable_binding_prekeys(requests=requests), + ) + analyses = _complete_inferred_bindings( + known_functions=known_functions, + known_types=known_types, + requests=requests, + analyses=analyses, + complete_binding_schemas=complete_binding_schemas, + inference_profile=inference_profile, + ) record_analysis_cache_metrics( batch_hits=compact_batch_hit_count, entry_hits=entry_cache_hit_count, misses=( - sum( - request.cache_key is None or request.cache_key not in cached_analyses - for request in requests - ) + len(requests) - compact_batch_hit_count - entry_cache_hit_count if analysis_cache is not None else 0 ), bypasses=(len(requests) if analysis_cache is None else 0), ) - analyses: tuple[_ModelSqlAnalysis, ...] = _analyze_model_sql_requests( - requests=requests, - cached_analyses=cached_analyses, - column_nullability_by_table=column_nullability_by_table, - column_types_by_table=column_types_by_table, - inference_profile=inference_profile, - allow_compact_analysis=allow_compact_analysis, - rich_type_inference=rich_type_inference, - on_compact_response=( - compact_analysis_batch_response_writer( - plan=compact_batch_plan, - count=len(requests), - ) - if compact_batch_plan is not None and not cached_analyses - else None - ), - ) - analyses = _complete_inferred_bindings( - requests=requests, - analyses=analyses, - complete_binding_schemas=complete_binding_schemas, - inference_profile=inference_profile, - ) if analysis_cache is None: return { _model_name(model_input): analysis @@ -757,7 +905,7 @@ def _analyze_model_sql_in_parallel( model_inputs=model_inputs, changed_names=changed_signature_names, ) - if invalidated_names: + if invalidated_names and not dependency_ordered: invalidated_requests: tuple[_ModelSqlAnalysisRequest, ...] = tuple( request for request in requests @@ -788,6 +936,8 @@ def _analyze_model_sql_in_parallel( for request, analysis in zip(requests, analyses, strict=True) ) analyses = _complete_inferred_bindings( + known_functions=known_functions, + known_types=known_types, requests=requests, analyses=analyses, complete_binding_schemas=complete_binding_schemas, @@ -854,6 +1004,7 @@ def _analyze_model_sql_requests( rich_type_inference: bool, cached_compact_batch: tuple[CompactBatchPreparation, object] | None = None, on_compact_response: CompactBatchResponseCallback | None = None, + shareable_prekeys: frozenset[str] | None = None, ) -> tuple[_ModelSqlAnalysis, ...]: if allow_compact_analysis: uncached: tuple[tuple[int, _ModelSqlAnalysisRequest], ...] = tuple( @@ -890,6 +1041,7 @@ def _analyze_model_sql_requests( binding_schemas=tuple( request.binding_schema for _, request in batch_requests ), + shareable_prekeys=shareable_prekeys, ), ) ) @@ -909,6 +1061,10 @@ def _analyze_model_sql_requests( sql=prepared_by_index[index].cleaned_sql, dialect=inference_profile.sql_analysis_dialect, schema=requests[index].binding_schema or {}, + known_functions=inference_profile.semantic_known_functions, + known_types=inference_profile.semantic_known_types, + quoted_identifiers_ignore_case=inference_profile.quoted_identifiers_ignore_case, + catalog=inference_profile.binding_catalog, ) for index in validation_indices ) @@ -960,73 +1116,134 @@ def analyze(request: _ModelSqlAnalysisRequest) -> _ModelSqlAnalysis: def _complete_inferred_bindings( *, + known_functions: tuple[str, ...], + known_types: tuple[str, ...], requests: tuple[_ModelSqlAnalysisRequest, ...], analyses: tuple[_ModelSqlAnalysis, ...], complete_binding_schemas: dict[str, dict[str, str]], inference_profile: ExpressionInferenceProfile, + supplied_relations: frozenset[str] = frozenset(), ) -> tuple[_ModelSqlAnalysis, ...]: - """Propagate complete derived model shapes without treating partial inputs as closed.""" + """Bind each scope independently, preserving open inputs as empty tables.""" complete_schemas: dict[str, dict[str, str]] = dict(complete_binding_schemas) results: list[_ModelSqlAnalysis] = list(analyses) - required_names_by_index: dict[int, frozenset[str]] = {} - dependents_by_name: dict[str, list[int]] = defaultdict(list) - for index, request in enumerate(requests): - required_names: frozenset[str] | None = _binding_required_names( - model_input=request.model_input + indexes: dict[str, int] = { + _model_name(request.model_input): index for index, request in enumerate(requests) + } + available_names: frozenset[str] = frozenset(indexes) + graph: dict[str, tuple[str, ...]] = { + name: _referenced_model_names( + model_input=requests[index].model_input, available_names=available_names ) + for name, index in indexes.items() + } + try: + ordered_names: tuple[str, ...] = tuple(TopologicalSorter(graph).static_order()) + except CycleError: + return analyses + for name in ordered_names: + index: int = indexes[name] + request: _ModelSqlAnalysisRequest = requests[index] + required_names: frozenset[str] | None = binding_required_names(request.model_input) if required_names is None: continue - required_names_by_index[index] = required_names - for required_name in required_names: - dependents_by_name[required_name].append(index) - ready: deque[int] = deque( - index - for index, required_names in required_names_by_index.items() - if required_names <= complete_schemas.keys() - ) - processed: set[int] = set() + analysis: PolyglotAnalysisResult = results[index].polyglot_analysis + inputs_known: bool = bool(required_names) and required_names <= complete_schemas.keys() + star_pending: bool = analysis.has_star and not analysis.star_resolved and inputs_known + stars_expanded_natively: bool = required_names <= supplied_relations + if star_pending and analysis.columns and stars_expanded_natively: + analysis = replace(analysis, star_resolved=True) + results[index] = replace(results[index], polyglot_analysis=analysis) + star_pending = False + if ( + not results[index].cached + and required_names + and (any(column.type is None for column in analysis.columns or ()) or star_pending) + and not re.search(r"\b(?:UNION|INTERSECT|EXCEPT)\b", request.query_sql, re.IGNORECASE) + ): + input_schemas: dict[str, dict[str, str]] = { + table: complete_schemas[table] + for table in required_names + if table in complete_schemas + } + nullability: dict[str, dict[str, InferredNullability]] = { + table: dict.fromkeys(columns, InferredNullability.UNKNOWN) + for table, columns in input_schemas.items() + } + enriched: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + query_sql=request.query_sql, + references=request.model_input.references, + placeholders=request.placeholders, + column_types_by_table=input_schemas, + column_nullability_by_table=nullability, + inference_profile=inference_profile, + allow_compact_analysis=True, + recover_cte_facts=_should_recover_cte_facts(request.model_input), + ) + recovered_types: dict[str, str | None] = { + column.name: column.type + for column in (*(enriched.columns or ()), *(analysis.columns or ())) + if column.type is not None + } + star_expanded: bool = star_pending and enriched.analysis_succeeded + if (not analysis.columns or star_expanded) and enriched.analysis_succeeded: + analysis = replace( + enriched, + binding_diagnostics=analysis.binding_diagnostics, + binding_validated=analysis.binding_validated, + ) + analysis = replace( + analysis, + columns=tuple( + replace(column, type=column.type or recovered_types.get(column.name)) + for column in analysis.columns or () + ), + star_resolved=analysis.star_resolved or star_expanded, + ) + results[index] = replace(results[index], polyglot_analysis=analysis) + star_known: bool = not analysis.has_star or (analysis.star_resolved and inputs_known) + if analysis.columns and star_known: + complete_schemas.setdefault( + name, + published_model_shape( + sql=results[index].cleaned_sql or request.query_sql, + profile=inference_profile, + columns={column.name: column.type or "UNKNOWN" for column in analysis.columns}, + inputs={table: complete_schemas.get(table, {}) for table in required_names}, + config_values=request.model_input.config.values, + ), + ) deferred_validation_indices: list[int] = [] deferred_validation_requests: list[SqlSchemaValidationRequest] = [] - while ready: - index: int = ready.popleft() - if index in processed: + for index, request in enumerate(requests): + if results[index].cached: continue - processed.add(index) - request: _ModelSqlAnalysisRequest = requests[index] - current: _ModelSqlAnalysis = results[index] - analysis: PolyglotAnalysisResult = current.polyglot_analysis - if not analysis.binding_validated: - binding_schema: dict[str, dict[str, str]] = { - name: complete_schemas[name] for name in required_names_by_index[index] - } - if binding_schema: - deferred_validation_indices.append(index) - deferred_validation_requests.append( - get_complete_schema_binding_request( - query_sql=request.query_sql, - placeholders=request.placeholders, - dialect=inference_profile.sql_analysis_dialect, - binding_schema=binding_schema, - ) - ) - else: - analysis = replace(analysis, binding_validated=True) - results[index] = replace(current, polyglot_analysis=analysis) - if not ( - required_names_by_index[index] - and analysis.analysis_succeeded - and analysis.columns is not None - and not analysis.has_star - ): + required_names = binding_required_names(request.model_input) + if required_names is None: continue - model_name: str = _model_name(request.model_input) - complete_schemas[model_name] = { - column.name: column.type or "UNKNOWN" for column in analysis.columns - } - for dependent_index in dependents_by_name.get(model_name, ()): - if required_names_by_index[dependent_index] <= complete_schemas.keys(): - ready.append(dependent_index) + if results[index].fused_binding_validated and ( + results[index].validated_schema or request.binding_schema + ) == {name: complete_schemas.get(name, {}) for name in required_names}: + continue + deferred_validation_indices.append(index) + deferred_validation_requests.append( + replace( + get_complete_schema_binding_request( + query_sql=request.query_sql, + cleaned_sql=results[index].cleaned_sql, + known_functions=known_functions, + known_types=known_types, + placeholders=request.placeholders, + dialect=inference_profile.sql_analysis_dialect, + binding_schema={ + name: complete_schemas.get(name, {}) for name in required_names + }, + ), + quoted_identifiers_ignore_case=inference_profile.quoted_identifiers_ignore_case, + catalog=inference_profile.binding_catalog, + ) + ) with record_compile_timing("binding_validation_ms"): validation_results: tuple[SqlBindingResult, ...] = get_schema_validations( requests=tuple(deferred_validation_requests) @@ -1034,7 +1251,7 @@ def _complete_inferred_bindings( for index, validation_result in zip( deferred_validation_indices, validation_results, strict=True ): - current = results[index] + current: _ModelSqlAnalysis = results[index] results[index] = replace( current, polyglot_analysis=replace( @@ -1046,63 +1263,6 @@ def _complete_inferred_bindings( return tuple(results) -def _binding_required_names(model_input: CompileModelInput) -> frozenset[str] | None: - if not model_input.sql_validation_enabled: - return None - names: set[str] = set() - for reference in model_input.references: - if reference.ref_kind == SqlReferenceKind.UDF: - continue - if reference.ref_kind == SqlReferenceKind.DBT_REF: - return None - names.add( - table_function_analysis_name(reference.ref_name) - if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION - else reference.ref_name - ) - if not names and re.search(r"\b(?:FROM|JOIN)\b", model_input.query_sql, re.IGNORECASE): - return None - return frozenset(names) - - -def _downstream_model_names( - *, - model_inputs: tuple[CompileModelInput, ...], - changed_names: set[str], -) -> set[str]: - downstream_by_name: dict[str, set[str]] = {} - for model_input in model_inputs: - model_name: str = _model_name(model_input) - for reference in model_input.references: - if reference.ref_kind == SqlReferenceKind.REF: - downstream_by_name.setdefault(reference.ref_name, set()).add(model_name) - downstream_names: set[str] = set() - pending: list[str] = list(changed_names) - while pending: - for downstream_name in downstream_by_name.get(pending.pop(), set()): - if downstream_name not in downstream_names and downstream_name not in changed_names: - downstream_names.add(downstream_name) - pending.append(downstream_name) - return downstream_names - - -def _referenced_model_names( - *, - model_input: CompileModelInput, - available_names: frozenset[str] | None = None, -) -> tuple[str, ...]: - return tuple( - sorted( - { - reference.ref_name - for reference in model_input.references - if reference.ref_kind == SqlReferenceKind.REF - and (available_names is None or reference.ref_name in available_names) - } - ) - ) - - def _dependency_signatures_by_key( *, requests: tuple[_ModelSqlAnalysisRequest, ...], @@ -1144,6 +1304,7 @@ def _analyze_model_sql( return _ModelSqlAnalysis( polyglot_analysis=cached_analysis, placeholders=request.placeholders, + cached=True, ) polyglot_analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( query_sql=request.query_sql, @@ -1159,7 +1320,11 @@ def _analyze_model_sql( ) return _ModelSqlAnalysis( polyglot_analysis=polyglot_analysis, + cleaned_sql=precomputed.cleaned_sql if precomputed is not None else None, placeholders=request.placeholders, + validated_schema=request.binding_schema, + fused_binding_validated=precomputed is not None + and precomputed.binding_diagnostics is not None, ) @@ -1176,7 +1341,7 @@ def _model_sql_analysis_request( sql=model_input.query_sql, cursor_type=model_input.config.values.get("cursor_type"), ) - binding_schema: dict[str, dict[str, str]] | None = _binding_schema_for_model( + binding_schema: dict[str, dict[str, str]] | None = binding_schema_for_model( model_input=model_input, complete_binding_schemas=complete_binding_schemas, ) @@ -1249,30 +1414,6 @@ def _build_column_nullability_by_table( return facts -def _build_column_types_by_table(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: - facts: dict[str, dict[str, str]] = {} - for model_input in inputs.model_inputs: - if model_input.schema_entry is not None: - facts[model_input.schema_entry.name] = { - column.name: column.type or "UNKNOWN" for column in model_input.schema_entry.columns - } - for seed_input in inputs.seed_inputs: - facts[seed_input.schema_entry.name] = { - column.name: column.type or "UNKNOWN" for column in seed_input.schema_entry.columns - } - for source_input in inputs.source_inputs: - facts[source_input.source_entry.name] = { - column.name: column.type or "UNKNOWN" for column in source_input.source_entry.columns - } - for function_input in inputs.sql_function_inputs: - if not function_input.return_columns: - continue - facts[table_function_analysis_name(function_input.name)] = { - column.name: column.type for column in function_input.return_columns - } - return facts - - def _build_dynamic_families_by_table( inputs: CompileProjectInputs, ) -> dict[str, tuple[SchemaDynamicColumnFamily, ...]]: @@ -1285,68 +1426,11 @@ def _build_dynamic_families_by_table( } -def _build_complete_binding_schemas( - inputs: CompileProjectInputs, -) -> dict[str, dict[str, str]]: - """Return only relation shapes whose missing columns are authoritative.""" - - schemas: dict[str, dict[str, str]] = {} - for model_input in inputs.model_inputs: - if ( - model_input.config.values.get("contract") == ContractPolicy.ENFORCED - and model_input.schema_entry is not None - and model_input.schema_entry.columns - and not model_input.schema_entry.dynamic_columns - ): - schemas[_model_name(model_input)] = _typed_schema_columns( - model_input.schema_entry.columns - ) - for source_input in inputs.source_inputs: - if ( - source_input.source_entry.contract == ContractPolicy.ENFORCED - and source_input.source_entry.columns - ): - schemas[source_input.source_entry.name] = _typed_schema_columns( - source_input.source_entry.columns - ) - for seed_input in inputs.seed_inputs: - if seed_input.schema_entry.columns: - schemas[seed_input.schema_entry.name] = _typed_schema_columns( - seed_input.schema_entry.columns - ) - for function_input in inputs.sql_function_inputs: - if function_input.return_columns: - schemas[table_function_analysis_name(function_input.name)] = { - column.name: column.type for column in function_input.return_columns - } - return schemas - - -def _typed_schema_columns(columns: tuple[SchemaColumn | SourceColumnEntry, ...]) -> dict[str, str]: - return {column.name: column.type or "UNKNOWN" for column in columns} - - -def _binding_schema_for_model( - *, - model_input: CompileModelInput, - complete_binding_schemas: dict[str, dict[str, str]], -) -> dict[str, dict[str, str]] | None: - required_names: frozenset[str] | None = _binding_required_names(model_input) - if not required_names: - return None - schema: dict[str, dict[str, str]] = {} - for table_name in required_names: - columns: dict[str, str] | None = complete_binding_schemas.get(table_name) - if columns is None: - return None - schema[table_name] = columns - return schema - - def _project_binding_diagnostics( *, model_inputs: tuple[CompileModelInput, ...], analyses_by_name: dict[str, _ModelSqlAnalysis], + dialect: str | None, ) -> tuple[CompilerDiagnostic, ...]: diagnostics: list[CompilerDiagnostic] = [] for model_input in model_inputs: @@ -1356,6 +1440,8 @@ def _project_binding_diagnostics( diagnostics.extend( _binding_compiler_diagnostics( model_input=model_input, + cleaned_sql=analysis.cleaned_sql, + dialect=dialect, diagnostics=analysis.polyglot_analysis.binding_diagnostics, ) ) @@ -1366,23 +1452,40 @@ def _binding_compiler_diagnostics( *, model_input: CompileModelInput, diagnostics: tuple[SqlBindingDiagnostic, ...], + dialect: str | None, + cleaned_sql: str | None = None, ) -> tuple[CompilerDiagnostic, ...]: - query_line_offset: int | None = _query_line_offset(model_input) + line_offset: int | None = query_line_offset(model_input) seen: set[tuple[str, str, int | None, int | None]] = set() result: list[CompilerDiagnostic] = [] for diagnostic in diagnostics: if diagnostic.code == BINDING_UNKNOWN_TABLE_INTERNAL_CODE: continue + if cleaned_sql is None: + cleaned_sql = get_complete_schema_binding_request( + query_sql=cursor_intrinsics_analysis_sql( + sql=model_input.query_sql, + cursor_type=model_input.config.values.get("cursor_type"), + ), + placeholders=_model_placeholders(model_input), + dialect=dialect, + binding_schema={}, + ).sql line: int | None column: int | None - line, column = _authored_binding_position( - model_input=model_input, + location: SourceLocation | None = get_authored_binding_location( + path=model_input.model_file.relative_path, + authored_sql=model_input.model_file.contents, + authored_query_sql=model_input.model_file.query_sql, + cleaned_sql=cleaned_sql, + expansion=model_input.sql_expansion, diagnostic=diagnostic, ) + line, column = (location.line, location.column) if location is not None else (None, None) if line is None: line = ( - diagnostic.line + query_line_offset - if diagnostic.line is not None and query_line_offset is not None + diagnostic.line + line_offset + if diagnostic.line is not None and line_offset is not None else diagnostic.line ) column = diagnostic.column @@ -1392,13 +1495,13 @@ def _binding_compiler_diagnostics( line, column, ) - if key in seen: + if key in seen and diagnostic.severity == DiagnosticSeverity.ERROR: continue seen.add(key) result.append( CompilerDiagnostic( phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.ERROR, + severity=DiagnosticSeverity(diagnostic.severity), code=diagnostic.code, message=diagnostic.message, resource_type=CompiledResourceType.MODEL, @@ -1406,39 +1509,12 @@ def _binding_compiler_diagnostics( path=model_input.model_file.relative_path, line=line, column=column, - help="correct the column reference or the authoritative upstream contract", + location=location, ) ) return tuple(result) -def _query_line_offset(model_input: CompileModelInput) -> int | None: - query_sql: str = model_input.model_file.query_sql - query_start: int = model_input.model_file.contents.find(query_sql) - if query_start < 0: - return None - return model_input.model_file.contents[:query_start].count("\n") - - -def _authored_binding_position( - *, model_input: CompileModelInput, diagnostic: SqlBindingDiagnostic -) -> tuple[int | None, int | None]: - match: re.Match[str] | None = re.search( - r"(?:Unknown column|Ambiguous column reference) '([^']+)'", diagnostic.message - ) - if match is None: - return None, None - identifier: str = match.group(1).rsplit(".", maxsplit=1)[-1] - contents: str = model_input.model_file.contents - occurrences: list[re.Match[str]] = list( - re.finditer(rf"(? dict[str, InferredNullability]: diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py new file mode 100644 index 0000000000..e1b7bfd68b --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_coverage.py @@ -0,0 +1,62 @@ +"""Explain the limits of offline semantic checks without per-model warnings.""" + +from __future__ import annotations + +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import TYPE_CHECKED_DIALECTS + + +def semantic_coverage(project: CompiledProject) -> dict[str, tuple[str, ...]]: + """Return partial-check reasons for machine reports.""" + + if not project.settings.sql_analysis: + return {} + shapes: dict[str, dict[str, str]] = semantic_shapes(project=project) + invalid: set[str] = set() + for model in project.models: + if any(item.is_error for item in model.binding_diagnostics): + invalid.add(model.name) + blocked_by: dict[str, set[str]] = {name: {name} for name in invalid} + if invalid: + for _ in project.models: + changed: bool = False + for model in project.models: + causes: set[str] = set(blocked_by.get(model.name, ())) + for reference in model.references: + causes.update(blocked_by.get(reference.ref_name, ())) + if causes and causes != blocked_by.get(model.name): + blocked_by[model.name] = causes + changed = True + if not changed: + break + reasons: dict[str, tuple[str, ...]] = {} + for model in project.models: + if model.config.values.get("sql_analysis") is False: + reasons[model.name] = ("SQL analysis disabled",) + continue + partial: list[str] = [ + f"open source: {reference.ref_name}" + if reference.ref_kind == SqlReferenceKind.SOURCE + else f"open input: {reference.ref_name}" + for reference in model.references + if reference.ref_kind + in {SqlReferenceKind.SOURCE, SqlReferenceKind.REF, SqlReferenceKind.DBT_REF} + and reference.ref_name not in shapes + ] + partial.extend( + f"depends on invalid model {name}" + for name in sorted(blocked_by.get(model.name, set()) - {model.name}) + ) + if not model.inferred_columns: + partial.append("output shape could not be inferred") + elif model.fast_lineage_has_star and model.name not in shapes: + partial.append("unresolved star over open inputs") + if any(column.type is None for column in model.inferred_columns or ()): + partial.append("some output types are unknown") + if (project.sql_analysis_dialect or "generic") not in TYPE_CHECKED_DIALECTS: + partial.append("type checking awaits dialect coercion support") + if partial: + reasons[model.name] = tuple(sorted(set(partial))) + return reasons diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py new file mode 100644 index 0000000000..c0216b843e --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -0,0 +1,311 @@ +"""Authoritative relation shapes shared by offline and warehouse validation.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any, cast + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.analysis.columns import table_function_analysis_name +from sqlbuild.compiler.compile._helpers.analysis.compact import ( + NativeCompactAnalysis, + analyze_columns_and_lineage_with_polyglot, + analyze_queries_with_compact_polyglot_batch, +) +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompileModelInput, + CompileProjectInputs, + CompileSqlReference, + PolyglotAnalysisResult, +) +from sqlbuild.compiler.planner.constants import ( + SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, + SNAPSHOT_DEFAULT_VALID_TO_COLUMN, +) +from sqlbuild.compiler.planner.types import ContractPolicy, MaterializationType +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import ( + CASE_SENSITIVE_BINDING_DIALECTS, + NATIVE_DIALECT_ALIASES, +) +from sqlbuild.compiler.sql_analysis.main._binding_catalog import create_binding_catalog +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql +from sqlbuild.spec.contracts.models import SourceEntry + + +def binding_relation_names(references: tuple[CompileSqlReference, ...]) -> frozenset[str]: + return frozenset( + table_function_analysis_name(reference.ref_name) + if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION + else reference.ref_name + for reference in references + if reference.ref_kind != SqlReferenceKind.UDF + ) + + +def binding_required_names(model_input: CompileModelInput) -> frozenset[str] | None: + if not model_input.sql_validation_enabled: + return None + return binding_relation_names(model_input.references) + + +def published_model_shape( + *, + sql: str, + columns: dict[str, str], + inputs: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, + config_values: dict[str, object], +) -> dict[str, str]: + """Return a model relation's shape, including snapshot validity columns.""" + shape: dict[str, str] = inferred_binding_shape( + sql=sql, columns=columns, inputs=inputs, profile=profile + ) + if config_values.get("materialized") == MaterializationType.SNAPSHOT: + for name in ( + config_values.get("valid_from_column") or SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, + config_values.get("valid_to_column") or SNAPSHOT_DEFAULT_VALID_TO_COLUMN, + ): + shape[str(name)] = "UNKNOWN" + return shape + + +def binding_schema_for_model( + *, model_input: CompileModelInput, complete_binding_schemas: dict[str, dict[str, str]] +) -> dict[str, dict[str, str]] | None: + required_names: frozenset[str] | None = binding_required_names(model_input) + if required_names is None: + return None + return {name: complete_binding_schemas.get(name) or {} for name in required_names} + + +def inferred_binding_shape( + *, + sql: str, + columns: dict[str, str], + inputs: dict[str, dict[str, str]], + profile: ExpressionInferenceProfile, +) -> dict[str, str]: + if ( + profile.quoted_identifiers_ignore_case + or profile.sql_analysis_dialect not in CASE_SENSITIVE_BINDING_DIALECTS + ): + return columns + catalog: Any = profile.binding_catalog or create_binding_catalog( + dialect=profile.sql_analysis_dialect or "generic", + quoted_ignore_case=False, + known_functions=profile.semantic_known_functions, + known_types=profile.semantic_known_types, + relations=inputs, + ) + return catalog.inferred_schema( + sql=normalize_analysis_sql(sql=sql, dialect=profile.sql_analysis_dialect), + columns=columns, + inputs=inputs, + ) + + +def build_declared_column_types(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: + """Collect declared type hints independently of closed-schema authority.""" + facts: dict[str, dict[str, str]] = {} + for model_input in inputs.model_inputs: + if model_input.schema_entry is not None: + facts[model_input.schema_entry.name] = { + column.name: column.type or "UNKNOWN" for column in model_input.schema_entry.columns + } + for seed_input in inputs.seed_inputs: + facts[seed_input.schema_entry.name] = { + column.name: column.type or "UNKNOWN" for column in seed_input.schema_entry.columns + } + for source_input in inputs.source_inputs: + facts[source_input.source_entry.name] = { + column.name: column.type or "UNKNOWN" for column in source_input.source_entry.columns + } + for function_input in inputs.sql_function_inputs: + if function_input.return_columns: + facts[table_function_analysis_name(function_input.name)] = { + column.name: column.type for column in function_input.return_columns + } + return facts + + +def build_complete_binding_schemas(inputs: CompileProjectInputs) -> dict[str, dict[str, str]]: + """Collect authoritative declared interfaces before SQL inference.""" + schemas: dict[str, dict[str, str]] = {} + for model in inputs.model_inputs: + if ( + model.config.values.get("contract") == ContractPolicy.ENFORCED + and model.schema_entry is not None + and model.schema_entry.columns + and not model.schema_entry.dynamic_columns + ): + schemas[model.model_file.file_path.stem] = { + column.name: column.type or "UNKNOWN" for column in model.schema_entry.columns + } + for source in inputs.source_inputs: + if source.source_entry.contract == ContractPolicy.ENFORCED and source.source_entry.columns: + schemas[source.source_entry.name] = { + column.name: column.type or "UNKNOWN" for column in source.source_entry.columns + } + for seed in inputs.seed_inputs: + if seed.schema_entry.columns: + schemas[seed.schema_entry.name] = { + column.name: column.type or "UNKNOWN" for column in seed.schema_entry.columns + } + for function in inputs.sql_function_inputs: + if function.return_columns: + schemas[table_function_analysis_name(function.name)] = { + column.name: column.type for column in function.return_columns + } + return schemas + + +def semantic_shapes( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile | None = None, +) -> dict[str, dict[str, str]]: + """Return closed shapes; absence means open, never an empty physical table.""" + + profile = profile or ExpressionInferenceProfile( + sql_analysis_dialect=project.sql_analysis_dialect + ) + profile = replace(profile, binding_catalog=project.binding_catalog) + shapes: dict[str, dict[str, str]] = {} + for seed in project.seeds: + shapes[seed.name] = { + column.name: column.type or "UNKNOWN" for column in seed.schema_entry.columns + } + for function in project.functions: + if function.return_columns: + shapes[table_function_analysis_name(function.name)] = { + column.name: column.type for column in function.return_columns + } + expression_sources: list[tuple[str, str]] = [] + for source in project.sources: + entry: SourceEntry = source.source_entry + if entry.contract == ContractPolicy.ENFORCED: + shapes[source.name] = { + column.name: column.type or "UNKNOWN" for column in entry.columns + } + elif entry.expression: + expression_sources.append((source.name, entry.expression)) + source_shapes: tuple[dict[str, str] | None, ...] = get_expression_source_shapes( + expressions=tuple(expression for _, expression in expression_sources), profile=profile + ) + for (source_name, _), source_shape in zip(expression_sources, source_shapes, strict=True): + if source_shape is not None: + shapes[source_name] = source_shape + pending: list[CompiledModel] = list(project.models) + while pending: + remaining: list[CompiledModel] = [] + for model in pending: + if ( + model.config.values.get("contract") == ContractPolicy.ENFORCED + and model.schema_entry + and not model.schema_entry.dynamic_columns + ): + shapes[model.name] = { + column.name: "UNKNOWN" + if column.name in model.unchecked_output_columns + else column.type or "UNKNOWN" + for column in model.schema_entry.columns + } + elif model.inferred_columns and ( + not model.fast_lineage_has_star + or ( + model.fast_lineage_star_resolved + and model.references + and all( + ( + table_function_analysis_name(reference.ref_name) + if reference.ref_kind == SqlReferenceKind.TABLE_FUNCTION + else reference.ref_name + ) + in shapes + for reference in model.references + if reference.ref_kind != SqlReferenceKind.UDF + ) + ) + ): + shapes[model.name] = published_model_shape( + sql=model.query_sql, + profile=replace(profile, binding_catalog=project.binding_catalog), + columns={ + column.name: column.type or "UNKNOWN" for column in model.inferred_columns + }, + inputs={ + name: shapes.get(name, {}) + for name in binding_relation_names(model.references) + }, + config_values=model.config.values, + ) + else: + remaining.append(model) + if len(remaining) == len(pending): + break + pending = remaining + return shapes + + +def get_expression_source_shapes( + *, expressions: tuple[str, ...], profile: ExpressionInferenceProfile | None +) -> tuple[dict[str, str] | None, ...]: + """Infer expression-source shapes in one native batch, reusing the compile catalog's answers.""" + effective_profile: ExpressionInferenceProfile = profile or ExpressionInferenceProfile() + effective_profile = replace( + effective_profile, + sql_analysis_dialect=NATIVE_DIALECT_ALIASES.get( + effective_profile.sql_analysis_dialect or "generic", + effective_profile.sql_analysis_dialect, + ), + ) + catalog: Any = effective_profile.binding_catalog + cache: dict[str, dict[str, str] | None] = ( + catalog.expression_shapes if catalog is not None else {} + ) + pending: tuple[str, ...] = tuple( + dict.fromkeys(expression for expression in expressions if expression not in cache) + ) + if pending: + prepared: tuple[NativeCompactAnalysis, ...] = analyze_queries_with_compact_polyglot_batch( + query_sqls=pending, + references=tuple(() for _ in pending), + placeholders=tuple(None for _ in pending), + column_nullability_by_table={}, + column_types_by_table={}, + inference_profile=effective_profile, + recover_cte_facts=tuple(True for _ in pending), + rich_type_inference=True, + ) + for expression, precomputed in zip(pending, prepared, strict=True): + cache[expression] = _expression_source_shape( + expression=expression, profile=effective_profile, precomputed=precomputed + ) + return tuple( + None if cache[expression] is None else dict(cast(dict[str, str], cache[expression])) + for expression in expressions + ) + + +def _expression_source_shape( + *, expression: str, profile: ExpressionInferenceProfile, precomputed: NativeCompactAnalysis +) -> dict[str, str] | None: + analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + query_sql=expression, + references=(), + inference_profile=profile, + allow_compact_analysis=True, + recover_cte_facts=True, + precomputed=precomputed, + ) + if not analysis.columns or analysis.has_star: + return None + return inferred_binding_shape( + sql=expression, + profile=profile, + columns={column.name: column.type or "UNKNOWN" for column in analysis.columns}, + inputs={}, + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py new file mode 100644 index 0000000000..1e0f573885 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/source_bindings.py @@ -0,0 +1,209 @@ +"""Rebind models to source columns already inspected by the planner.""" + +from __future__ import annotations + +from dataclasses import replace +from graphlib import TopologicalSorter +from typing import Any + +from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.analysis.compact import ( + analyze_columns_and_lineage_with_polyglot, + get_complete_schema_binding_request, +) +from sqlbuild.compiler.compile._helpers.assembly.binding_positions import ( + get_authored_binding_position, +) +from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( + known_declared_types, + known_function_names, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( + binding_relation_names, + published_model_shape, + semantic_shapes, +) +from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import ( + cursor_intrinsics_analysis_sql, +) +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledObjectKey, + CompiledProject, + CompiledSqlExpansion, + CompilerDiagnostic, + PolyglotAnalysisResult, +) +from sqlbuild.compiler.compile.types import ( + CompiledResourceType, + DiagnosticPhase, + DiagnosticSeverity, +) +from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import ( + BINDING_UNKNOWN_TABLE_INTERNAL_CODE, + CASE_SENSITIVE_BINDING_DIALECTS, + NATIVE_DIALECT_ALIASES, +) +from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case +from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations +from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest + + +def _model_shapes( + *, model: CompiledModel, shapes: dict[str, dict[str, str]] +) -> dict[str, dict[str, str]]: + return {name: shapes.get(name, {}) for name in binding_relation_names(model.references)} + + +def get_source_binding_diagnostics( + *, + project: CompiledProject, + columns: dict[str, tuple[ColumnInfo, ...]], + profile: ExpressionInferenceProfile, + selected_keys: frozenset[CompiledObjectKey], +) -> tuple[CompilerDiagnostic, ...]: + """Validate selected source readers without executing or inspecting warehouse SQL.""" + + if not project.settings.sql_analysis or not columns: + return () + profile = replace( + profile, + sql_analysis_dialect=NATIVE_DIALECT_ALIASES.get( + profile.sql_analysis_dialect or "generic", profile.sql_analysis_dialect + ), + quoted_identifiers_ignore_case=ignores_quoted_case( + connection=project.effective_connection, dialect=profile.sql_analysis_dialect + ), + ) + shapes: dict[str, dict[str, str]] = semantic_shapes(project=project, profile=profile) + changed_sources: set[str] = set() + for name, value in columns.items(): + if value: + raw_shape: dict[str, str] = {column.name: column.type for column in value} + if raw_shape != shapes.get(name) or ( + profile.sql_analysis_dialect in CASE_SENSITIVE_BINDING_DIALECTS + and not profile.quoted_identifiers_ignore_case + ): + changed_sources.add(name) + shapes[name] = { + '"' + column.name.replace('"', '""') + '"': column.type for column in value + } + physical_catalog: Any | None = ( + project.binding_catalog.with_relations(shapes) + if project.binding_catalog is not None + else None + ) + profile = replace(profile, binding_catalog=physical_catalog) + by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + graph: dict[str, tuple[str, ...]] = {} + for model in project.models: + graph[model.name] = tuple( + reference.ref_name + for reference in model.references + if reference.ref_kind == SqlReferenceKind.REF and reference.ref_name in by_name + ) + affected: set[str] = set() + models: list[CompiledModel] = [] + for name in TopologicalSorter(graph).static_order(): + model: CompiledModel = by_name[name] + if model.key not in selected_keys or model.config.values.get("sql_analysis") is False: + continue + if any( + ( + reference.ref_kind == SqlReferenceKind.SOURCE + and reference.ref_name in changed_sources + ) + or (reference.ref_kind == SqlReferenceKind.REF and reference.ref_name in affected) + for reference in model.references + ): + models.append(model) + affected.add(model.name) + inputs: dict[str, dict[str, str]] = _model_shapes(model=model, shapes=shapes) + nullability: dict[str, dict[str, InferredNullability]] = { + table: dict.fromkeys(shape, InferredNullability.UNKNOWN) + for table, shape in inputs.items() + } + analysis: PolyglotAnalysisResult = analyze_columns_and_lineage_with_polyglot( + query_sql=model.query_sql, + references=model.references, + placeholders=_placeholders(model), + column_types_by_table=inputs, + column_nullability_by_table=nullability, + inference_profile=profile, + allow_compact_analysis=True, + recover_cte_facts=True, + ) + if analysis.columns and (not analysis.has_star or all(inputs.values())): + shapes[model.name] = published_model_shape( + sql=model.query_sql, + profile=profile, + inputs=inputs, + columns={column.name: column.type or "UNKNOWN" for column in analysis.columns}, + config_values=model.config.values, + ) + else: + shapes.pop(model.name, None) + if physical_catalog is not None: + physical_catalog = physical_catalog.with_relations(shapes) + functions: tuple[str, ...] = known_function_names(project.functions) + types: tuple[str, ...] = known_declared_types(functions=project.functions, column_types=shapes) + requests: tuple[SqlSchemaValidationRequest, ...] = tuple( + replace( + get_complete_schema_binding_request( + known_functions=functions, + known_types=types, + query_sql=cursor_intrinsics_analysis_sql( + sql=model.query_sql, cursor_type=model.config.values.get("cursor_type") + ), + placeholders=_placeholders(model), + dialect=profile.sql_analysis_dialect, + binding_schema=_model_shapes(model=model, shapes=shapes), + ), + catalog=physical_catalog, + quoted_identifiers_ignore_case=ignores_quoted_case( + connection=project.effective_connection, dialect=profile.sql_analysis_dialect + ), + ) + for model in models + ) + results: tuple[SqlBindingResult, ...] = get_schema_validations(requests=requests) + expansions: dict[str, CompiledSqlExpansion] = { + path.stem: expansion for path, expansion in project.sql_expansions.items() + } + diagnostics: list[CompilerDiagnostic] = [] + for model, request, result in zip(models, requests, results, strict=True): + for diagnostic in result.diagnostics: + if diagnostic.code == BINDING_UNKNOWN_TABLE_INTERNAL_CODE: + continue + line: int | None + column: int | None + line, column = get_authored_binding_position( + authored_sql=model.authored_sql, + authored_query_sql=model.authored_query_sql, + cleaned_sql=request.sql, + diagnostic=diagnostic, + expansion=expansions.get(model.name), + ) + diagnostics.append( + CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity(diagnostic.severity), + code=diagnostic.code, + message=diagnostic.message, + resource_type=CompiledResourceType.MODEL, + resource_name=model.name, + path=model.relative_path, + line=line, + column=column, + ) + ) + return tuple(diagnostics) + + +def _placeholders(model: CompiledModel) -> dict[str, str] | None: + values: object = model.config.values.get("placeholders") + if isinstance(values, dict): + return {str(key): str(value) for key, value in values.items()} + return None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py index 863629bf07..4e090e3bad 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/model_config.py @@ -50,6 +50,7 @@ ) _MODEL_HOOK_KEYS: frozenset[str] = frozenset({"pre_hooks", "post_hooks"}) +_REMOVED_MODEL_KEYS: frozenset[str] = frozenset({"run_despite_unchanged"}) def _contains_template_data_cached(*, value: object, cache: IdentityPresenceCache | None) -> bool: @@ -133,7 +134,14 @@ def build_layered_model_values( values: dict[str, object] = project_defaults_to_mapping(defaults) if matched_path_default is not None: values = _merged_with_tag_union(base=values, overlay=path_defaults[matched_path_default]) - return _merged_with_tag_union(base=values, overlay=model_header_values) + values = _merged_with_tag_union(base=values, overlay=model_header_values) + removed: list[str] = sorted(_REMOVED_MODEL_KEYS.intersection(values)) + if removed: + raise CompileInputError( + f"MODEL option(s) {', '.join(removed)} were removed with virtual environments; " + "projects run in direct mode" + ) + return values def _merged_with_tag_union( @@ -584,8 +592,6 @@ def project_defaults_to_mapping(defaults: DefaultsConfig) -> dict[str, object]: values["unaccounted_partition_policy"] = defaults.unaccounted_partition_policy if defaults.replay_on_change is not None: values["replay_on_change"] = defaults.replay_on_change - if defaults.run_despite_unchanged is not None: - values["run_despite_unchanged"] = defaults.run_despite_unchanged if defaults.row_diff_exclude_columns: values["row_diff_exclude_columns"] = defaults.row_diff_exclude_columns if defaults.row_diff_tolerances: diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py new file mode 100644 index 0000000000..b1eb7de178 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/details.py @@ -0,0 +1,333 @@ +"""Explain semantic diagnostics using the query's actual input schemas.""" + +from __future__ import annotations + +import re +from dataclasses import replace +from typing import Any + +from sqlbuild.compiler.compile._helpers.analysis.compact import get_complete_schema_binding_request +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile._helpers.diagnostics.help import ( + comparison_help, + semantic_help, +) +from sqlbuild.compiler.compile.models import CompiledModel, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest +from sqlbuild.spec.contracts.models import SourceLocation + +_UNKNOWN_COLUMN: str = "B002" +_COMPARISON: str = "B217" +_MIN_ABBREVIATION_LENGTH: int = 3 +_MAX_EDIT_DISTANCE: int = 2 +_DISPLAY_LIMIT: int = 10 +_QUALIFIER_SEPARATOR: str = "." +_COLUMN_KIND: str = "column" +_TYPE_CODES: frozenset[str] = frozenset(f"B21{i}" for i in range(10)) +_TYPE_WORDS: str = ( + r"\b(timestamp|integer|varchar|boolean|date|interval|double|float|decimal|numeric|" + r"bigint|smallint|text|time|string|binary|array|struct)\b" +) +_MISSING: re.Pattern[str] = re.compile(r"Unknown column '([^']+)'(?: in table '([^']+)')?") +_OPERAND: str = ( + r"(?:TIMESTAMP\s+'[^']*'|DATE\s+'[^']*'|'(?:[^']|'')*'|-?\d+(?:\.\d+)?|" + r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)?)" +) +_BINARY: re.Pattern[str] = re.compile( + rf"(?P{_OPERAND})\s*(?:>=|<=|<>|!=|=|>|<|\+|\*|/|-)\s*(?P{_OPERAND})", + re.IGNORECASE, +) + + +def sentence_message(message: str) -> str: + text: str = re.sub(r" \(context: [^)]*\)$", "", message) + pieces: list[str] = re.split(r"('[^']*')", text) + for index in range(0, len(pieces), 2): + pieces[index] = re.sub( + _TYPE_WORDS, lambda match: match.group().upper(), pieces[index], flags=re.IGNORECASE + ) + return "".join(pieces) + + +def column_distance(*, left: str, right: str) -> int: + """Levenshtein distance, also used to order schema evidence.""" + left, right = left.casefold(), right.casefold() + row: list[int] = list(range(len(right) + 1)) + for i, a in enumerate(left, 1): + following: list[int] = [i] + for j, b in enumerate(right, 1): + following.append(min(following[-1] + 1, row[j] + 1, row[j - 1] + (a != b))) + row = following + return row[-1] + + +def ordered_columns(*, name: str, columns: dict[str, str]) -> list[str]: + return sorted( + columns, + key=lambda value: ( + column_distance(left=name, right=value) / max(len(name), len(value), 1), + value, + ), + ) + + +def closest_column(*, name: str, columns: dict[str, str]) -> str | None: + candidates: list[str] = ordered_columns(name=name, columns=columns) + for candidate in candidates: + distance: int = column_distance(left=name, right=candidate) + remaining: Any = iter(candidate.casefold()) + abbreviation: bool = ( + len(name) >= _MIN_ABBREVIATION_LENGTH + and candidate[:1].casefold() == name[:1].casefold() + and candidate[-1:].casefold() == name[-1:].casefold() + and all(character in remaining for character in name.casefold()) + ) + if distance <= _MAX_EDIT_DISTANCE or abbreviation: + return candidate + return None + + +def missing_column(message: str) -> tuple[str, str | None] | None: + match: re.Match[str] | None = _MISSING.search(message) + return (match.group(1), match.group(2)) if match else None + + +def input_aliases(*, model: CompiledModel, dialect: str | None) -> dict[str, str]: + parsed: Any = _parsed_model(model=model, dialect=dialect) + if parsed is None: + return {} + aliases: dict[str, set[str]] = {} + for table in parsed.find_all("table"): + value: dict[str, Any] = table.to_dict().get("table", {}) + name: str = str(value.get("name", {}).get("name", "")) + aliases.setdefault(name, set()).add(name) + alias: dict[str, Any] = value.get("alias") or {} + if alias: + aliases.setdefault(str(alias["name"]), set()).add(name) + return {alias: next(iter(names)) for alias, names in aliases.items() if len(names) == 1} + + +def unaliased_output_columns(*, model: CompiledModel, dialect: str | None) -> set[str]: + """Only an unaliased missing projection can lose its corrected output name.""" + parsed: Any = _parsed_model(model=model, dialect=dialect) + if parsed is None: + return set() + projections: list[dict[str, Any]] = parsed.to_dict().get("select", {}).get("expressions", []) + return { + str(expression["column"]["name"]["name"]) + for expression in projections + if _COLUMN_KIND in expression + } + + +def _parsed_model(*, model: CompiledModel, dialect: str | None) -> Any: + request: SqlSchemaValidationRequest = get_complete_schema_binding_request( + query_sql=model.query_sql, placeholders=None, dialect=dialect, binding_schema={} + ) + module: Any = import_polyglot_sql() + try: + return module.parse_one(request.sql, dialect=dialect) + except module.PolyglotError: + return None + + +def explain_diagnostics(project: CompiledProject) -> CompiledProject: + """Enrich errors only; successful models require no extra SQL parsing.""" + if not project.diagnostics: + return project + shapes: dict[str, dict[str, str]] = semantic_shapes(project=project) + models: dict[str, CompiledModel] = {model.name: model for model in project.models} + aliases: dict[str, dict[str, str]] = {} + diagnostics: list[CompilerDiagnostic] = [] + diagnostic: CompilerDiagnostic + for diagnostic in project.diagnostics: + model: CompiledModel | None = models.get(diagnostic.resource_name or "") + if model is not None and diagnostic.code in {_UNKNOWN_COLUMN, *_TYPE_CODES}: + if model.name not in aliases: + aliases[model.name] = input_aliases( + model=model, dialect=project.sql_analysis_dialect + ) + diagnostic = _explain_model( + diagnostic=diagnostic, + model=model, + aliases=aliases[model.name], + shapes=shapes, + dialect=project.sql_analysis_dialect, + ) + elif semantic_help(diagnostic.code) is not None: + diagnostic = replace( + diagnostic, + message=sentence_message(diagnostic.message), + help=semantic_help(diagnostic.code) or diagnostic.help, + ) + diagnostics.append(diagnostic) + return replace( + project, + diagnostics=tuple(diagnostics), + models=update_binding_models(models=project.models, diagnostics=tuple(diagnostics)), + ) + + +def update_binding_models( + *, models: tuple[CompiledModel, ...], diagnostics: tuple[CompilerDiagnostic, ...] +) -> tuple[CompiledModel, ...]: + """Keep model binding facts distinct from metadata and SQL-test diagnostics.""" + by_model: dict[str, list[CompilerDiagnostic]] = {} + for diagnostic in diagnostics: + if diagnostic.resource_type == CompiledResourceType.MODEL: + by_model.setdefault(diagnostic.resource_name or "", []).append(diagnostic) + updated: list[CompiledModel] = [] + for model in models: + codes: set[str] = {item.code for item in model.binding_diagnostics} + bindings: tuple[CompilerDiagnostic, ...] = tuple( + item for item in by_model.get(model.name, ()) if item.code in codes + ) + updated.append(replace(model, binding_diagnostics=bindings)) + return tuple(updated) + + +def _binary_at(*, sql: str, offset: int) -> re.Match[str] | None: + """Matches are ordered and disjoint, so none after the offset can contain it.""" + for match in _BINARY.finditer(sql): + if match.start() > offset: + return None + if offset < match.end(): + return match + return None + + +def _explain_model( + *, + diagnostic: CompilerDiagnostic, + model: CompiledModel, + aliases: dict[str, str], + shapes: dict[str, dict[str, str]], + dialect: str | None, +) -> CompilerDiagnostic: + message: str = sentence_message(diagnostic.message) + help_text: str | None = semantic_help(diagnostic.code) + if diagnostic.code == _COMPARISON: + help_text = comparison_help( + types=tuple( + value.upper() + for value in re.findall(_TYPE_WORDS, diagnostic.message, re.IGNORECASE) + ), + dialect=dialect, + ) + notes: list[str] = list(diagnostic.notes) + location: SourceLocation | None = diagnostic.location + missing: tuple[str, str | None] | None = missing_column(diagnostic.message) + if missing is not None: + name, table = missing + columns: dict[str, str] = shapes.get(table or "", {}) + suggestion: str | None = closest_column(name=name, columns=columns) + help_text = f"did you mean '{suggestion}'?" if suggestion else None + message = f"Unknown column '{name}'" + (f" in {table}" if table else "") + if location is not None: + lines: list[str] = model.authored_sql.splitlines() + prefix: str = lines[location.line - 1][: location.column - 1] + qualifier: re.Match[str] | None = re.search( + r'(?:"((?:[^"]|"")+)"|([A-Za-z_]\w*))\.$', prefix + ) + alias: str | None = ( + (qualifier.group(1) or qualifier.group(2)).replace('""', '"') if qualifier else None + ) + if alias and aliases.get(alias) == table: + message += f" (as {alias})" + available: list[str] = ordered_columns(name=name, columns=columns) + if available: + suffix: str = ( + f", and {len(available) - _DISPLAY_LIMIT} more" + if len(available) > _DISPLAY_LIMIT + else "" + ) + notes.insert(0, f"{table} has: {', '.join(available[:10])}{suffix}") + if diagnostic.code in _TYPE_CODES and location is not None: + offset: int = ( + sum( + len(line) + for line in model.authored_sql.splitlines(keepends=True)[: location.line - 1] + ) + + location.column + - 1 + ) + binary: re.Match[str] | None = _binary_at(sql=model.authored_sql, offset=offset) + if binary: + left: str = binary.group("left") + right: str = binary.group("right") + types: tuple[str | None, str | None] = ( + _operand_type(text=left, aliases=aliases, shapes=shapes), + _operand_type(text=right, aliases=aliases, shapes=shapes), + ) + if all(types): + notes.append(f"{left} is {types[0]}, {right} is {types[1]}") + if diagnostic.code == _COMPARISON: + help_text = comparison_help( + types=tuple(value for value in types if value), dialect=dialect + ) + end: int = binary.end() + if location.end_line is not None and location.end_column is not None: + native_end: int = ( + sum( + len(line) + for line in model.authored_sql.splitlines(keepends=True)[ + : location.end_line - 1 + ] + ) + + location.end_column + - 1 + ) + end = max(end, native_end) + begin: int = min(binary.start(), offset) + location = replace( + location, + line=model.authored_sql.count("\n", 0, begin) + 1, + column=begin - model.authored_sql.rfind("\n", 0, begin), + end_line=model.authored_sql.count("\n", 0, end) + 1, + end_column=end - model.authored_sql.rfind("\n", 0, end), + ) + else: + operand: re.Match[str] | None = re.compile(_OPERAND, re.IGNORECASE).match( + model.authored_sql, offset + ) + if operand: + operand_type: str | None = _operand_type( + text=operand.group(), aliases=aliases, shapes=shapes + ) + if operand_type: + notes.append(f"{operand.group()} is {operand_type}") + return replace( + diagnostic, + message=message, + help=help_text, + notes=tuple(notes), + location=location, + line=location.line if location is not None else diagnostic.line, + column=location.column if location is not None else diagnostic.column, + ) + + +def _operand_type( + *, text: str, aliases: dict[str, str], shapes: dict[str, dict[str, str]] +) -> str | None: + temporal: re.Match[str] | None = re.match(r"(TIMESTAMP|DATE)\s*'", text, re.IGNORECASE) + if temporal: + return temporal.group(1).upper() + if re.fullmatch(r"-?\d+", text): + return "INTEGER" + if re.fullmatch(r"-?\d+\.\d+", text): + return "DECIMAL" + if text.startswith("'"): + return "VARCHAR" + if _QUALIFIER_SEPARATOR in text: + alias, column = text.split(".", 1) + value: str | None = shapes.get(aliases.get(alias, alias), {}).get(column) + return value.upper() if value else None + candidates: set[str] = { + shapes[table][text].upper() + for table in set(aliases.values()) + if table in shapes and text in shapes[table] + } + return next(iter(candidates)) if len(candidates) == 1 else None diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py new file mode 100644 index 0000000000..904fa0380c --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/help.py @@ -0,0 +1,52 @@ +"""Code-owned semantic remediation; no catch-all help for unrelated diagnostics.""" + +_TIMESTAMP: str = "TIMESTAMP" +_DATE: str = "DATE" +_BIGQUERY: str = "bigquery" +_SEMANTIC_HELP: dict[str, str] = { + "B003": "qualify this column with the intended input alias", + "B004": "give each referenced relation a visible alias in this scope", + "B005": "match the relation alias list to its available output columns", + "B101": "check the function spelling or declare the project function before using it", + "B102": "pass the number of arguments required by this function's signature", + "B210": "use an expression whose inferred type satisfies this SQL construct", + "B211": "use a Boolean predicate in this condition", + "B212": "use operands supported by this arithmetic operator, or convert them explicitly", + "B213": "supply argument types supported by this function or conditional expression", + "B214": "make the assigned expression compatible with the destination type", + "B215": "align the corresponding types in each set-operation branch", + "B216": "return the same number of columns on both sides of this operation", + "B217": "compare compatible types using a correctly typed literal or explicit conversion", + "B218": "choose a supported source-to-target cast for this dialect", + "B219": "declare or cast this expression's unknown type so it can be checked", + "B230": "add this expression to GROUP BY or use it inside an aggregate", + "B231": "move this aggregate to SELECT or HAVING, or compute it in an input CTE", + "B232": "use this window function with a valid OVER clause and window context", + "B233": "use distinct names for relations or CTEs within this scope", + "B234": "use a valid constant bound for LIMIT or OFFSET", + "B300": "change the metadata column name to an existing output column", + "B301": "align this declared metadata or audit value with the column or argument type", + "B302": "align the SQL-test fixture or expected columns with the tested resource's output", +} + + +def semantic_help_catalogue() -> dict[str, str]: + """Return the code-owned remediation catalogue.""" + return dict(_SEMANTIC_HELP) + + +def semantic_help(code: str) -> str | None: + """Return help only for a code with its own remediation.""" + return _SEMANTIC_HELP.get(code) + + +def comparison_help(*, types: tuple[str, ...], dialect: str | None) -> str: + """Offer a concrete literal for a proven temporal comparison mismatch.""" + if any(_TIMESTAMP in value for value in types): + literal: str = "TIMESTAMP '2026-04-01'" + if dialect == _BIGQUERY: + literal = "TIMESTAMP '2026-04-01 00:00:00+00'" + return f"compare with a timestamp, for example {literal}" + if _DATE in types: + return "compare with a date, for example DATE '2026-04-01'" + return _SEMANTIC_HELP["B217"] diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py new file mode 100644 index 0000000000..32b29aa426 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/recovery.py @@ -0,0 +1,134 @@ +"""Recover poisoned output names without suppressing unrelated downstream errors.""" + +from __future__ import annotations + +import re +from collections import Counter +from dataclasses import replace + +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.assembly.metadata_validation import ( + get_semantic_metadata_diagnostics, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile._helpers.diagnostics.details import ( + closest_column, + explain_diagnostics, + missing_column, + unaliased_output_columns, + update_binding_models, +) +from sqlbuild.compiler.compile._helpers.diagnostics.type_recovery import recover_output_types +from sqlbuild.compiler.compile.models import ( + CompiledLineageSourceFact, + CompiledModel, + CompiledProject, + CompilerDiagnostic, +) +from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic + +_MISSING_CODE: str = "B002" +_TEST_CODE: str = "B302" + + +def complete_semantic_diagnostics( + *, + project: CompiledProject, + profile: ExpressionInferenceProfile, + binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]], +) -> CompiledProject: + """Recover type facts before metadata checks, then explain retained root diagnostics.""" + project = recover_output_types(project=project, binding_results=binding_results) + project = replace( + project, + diagnostics=( + *project.diagnostics, + *get_semantic_metadata_diagnostics(project=project, profile=profile), + ), + ) + return explain_diagnostics(recover_diagnostics(project)) + + +def recover_diagnostics(project: CompiledProject) -> CompiledProject: + """Recover poisoned uses while retaining independent errors on unaffected outputs.""" + roots: list[CompilerDiagnostic] = [ + item for item in project.diagnostics if item.code == _MISSING_CODE + ] + if not roots: + return project + shapes: dict[str, dict[str, str]] = semantic_shapes(project=project) + models: dict[str, CompiledModel] = {model.name: model for model in project.models} + poisoned: dict[tuple[str, str], CompilerDiagnostic] = {} + origins: dict[CompilerDiagnostic, tuple[str, str]] = {} + projections: dict[str, set[str]] = {} + for diagnostic in roots: + model: CompiledModel | None = models.get(diagnostic.resource_name or "") + missing: tuple[str, str | None] | None = missing_column(diagnostic.message) + if model is None or missing is None: + continue + column, table = missing + if column not in {item.name for item in model.inferred_columns or ()}: + continue + if model.name not in projections: + projections[model.name] = unaliased_output_columns( + model=model, dialect=project.sql_analysis_dialect + ) + if column not in projections[model.name]: + continue + upstream_columns: list[CompiledLineageSourceFact] = [] + for output in model.fast_lineage_columns or (): + if output.output_column == column: + upstream_columns.extend(output.upstream_columns) + if not any( + source.resource_name == table and source.column_name == column + for source in upstream_columns + ): + continue + suggestion: str | None = closest_column(name=column, columns=shapes.get(table or "", {})) + if suggestion is not None: + poisoned[(model.name, suggestion)] = diagnostic + origins[diagnostic] = model.name, suggestion + for _ in project.models: + previous: int = len(poisoned) + for model in project.models: + for column in model.fast_lineage_columns or (): + for source in column.upstream_columns: + root: CompilerDiagnostic | None = poisoned.get( + (source.resource_name, source.column_name) + ) + if root is not None: + poisoned.setdefault((model.name, column.output_column), root) + if len(poisoned) == previous: + break + skipped: Counter[tuple[CompilerDiagnostic, str, str]] = Counter() + kept: list[CompilerDiagnostic] = [] + for diagnostic in project.diagnostics: + missing = missing_column(diagnostic.message) if diagnostic.code == _MISSING_CODE else None + target: tuple[str, str] | None = ( + (missing[1], missing[0]) if missing and missing[1] else None + ) + if diagnostic.code == _TEST_CODE: + match: re.Match[str] | None = re.search( + r"'__(?:expected|ref)__(.+)' names unknown column '([^']+)'", diagnostic.message + ) + if match: + target = match.group(1), match.group(2) + root = poisoned.get(target) if target is not None else None + if root is not None and root != diagnostic and target is not None: + skipped[(root, *origins[root])] += 1 + else: + kept.append(diagnostic) + notes: dict[CompilerDiagnostic, list[str]] = {} + for (root, model_name, column_name), count in skipped.items(): + notes.setdefault(root, []).append( + f"{count} downstream uses of {model_name}.{column_name} " + "were not checked because of this error" + ) + diagnostics: tuple[CompilerDiagnostic, ...] = tuple( + replace(item, notes=(*item.notes, *notes.get(item, ()))) for item in kept + ) + return replace( + project, + diagnostics=diagnostics, + models=update_binding_models(models=project.models, diagnostics=diagnostics), + ) diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/type_recovery.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/type_recovery.py new file mode 100644 index 0000000000..f0468fc25f --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/type_recovery.py @@ -0,0 +1,283 @@ +"""Propagate unknown output types from located root errors without opening other columns.""" + +from collections import Counter, deque +from dataclasses import replace +from typing import Any + +from sqlbuild.compiler.compile._helpers.analysis.compact import get_complete_schema_binding_request +from sqlbuild.compiler.compile._helpers.assembly.native_declarations import ( + known_declared_types, + known_function_names, +) +from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import semantic_shapes +from sqlbuild.compiler.compile._helpers.diagnostics.details import update_binding_models +from sqlbuild.compiler.compile.models import ( + CompiledModel, + CompiledProject, + CompilerDiagnostic, + InferredColumn, +) +from sqlbuild.compiler.sql_analysis.main._identifier_case import ignores_quoted_case +from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations +from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql +from sqlbuild.compiler.sql_analysis.models import ( + SqlBindingDiagnostic, + SqlBindingResult, + SqlSchemaValidationRequest, +) + +_SELECT: str = "SELECT" +_PROJECTION_END: frozenset[str] = frozenset( + { + "FROM", + "WHERE", + "GROUP_BY", + "HAVING", + "QUALIFY", + "ORDER_BY", + "LIMIT", + "UNION", + "INTERSECT", + "EXCEPT", + } +) +_OPEN: str = "(" +_CLOSE: str = ")" +_COMMA: str = "," + + +def recover_output_types( + *, project: CompiledProject, binding_results: dict[str, tuple[SqlBindingDiagnostic, ...]] +) -> CompiledProject: + """Recheck only failed dependants using unknown types for poisoned lineage outputs.""" + failed: list[CompiledModel] = [model for model in project.models if model.binding_diagnostics] + blocking: list[CompilerDiagnostic] = [] + for model in failed: + blocking.extend(item for item in model.binding_diagnostics if item.is_error) + if not blocking: + return project + shapes: dict[str, dict[str, str]] = semantic_shapes(project=project) + requests: tuple[SqlSchemaValidationRequest, ...] = tuple( + _request(project=project, model=model, shapes=shapes) for model in failed + ) + results: tuple[SqlBindingResult, ...] = tuple( + SqlBindingResult(diagnostics=binding_results.get(model.name, ())) for model in failed + ) + poisoned: dict[tuple[str, str], CompilerDiagnostic] = {} + roots_by_model: dict[str, dict[SqlBindingDiagnostic, CompilerDiagnostic]] = {} + outputs_by_diagnostic: dict[CompilerDiagnostic, set[str]] = {} + for model, request, result in zip(failed, requests, results, strict=True): + roots_by_model[model.name] = _root_bindings(model=model, diagnostics=result.diagnostics) + spans: list[tuple[int, int]] = _projection_spans(sql=request.sql, dialect=request.dialect) + columns: tuple[InferredColumn, ...] = model.inferred_columns or () + if len(spans) != len(columns): + continue + for raw in result.diagnostics: + if raw.start is None: + continue + root: CompilerDiagnostic | None = roots_by_model[model.name].get(raw) + if root is None or not root.is_error: + continue + for column, (start, end) in zip(columns, spans, strict=True): + if start <= raw.start < end: + poisoned[(model.name, column.name)] = root + outputs_by_diagnostic.setdefault(root, set()).add(column.name) + if not poisoned: + return project + for _ in project.models: + previous: int = len(poisoned) + for model in project.models: + for output in model.fast_lineage_columns or (): + for source in output.upstream_columns: + root = poisoned.get((source.resource_name, source.column_name)) + if root is not None: + poisoned.setdefault((model.name, output.output_column), root) + if len(poisoned) == previous: + break + unknown_shapes: dict[str, dict[str, str]] = { + name: dict(columns) for name, columns in shapes.items() + } + for name, column_name in poisoned: + if name in unknown_shapes and column_name in unknown_shapes[name]: + unknown_shapes[name][column_name] = "UNKNOWN" + skipped: set[CompilerDiagnostic] = set() + redirected: dict[CompilerDiagnostic, set[CompilerDiagnostic]] = {} + for model, original in zip(failed, results, strict=True): + causes: set[CompilerDiagnostic] = set() + for reference in model.references: + causes.update( + root for (name, _), root in poisoned.items() if name == reference.ref_name + ) + if not causes: + continue + revised: SqlBindingResult = get_schema_validations( + requests=(_request(project=project, model=model, shapes=unknown_shapes),) + )[0] + remaining: set[tuple[str, int | None, int | None]] = { + (item.code, item.start, item.end) for item in revised.diagnostics + } + for raw in original.diagnostics: + diagnostic: CompilerDiagnostic | None = roots_by_model[model.name].get(raw) + if ( + diagnostic is None + or diagnostic in causes + or (raw.code, raw.start, raw.end) in remaining + ): + continue + skipped.add(diagnostic) + redirected[diagnostic] = ( + _output_causes( + model=model, + outputs=outputs_by_diagnostic.get(diagnostic, set()), + poisoned=poisoned, + ) + or causes + ) + counts: Counter[CompilerDiagnostic] = _unchecked_uses( + project=project, poisoned=poisoned, redirected=redirected + ) + diagnostics: tuple[CompilerDiagnostic, ...] = tuple( + replace( + item, + notes=( + *item.notes, + *( + ( + f"{counts[item]} downstream output uses were not type-checked " + "because of this error", + ) + if counts[item] + else () + ), + ), + ) + for item in project.diagnostics + if item not in skipped + ) + models: list[CompiledModel] = [] + for model in update_binding_models(models=project.models, diagnostics=diagnostics): + updated: tuple[InferredColumn, ...] | None = None + if model.inferred_columns is not None: + updated = tuple( + replace(column, type=None) if (model.name, column.name) in poisoned else column + for column in model.inferred_columns + ) + models.append( + replace( + model, + inferred_columns=updated, + unchecked_output_columns=frozenset( + column for name, column in poisoned if name == model.name + ), + ) + ) + return replace(project, models=tuple(models), diagnostics=diagnostics) + + +def _unchecked_uses( + *, + project: CompiledProject, + poisoned: dict[tuple[str, str], CompilerDiagnostic], + redirected: dict[CompilerDiagnostic, set[CompilerDiagnostic]], +) -> Counter[CompilerDiagnostic]: + """Attribute transitive unchecked output uses to retained root errors.""" + counts: Counter[CompilerDiagnostic] = Counter() + for model in project.models: + for output in model.fast_lineage_columns or (): + pending: list[CompilerDiagnostic] = [] + for source in output.upstream_columns: + root: CompilerDiagnostic | None = poisoned.get( + (source.resource_name, source.column_name) + ) + if root is not None: + pending.append(root) + visited: set[CompilerDiagnostic] = set() + while pending: + root = pending.pop() + if root in visited: + continue + visited.add(root) + if root in redirected: + pending.extend(redirected[root]) + else: + counts[root] += 1 + return counts + + +def _root_bindings( + *, model: CompiledModel, diagnostics: tuple[SqlBindingDiagnostic, ...] +) -> dict[SqlBindingDiagnostic, CompilerDiagnostic]: + """Pair repeated messages by occurrence, preserving distinct native spans.""" + pending: dict[tuple[str, str], deque[CompilerDiagnostic]] = {} + for diagnostic in model.binding_diagnostics: + pending.setdefault((diagnostic.code, diagnostic.message), deque()).append(diagnostic) + result: dict[SqlBindingDiagnostic, CompilerDiagnostic] = {} + for raw in diagnostics: + matches: deque[CompilerDiagnostic] | None = pending.get((raw.code, raw.message)) + if matches: + result[raw] = matches.popleft() + return result + + +def _output_causes( + *, model: CompiledModel, outputs: set[str], poisoned: dict[tuple[str, str], CompilerDiagnostic] +) -> set[CompilerDiagnostic]: + """Redirect only the failing projection's lineage instead of every poisoned input.""" + result: set[CompilerDiagnostic] = set() + for output in model.fast_lineage_columns or (): + if output.output_column in outputs: + for source in output.upstream_columns: + root: CompilerDiagnostic | None = poisoned.get( + (source.resource_name, source.column_name) + ) + if root is not None: + result.add(root) + return result + + +def _request( + *, project: CompiledProject, model: CompiledModel, shapes: dict[str, dict[str, str]] +) -> SqlSchemaValidationRequest: + return replace( + get_complete_schema_binding_request( + query_sql=model.query_sql, + placeholders=None, + dialect=project.sql_analysis_dialect, + known_functions=known_function_names(project.functions), + known_types=known_declared_types(functions=project.functions, column_types=shapes), + binding_schema={ + reference.ref_name: shapes.get(reference.ref_name, {}) + for reference in model.references + }, + ), + catalog=project.binding_catalog, + quoted_identifiers_ignore_case=ignores_quoted_case( + connection=project.effective_connection, dialect=project.sql_analysis_dialect + ), + ) + + +def _projection_spans(*, sql: str, dialect: str | None) -> list[tuple[int, int]]: + module: Any = import_polyglot_sql() + tokens: list[dict[str, Any]] = module.tokenize(sql, dialect=dialect) + depth: int = 0 + start: int | None = None + spans: list[tuple[int, int]] = [] + for token in tokens: + text: str = token["text"] + if text == _OPEN: + depth += 1 + elif text == _CLOSE: + depth -= 1 + elif depth == 0: + if start is None and token["token_type"] == _SELECT: + start = token["span"]["end"] + elif start is not None and token["token_type"] in _PROJECTION_END: + spans.append((start, token["span"]["start"])) + return spans + elif start is not None and text == _COMMA: + spans.append((start, token["span"]["start"])) + start = token["span"]["end"] + if start is not None: + spans.append((start, len(sql))) + return spans diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index 80f496ab11..788cca6555 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -8,6 +8,7 @@ from pathlib import Path import sqlbuild._native as _native +from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile._helpers.render.declarations import ( expand_declaration_references_result, expand_declaration_references_with_spans, @@ -23,7 +24,6 @@ SQL_QUOTE_TOKENS, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, DeclarationExpansionResult, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 007fde4d3f..3bd6eb68c0 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -7,6 +7,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass +from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.constants import ( MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, @@ -25,7 +26,6 @@ TEMPLATE_TRUE_LITERAL, ) from sqlbuild.compiler.compile.exceptions import CompileInputError -from sqlbuild.compiler.compile.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.types import TemplateNamespace diff --git a/src/sqlbuild/compiler/compile/_helpers/sharing/binding.py b/src/sqlbuild/compiler/compile/_helpers/sharing/binding.py new file mode 100644 index 0000000000..dad614ff6b --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/sharing/binding.py @@ -0,0 +1,365 @@ +"""Exact sharing of binding-mode native analyses between models with equal inputs.""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import replace +from typing import Any, cast + +from sqlbuild.compiler.compile._helpers.analysis.columns import ( + _analysis_reference_name, + _lineage_resource_type, + _qualified_reference_names, +) +from sqlbuild.compiler.compile.constants import ( + COMPACT_REFERENCE_MARKER_PATTERN, + COMPACT_RELATION_STUB_PREFIX, + COMPACT_UNSTUBBED_REFERENCE_KIND, + MIN_SHARED_BINDING_QUERY_MEMBERS, +) +from sqlbuild.compiler.compile.models import ( + CompactBatchContext, + CompactBatchInputs, + CompileSqlReference, + ModelSqlAnalysisRequest, + NativeCompactAnalysis, + PreparedBindingQuery, + SharedBindingQuery, +) +from sqlbuild.compiler.compile.types import CompiledResourceType +from sqlbuild.compiler.sql_analysis.constants import SQL_QUOTED_IDENTIFIER_DELIMITER +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql +from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest + + +def lineage_reference_map( + references: tuple[CompileSqlReference, ...], +) -> dict[str, tuple[CompiledResourceType, str]]: + reference_map: dict[str, tuple[CompiledResourceType, str]] = {} + reference: CompileSqlReference + for reference in references: + resource_type: CompiledResourceType | None = _lineage_resource_type(reference) + if resource_type is None: + continue + reference_map[_analysis_reference_name(reference)] = ( + resource_type, + reference.ref_name, + ) + return reference_map + + +def binding_share_prekey(*, query_sql: str, references: tuple[CompileSqlReference, ...]) -> str: + """Return authored SQL with lineage relation markers replaced by their stub positions.""" + + positions: dict[str, int] = { + name: index for index, name in enumerate(lineage_reference_map(references)) + } + + def marker(match: re.Match[str]) -> str: + position: int | None = positions.get(match.group(2)) + if position is None or match.group(1) == COMPACT_UNSTUBBED_REFERENCE_KIND: + return match.group(0) + return f"{COMPACT_RELATION_STUB_PREFIX}{position}" + + return COMPACT_REFERENCE_MARKER_PATTERN.sub(marker, query_sql) + + +def shareable_binding_prekeys(*, requests: Sequence[ModelSqlAnalysisRequest]) -> frozenset[str]: + """Return binding pre-keys shared by at least two binding requests of a compile.""" + + counts: Counter[str] = Counter( + binding_share_prekey(query_sql=request.query_sql, references=request.model_input.references) + for request in requests + if request.binding_schema is not None + ) + return frozenset( + prekey for prekey, count in counts.items() if count >= MIN_SHARED_BINDING_QUERY_MEMBERS + ) + + +def prepare_binding_queries( + *, + binding_catalog: Any, + inputs: CompactBatchInputs, + cleaned_sqls: list[str], + share: bool, + shareable_prekeys: frozenset[str] | None, +) -> tuple[PreparedBindingQuery | None, ...]: + """Prepare binding references in catalog order, with shared forms where keys can match.""" + + binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] = inputs.binding_schemas or () + dialect: str | None = inputs.inference_profile.sql_analysis_dialect + prekeys: list[str | None] = [ + binding_share_prekey(query_sql=query_sql, references=query_references) + if share and binding_schema is not None + else None + for query_sql, query_references, binding_schema in zip( + inputs.query_sqls, inputs.references, binding_schemas, strict=True + ) + ] + candidates: frozenset[str] = ( + shareable_prekeys + if shareable_prekeys is not None + else frozenset( + prekey + for prekey, count in Counter(prekeys).items() + if prekey is not None and count >= MIN_SHARED_BINDING_QUERY_MEMBERS + ) + ) + prepared: list[PreparedBindingQuery | None] = [] + for index, binding_schema in enumerate(binding_schemas): + if binding_schema is None: + prepared.append(None) + continue + _, binding_references, overrides = binding_catalog.prepare( + [ + SqlSchemaValidationRequest( + sql=cleaned_sqls[index], dialect=dialect, schema=binding_schema + ) + ] + )[0] + prekey: str | None = prekeys[index] + shared: SharedBindingQuery | None = None + if prekey is not None and prekey in candidates: + shared = _shared_binding_query( + binding_catalog=binding_catalog, + query=SqlSchemaValidationRequest( + sql=inputs.query_sqls[index], dialect=dialect, schema=binding_schema + ), + cleaned_sql=cleaned_sqls[index], + references=inputs.references[index], + placeholders=inputs.placeholders[index], + overrides=overrides, + ) + prepared.append( + PreparedBindingQuery(references=binding_references, overrides=overrides, shared=shared) + ) + if shareable_prekeys is not None: + return tuple(prepared) + counts: Counter[tuple[object, ...]] = Counter( + query.shared.key for query in prepared if query is not None and query.shared is not None + ) + return tuple( + ( + PreparedBindingQuery(references=query.references, overrides=query.overrides) + if query is not None + and query.shared is not None + and counts[query.shared.key] < MIN_SHARED_BINDING_QUERY_MEMBERS + else query + ) + for query in prepared + ) + + +def binding_query_fields( + *, + binding_query: PreparedBindingQuery, + binding_catalog: Any, +) -> tuple[dict[str, object], dict[str, object]]: + """Return native binding query fields and their identity payload.""" + + binding_references: list[tuple[str, bool]] = binding_query.references + fields: dict[str, object] = {} + shared: SharedBindingQuery | None = binding_query.shared + if shared is not None: + stubs: dict[str, str] = shared.stubs + binding_references = sorted( + binding_references, key=lambda reference: stubs.get(reference[0], reference[0]) + ) + fields["binding_aliases"] = { + name: stubs[name] for name, _ in binding_references if name in stubs + } + binding_payload: dict[str, object] = {"references": binding_references} + fields["binding_references"] = binding_references + if binding_query.overrides: + override_id: int = binding_catalog.native.register_override(binding_query.overrides) + fields["binding_override"] = override_id + binding_payload["override"] = override_id + return fields, binding_payload + + +def shared_result_is_exact(*, validation: object, template: object) -> bool: + """Only a successful analysis without diagnostics is independent of relation names.""" + + if isinstance(template, str): + return False + if validation is None: + return True + return isinstance(validation, dict) and cast(dict[str, object], validation).get("errors") == [] + + +def _shared_reference_names( + *, + cleaned_sql: str, + references: tuple[CompileSqlReference, ...], + binding_schema: Mapping[str, Mapping[str, str]], +) -> list[str] | None: + names: list[str] = list(lineage_reference_map(references)) + if not names or len({name.casefold() for name in names}) != len(names): + return None + if any(name.startswith(COMPACT_RELATION_STUB_PREFIX) for name in names): + return None + if COMPACT_RELATION_STUB_PREFIX.casefold() in cleaned_sql.casefold(): + return None + if SQL_QUOTED_IDENTIFIER_DELIMITER in cleaned_sql: + return None + for relation, columns in binding_schema.items(): + if SQL_QUOTED_IDENTIFIER_DELIMITER in relation or any( + SQL_QUOTED_IDENTIFIER_DELIMITER in column for column in columns + ): + return None + if _qualified_reference_names(query_sql=cleaned_sql, reference_names=names): + return None + return names + + +def _shared_binding_query( + *, + binding_catalog: Any, + query: SqlSchemaValidationRequest, + cleaned_sql: str, + references: tuple[CompileSqlReference, ...], + placeholders: dict[str, str] | None, + overrides: Mapping[str, Mapping[str, str]], +) -> SharedBindingQuery | None: + names: list[str] | None = _shared_reference_names( + cleaned_sql=cleaned_sql, references=references, binding_schema=query.schema + ) + if names is None: + return None + stubs: dict[str, str] = { + name: f"{COMPACT_RELATION_STUB_PREFIX}{index}" for index, name in enumerate(names) + } + sql: str = normalize_analysis_sql( + sql=query.sql, dialect=query.dialect, stubs=stubs, placeholders=placeholders + ) + analysis_shapes: list[tuple[object, ...]] = [] + for name in names: + types, nullability = binding_catalog.analysis_shapes.get(name, ({}, {})) + analysis_shapes.append((stubs[name], tuple(types.items()), tuple(nullability.items()))) + binding_shapes: list[tuple[object, ...]] = [] + for relation, columns in query.schema.items(): + effective: Mapping[str, str] = ( + overrides[relation] + if relation in overrides + else binding_catalog.schemas[relation] + if columns + else {} + ) + binding_shapes.append( + (stubs.get(relation, relation), bool(columns), tuple(effective.items())) + ) + return SharedBindingQuery( + sql=sql, + stubs=stubs, + key=( + sql, + query.dialect or "generic", + tuple(analysis_shapes), + tuple(sorted(binding_shapes)), + ), + ) + + +def shared_result_keys( + *, inputs: CompactBatchInputs, context: CompactBatchContext +) -> tuple[tuple[object, ...] | None, ...]: + """Identify each shared member's complete native input, including its template.""" + + function_return_types: tuple[tuple[str, str], ...] = tuple( + inputs.inference_profile.function_return_types.items() + ) + keys: list[tuple[object, ...] | None] = [] + for index, binding_query in enumerate(context.binding_queries): + shared: SharedBindingQuery | None = ( + binding_query.shared if binding_query is not None else None + ) + if shared is None: + keys.append(None) + continue + query_references: tuple[CompileSqlReference, ...] = inputs.references[index] + declared_column_order: tuple[str, ...] = ( + tuple( + inputs.column_nullability_by_table.get( + _analysis_reference_name(query_references[0]), {} + ) + ) + if len(query_references) == 1 + else () + ) + reference_types: tuple[tuple[str, str], ...] = tuple( + (shared.stubs[name], resource_type.value) + for name, (resource_type, _) in lineage_reference_map(query_references).items() + ) + keys.append( + ( + shared.key, + inputs.recover_cte_facts[index], + inputs.rich_type_inference, + function_return_types, + declared_column_order, + reference_types, + ) + ) + return tuple(keys) + + +def reused_shared_results( + *, + inputs: CompactBatchInputs, + context: CompactBatchContext, + keys: tuple[tuple[object, ...] | None, ...], + memo: dict[object, object], +) -> dict[int, NativeCompactAnalysis]: + """Project remembered exact shared results onto members with the same key.""" + + reused: dict[int, NativeCompactAnalysis] = {} + for index, key in enumerate(keys): + representative: object = memo.get(key) if key is not None else None + binding_query: PreparedBindingQuery | None = context.binding_queries[index] + if not isinstance(representative, NativeCompactAnalysis) or binding_query is None: + continue + shared: SharedBindingQuery | None = binding_query.shared + if shared is None: + continue + resource_names_by_stub: dict[str, str] = { + shared.stubs[name]: resource_name + for name, (_, resource_name) in lineage_reference_map(inputs.references[index]).items() + } + resource_names: dict[int, str] = {} + for canonical_index in representative.resource_name_indexes: + resource_name: str | None = resource_names_by_stub.get( + representative.string_pool[canonical_index] + ) + if resource_name is None: + break + resource_names[canonical_index] = resource_name + else: + reused[index] = replace( + representative, + cleaned_sql=context.normalized_sqls[index], + resource_names=resource_names, + ) + return reused + + +def remembered_shared_results( + *, + keys: tuple[tuple[object, ...] | None, ...], + results: tuple[NativeCompactAnalysis, ...], +) -> dict[object, NativeCompactAnalysis]: + """Select exact, fully projected shared results to remember for later batches.""" + + remembered: dict[object, NativeCompactAnalysis] = {} + for key, result in zip(keys, results, strict=True): + if ( + key is not None + and key not in remembered + and result.projected + and result.analysis is not None + and result.compact_rows is not None + ): + remembered[key] = result + return remembered diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 38c91398ea..0637c627fd 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -129,6 +129,12 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPACT_ANALYSIS_RESPONSE_LENGTH: int = 2 +COMPACT_RELATION_STUB_PREFIX: str = "__sqlbuild_project_input_" +COMPACT_REFERENCE_MARKER_PATTERN: re.Pattern[str] = re.compile( + r'__(ref|seed|source|dbt_ref)\((?:"[^"]+"\s*,\s*)?"([^"]+)"\)' +) +COMPACT_UNSTUBBED_REFERENCE_KIND: str = "dbt_ref" +MIN_SHARED_BINDING_QUERY_MEMBERS: int = 2 COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH: int = 3 COMPACT_ANALYSIS_FACT_LENGTH: int = 6 COMPACT_ANALYSIS_SOURCE_LENGTH: int = 3 diff --git a/src/sqlbuild/compiler/compile/main/_source_bindings.py b/src/sqlbuild/compiler/compile/main/_source_bindings.py new file mode 100644 index 0000000000..478fd49b03 --- /dev/null +++ b/src/sqlbuild/compiler/compile/main/_source_bindings.py @@ -0,0 +1,20 @@ +"""Planner entrypoint for warehouse-backed semantic binding.""" + +from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile +from sqlbuild.compiler.compile._helpers.assembly.source_bindings import ( + get_source_binding_diagnostics as _get_source_binding_diagnostics, +) +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompilerDiagnostic + + +def get_source_binding_diagnostics( + *, + project: CompiledProject, + columns: dict[str, tuple[ColumnInfo, ...]], + profile: ExpressionInferenceProfile, + selected_keys: frozenset[CompiledObjectKey], +) -> tuple[CompilerDiagnostic, ...]: + """Bind selected source readers to already-inspected warehouse columns.""" + return _get_source_binding_diagnostics( + project=project, columns=columns, profile=profile, selected_keys=selected_keys + ) diff --git a/src/sqlbuild/compiler/compile/main/semantic_coverage.py b/src/sqlbuild/compiler/compile/main/semantic_coverage.py new file mode 100644 index 0000000000..33b28784a6 --- /dev/null +++ b/src/sqlbuild/compiler/compile/main/semantic_coverage.py @@ -0,0 +1,9 @@ +"""Public semantic-check coverage report.""" + +from sqlbuild.compiler.compile._helpers.assembly.semantic_coverage import semantic_coverage +from sqlbuild.compiler.compile.models import CompiledProject + + +def get_semantic_coverage(project: CompiledProject) -> dict[str, tuple[str, ...]]: + """Return per-model partial-check reasons.""" + return semantic_coverage(project) diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index ddddb2584f..6ccb97ed87 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -107,6 +107,8 @@ class CompilerDiagnostic: location: SourceLocation | None = None related_locations: tuple[RelatedLocation, ...] = field(default_factory=tuple) help: str | None = None + notes: tuple[str, ...] = () + affected_rules: tuple[str, ...] = () def __post_init__(self) -> None: object.__setattr__(self, "phase", DiagnosticPhase(self.phase)) @@ -522,6 +524,7 @@ class CompactLineageFacts(Sequence[CompiledLineageColumnFact]): ..., ] resource_name_indexes: dict[int, int] = field(default_factory=dict) + resource_names: dict[int, str] = field(default_factory=dict) _cache: dict[int, CompiledLineageColumnFact] = field( default_factory=dict, init=False, @@ -577,6 +580,9 @@ def __eq__(self, other: object) -> bool: def resource_name(self, index: int) -> str: """Resolve a canonical native relation index to this model's resource name.""" + name: str | None = self.resource_names.get(index) + if name is not None: + return name return self.string_pool[self.resource_name_indexes.get(index, index)] @staticmethod @@ -607,6 +613,7 @@ class PolyglotAnalysisResult: columns: tuple[InferredColumn, ...] | None = None lineage_columns: Sequence[CompiledLineageColumnFact] = field(default_factory=tuple) has_star: bool = False + star_resolved: bool = False binding_diagnostics: tuple[SqlBindingDiagnostic, ...] = field(default_factory=tuple) binding_validated: bool = False @@ -732,6 +739,29 @@ class CompileModelInput: sql_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) +@dataclass(frozen=True) +class ModelSqlAnalysis: + """Completed model analysis and its in-memory binding proof.""" + + polyglot_analysis: PolyglotAnalysisResult + placeholders: dict[str, str] | None + cached: bool = False + fused_binding_validated: bool = False + cleaned_sql: str | None = None + validated_schema: dict[str, dict[str, str]] | None = None + + +@dataclass(frozen=True) +class ModelSqlAnalysisRequest: + """Inputs to one model's SQL analysis.""" + + model_input: CompileModelInput + query_sql: str + placeholders: dict[str, str] | None + cache_key: str | None + binding_schema: dict[str, dict[str, str]] | None + + @dataclass(frozen=True) class CompileSeedInput: """One discovered seed file with its attached seed metadata.""" @@ -863,6 +893,7 @@ class CompiledModel: inferred_columns: tuple[InferredColumn, ...] | None = None fast_lineage_columns: Sequence[CompiledLineageColumnFact] | None = None fast_lineage_has_star: bool = False + fast_lineage_star_resolved: bool = False authored_sql: str = "" authored_query_sql: str = "" output_column_locations: dict[str, SourceLocation] = field(default_factory=dict) @@ -874,6 +905,7 @@ class CompiledModel: binding_diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) binding_validated: bool = False dynamic_column_contract: DynamicColumnContractProof | None = None + unchecked_output_columns: frozenset[str] = frozenset() @dataclass(frozen=True) @@ -998,8 +1030,10 @@ class CompiledProject: effective_target_name: str | None effective_connection: dict[str, object] effective_vars: dict[str, object] + binding_catalog: Any | None = field(default=None, repr=False, compare=False) effective_target_database: str | None = None effective_target_schema: str | None = None + sql_analysis_dialect: str | None = None compile_cache_dir: Path | None = None settings: SettingsConfig = field(default_factory=SettingsConfig) scenario: ScenarioConfig = field(default_factory=ScenarioConfig) @@ -1258,6 +1292,7 @@ class CompactBatchExecutionOptions: binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] | None = None on_response: CompactBatchResponseCallback | None = None + shareable_prekeys: frozenset[str] | None = None @dataclass(frozen=True) @@ -1268,6 +1303,50 @@ class CompactBatchPreparation: queries: tuple[dict[str, object], ...] templates: tuple[dict[str, object], ...] projections: tuple[dict[str, object], ...] + binding_catalog: Any | None = field(default=None, repr=False, compare=False) + shared_query_indexes: frozenset[int] = frozenset() + + +@dataclass(frozen=True) +class CompactBatchInputs: + """Ordered per-model inputs of one compact native analysis batch.""" + + query_sqls: tuple[str, ...] + references: tuple[tuple[CompileSqlReference, ...], ...] + placeholders: tuple[dict[str, str] | None, ...] + column_nullability_by_table: dict[str, dict[str, InferredNullability]] + column_types_by_table: dict[str, dict[str, str]] + inference_profile: Any + recover_cte_facts: tuple[bool, ...] + rich_type_inference: bool + binding_schemas: tuple[dict[str, dict[str, str]] | None, ...] | None + + +@dataclass(frozen=True) +class CompactBatchContext: + """Binding catalog, cleaned SQL, and binding references prepared for one batch.""" + + binding_catalog: Any | None + normalized_sqls: list[str] + binding_queries: tuple[PreparedBindingQuery | None, ...] + + +@dataclass(frozen=True) +class SharedBindingQuery: + """Relation-stubbed binding query shared by models with identical input shapes.""" + + sql: str + stubs: dict[str, str] + key: tuple[object, ...] + + +@dataclass(frozen=True) +class PreparedBindingQuery: + """Native binding references for one model and its optional shared form.""" + + references: list[tuple[str, bool]] + overrides: dict[str, Mapping[str, str]] + shared: SharedBindingQuery | None = None @dataclass(frozen=True) @@ -1356,6 +1435,7 @@ class ProjectedAnalysisRequest: caches: CompactProjectionCaches = field(default_factory=CompactProjectionCaches) template_index: int | None = None resource_name_indexes: dict[int, int] = field(default_factory=dict) + resource_names: dict[int, str] = field(default_factory=dict) @dataclass(frozen=True) @@ -1380,6 +1460,7 @@ class NativeCompactAnalysis: | None ) = None resource_name_indexes: dict[int, int] = field(default_factory=dict) + resource_names: dict[int, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/contracts/_helpers/columns.py b/src/sqlbuild/compiler/contracts/_helpers/columns.py index ab20f551c9..d718379e33 100644 --- a/src/sqlbuild/compiler/contracts/_helpers/columns.py +++ b/src/sqlbuild/compiler/contracts/_helpers/columns.py @@ -332,7 +332,7 @@ def _type_diagnostics( model.schema_entry.type_enforcement ) if inferred_column.type is None: - if not type_enforcement: + if not type_enforcement or inferred_column.name in model.unchecked_output_columns: return () return ( CompilerDiagnostic( diff --git a/src/sqlbuild/compiler/dag/_helpers/artifact.py b/src/sqlbuild/compiler/dag/_helpers/artifact.py index 36b831748c..1feb635ad2 100644 --- a/src/sqlbuild/compiler/dag/_helpers/artifact.py +++ b/src/sqlbuild/compiler/dag/_helpers/artifact.py @@ -7,7 +7,6 @@ from enum import StrEnum from sqlbuild.compiler.auditing.types import AuditSeverity -from sqlbuild.compiler.compile.main.function_node_type import function_node_type from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, @@ -37,6 +36,7 @@ from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph from sqlbuild.compiler.python_nodes.types import PythonNodeKind +from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.python_nodes.models import ColumnLineageRef, SqlResourceRef from sqlbuild.python_nodes.types import SqlResourceRefKind from sqlbuild.spec.contracts.main.loader_destination_parts import loader_destination_parts diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 7c85fa536f..842e074f83 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -2,6 +2,7 @@ from __future__ import annotations +import ast import importlib.util import inspect import sys @@ -1903,7 +1904,24 @@ def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) def _load_materialization_module(*, file_path: Path, project_dir: Path) -> ModuleType: - return _exec_project_module( + removed_hook_name: str = "prepare_version" + removed_hook_message: str = ( + f"materialization '{file_path.stem}' at {file_path} defines " + "'prepare_version', which was removed with virtual environments; " + "projects run in direct mode" + ) + try: + module_ast: ast.Module = ast.parse(file_path.read_text(encoding="utf-8")) + except SyntaxError as error: + raise PythonNodeDiscoveryError( + f"Failed to import materialization file {file_path.relative_to(project_dir)}: {error}" + ) from error + if any( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == removed_hook_name + for node in module_ast.body + ): + raise PythonNodeDiscoveryError(removed_hook_message) + module: ModuleType = _exec_project_module( module_name=".".join( _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ), @@ -1912,6 +1930,9 @@ def _load_materialization_module(*, file_path: Path, project_dir: Path) -> Modul file_label="materialization", error_type=PythonNodeDiscoveryError, ) + if hasattr(module, removed_hook_name): + raise PythonNodeDiscoveryError(removed_hook_message) + return module def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py index 0662eecb27..06d0b4b09f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/project.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/project.py @@ -37,9 +37,12 @@ from sqlbuild.cursor_algebra.constants import DURATION_DAY_UNIT from sqlbuild.cursor_algebra.models import Duration from sqlbuild.spec.contracts.constants import ( + SCENARIO_RUN_NAMESPACE_CONFIG_KEY, TIME_TRAVEL_RETENTION_MATERIALIZATIONS, ZERO_DAY_CURSOR_DURATION, ) +from sqlbuild.spec.contracts.exceptions import SpecConfigError +from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import parse_scenario_run_namespace from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ClonePolicy, @@ -57,7 +60,6 @@ LocalClonePolicy, LocalConfig, LocalDbtConfig, - LocalStateConfig, LocalTargetConfig, MaterializationDefaultsConfig, MaterializationRetentionDefaults, @@ -71,7 +73,6 @@ SinksConfig, SnapshotsConfig, StartCursorsConfig, - StateConfig, TargetConfig, ) from sqlbuild.spec.contracts.types import ( @@ -469,6 +470,12 @@ def _load_settings(*, payload: object, file_path: Path) -> SettingsConfig: mapping: dict[str, object] = _coerce_mapping( payload=payload, label="settings", file_path=file_path ) + _reject_removed_virtual_keys( + mapping=mapping, + removed_keys=frozenset({"virtual_environments", "changes_only"}), + label="settings", + file_path=file_path, + ) canonical_setting_names: frozenset[str] = frozenset( field.name for field in fields(SettingsConfig) ) @@ -512,12 +519,6 @@ def _load_settings(*, payload: object, file_path: Path) -> SettingsConfig: "settings.column_contract_mode must be one of: implicit, explicit" ) from error auto_load_sources: bool = _optional_bool(mapping=mapping, key="auto_load_sources", default=True) - virtual_environments: bool = _optional_bool( - mapping=mapping, - key="virtual_environments", - default=False, - ) - changes_only: bool = _optional_bool(mapping=mapping, key="changes_only", default=False) microbatch_concurrency: bool = _optional_bool( mapping=mapping, key="microbatch_concurrency", default=False ) @@ -563,8 +564,6 @@ def _load_settings(*, payload: object, file_path: Path) -> SettingsConfig: column_contract_mode=column_contract_mode, concurrency=concurrency, auto_load_sources=auto_load_sources, - changes_only=changes_only, - virtual_environments=virtual_environments, microbatch_concurrency=microbatch_concurrency, microbatch_unaccounted_partition_policy=microbatch_unaccounted_partition_policy, table_promotion_mode=table_promotion_mode, @@ -817,6 +816,12 @@ def _load_defaults(*, payload: object, file_path: Path) -> DefaultsConfig: mapping: dict[str, object] = _coerce_mapping( payload=payload, label="defaults", file_path=file_path ) + _reject_removed_virtual_keys( + mapping=mapping, + removed_keys=frozenset({"run_despite_unchanged"}), + label="defaults", + file_path=file_path, + ) row_diff_exclude_columns: tuple[str, ...] = tuple( _load_string_sequence( payload=mapping.get("row_diff_exclude_columns"), @@ -883,7 +888,6 @@ def _load_defaults(*, payload: object, file_path: Path) -> DefaultsConfig: payload=mapping, key="unaccounted_partition_policy" ), replay_on_change=_optional_str(payload=mapping, key="replay_on_change"), - run_despite_unchanged=_optional_str(payload=mapping, key="run_despite_unchanged"), row_diff_exclude_columns=row_diff_exclude_columns, row_diff_tolerances=row_diff_tolerances, row_diff_sample_rows=_optional_non_negative_int( @@ -905,6 +909,12 @@ def _load_path_defaults(*, payload: object, file_path: Path) -> dict[str, dict[s if not isinstance(path_value, dict): raise ProjectConfigError(f"{file_path} path_defaults['{path_key}'] must be a mapping") path_dict: dict[str, object] = cast(dict[str, object], path_value) + _reject_removed_virtual_keys( + mapping=path_dict, + removed_keys=frozenset({"run_despite_unchanged"}), + label="path_defaults", + file_path=file_path, + ) _validate_path_default_tags(path_dict=path_dict, path_key=path_key, file_path=file_path) normalized_path_key: str = _normalize_path_default_key( path_key=path_key, file_path=file_path @@ -1107,14 +1117,6 @@ def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig _validate_clone_keys( clone_mapping=clone_mapping, target_name=target_name, file_path=file_path ) - state_mapping: dict[str, object] = _coerce_mapping( - payload=target_mapping.get("state"), - label=f"targets.{target_name}.state", - file_path=file_path, - ) - _validate_state_keys( - state_mapping=state_mapping, target_name=target_name, file_path=file_path - ) execution_limits: ExecutionLimitsConfig = _load_execution_limits( payload=target_mapping.get("execution_limits"), label=f"targets.{target_name}.execution_limits", @@ -1132,7 +1134,6 @@ def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig loader_schema=_optional_str(payload=target_mapping, key="loader_schema"), defer_sources_to=_optional_str(payload=target_mapping, key="defer_sources_to"), defer_clone_from=_optional_str(payload=target_mapping, key="defer_clone_from"), - changes_only=_optional_nullable_bool(mapping=target_mapping, key="changes_only"), compile_cache=_optional_nullable_bool(mapping=target_mapping, key="compile_cache"), time_travel_retention=_optional_target_retention_default( mapping=target_mapping, target_name=target_name, file_path=file_path @@ -1179,20 +1180,6 @@ def _load_targets(*, payload: object, file_path: Path) -> dict[str, TargetConfig default=False, ), ), - state=StateConfig( - backend=_optional_str(payload=state_mapping, key="backend"), - schema=_optional_str(payload=state_mapping, key="schema"), - connection=_optional_mapping(payload=state_mapping, key="connection"), - allow_reset=_optional_bool( - mapping=state_mapping, - key="allow_reset", - default=False, - ), - unsuffixed_virtual_env=_optional_str( - payload=state_mapping, - key="unsuffixed_virtual_env", - ), - ), ) return targets @@ -1221,14 +1208,6 @@ def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalT _validate_clone_keys( clone_mapping=clone_mapping, target_name=target_name, file_path=file_path ) - state_mapping: dict[str, object] = _coerce_mapping( - payload=target_mapping.get("state"), - label=f"targets.{target_name}.state", - file_path=file_path, - ) - _validate_state_keys( - state_mapping=state_mapping, target_name=target_name, file_path=file_path - ) execution_limits: ExecutionLimitsConfig = _load_execution_limits( payload=target_mapping.get("execution_limits"), label=f"targets.{target_name}.execution_limits", @@ -1246,7 +1225,6 @@ def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalT loader_schema=_optional_str(payload=target_mapping, key="loader_schema"), defer_sources_to=_optional_str(payload=target_mapping, key="defer_sources_to"), defer_clone_from=_optional_str(payload=target_mapping, key="defer_clone_from"), - changes_only=_optional_nullable_bool(mapping=target_mapping, key="changes_only"), compile_cache=_optional_nullable_bool(mapping=target_mapping, key="compile_cache"), time_travel_retention=_optional_target_retention_default( mapping=target_mapping, target_name=target_name, file_path=file_path @@ -1288,19 +1266,6 @@ def _load_local_targets(*, payload: object, file_path: Path) -> dict[str, LocalT key="allow_as_clone_destination", ), ), - state=LocalStateConfig( - backend=_optional_str(payload=state_mapping, key="backend"), - schema=_optional_str(payload=state_mapping, key="schema"), - connection=_optional_mapping(payload=state_mapping, key="connection"), - allow_reset=_optional_nullable_bool( - mapping=state_mapping, - key="allow_reset", - ), - unsuffixed_virtual_env=_optional_str( - payload=state_mapping, - key="unsuffixed_virtual_env", - ), - ), ) return targets @@ -1337,6 +1302,12 @@ def _load_target_connection( def _validate_target_keys( *, target_mapping: dict[str, object], target_name: str, file_path: Path ) -> None: + _reject_removed_virtual_keys( + mapping=target_mapping, + removed_keys=frozenset({"state", "changes_only"}), + label=f"targets.{target_name}", + file_path=file_path, + ) _validate_allowed_keys( mapping=target_mapping, allowed_keys=frozenset( @@ -1348,7 +1319,6 @@ def _validate_target_keys( "loader_schema", "defer_sources_to", "defer_clone_from", - "changes_only", "compile_cache", "time_travel_retention", "owns_time_travel_retention_namespace", @@ -1356,7 +1326,6 @@ def _validate_target_keys( "table_type_downgrade", "time_travel_retention_decrease", "clone", - "state", "execution_limits", } ), @@ -1409,29 +1378,21 @@ def _validate_clone_keys( ) -def _validate_state_keys( - *, state_mapping: dict[str, object], target_name: str, file_path: Path -) -> None: - _validate_allowed_keys( - mapping=state_mapping, - allowed_keys=frozenset( - {"backend", "schema", "connection", "allow_reset", "unsuffixed_virtual_env"} - ), - label=f"targets.{target_name}.state", - file_path=file_path, - ) - - def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: mapping: dict[str, object] = _coerce_mapping( payload=payload, label="janitor", file_path=file_path ) + _reject_removed_virtual_keys( + mapping=mapping, + removed_keys=frozenset({"max_checkpoints"}), + label="janitor", + file_path=file_path, + ) enabled: bool = _optional_bool(mapping=mapping, key="enabled", default=False) retention_days: int | None = _optional_nullable_int(mapping=mapping, key="retention_days") archive_retention_days: int = _optional_int( mapping=mapping, key="archive_retention_days", default=14 ) - max_checkpoints: int = _optional_int(mapping=mapping, key="max_checkpoints", default=20) direct_state_history_versions: int = _optional_int( mapping=mapping, key="direct_state_history_versions", @@ -1453,21 +1414,33 @@ def _load_janitor(*, payload: object, file_path: Path) -> JanitorConfig: raise ProjectConfigError(f"{file_path} janitor.retention_days must be >= 0") if archive_retention_days < 0: raise ProjectConfigError(f"{file_path} janitor.archive_retention_days must be >= 0") - if max_checkpoints < 1: - raise ProjectConfigError(f"{file_path} janitor.max_checkpoints must be >= 1") if direct_state_history_versions < 0: raise ProjectConfigError(f"{file_path} janitor.direct_state_history_versions must be >= 0") return JanitorConfig( enabled=enabled, retention_days=retention_days, archive_retention_days=archive_retention_days, - max_checkpoints=max_checkpoints, direct_state_history_versions=direct_state_history_versions, delete_tracked_only=delete_tracked_only, exclude_patterns=exclude_patterns, ) +def _reject_removed_virtual_keys( + *, + mapping: dict[str, object], + removed_keys: frozenset[str], + label: str, + file_path: Path, +) -> None: + removed: list[str] = sorted(removed_keys.intersection(mapping)) + if removed: + raise ProjectConfigError( + f"{file_path} [{label}] option(s) {', '.join(removed)} were removed " + "with virtual environments; projects run in direct mode" + ) + + def _load_snapshots(*, payload: object, file_path: Path) -> SnapshotsConfig: mapping: dict[str, object] = _coerce_mapping( payload=payload, label="snapshots", file_path=file_path @@ -1515,7 +1488,9 @@ def _load_scenario(*, payload: object, file_path: Path) -> ScenarioConfig: ) _validate_allowed_keys( mapping=mapping, - allowed_keys=frozenset({"local_type_overrides", "snapshot_limits"}), + allowed_keys=frozenset( + {"local_type_overrides", "snapshot_limits", SCENARIO_RUN_NAMESPACE_CONFIG_KEY} + ), label="scenario", file_path=file_path, ) @@ -1536,7 +1511,14 @@ def _load_scenario(*, payload: object, file_path: Path) -> ScenarioConfig: payload=rules_payload, file_path=file_path, ) + run_namespace: str | None = None + if SCENARIO_RUN_NAMESPACE_CONFIG_KEY in mapping: + try: + run_namespace = parse_scenario_run_namespace(mapping[SCENARIO_RUN_NAMESPACE_CONFIG_KEY]) + except SpecConfigError as exc: + raise ProjectConfigError(f"{file_path} {exc}") from exc return ScenarioConfig( + run_namespace=run_namespace, local_type_overrides=local_type_overrides, snapshot_limits=_load_scenario_snapshot_limits( payload=mapping.get("snapshot_limits"), diff --git a/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py b/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py index 63b5c8e5e2..aa941ff376 100644 --- a/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py +++ b/src/sqlbuild/compiler/lineage/_helpers/fast_columns.py @@ -68,7 +68,7 @@ def build_fast_project_column_lineage( continue if model.fast_lineage_columns is not None: columns: Sequence[CompiledLineageColumnFact] = model.fast_lineage_columns - if model.fast_lineage_has_star: + if model.fast_lineage_has_star and not model.fast_lineage_star_resolved: normalized_sql: str physical_resources: tuple[PhysicalResource, ...] normalized_sql, physical_resources = _normalize_sqlbuild_refs(model.query_sql) @@ -147,7 +147,7 @@ def _build_polyglot_fast_model_column_lineage( confidence=fact.confidence, ) ) - if model.fast_lineage_has_star: + if model.fast_lineage_has_star and not model.fast_lineage_star_resolved: normalized_sql: str physical_resources: tuple[PhysicalResource, ...] normalized_sql, physical_resources = _normalize_sqlbuild_refs(model.query_sql) diff --git a/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py b/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py index 3bb117943f..b1dddc1b06 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/deferred_locations.py @@ -8,8 +8,8 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import RelationInfo +from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.constants import PRESERVE_TARGET_VALUE -from sqlbuild.compiler.compile.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledProject, diff --git a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py index 24a01f8ce2..a03e62ffdc 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/materializations.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/materializations.py @@ -36,31 +36,3 @@ def load_custom_materializations( ) registry[mat_file.name] = materialize_fn return registry - - -def load_custom_prepare_version_functions( - materialization_files: tuple[DiscoveredMaterializationFile, ...], -) -> dict[str, Callable[..., None]]: - """Import discovered materialization modules and extract optional prepare_version callables.""" - - registry: dict[str, Callable[..., None]] = {} - mat_file: DiscoveredMaterializationFile - for mat_file in materialization_files: - spec: Any = importlib.util.spec_from_file_location(mat_file.name, mat_file.file_path) - if spec is None or spec.loader is None: - raise PlannerInputError( - f"materialization '{mat_file.name}' at {mat_file.file_path} " - f"could not be loaded as a Python module" - ) - module: ModuleType = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - prepare_version_fn: Any = getattr(module, "prepare_version", None) - if prepare_version_fn is None: - continue - if not callable(prepare_version_fn): - raise PlannerInputError( - f"materialization '{mat_file.name}' at {mat_file.file_path} " - f"defines a non-callable 'prepare_version' attribute" - ) - registry[mat_file.name] = prepare_version_fn - return registry diff --git a/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py b/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py index 3b5871e02a..c2b8470137 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/python_plan_entries.py @@ -7,12 +7,9 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.graph.main.transitive_closure import transitive_closure -from sqlbuild.compiler.pipeline._helpers.python_stale_selection import ( - filter_python_node_names_for_selected_sql, -) from sqlbuild.compiler.pipeline.models import PythonPlanEntry, PythonRunPlanOutputs from sqlbuild.compiler.planner.models import PlanOutput, PlanWarning -from sqlbuild.compiler.planner.types import WarningSeverity, WorkSelectionPolicy +from sqlbuild.compiler.planner.types import WarningSeverity from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle from sqlbuild.compiler.python_nodes.models import ( @@ -34,7 +31,6 @@ def build_python_run_plan_outputs( plan_output: PlanOutput, run_selection: PythonSqlRunSelection | None, selected_python_node_names: frozenset[str], - work_selection_policy: WorkSelectionPolicy, ) -> PythonRunPlanOutputs: """Attach python-aware selection warnings and plan entries to the plan output.""" @@ -47,12 +43,6 @@ def build_python_run_plan_outputs( python_graph: PythonNodeGraph = build_discovered_python_node_graph( discovered_inputs=discovered_inputs ) - if work_selection_policy == WorkSelectionPolicy.STALE_ONLY: - selected_python_node_names = filter_python_node_names_for_selected_sql( - python_graph=python_graph, - python_node_names=selected_python_node_names, - selected_sql_keys=plan_output.selected_keys, - ) plan_output = replace( plan_output, warnings=( diff --git a/src/sqlbuild/compiler/pipeline/_helpers/python_stale_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/python_stale_selection.py deleted file mode 100644 index c5ec54ce52..0000000000 --- a/src/sqlbuild/compiler/pipeline/_helpers/python_stale_selection.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Python-node selection pruning for stale-only planning.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.python_nodes.types import SqlResourceRefKind - - -def filter_python_node_names_for_selected_sql( - *, - python_graph: PythonNodeGraph, - python_node_names: frozenset[str], - selected_sql_keys: frozenset[CompiledObjectKey], -) -> frozenset[str]: - """Drop read-side Python nodes whose selected SQL dependencies were pruned.""" - - selected_sql_refs_set: set[SqlResourceRef] = set() - key: CompiledObjectKey - for key in selected_sql_keys: - sql_ref: SqlResourceRef | None = _sql_ref_from_key(key) - if sql_ref is not None: - selected_sql_refs_set.add(sql_ref) - selected_sql_refs: frozenset[SqlResourceRef] = frozenset(selected_sql_refs_set) - kept: set[str] = set() - node_name: str - for node_name in python_node_names: - node: DiscoveredPythonNode = python_graph.nodes_by_name[node_name] - if node.kind == PythonNodeKind.LOADER: - kept.add(node_name) - continue - if all(sql_dep in selected_sql_refs for sql_dep in node.sql_deps): - kept.add(node_name) - - changed: bool = True - while changed: - changed = False - for node_name in tuple(kept): - upstream_name: str - for upstream_name in python_graph.upstream_deps.get(node_name, ()): - if upstream_name in python_node_names and upstream_name not in kept: - kept.remove(node_name) - changed = True - break - return frozenset(kept) - - -def _sql_ref_from_key(key: CompiledObjectKey) -> SqlResourceRef | None: - if key.resource_type == CompiledResourceType.MODEL: - return SqlResourceRef(kind=SqlResourceRefKind.MODEL, name=key.name) - if key.resource_type == CompiledResourceType.SOURCE: - return SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=key.name) - return None diff --git a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py index a9f2f7c03e..ebd233917e 100644 --- a/src/sqlbuild/compiler/pipeline/main/_compile_phase.py +++ b/src/sqlbuild/compiler/pipeline/main/_compile_phase.py @@ -6,8 +6,10 @@ from collections.abc import Callable from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main.effective_config import build_effective_connection_config -from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline._helpers.analysis_selection import ( resolve_compile_analysis_selection, @@ -58,6 +60,26 @@ def compile_project_phase( timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() if timing_tracker is not None: timing_tracker.compile_seconds = compile_seconds + errors: tuple[CompilerDiagnostic, ...] = tuple( + diagnostic + for diagnostic in project.diagnostics + if diagnostic.is_error + and ( + resolved_options.plan_sql_tests + or diagnostic.resource_type != CompiledResourceType.SQL_TEST + ) + ) + if errors: + if on_progress is not None: + on_progress("Project compilation failed.") + raise CompileInputError( + "\n".join( + f"[{diagnostic.code}] {diagnostic.path}:" + f"{diagnostic.line or 1}:{diagnostic.column or 1}: {diagnostic.message}" + for diagnostic in errors + ), + code=errors[0].code, + ) if on_progress is not None: on_progress(f"Compiled project. ({compile_seconds:.2f}s)") return CompiledProjectPhaseResult( diff --git a/src/sqlbuild/compiler/pipeline/main/compile.py b/src/sqlbuild/compiler/pipeline/main/compile.py index 91f2e04283..03bb146649 100644 --- a/src/sqlbuild/compiler/pipeline/main/compile.py +++ b/src/sqlbuild/compiler/pipeline/main/compile.py @@ -55,7 +55,6 @@ PlannerSelection, PlanOutput, ) -from sqlbuild.compiler.planner.types import WorkSelectionPolicy from sqlbuild.compiler.python_nodes.main._run_selection import ( resolve_python_sql_run_selection_from_inputs, ) @@ -131,6 +130,7 @@ def run_compile_pipeline( result=RulesResult( findings=rules_result.findings, evaluated_models=rules_result.evaluated_models, + unevaluated_resources=rules_result.unevaluated_resources, cache_hits=rules_result.cache_hits, cache_misses=rules_result.cache_misses, ), @@ -252,7 +252,6 @@ def _build_result( plan_output=plan_output, run_selection=run_selection, selected_python_node_names=selected_python_node_names, - work_selection_policy=WorkSelectionPolicy.ALL_SELECTED, ) return CompilePipelineResult( diff --git a/src/sqlbuild/compiler/pipeline/main/materializations.py b/src/sqlbuild/compiler/pipeline/main/materializations.py deleted file mode 100644 index 018e2ccf33..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/materializations.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Public entrypoint for custom materialization loading.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.discovery.models import DiscoveredMaterializationFile -from sqlbuild.compiler.pipeline._helpers.materializations import ( - load_custom_materializations as _load_custom_materializations, -) -from sqlbuild.executor.custom.models import MaterializationContext, MaterializationResult - - -def load_custom_materializations( - materialization_files: tuple[DiscoveredMaterializationFile, ...], -) -> dict[str, Callable[[MaterializationContext], MaterializationResult]]: - """Load custom materialization callables from discovered project files.""" - - return _load_custom_materializations(materialization_files) diff --git a/src/sqlbuild/compiler/pipeline/main/prepare_versions.py b/src/sqlbuild/compiler/pipeline/main/prepare_versions.py deleted file mode 100644 index f949777c5c..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/prepare_versions.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public entrypoint for custom prepare_version loading.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.compiler.discovery.models import DiscoveredMaterializationFile -from sqlbuild.compiler.pipeline._helpers.materializations import ( - load_custom_prepare_version_functions as _load_custom_prepare_version_functions, -) - - -def load_custom_prepare_version_functions( - materialization_files: tuple[DiscoveredMaterializationFile, ...], -) -> dict[str, Callable[..., None]]: - """Load optional custom prepare_version callables from discovered project files.""" - - return _load_custom_prepare_version_functions(materialization_files) diff --git a/src/sqlbuild/compiler/pipeline/main/python_plan_entries.py b/src/sqlbuild/compiler/pipeline/main/python_plan_entries.py deleted file mode 100644 index f67b614cef..0000000000 --- a/src/sqlbuild/compiler/pipeline/main/python_plan_entries.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public entrypoint for Python-node plan display entries.""" - -from __future__ import annotations - -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline._helpers.python_plan_entries import ( - build_python_plan_entries as _build_python_plan_entries, -) -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunLifecyclePlan - - -def build_python_plan_entries( - *, - lifecycle_plan: PythonSqlRunLifecyclePlan, - python_graph: PythonNodeGraph, - previous_identities: dict[tuple[str, str], Fingerprint] | None = None, -) -> tuple[PythonPlanEntry, ...]: - """Return display entries for a lifecycle-aware Python-node plan.""" - - return _build_python_plan_entries( - lifecycle_plan=lifecycle_plan, - python_graph=python_graph, - previous_identities=previous_identities, - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py b/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py index 39f0623cc6..504916e1c4 100644 --- a/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py +++ b/src/sqlbuild/compiler/planner/_helpers/graph/model_closure.py @@ -17,16 +17,6 @@ def build_downstream_model_name_closure( return _build_model_name_closure(start_keys=start_keys, deps=downstream_deps) -def build_upstream_model_name_closure( - *, - start_keys: tuple[CompiledObjectKey, ...], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - """Return model names reachable upstream from the given keys, including model roots.""" - - return _build_model_name_closure(start_keys=start_keys, deps=upstream_deps) - - def _build_model_name_closure( *, start_keys: tuple[CompiledObjectKey, ...], diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py index 47be2dcf1f..c6774c1998 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/direct.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/direct.py @@ -6,6 +6,7 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredHookFunction from sqlbuild.compiler.planner._helpers.identity.hashing import ( + build_model_local_identity_hash, compose_native_graph_identity, graph_key_for_compiled_resource, ) @@ -16,9 +17,6 @@ from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( build_function_local_hashes, ) -from sqlbuild.compiler.planner.main.identity.version_identity_local_hash import ( - build_model_local_identity_hash, -) from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( build_model_version_identity_metadata_json, ) diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py b/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py index efaf436f95..62c2ab6156 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/hashing.py @@ -4,7 +4,6 @@ import hashlib -from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.fingerprints.constants import ( NODE_TYPE_MODEL, @@ -27,37 +26,6 @@ def build_model_local_identity_hash(*, query_sql: str, metadata_json: str) -> st return stable_version_identity_hash("\n".join((query_sql, metadata_json))) -def build_model_version_identity_hash( - *, - local_hash: str, - upstream_deps: tuple[CompiledObjectKey, ...], - upstream_version_hashes: dict[str, str], - source_version_hashes: dict[str, str] | None = None, -) -> str: - """Build a composed model version hash from local and upstream identities.""" - - source_hashes: dict[str, str] = source_version_hashes or {} - upstream_hashes: list[str] = [] - upstream_key: CompiledObjectKey - for upstream_key in upstream_deps: - if upstream_key.resource_type == CompiledResourceType.SOURCE: - source_hash: str | None = source_hashes.get(upstream_key.name) - if source_hash is not None: - upstream_hashes.append(source_hash) - continue - if upstream_key.resource_type not in ( - CompiledResourceType.MODEL, - CompiledResourceType.UDF, - CompiledResourceType.TABLE_FN, - CompiledResourceType.SEED, - ): - continue - upstream_hash: str | None = upstream_version_hashes.get(upstream_key.name) - if upstream_hash is not None: - upstream_hashes.append(upstream_hash) - return stable_version_identity_hash("\n".join((local_hash, *upstream_hashes))) - - def graph_key_for_compiled_resource( *, resource_type: str | CompiledResourceType, name: str ) -> GraphNodeKey: diff --git a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py b/src/sqlbuild/compiler/planner/_helpers/identity/honest.py index fb9f2b3e34..368666166b 100644 --- a/src/sqlbuild/compiler/planner/_helpers/identity/honest.py +++ b/src/sqlbuild/compiler/planner/_helpers/identity/honest.py @@ -25,7 +25,7 @@ ResolvedModelAction, WarehouseSnapshot, ) -from sqlbuild.compiler.planner.types import ChangeKind, GraphResourceKind +from sqlbuild.compiler.planner.types import GraphResourceKind def with_honest_model_write_hashes( @@ -115,19 +115,10 @@ def _merged_recomputed_change( resolved_change: ChangeDetectionResult, recomputed_change: ChangeDetectionResult | None, ) -> ChangeDetectionResult: - """Prefer the recomputed change but never downgrade RUN_DESPITE_UNCHANGED to NO_CHANGE.""" + """Prefer a recomputed change when available.""" if recomputed_change is None: return resolved_change - if ( - resolved_change.change_kind == ChangeKind.RUN_DESPITE_UNCHANGED - and recomputed_change.change_kind == ChangeKind.NO_CHANGE - ): - return replace( - recomputed_change, - change_kind=ChangeKind.RUN_DESPITE_UNCHANGED, - backfill=resolved_change.backfill, - ) return recomputed_change diff --git a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py index f9a6ea1a8a..9705d0e002 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/plan_entry.py @@ -12,6 +12,7 @@ from sqlbuild.compiler.compile.constants import CURSOR_INPUTS_CONFIG_KEY from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.main._cursor_roles import resolve_cursor_input_roles +from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, @@ -20,6 +21,7 @@ CompiledProject, CompiledRelationLocation, CompiledSeed, + CompilerDiagnostic, CompileSqlReference, CursorInputRoles, ) @@ -62,6 +64,9 @@ build_seed_locations, ) from sqlbuild.compiler.planner._helpers.resolve.resolve import resolve_model_sql +from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( + get_semantic_source_columns, +) from sqlbuild.compiler.planner._helpers.warehouse.source_deferral import build_source_read_map from sqlbuild.compiler.planner.constants import ( METADATA_NAME_FILTER_LIMIT, @@ -102,8 +107,6 @@ PlannerScope, PlanWarning, ResolvedModelAction, - RunDespiteUnchangedDecision, - RunDespiteUnchangedPlanningResult, SchemaAction, WarehouseSnapshot, ) @@ -249,12 +252,19 @@ def build_planner_relations_context( function_locations=function_locations, source_map=source_map, source_read_map=source_read_map, - source_warehouse_columns=_resolve_source_warehouse_columns( + source_warehouse_columns=get_semantic_source_columns( project=project, adapter=adapter, connection=connection, source_read_map=source_read_map, - known_source_columns=known_source_columns, + selected_keys=scope.selected_keys, + columns=_resolve_source_warehouse_columns( + project=project, + adapter=adapter, + connection=connection, + source_read_map=source_read_map, + known_source_columns=known_source_columns, + ), ), star_exclude_keyword=adapter.star_exclude_keyword(), ) @@ -338,7 +348,31 @@ def build_plan_entries( inputs: PlanEntryBuildInputs = ( build_inputs if build_inputs is not None else PlanEntryBuildInputs() ) - run_despite_unchanged: RunDespiteUnchangedPlanningResult | None = inputs.run_despite_unchanged + binding_diagnostics: tuple[CompilerDiagnostic, ...] = ( + *( + diagnostic + for diagnostic in project.diagnostics + if diagnostic.is_error and diagnostic.resource_type != CompiledResourceType.SQL_TEST + ), + *get_source_binding_diagnostics( + project=project, + columns=relations.source_warehouse_columns, + profile=adapter.expression_inference_profile(), + selected_keys=scope.selected_keys, + ), + ) + binding_diagnostics = tuple( + diagnostic for diagnostic in binding_diagnostics if diagnostic.is_error + ) + if binding_diagnostics: + raise PlannerInputError( + "\n".join( + f"error[{diagnostic.code}]: {diagnostic.message}\n" + f" --> {diagnostic.path}:{diagnostic.line or 1}:{diagnostic.column or 1}" + for diagnostic in binding_diagnostics + ), + code=binding_diagnostics[0].code, + ) source_freshness_blocked_model_names: frozenset[str] = ( inputs.source_freshness_blocked_model_names ) @@ -418,12 +452,6 @@ def build_plan_entries( action=PlanAction.SKIP, reason=PlanReason.EXTERNAL_UPSTREAM_FAILED, ) - if run_despite_unchanged is not None: - run_decision: RunDespiteUnchangedDecision | None = run_despite_unchanged.decisions.get( - entry.name - ) - if run_decision is not None: - entry = replace(entry, run_despite_unchanged=run_decision) if entry.incremental_mode == IncrementalMode.MICROBATCH: entry = replace( entry, diff --git a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py index 58f63a461f..e4b80ada11 100644 --- a/src/sqlbuild/compiler/planner/_helpers/output/strategy.py +++ b/src/sqlbuild/compiler/planner/_helpers/output/strategy.py @@ -319,8 +319,6 @@ def _table_action( return PlanAction.CREATE_TABLE, PlanReason.CONFIG_CHANGED if change_result.change_kind == ChangeKind.SCHEMA_CHANGED: return PlanAction.CREATE_TABLE, PlanReason.SCHEMA_CHANGED - if change_result.change_kind == ChangeKind.RUN_DESPITE_UNCHANGED: - return PlanAction.CREATE_TABLE, PlanReason.RUN_DESPITE_UNCHANGED return PlanAction.CREATE_TABLE, PlanReason.NO_CHANGE diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/entries.py b/src/sqlbuild/compiler/planner/_helpers/planning/entries.py index 7cd7ce8543..7f65552afb 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/entries.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/entries.py @@ -57,7 +57,6 @@ def build_planner_entry_results( cursor_overrides=overrides.cursor_overrides, full_refresh=overrides.full_refresh, build_inputs=PlanEntryBuildInputs( - run_despite_unchanged=pruning.run_despite_unchanged, source_freshness_blocked_model_names=( source_freshness.propagation.blocked_model_names if source_freshness.propagation is not None diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py index fe0d02ed5e..271fac940e 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/output_assembly.py @@ -26,7 +26,6 @@ PlanOutput, PlanOutputExtras, PlanWarning, - RunDespiteUnchangedPlanningResult, ) from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult from sqlbuild.compiler.source_freshness.types import SourceFreshnessAgeStatus @@ -147,10 +146,7 @@ def with_plan_metadata( direct_remaining_stale_model_names: tuple[str, ...] = tuple( sorted( - ( - pruning.direct_identity_stale_model_names - | pruning.run_despite_unchanged.stale_model_names - ) + pruning.direct_identity_stale_model_names - frozenset( key.name for key in pruning.inspection_scope.selected_keys @@ -166,9 +162,6 @@ def with_plan_metadata( source_freshness ), "direct_remaining_stale_model_names": direct_remaining_stale_model_names, - "direct_run_despite_unchanged": _serialize_run_despite_unchanged_metadata( - pruning.run_despite_unchanged - ), "selection_diagnostics": { "mode": "direct", "enabled": policies.selection_diagnostics, @@ -223,21 +216,3 @@ def _serialize_direct_source_freshness_metadata( "stale_model_names": stale_model_names, "blocked_model_names": blocked_model_names, } - - -def _serialize_run_despite_unchanged_metadata( - run_despite_unchanged: RunDespiteUnchangedPlanningResult, -) -> dict[str, object]: - return { - "root_model_names": tuple(sorted(run_despite_unchanged.root_model_names)), - "stale_model_names": tuple(sorted(run_despite_unchanged.stale_model_names)), - "decisions": { - model_name: { - "mode": decision.mode.value, - "duration": decision.duration, - "newest_source_name": decision.newest_source_name, - "newest_source_data_age_seconds": (decision.newest_source_data_age_seconds), - } - for model_name, decision in sorted(run_despite_unchanged.decisions.items()) - }, - } diff --git a/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py b/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py index 781f33daa2..7386ed1cd9 100644 --- a/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py +++ b/src/sqlbuild/compiler/planner/_helpers/planning/scope_pruning.py @@ -7,7 +7,6 @@ PlannerScope, PlannerScopePruningResult, PlannerScopeResolution, - RunDespiteUnchangedPlanningResult, ) @@ -26,5 +25,4 @@ def prune_planner_execution_scope( resolved_actions=resolved_actions, pruned_direct_model_names=(), direct_identity_stale_model_names=frozenset(), - run_despite_unchanged=RunDespiteUnchangedPlanningResult(), ) diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/run_despite_unchanged.py b/src/sqlbuild/compiler/planner/_helpers/pruning/run_despite_unchanged.py deleted file mode 100644 index 23346cc561..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/run_despite_unchanged.py +++ /dev/null @@ -1,253 +0,0 @@ -"""Changes-only run_despite_unchanged planning helpers.""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta - -from sqlbuild.compiler.compile.models import CompiledModel, CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.model_downstream_closure import ( - build_downstream_model_names, -) -from sqlbuild.compiler.planner.models import ( - PlannerScope, - RunDespiteUnchangedDecision, - RunDespiteUnchangedPlanningResult, -) -from sqlbuild.compiler.planner.types import MaterializationType, RunDespiteUnchangedMode -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessRecord, -) -from sqlbuild.cursor_algebra.constants import MINUTE_TO_DAY_DURATION_UNITS -from sqlbuild.cursor_algebra.models import Duration -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind - - -def build_run_despite_unchanged_planning_result( - *, - scope: PlannerScope, - source_freshness: DirectSourceFreshnessPlanningResult, - already_stale_model_names: frozenset[str], - now: datetime, -) -> RunDespiteUnchangedPlanningResult: - """Return configured table roots that should run despite unchanged inputs.""" - - decisions: dict[str, RunDespiteUnchangedDecision] = {} - downstream_root_causes: dict[str, str] = {} - stale_model_names: set[str] = set() - key: CompiledObjectKey - for key in scope.selected_keys: - if key.resource_type != CompiledResourceType.MODEL: - continue - model: CompiledModel | None = scope.models_by_name.get(key.name) - if model is None: - continue - raw_value: object | None = model.config.values.get("run_despite_unchanged") - if raw_value is None: - continue - _validate_supported_model(model=model, raw_value=raw_value) - if model.name in already_stale_model_names: - _validate_duration_prerequisites_if_needed( - model=model, - scope=scope, - source_freshness=source_freshness, - now=now, - raw_value=raw_value, - ) - decision: RunDespiteUnchangedDecision | None = _decision_for_model( - model=model, - scope=scope, - source_freshness=source_freshness, - now=now, - raw_value=raw_value, - ) - if decision is None: - continue - decisions[model.name] = decision - downstream_names: frozenset[str] = build_downstream_model_names( - start_keys=(model.key,), - downstream_deps=scope.downstream_deps, - ) - stale_model_names.update(downstream_names) - downstream_name: str - for downstream_name in sorted(downstream_names - {model.name}): - downstream_root_causes.setdefault(downstream_name, model.name) - - return RunDespiteUnchangedPlanningResult( - root_model_names=frozenset(decisions), - stale_model_names=frozenset(stale_model_names), - decisions=decisions, - downstream_root_causes=downstream_root_causes, - ) - - -def _validate_supported_model(*, model: CompiledModel, raw_value: object) -> None: - if not isinstance(raw_value, str): - raise PlannerInputError( - f"model '{model.name}' run_despite_unchanged must be 'always' or a positive " - "duration like 30d, 12h, or 90m" - ) - materialized: object | None = model.config.values.get("materialized") - if materialized != MaterializationType.TABLE: - raise PlannerInputError( - f"model '{model.name}' has run_despite_unchanged, but only table " - "materializations support it" - ) - - -def _validate_duration_prerequisites_if_needed( - *, - model: CompiledModel, - scope: PlannerScope, - source_freshness: DirectSourceFreshnessPlanningResult, - now: datetime, - raw_value: object, -) -> None: - value: str = str(raw_value).strip().lower() - if value == RunDespiteUnchangedMode.ALWAYS.value: - return - _decision_for_model( - model=model, - scope=scope, - source_freshness=source_freshness, - now=now, - raw_value=raw_value, - ) - - -def _decision_for_model( - *, - model: CompiledModel, - scope: PlannerScope, - source_freshness: DirectSourceFreshnessPlanningResult, - now: datetime, - raw_value: object, -) -> RunDespiteUnchangedDecision | None: - value: str = str(raw_value).strip().lower() - if value == RunDespiteUnchangedMode.ALWAYS.value: - return RunDespiteUnchangedDecision( - model_name=model.name, - mode=RunDespiteUnchangedMode.ALWAYS, - ) - - duration: timedelta = _parse_duration(model_name=model.name, value=value) - records: tuple[SourceFreshnessRecord, ...] = _upstream_source_records( - model=model, - scope=scope, - source_freshness=source_freshness, - ) - if not records: - raise PlannerInputError( - f"model '{model.name}' has run_despite_unchanged = {value}, but SQLBuild " - "cannot determine upstream source freshness age. Configure timestamp source " - "freshness for at least one upstream source, use run_despite_unchanged always, " - "or remove the setting." - ) - - newest_record: SourceFreshnessRecord = max( - records, - key=lambda record: _timestamp_data_version( - model_name=model.name, - record=record, - ), - ) - newest_timestamp: datetime = _timestamp_data_version( - model_name=model.name, - record=newest_record, - ) - age_seconds: int = max(0, int((_normalize_now(now) - newest_timestamp).total_seconds())) - if timedelta(seconds=age_seconds) > duration: - return None - return RunDespiteUnchangedDecision( - model_name=model.name, - mode=RunDespiteUnchangedMode.DURATION, - duration=value, - newest_source_name=newest_record.source_name, - newest_source_data_age_seconds=age_seconds, - ) - - -def _parse_duration(*, model_name: str, value: str) -> timedelta: - duration: Duration | None = Duration.parse(value) - if duration is None or not duration.is_single_unit_in(MINUTE_TO_DAY_DURATION_UNITS): - raise PlannerInputError( - f"model '{model_name}' run_despite_unchanged must be 'always' or a positive " - "duration like 30d, 12h, or 90m" - ) - return timedelta(seconds=duration.fixed_seconds) - - -def _upstream_source_records( - *, - model: CompiledModel, - scope: PlannerScope, - source_freshness: DirectSourceFreshnessPlanningResult, -) -> tuple[SourceFreshnessRecord, ...]: - upstream_source_names: frozenset[str] = _upstream_source_names( - start_key=model.key, - upstream_deps=scope.upstream_deps, - ) - records: list[SourceFreshnessRecord] = [] - record: SourceFreshnessRecord - for record in source_freshness.observed_records: - if record.source_name not in upstream_source_names: - continue - if record.value_kind != SourceFreshnessValueKind.TIMESTAMP.value: - raise PlannerInputError( - f"model '{model.name}' has run_despite_unchanged = " - f"{model.config.values.get('run_despite_unchanged')}, but upstream source " - f"'{record.source_name}' uses {record.value_kind} freshness. Duration mode " - "requires timestamp source freshness; use run_despite_unchanged always or " - "configure timestamp source freshness." - ) - records.append(record) - return tuple(records) - - -def _upstream_source_names( - *, - start_key: CompiledObjectKey, - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - result: set[str] = set() - visited: set[CompiledObjectKey] = set() - stack: list[CompiledObjectKey] = [start_key] - while stack: - current: CompiledObjectKey = stack.pop() - if current in visited: - continue - visited.add(current) - if current.resource_type == CompiledResourceType.SOURCE: - result.add(current.name) - continue - neighbor: CompiledObjectKey - for neighbor in upstream_deps.get(current, ()): - stack.append(neighbor) - return frozenset(result) - - -def _timestamp_data_version(*, model_name: str, record: SourceFreshnessRecord) -> datetime: - if record.data_version is None: - raise PlannerInputError( - f"model '{model_name}' cannot determine source freshness age because source " - f"'{record.source_name}' has no data_version" - ) - try: - timestamp: datetime = datetime.fromisoformat(record.data_version) - except ValueError as exc: - raise PlannerInputError( - f"model '{model_name}' cannot determine source freshness age because source " - f"'{record.source_name}' has invalid timestamp data_version " - f"'{record.data_version}'" - ) from exc - if timestamp.tzinfo is None: - return timestamp.replace(tzinfo=UTC) - return timestamp.astimezone(UTC) - - -def _normalize_now(now: datetime) -> datetime: - if now.tzinfo is None: - return now.replace(tzinfo=UTC) - return now.astimezone(UTC) diff --git a/src/sqlbuild/compiler/planner/_helpers/pruning/version_staleness.py b/src/sqlbuild/compiler/planner/_helpers/pruning/version_staleness.py deleted file mode 100644 index 224b1d1829..0000000000 --- a/src/sqlbuild/compiler/planner/_helpers/pruning/version_staleness.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Pure expected-vs-built model version staleness helpers.""" - -from __future__ import annotations - -from collections.abc import Mapping - - -def build_stale_model_names_from_version_identities( - *, - model_names: tuple[str, ...], - expected_version_hashes: Mapping[str, str], - built_version_hashes: Mapping[str, str | None], - forced_stale_model_names: tuple[str, ...] = (), -) -> tuple[str, ...]: - """Return models with missing/mismatched built hashes or external stale markers.""" - - forced_stale: set[str] = set(forced_stale_model_names) - return tuple( - model_name - for model_name in model_names - if model_name in forced_stale - or built_version_hashes.get(model_name) != expected_version_hashes.get(model_name) - ) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py b/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py index a15b2512c9..f55697843f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/artifacts.py @@ -29,6 +29,7 @@ def compute_scenario_hash_prefix( *, project_name: str, scenario_name: str, + run_namespace: str | None = None, prefix_length: int = SCENARIO_HASH_PREFIX_LENGTH, ) -> str: """Return a stable scenario artifact hash prefix.""" @@ -39,6 +40,8 @@ def compute_scenario_hash_prefix( code=SCENARIO_PLAN_INVALID_HASH_PREFIX, ) hash_input: str = f"{project_name}:{scenario_name}" + if run_namespace is not None: + hash_input = f"{project_name}:{run_namespace}:{scenario_name}" return hashlib.sha256(hash_input.encode("utf-8")).hexdigest()[:prefix_length] @@ -46,6 +49,7 @@ def build_scenario_hash_index( *, project_name: str, scenarios: tuple[CompiledSqlScenario, ...], + run_namespace: str | None = None, prefix_length: int = SCENARIO_HASH_PREFIX_LENGTH, ) -> dict[str, str]: """Return scenario name to hash prefix, failing clearly on prefix collisions.""" @@ -57,6 +61,7 @@ def build_scenario_hash_index( prefix: str = compute_scenario_hash_prefix( project_name=project_name, scenario_name=scenario.name, + run_namespace=run_namespace, prefix_length=prefix_length, ) existing: CompiledSqlScenario | None = by_prefix.get(prefix) diff --git a/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py b/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py index 81418f324d..cb65ad336c 100644 --- a/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py +++ b/src/sqlbuild/compiler/planner/_helpers/scenario/cli.py @@ -50,6 +50,7 @@ def build_cli_scenario_plan( hash_index: dict[str, str] = build_scenario_hash_index( project_name=project_name, scenarios=pipeline_result.project.sql_scenarios, + run_namespace=pipeline_result.project.scenario.run_namespace, ) relation_map: ScenarioRelationMap = build_scenario_relation_map( scenario_name=scenario.name, diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py new file mode 100644 index 0000000000..1d0be9b2f1 --- /dev/null +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py @@ -0,0 +1,76 @@ +"""Complete missing source metadata in bounded, database-grouped batches.""" + +from __future__ import annotations + +from typing import Any + +from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.planner.types import ContractPolicy +from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.spec.contracts.models import SourceEntry + + +def get_semantic_source_columns( + *, + project: CompiledProject, + adapter: BaseAdapter, + connection: Any, + source_read_map: dict[str, SourceEntry], + columns: dict[str, tuple[ColumnInfo, ...]], + selected_keys: frozenset[CompiledObjectKey], +) -> dict[str, tuple[ColumnInfo, ...]]: + """Reuse collected columns and inspect only selected, unresolved table sources.""" + if not project.settings.sql_analysis or connection is None: + return columns + required: set[str] = set() + for model in project.models: + if model.key in selected_keys and model.config.values.get("sql_analysis") is not False: + required.update( + reference.ref_name + for reference in model.references + if reference.ref_kind == SqlReferenceKind.SOURCE + ) + by_database: dict[str | None, list[SourceEntry]] = {} + for name in sorted(required - columns.keys()): + entry: SourceEntry | None = source_read_map.get(name) + if ( + entry is None + or entry.expression is not None + or entry.contract == ContractPolicy.ENFORCED + ): + continue + by_database.setdefault(entry.database, []).append(entry) + result: dict[str, tuple[ColumnInfo, ...]] = dict(columns) + for database, entries in by_database.items(): + relations: tuple[RelationInfo, ...] = adapter.list_relations( + connection=connection, + database=database, + schemas=None, + names=tuple(sorted({entry.table or entry.name for entry in entries})), + ) + inspected: dict[tuple[str | None, str | None, str], tuple[ColumnInfo, ...]] = ( + adapter.get_columns_for_relations(connection=connection, relations=relations) + ) + for entry in entries: + candidates: tuple[RelationInfo, ...] = tuple( + relation + for relation in relations + if relation.name.casefold() == (entry.table or entry.name).casefold() + and ( + entry.schema is None + or (relation.schema or "").casefold() == entry.schema.casefold() + ) + ) + if len(candidates) != 1: + continue + relation: RelationInfo = candidates[0] + identity: tuple[str | None, str | None, str] = ( + relation.database.lower() if relation.database else None, + relation.schema.lower() if relation.schema else None, + relation.name.lower(), + ) + if identity in inspected: + result[entry.name] = inspected[identity] + return result diff --git a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py index fd54fabb7a..c843f4b21f 100644 --- a/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py +++ b/src/sqlbuild/compiler/planner/_helpers/warehouse/source_deferral.py @@ -5,7 +5,7 @@ import re from dataclasses import replace -from sqlbuild.compiler.compile.main._project_var_values import render_project_var_text +from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.models import ( CompiledAudit, CompiledFunction, diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 88de87825d..73829c5887 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -78,3 +78,5 @@ "snapshot_schema_change", } ) +SNAPSHOT_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" +SNAPSHOT_DEFAULT_VALID_TO_COLUMN: str = "valid_to" diff --git a/src/sqlbuild/compiler/planner/main/changes/replay_on_change.py b/src/sqlbuild/compiler/planner/main/changes/replay_on_change.py deleted file mode 100644 index a3e2a3afce..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/replay_on_change.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Replay-on-change policy public entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.changes.policy import resolve_replay_on_change as _resolve -from sqlbuild.compiler.planner.models import BackfillResult - - -def resolve_replay_on_change(*, replay_on_change: str | None) -> BackfillResult: - """Resolve a configured replay-on-change policy.""" - - return _resolve(replay_on_change=replay_on_change) diff --git a/src/sqlbuild/compiler/planner/main/changes/run_despite_unchanged.py b/src/sqlbuild/compiler/planner/main/changes/run_despite_unchanged.py deleted file mode 100644 index a92602478f..0000000000 --- a/src/sqlbuild/compiler/planner/main/changes/run_despite_unchanged.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Public planner entrypoint for run_despite_unchanged planning.""" - -from __future__ import annotations - -from datetime import datetime - -from sqlbuild.compiler.planner._helpers.pruning.run_despite_unchanged import ( - build_run_despite_unchanged_planning_result as _build_result, -) -from sqlbuild.compiler.planner.models import PlannerScope, RunDespiteUnchangedPlanningResult -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult - - -def build_run_despite_unchanged_planning_result( - *, - scope: PlannerScope, - source_freshness: DirectSourceFreshnessPlanningResult, - already_stale_model_names: frozenset[str], - now: datetime, -) -> RunDespiteUnchangedPlanningResult: - """Return configured table roots that should run despite unchanged inputs.""" - - return _build_result( - scope=scope, - source_freshness=source_freshness, - already_stale_model_names=already_stale_model_names, - now=now, - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/plan_entry.py b/src/sqlbuild/compiler/planner/main/execution/plan_entry.py deleted file mode 100644 index 93e63bf95f..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/plan_entry.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Public model plan-entry phase entrypoint.""" - -from __future__ import annotations - -from dataclasses import replace -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.planner._helpers.changes.detect import detect_changes -from sqlbuild.compiler.planner._helpers.output.plan_entry import ( - build_plan_entries, - build_planner_relations_context, -) -from sqlbuild.compiler.planner._helpers.output.plan_output import build_plan_output -from sqlbuild.compiler.planner._helpers.pruning.cascade import resolve_cascades -from sqlbuild.compiler.planner.models import ( - ChangeDetectionResult, - DeferralInputs, - ModelChangesPlanInputs, - PlanEntryBuildInputs, - PlannedSqlTests, - PlannerChangeResults, - PlannerModelEntryResults, - PlannerRelationsContext, - PlannerResolvedActions, - PlannerScope, - PlanOutput, - PlanOutputExtras, - WarehouseSnapshot, -) - - -def build_plan_output_from_model_changes_phase( - *, - project: CompiledProject, - adapter: BaseAdapter, - connection: Any, - scope: PlannerScope, - snapshot: WarehouseSnapshot, - model_changes: dict[str, ChangeDetectionResult], - inputs: ModelChangesPlanInputs | None = None, -) -> PlanOutput: - resolved: ModelChangesPlanInputs = inputs if inputs is not None else ModelChangesPlanInputs() - relations: PlannerRelationsContext = build_planner_relations_context( - project=project, - adapter=adapter, - connection=connection, - scope=scope, - deferral=DeferralInputs( - deferred_locations=resolved.deferred_locations, - defer_sources_to=resolved.defer_sources_to, - source_deferral_enabled=resolved.source_deferral_enabled, - ), - project_config=resolved.project_config, - local_config=resolved.local_config, - ) - detected_changes: PlannerChangeResults = detect_changes( - project=project, - scope=scope, - snapshot=snapshot, - full_refresh=resolved.full_refresh, - ) - changes: PlannerChangeResults = replace(detected_changes, models=model_changes) - resolved_actions: PlannerResolvedActions = resolve_cascades( - scope=scope, - changes=changes, - ) - model_entry_results: PlannerModelEntryResults = build_plan_entries( - project=project, - adapter=adapter, - scope=scope, - snapshot=snapshot, - relations=relations, - resolved_actions=resolved_actions, - cursor_overrides=resolved.cursor_overrides, - full_refresh=resolved.full_refresh, - build_inputs=PlanEntryBuildInputs( - future_cursor_config=( - resolved.project_config.cursors.future - if resolved.project_config is not None - else None - ), - start_cursor_config=( - resolved.project_config.cursors.start - if resolved.project_config is not None - else None - ), - max_microbatches=( - resolved.max_microbatches - if resolved.max_microbatches is not None - else ( - resolved.project_config.microbatches.limits.max_batches - if resolved.project_config is not None - else None - ) - ), - max_microbatches_is_override=resolved.max_microbatches is not None, - microbatch_limit_action=( - resolved.project_config.microbatches.limits.action - if resolved.project_config is not None - else PlanEntryBuildInputs().microbatch_limit_action - ), - ), - ) - return build_plan_output( - project=project, - adapter=adapter, - scope=scope, - snapshot=snapshot, - relations=relations, - changes=changes, - model_entry_results=model_entry_results, - reload_sources=resolved.reload_sources, - extras=PlanOutputExtras( - seed_version_hashes=resolved.seed_version_hashes, - seed_metadata_jsons=resolved.seed_metadata_jsons, - seed_plan_reasons=resolved.seed_plan_reasons, - planned_sql_tests=( - None - if resolved.plan_sql_tests - else PlannedSqlTests(selected_keys=scope.selected_keys) - ), - ), - ) diff --git a/src/sqlbuild/compiler/planner/main/execution/warehouse_snapshot.py b/src/sqlbuild/compiler/planner/main/execution/warehouse_snapshot.py deleted file mode 100644 index f6de33f1b9..0000000000 --- a/src/sqlbuild/compiler/planner/main/execution/warehouse_snapshot.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Public warehouse discovery planner phase entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation -from sqlbuild.compiler.planner._helpers.graph.scope import build_planner_scope -from sqlbuild.compiler.planner._helpers.warehouse.snapshot import build_warehouse_snapshot -from sqlbuild.compiler.planner.models import ( - PlannerScope, - PlannerWarehouseSnapshotResult, - WarehouseSnapshot, -) - - -def build_warehouse_snapshot_phase( - *, - project: CompiledProject, - adapter: BaseAdapter, - connection: Any, - select: tuple[str, ...] = (), - exclude: tuple[str, ...] = (), - auto_load_sources: bool = False, - full_refresh: bool = False, - on_progress: Callable[[str], None] | None = None, - deferred_locations: dict[str, CompiledRelationLocation] | None = None, - deferred_relations: dict[str, RelationInfo] | None = None, -) -> PlannerWarehouseSnapshotResult: - scope: PlannerScope = build_planner_scope( - project=project, - select=select, - exclude=exclude, - auto_load_sources=auto_load_sources, - ) - snapshot: WarehouseSnapshot = build_warehouse_snapshot( - project=project, - adapter=adapter, - connection=connection, - scope=scope, - full_refresh=full_refresh, - on_progress=on_progress, - deferred_locations=deferred_locations, - deferred_relations=deferred_relations, - ) - return PlannerWarehouseSnapshotResult(scope=scope, snapshot=snapshot) diff --git a/src/sqlbuild/compiler/planner/main/identity/seed_identity.py b/src/sqlbuild/compiler/planner/main/identity/seed_identity.py deleted file mode 100644 index 4903c1ddbd..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/seed_identity.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public entrypoint for seed version identity hashing.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledSeed -from sqlbuild.compiler.planner._helpers.identity.seed import ( - build_seed_identity as _build_seed_identity, -) - - -def build_seed_identity(seed: CompiledSeed) -> tuple[str, str]: - """Return ``(identity_hash, metadata_json)`` for one compiled seed.""" - - return _build_seed_identity(seed) diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_local_hash.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_local_hash.py deleted file mode 100644 index 29a3a9eb14..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_local_hash.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public entrypoint for local model version identity hashing.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - build_model_local_identity_hash as _build_model_local_identity_hash, -) - - -def build_model_local_identity_hash(*, query_sql: str, metadata_json: str) -> str: - """Build a model's local identity hash from query SQL and non-query metadata.""" - - return _build_model_local_identity_hash(query_sql=query_sql, metadata_json=metadata_json) diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_stale_model_names.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_stale_model_names.py deleted file mode 100644 index 75438332cd..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_stale_model_names.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public wrapper for expected-vs-built model version staleness.""" - -from __future__ import annotations - -from collections.abc import Mapping - -from sqlbuild.compiler.planner._helpers.pruning.version_staleness import ( - build_stale_model_names_from_version_identities, -) - - -def build_version_identity_stale_model_names( - *, - model_names: tuple[str, ...], - expected_version_hashes: Mapping[str, str], - built_version_hashes: Mapping[str, str | None], - forced_stale_model_names: tuple[str, ...] = (), -) -> tuple[str, ...]: - return build_stale_model_names_from_version_identities( - model_names=model_names, - expected_version_hashes=expected_version_hashes, - built_version_hashes=built_version_hashes, - forced_stale_model_names=forced_stale_model_names, - ) diff --git a/src/sqlbuild/compiler/planner/main/identity/version_identity_version_hash.py b/src/sqlbuild/compiler/planner/main/identity/version_identity_version_hash.py deleted file mode 100644 index c51f3984d1..0000000000 --- a/src/sqlbuild/compiler/planner/main/identity/version_identity_version_hash.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Public entrypoint for composed model version identity hashing.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.identity.hashing import ( - build_model_version_identity_hash as _build_model_version_identity_hash, -) - - -def build_model_version_identity_hash( - *, - local_hash: str, - upstream_deps: tuple[CompiledObjectKey, ...], - upstream_version_hashes: dict[str, str], - source_version_hashes: dict[str, str] | None = None, -) -> str: - """Build a composed model version hash from local and upstream identities.""" - - return _build_model_version_identity_hash( - local_hash=local_hash, - upstream_deps=upstream_deps, - upstream_version_hashes=upstream_version_hashes, - source_version_hashes=source_version_hashes, - ) diff --git a/src/sqlbuild/compiler/planner/main/selection/build_resources.py b/src/sqlbuild/compiler/planner/main/selection/_build_resources.py similarity index 93% rename from src/sqlbuild/compiler/planner/main/selection/build_resources.py rename to src/sqlbuild/compiler/planner/main/selection/_build_resources.py index d63c5b39c8..c8f1499525 100644 --- a/src/sqlbuild/compiler/planner/main/selection/build_resources.py +++ b/src/sqlbuild/compiler/planner/main/selection/_build_resources.py @@ -1,4 +1,4 @@ -"""Public required-build-resource expansion entrypoint.""" +"""Compiler-internal required-build-resource expansion entrypoint.""" from __future__ import annotations diff --git a/src/sqlbuild/compiler/planner/main/selection/model_downstream_closure.py b/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py similarity index 88% rename from src/sqlbuild/compiler/planner/main/selection/model_downstream_closure.py rename to src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py index f9745944b1..ce4bd03d3d 100644 --- a/src/sqlbuild/compiler/planner/main/selection/model_downstream_closure.py +++ b/src/sqlbuild/compiler/planner/main/selection/_model_downstream_closure.py @@ -1,4 +1,4 @@ -"""Public wrapper for downstream model-name closure.""" +"""Compiler-internal downstream model-name closure entrypoint.""" from __future__ import annotations diff --git a/src/sqlbuild/compiler/planner/main/selection/model_upstream_closure.py b/src/sqlbuild/compiler/planner/main/selection/model_upstream_closure.py deleted file mode 100644 index f653123179..0000000000 --- a/src/sqlbuild/compiler/planner/main/selection/model_upstream_closure.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Public wrapper for upstream model-name closure.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.planner._helpers.graph.model_closure import build_upstream_model_name_closure - - -def build_upstream_model_names( - *, - start_keys: tuple[CompiledObjectKey, ...], - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]], -) -> frozenset[str]: - return build_upstream_model_name_closure( - start_keys=start_keys, - upstream_deps=upstream_deps, - ) diff --git a/src/sqlbuild/compiler/planner/models.py b/src/sqlbuild/compiler/planner/models.py index 8d1b1ec04c..8b9b6afd18 100644 --- a/src/sqlbuild/compiler/planner/models.py +++ b/src/sqlbuild/compiler/planner/models.py @@ -51,7 +51,6 @@ PlanReason, RetentionDirection, RetentionPlanPhase, - RunDespiteUnchangedMode, ScenarioArtifactKind, SchemaActionKind, SchemaChangeKind, @@ -677,27 +676,6 @@ class BackfillResult: duration: str | None = None -@dataclass(frozen=True) -class RunDespiteUnchangedDecision: - """One model-level changes-only override decision.""" - - model_name: str - mode: RunDespiteUnchangedMode - duration: str | None = None - newest_source_name: str | None = None - newest_source_data_age_seconds: int | None = None - - -@dataclass(frozen=True) -class RunDespiteUnchangedPlanningResult: - """Planner-time changes-only override roots and propagated stale models.""" - - root_model_names: frozenset[str] = frozenset() - stale_model_names: frozenset[str] = frozenset() - decisions: dict[str, RunDespiteUnchangedDecision] = field(default_factory=dict) - downstream_root_causes: dict[str, str] = field(default_factory=dict) - - @dataclass(frozen=True) class CascadeCause: """One upstream model that contributed to a backfill cascade.""" @@ -752,14 +730,6 @@ class PlannerScope: user_selected_keys: frozenset[CompiledObjectKey] = frozenset() -@dataclass(frozen=True) -class PlannerWarehouseSnapshotResult: - """Warehouse discovery phase output with its resolved planning scope.""" - - scope: PlannerScope - snapshot: WarehouseSnapshot - - @dataclass(frozen=True) class PlannerRelationsContext: """Resolved relation and source inputs for plan entry construction.""" @@ -799,30 +769,10 @@ class CursorOverridePair: end_cursor_override: str | None = None -@dataclass(frozen=True) -class ModelChangesPlanInputs: - """Optional planning inputs for building a plan output from model changes.""" - - cursor_overrides: CursorOverrides | None = None - full_refresh: bool = False - reload_sources: bool = False - deferred_locations: dict[str, CompiledRelationLocation] | None = None - project_config: ProjectConfig | None = None - local_config: LocalConfig | None = None - defer_sources_to: str | None = None - source_deferral_enabled: bool = True - seed_version_hashes: dict[str, str] | None = None - seed_metadata_jsons: dict[str, str] | None = None - seed_plan_reasons: dict[str, PlanReason] | None = None - max_microbatches: int | None = None - plan_sql_tests: bool = True - - @dataclass(frozen=True) class PlanEntryBuildInputs: """Blocked models and cursor overrides for plan entry building.""" - run_despite_unchanged: RunDespiteUnchangedPlanningResult | None = None source_freshness_blocked_model_names: frozenset[str] = frozenset() external_blocked_model_names: frozenset[str] = frozenset() start_cursor_override: str | None = None @@ -1086,7 +1036,6 @@ class ModelPlanEntry: custom_materialization_name: str | None = None custom_config: dict[str, object] = field(default_factory=dict) custom_placeholders: dict[str, str] = field(default_factory=dict) - run_despite_unchanged: RunDespiteUnchangedDecision | None = None @dataclass(frozen=True) @@ -1549,7 +1498,6 @@ class PlannerScopePruningResult: resolved_actions: PlannerResolvedActions pruned_direct_model_names: tuple[str, ...] direct_identity_stale_model_names: frozenset[str] - run_despite_unchanged: RunDespiteUnchangedPlanningResult @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/planner/types.py b/src/sqlbuild/compiler/planner/types.py index 5dd189cf1e..b41eae84bb 100644 --- a/src/sqlbuild/compiler/planner/types.py +++ b/src/sqlbuild/compiler/planner/types.py @@ -44,11 +44,6 @@ class SelectorKind(StrEnum): PATH = "path" -class WorkSelectionPolicy(StrEnum): - ALL_SELECTED = "all_selected" - STALE_ONLY = "stale_only" - - class GraphResourceKind(StrEnum): MODEL = "model" SEED = "seed" @@ -62,7 +57,6 @@ class ChangeKind(StrEnum): QUERY_CHANGED = "query_changed" CONFIG_CHANGED = "config_changed" SCHEMA_CHANGED = "schema_changed" - RUN_DESPITE_UNCHANGED = "run_despite_unchanged" NO_CHANGE = "no_change" @@ -130,7 +124,6 @@ class PlanReason(StrEnum): FUNCTION_CHANGED = "function_changed" CONFIG_CHANGED = "config_changed" SCHEMA_CHANGED = "schema_changed" - RUN_DESPITE_UNCHANGED = "run_despite_unchanged" UPSTREAM_CHANGED = "upstream_changed" NORMAL_INCREMENTAL = "normal_incremental" NO_CHANGE = "no_change" @@ -139,11 +132,6 @@ class PlanReason(StrEnum): EXTERNAL_UPSTREAM_FAILED = "external_upstream_failed" -class RunDespiteUnchangedMode(StrEnum): - ALWAYS = "always" - DURATION = "duration" - - class IncrementalStrategy(StrEnum): APPEND = "append" DELETE_INSERT = "delete_insert" diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py index 11d76c279e..eda7d19cca 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/unified_selectors.py @@ -11,7 +11,9 @@ from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.planner.constants import PATH_SELECTOR_EXPLICIT_ROOT_ERROR from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.build_resources import expand_build_resource_selection +from sqlbuild.compiler.planner.main.selection._build_resources import ( + expand_build_resource_selection, +) from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors from sqlbuild.compiler.planner.main.selection.selector_parse import parse_project_selector from sqlbuild.compiler.planner.models import ParsedSelector, PathSelector diff --git a/src/sqlbuild/compiler/python_nodes/main/run_selection_graph.py b/src/sqlbuild/compiler/python_nodes/main/run_selection_graph.py deleted file mode 100644 index 6dbb326159..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/run_selection_graph.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Run-command Python-node selection entrypoint for prebuilt graphs.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.run_selectors import resolve_python_sql_run_selectors -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlRunSelection - - -def resolve_python_sql_run_selection_from_graph( - *, - select: tuple[str, ...], - exclude: tuple[str, ...], - project_graph: ProjectGraph, - python_graph: PythonNodeGraph, -) -> PythonSqlRunSelection: - """Resolve run selectors with an already-built Python-node graph.""" - - return resolve_python_sql_run_selectors( - select=select, - exclude=exclude, - project_graph=project_graph, - python_graph=python_graph, - ) diff --git a/src/sqlbuild/compiler/python_nodes/main/selection_validation.py b/src/sqlbuild/compiler/python_nodes/main/selection_validation.py deleted file mode 100644 index 12a1bbbcd2..0000000000 --- a/src/sqlbuild/compiler/python_nodes/main/selection_validation.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public entrypoint for SQL/Python selection validation.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.python_nodes._helpers.unified_selectors import ( - validate_python_sql_selection_dependencies, -) -from sqlbuild.compiler.python_nodes.models import PythonNodeGraph, PythonSqlSelection - - -def validate_python_sql_selection( - *, selection: PythonSqlSelection, project_graph: ProjectGraph, python_graph: PythonNodeGraph -) -> None: - """Validate a selected SQL/Python graph boundary.""" - - validate_python_sql_selection_dependencies( - selection=selection, - project_graph=project_graph, - python_graph=python_graph, - ) diff --git a/src/sqlbuild/compiler/compile/main/function_node_type.py b/src/sqlbuild/compiler/resource_names/main/function_node_type.py similarity index 100% rename from src/sqlbuild/compiler/compile/main/function_node_type.py rename to src/sqlbuild/compiler/resource_names/main/function_node_type.py diff --git a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py index 8a8c3c3465..da64eb1d77 100644 --- a/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py +++ b/src/sqlbuild/compiler/source_freshness/_helpers/propagation.py @@ -4,7 +4,7 @@ from sqlbuild.compiler.compile.models import CompiledObjectKey from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.main.selection.model_downstream_closure import ( +from sqlbuild.compiler.planner.main.selection._model_downstream_closure import ( build_downstream_model_names, ) from sqlbuild.compiler.planner.models import PlannerScope diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/identifier_case.py b/src/sqlbuild/compiler/sql_analysis/_helpers/identifier_case.py new file mode 100644 index 0000000000..e4f1ad0d6c --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/identifier_case.py @@ -0,0 +1,16 @@ +"""Effective target identifier semantics for native schema validation.""" + +_SNOWFLAKE: str = "snowflake" +_PARAMETER: str = "QUOTED_IDENTIFIERS_IGNORE_CASE" + + +def ignores_quoted_case(*, connection: dict[str, object], dialect: str | None) -> bool: + """Read only an explicitly enabled setting from the resolved connection.""" + if dialect != _SNOWFLAKE: + return False + parameters: object = connection.get("session_parameters") + if not isinstance(parameters, dict): + return False + return any( + str(key).upper() == _PARAMETER and value is True for key, value in parameters.items() + ) diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot_normalization.py b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot_normalization.py deleted file mode 100644 index 12dd8a3202..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot_normalization.py +++ /dev/null @@ -1,155 +0,0 @@ -"""Dialect compatibility normalization for Polyglot analysis input.""" - -from __future__ import annotations - -import re - -from sqlbuild.compiler.sql_analysis._helpers.scanning import ( - skip_block_comment_impl, - skip_line_comment_impl, - skip_quoted_text_impl, -) -from sqlbuild.compiler.sql_analysis.constants import ( - SNOWFLAKE_DIALECT_NAME, - SQL_CLOSE_BRACKET, - SQL_DOLLAR_QUOTE_DELIMITER, - SQL_ESCAPE_CHARACTER, - SQL_OPEN_BRACKET, - SQL_QUOTE_CHARACTERS, - SQL_STRING_QUOTE_CHARACTER, -) - -_IDENTIFIER: str = r'(?:[A-Za-z_][A-Za-z0-9_$]*|"(?:""|[^"])+")' -_VARIANT_PATH_SUFFIX: re.Pattern[str] = re.compile( - rf"(?P{_IDENTIFIER}(?:\.{_IDENTIFIER})*:{_IDENTIFIER}" - rf"(?:(?::|\.){_IDENTIFIER})*)$" -) -_SIMPLE_BRACKET_KEY: re.Pattern[str] = re.compile(rf"\s*(?:{_IDENTIFIER}|[0-9]+|'(?:''|[^'])*')\s*") -_SCAN_SPECIAL: re.Pattern[str] = re.compile(r"[-/$'\"`\[]") - - -def normalize_sql_for_polyglot_impl(*, sql: str, dialect: str | None) -> str: - """Return analysis-equivalent SQL for Polyglot's supported dialect surface.""" - - if dialect is None or dialect.casefold() != SNOWFLAKE_DIALECT_NAME: - return sql - replacements: list[tuple[int, int, str]] = [] - index: int = 0 - while index < len(sql): - special: re.Match[str] | None = _SCAN_SPECIAL.search(sql, index) - if special is None: - break - index = special.start() - if sql.startswith("--", index): - index = skip_line_comment_impl(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment_impl( - sql=sql, - start=index, - context="Snowflake SQL analysis", - ) - continue - if sql.startswith(SQL_DOLLAR_QUOTE_DELIMITER, index): - index = _skip_dollar_quoted_text(sql=sql, start=index) - continue - if sql[index] in SQL_QUOTE_CHARACTERS: - index = _skip_snowflake_quoted_text(sql=sql, start=index) - continue - if sql[index] != SQL_OPEN_BRACKET: - index += 1 - continue - close_bracket: int | None = _matching_bracket(sql=sql, open_bracket=index) - if close_bracket is None: - index += 1 - continue - key_sql: str = sql[index + 1 : close_bracket] - if _SIMPLE_BRACKET_KEY.fullmatch(key_sql) is not None: - index = close_bracket + 1 - continue - normalized_key: str = normalize_sql_for_polyglot_impl(sql=key_sql, dialect=dialect) - base_end: int = _trailing_trivia_start(sql=sql, end=index) - base_match: re.Match[str] | None = _VARIANT_PATH_SUFFIX.search(sql[:base_end]) - if base_match is not None: - base_sql: str = base_match.group("base") - trivia: str = sql[base_end:index] - replacements.append( - ( - base_match.start("base"), - close_bracket + 1, - f"GET({base_sql}{trivia}, {normalized_key})", - ) - ) - elif normalized_key != key_sql: - replacements.append((index + 1, close_bracket, normalized_key)) - index = close_bracket + 1 - normalized: str = sql - for start, end, replacement in reversed(replacements): - normalized = f"{normalized[:start]}{replacement}{normalized[end:]}" - return normalized - - -def _matching_bracket(*, sql: str, open_bracket: int) -> int | None: - depth: int = 1 - index: int = open_bracket + 1 - while index < len(sql): - if sql.startswith("--", index): - index = skip_line_comment_impl(sql=sql, start=index) - continue - if sql.startswith("/*", index): - index = skip_block_comment_impl( - sql=sql, - start=index, - context="Snowflake SQL analysis", - ) - continue - if sql.startswith(SQL_DOLLAR_QUOTE_DELIMITER, index): - index = _skip_dollar_quoted_text(sql=sql, start=index) - continue - if sql[index] in SQL_QUOTE_CHARACTERS: - index = _skip_snowflake_quoted_text(sql=sql, start=index) - continue - if sql[index] == SQL_OPEN_BRACKET: - depth += 1 - elif sql[index] == SQL_CLOSE_BRACKET: - depth -= 1 - if depth == 0: - return index - index += 1 - return None - - -def _skip_snowflake_quoted_text(*, sql: str, start: int) -> int: - if sql[start] != SQL_STRING_QUOTE_CHARACTER: - return skip_quoted_text_impl(sql=sql, start=start, context="Snowflake SQL analysis") - index: int = start + 1 - while index < len(sql): - if sql[index] == SQL_ESCAPE_CHARACTER: - index += 2 - continue - if sql[index] == SQL_STRING_QUOTE_CHARACTER: - if index + 1 < len(sql) and sql[index + 1] == SQL_STRING_QUOTE_CHARACTER: - index += 2 - continue - return index + 1 - index += 1 - return len(sql) - - -def _skip_dollar_quoted_text(*, sql: str, start: int) -> int: - closing: int = sql.find(SQL_DOLLAR_QUOTE_DELIMITER, start + len(SQL_DOLLAR_QUOTE_DELIMITER)) - return len(sql) if closing == -1 else closing + len(SQL_DOLLAR_QUOTE_DELIMITER) - - -def _trailing_trivia_start(*, sql: str, end: int) -> int: - current: int = end - while True: - trimmed: int = current - while trimmed > 0 and sql[trimmed - 1].isspace(): - trimmed -= 1 - if not sql[:trimmed].endswith("*/"): - return trimmed - opening: int = sql.rfind("/*", 0, trimmed - 2) - if opening == -1: - return trimmed - current = opening diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py b/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py index e37d8ac421..eab2c51cf8 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/schema_validation.py @@ -1,458 +1,116 @@ -"""Native Polyglot schema-validation implementation.""" - -from __future__ import annotations +"""Orchestrate compile-owned catalogs; native code owns schemas and diagnostics.""" import json -import re from typing import Any, cast import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog +from sqlbuild.compiler.sql_analysis.constants import BINDING_SEVERITIES, NATIVE_DIALECT_ALIASES from sqlbuild.compiler.sql_analysis.exceptions import SqlAnalysisBoundaryError -from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql from sqlbuild.compiler.sql_analysis.models import ( SqlBindingDiagnostic, SqlBindingResult, SqlSchemaValidationRequest, ) -from sqlbuild.compiler.sql_analysis.types import NativeValidationModule - -_SQLBUILD_CODE_BY_NATIVE_CODE: dict[str, str] = { - "E200": "B000", - "E201": "B002", - "E221": "B003", - "E222": "B004", - "E223": "B005", -} -_ERROR_SEVERITY: str = "error" -_NATIVE_UNKNOWN_COLUMN_CODE: str = "E201" -_NATIVE_AMBIGUOUS_COLUMN_CODE: str = "E221" -_ORDER_BY_CONTEXT: str = "ORDER BY" -_WINDOW_ORDER_BY_CONTEXT: str = "WINDOW ORDER BY" -_SNOWFLAKE_DIALECT: str = "snowflake" -_SELECT_KIND: str = "select" -_IMPLICIT_VALUES_COLUMN_PATTERN: re.Pattern[str] = re.compile( - r"^Unknown column '(column([1-9][0-9]*))'$", - flags=re.IGNORECASE, -) -_UNKNOWN_COLUMN_PATTERN: re.Pattern[str] = re.compile( - r"^Unknown column '([^']+)'(?: in table '[^']+'| \(not found in any referenced table\))$" -) -_AMBIGUOUS_UNQUALIFIED_COLUMN_PATTERN: re.Pattern[str] = re.compile( - r"^Ambiguous unqualified column '([^']+)' found in [0-9]+ referenced tables$" +from sqlbuild.compiler.sql_analysis.types import ( + NativeDiagnosticRow, + NativePositionsModule, + NativeValidationModule, ) def get_schema_validations( *, requests: tuple[SqlSchemaValidationRequest, ...] ) -> tuple[SqlBindingResult, ...]: - """Validate a SQL batch without exposing Polyglot payloads.""" - - payloads: list[dict[str, object]] = [_request_payload(request=request) for request in requests] - responses: Any = json.loads( - cast(NativeValidationModule, _native).validate_sql_with_schemas_json( - json.dumps(payloads, separators=(",", ":"), sort_keys=True) + if not requests: + return () + catalogs: dict[tuple[str, bool, tuple[str, ...], tuple[str, ...]], BindingCatalog] = {} + groups: dict[BindingCatalog, list[tuple[int, SqlSchemaValidationRequest]]] = {} + for index, request in enumerate(requests): + dialect: str = NATIVE_DIALECT_ALIASES.get( + request.dialect or "generic", request.dialect or "generic" ) - ) - if not isinstance(responses, list) or len(responses) != len(requests): - raise SqlAnalysisBoundaryError( - "native SQL schema validation returned an invalid batch response" + key: tuple[str, bool, tuple[str, ...], tuple[str, ...]] = ( + dialect, + request.quoted_identifiers_ignore_case, + request.known_functions, + request.known_types, ) - return tuple( - _binding_result(sql=request.sql, dialect=request.dialect, response=response) - for request, response in zip(requests, responses, strict=True) - ) - + catalog: BindingCatalog | None = request.catalog or catalogs.get(key) + if catalog is None: + catalog = BindingCatalog( + dialect=dialect, + quoted_ignore_case=request.quoted_identifiers_ignore_case, + known_functions=request.known_functions, + known_types=request.known_types, + relations=request.schema, + ) + catalogs[key] = catalog + groups.setdefault(catalog, []).append((index, request)) + results: list[SqlBindingResult] = [SqlBindingResult()] * len(requests) + for catalog, group in groups.items(): + batch: list[SqlSchemaValidationRequest] = [request for _, request in group] + if all(request.catalog is None for request in batch): + payload: str = catalog.native.validation_payload(catalog.prepare(batch)) + responses: object = json.loads( + cast(NativeValidationModule, _native).validate_sql_with_schemas_json(payload) + ) + if not isinstance(responses, list) or len(responses) != len(batch): + raise SqlAnalysisBoundaryError( + "native SQL schema validation returned an invalid batch response" + ) + for (index, request), response in zip(group, responses, strict=True): + results[index] = _binding_result( + sql=request.sql, dialect=request.dialect, response=response + ) + continue + rows: list[list[NativeDiagnosticRow]] = catalog.native.binding_results( + catalog.prepare(batch) + ) + if len(rows) != len(batch): + raise SqlAnalysisBoundaryError( + "native SQL schema validation returned an invalid batch response" + ) + for (index, _), diagnostics in zip(group, rows, strict=True): + results[index] = _result(diagnostics) + return tuple(results) -def _request_payload(*, request: SqlSchemaValidationRequest) -> dict[str, object]: - tables: list[dict[str, object]] = [] - for table_name, columns in sorted(request.schema.items()): - table_columns: list[dict[str, str]] = [] - for column_name, column_type in columns.items(): - table_columns.append({"name": column_name, "type": column_type or "UNKNOWN"}) - tables.append({"name": table_name, "columns": table_columns}) - return { - "sql": request.sql, - "dialect": request.dialect or "generic", - "schema": { - "strict": True, - "tables": tables, - }, - "options": { - "check_types": False, - "check_references": True, - "strict": True, - "semantic": False, - "strict_syntax": False, - }, - } +def _result(rows: list[NativeDiagnosticRow]) -> SqlBindingResult: + return SqlBindingResult(diagnostics=tuple(SqlBindingDiagnostic(*row) for row in rows)) def _binding_result(*, sql: str, dialect: str | None, response: object) -> SqlBindingResult: - response_dict: dict[str, object] | None = ( - cast(dict[str, object], response) if isinstance(response, dict) else None - ) - errors: object = response_dict.get("errors") if response_dict is not None else None - if not isinstance(errors, list): + if not isinstance(response, dict): raise SqlAnalysisBoundaryError("native SQL schema validation returned an invalid response") - diagnostics: list[SqlBindingDiagnostic] = [] - for value in errors: - if not isinstance(value, dict): - continue - value_dict: dict[str, object] = cast(dict[str, object], value) - if value_dict.get("severity") != _ERROR_SEVERITY: + payload: dict[str, Any] = cast(dict[str, Any], response) + if not isinstance(payload.get("errors"), list): + raise SqlAnalysisBoundaryError("native SQL schema validation returned an invalid response") + rows: list[NativeDiagnosticRow] = [] + for value in payload["errors"]: + if not isinstance(value, dict) or value.get("severity") not in BINDING_SEVERITIES: continue - native_code: object = value_dict.get("code") - if not isinstance(native_code, str) or native_code not in _SQLBUILD_CODE_BY_NATIVE_CODE: + if not isinstance(value.get("code"), str): continue - raw_message: str = str(value_dict.get("message") or "SQL binding failed") - context: str | None = _diagnostic_context(sql=sql, message=raw_message) - message: str = f"{raw_message} (context: {context})" if context is not None else raw_message - line, column = _diagnostic_position( - sql=sql, - message=message, - line=_optional_int(value_dict.get("line")), - column=_optional_int(value_dict.get("column")), - ) - if _is_proven_snowflake_values_column( - sql=sql, - dialect=dialect, - message=raw_message, - line=line, - column=column, - ) or _is_proven_snowflake_output_alias( - sql=sql, - dialect=dialect, - native_code=native_code, - message=raw_message, - ): - continue - diagnostics.append( - SqlBindingDiagnostic( - code=_SQLBUILD_CODE_BY_NATIVE_CODE[native_code], - message=message, - line=line, - column=column, - start=_optional_int(value_dict.get("start")), - end=_optional_int(value_dict.get("end")), + rows.append( + ( + value["code"], + str(value.get("message") or "SQL binding failed"), + _optional_int(value.get("line")), + _optional_int(value.get("column")), + _optional_int(value.get("start")), + _optional_int(value.get("end")), + str(value["severity"]), ) ) - return SqlBindingResult(diagnostics=tuple(diagnostics)) - - -def _is_proven_snowflake_values_column( - *, - sql: str, - dialect: str | None, - message: str, - line: int | None, - column: int | None, -) -> bool: - if (dialect or "").lower() != _SNOWFLAKE_DIALECT or line is None or column is None: - return False - match: re.Match[str] | None = _IMPLICIT_VALUES_COLUMN_PATTERN.fullmatch(message) - if match is None: - return False - column_name: str = match.group(1) - ordinal: int = int(match.group(2)) - polyglot_module: Any = import_polyglot_sql() - try: - parsed: Any = polyglot_module.parse_one(sql, dialect=_SNOWFLAKE_DIALECT) - except polyglot_module.PolyglotError: - return False - selects: list[Any] = [] - if str(getattr(parsed, "kind", "")) == _SELECT_KIND: - selects.append(parsed) - selects.extend(parsed.find_all(_SELECT_KIND)) - return any( - _select_proves_implicit_values_column( - select=select, - sql=sql, - column_name=column_name, - ordinal=ordinal, - line=line, - column=column, - ) - for select in selects - ) - - -def _select_proves_implicit_values_column( - *, - select: Any, - sql: str, - column_name: str, - ordinal: int, - line: int, - column: int, -) -> bool: - payload: object = select.to_dict().get(_SELECT_KIND) - if not isinstance(payload, dict): - return False - select_payload: dict[str, object] = cast(dict[str, object], payload) - if select_payload.get("joins"): - return False - from_payload: object = select_payload.get("from") - if not isinstance(from_payload, dict): - return False - relations: object = cast(dict[str, object], from_payload).get("expressions") - if not isinstance(relations, list) or len(relations) != 1: - return False - relation: object = relations[0] - if not isinstance(relation, dict): - return False - values: object = cast(dict[str, object], relation).get("values") - if not isinstance(values, dict): - return False - values_payload: dict[str, object] = cast(dict[str, object], values) - if values_payload.get("alias") is not None or values_payload.get("column_aliases"): - return False - rows: object = values_payload.get("expressions") - if not isinstance(rows, list) or not rows: - return False - row_widths: list[int] = [] - for row in rows: - if not isinstance(row, dict): - return False - expressions: object = cast(dict[str, object], row).get("expressions") - if not isinstance(expressions, list): - return False - row_widths.append(len(expressions)) - if any(width < ordinal for width in row_widths): - return False - projections: object = select_payload.get("expressions") - if not isinstance(projections, list): - return False - return any( - _projection_is_implicit_values_column( - projection=projection, + return _result( + cast(NativePositionsModule, _native).binding_diagnostics( sql=sql, - column_name=column_name, - line=line, - column=column, + dialect=NATIVE_DIALECT_ALIASES.get(dialect or "generic", dialect or "generic"), + rows=rows, ) - for projection in projections - ) - - -def _projection_is_implicit_values_column( - *, projection: object, sql: str, column_name: str, line: int, column: int -) -> bool: - if not isinstance(projection, dict): - return False - expression: object = projection - alias_payload: object = cast(dict[str, object], projection).get("alias") - if isinstance(alias_payload, dict): - expression = cast(dict[str, object], alias_payload).get("this") - if not isinstance(expression, dict): - return False - column_payload: object = cast(dict[str, object], expression).get("column") - if not isinstance(column_payload, dict): - return False - column_dict: dict[str, object] = cast(dict[str, object], column_payload) - if column_dict.get("table") is not None: - return False - name_payload: object = column_dict.get("name") - if not isinstance(name_payload, dict): - return False - name_dict: dict[str, object] = cast(dict[str, object], name_payload) - if ( - name_dict.get("quoted") is True - or str(name_dict.get("name") or "").lower() != column_name.lower() - ): - return False - span: object = column_dict.get("span") - if not isinstance(span, dict): - return False - start: object = cast(dict[str, object], span).get("start") - if not isinstance(start, int): - return False - candidate_line: int = sql.count("\n", 0, start) + 1 - candidate_column: int = start - sql.rfind("\n", 0, start) - return (candidate_line, candidate_column) == (line, column) - - -def _is_proven_snowflake_output_alias( - *, sql: str, dialect: str | None, native_code: str, message: str -) -> bool: - if (dialect or "").lower() != _SNOWFLAKE_DIALECT: - return False - column_name: str | None = _output_alias_diagnostic_column( - native_code=native_code, - message=message, - ) - if column_name is None: - return False - polyglot_module: Any = import_polyglot_sql() - try: - parsed: Any = polyglot_module.parse_one(sql, dialect=_SNOWFLAKE_DIALECT) - except polyglot_module.PolyglotError: - return False - selects: list[Any] = [] - if str(getattr(parsed, "kind", "")) == _SELECT_KIND: - selects.append(parsed) - selects.extend(parsed.find_all(_SELECT_KIND)) - return any( - _select_proves_output_alias( - select=select, - column_name=column_name, - native_code=native_code, - ) - for select in selects - ) - - -def _output_alias_diagnostic_column(*, native_code: str, message: str) -> str | None: - pattern: re.Pattern[str] - if native_code == _NATIVE_UNKNOWN_COLUMN_CODE: - pattern = _UNKNOWN_COLUMN_PATTERN - elif native_code == _NATIVE_AMBIGUOUS_COLUMN_CODE: - pattern = _AMBIGUOUS_UNQUALIFIED_COLUMN_PATTERN - else: - return None - match: re.Match[str] | None = pattern.fullmatch(message) - return match.group(1) if match is not None else None - - -def _select_proves_output_alias(*, select: Any, column_name: str, native_code: str) -> bool: - payload: object = select.to_dict().get(_SELECT_KIND) - if not isinstance(payload, dict): - return False - select_payload: dict[str, object] = cast(dict[str, object], payload) - projections: object = select_payload.get("expressions") - if not isinstance(projections, list): - return False - aliases: set[str] = set() - for projection in projections: - expression, alias = _projection_expression_and_alias(projection) - if native_code == _NATIVE_UNKNOWN_COLUMN_CODE and column_name.lower() in aliases: - if _payload_contains_unqualified_column( - payload=expression, - column_name=column_name, - ): - return True - if alias is not None: - aliases.add(alias.lower()) - if column_name.lower() not in aliases: - return False - if native_code == _NATIVE_UNKNOWN_COLUMN_CODE: - return _payload_contains_unqualified_column( - payload=select_payload.get("where_clause"), - column_name=column_name, - ) - return _payload_contains_unqualified_column( - payload=select_payload.get("order_by"), - column_name=column_name, - ) - - -def _projection_expression_and_alias(projection: object) -> tuple[object, str | None]: - if not isinstance(projection, dict): - return projection, None - alias_payload: object = cast(dict[str, object], projection).get("alias") - if not isinstance(alias_payload, dict): - return projection, None - alias_dict: dict[str, object] = cast(dict[str, object], alias_payload) - name_payload: object = alias_dict.get("alias") - if not isinstance(name_payload, dict): - return alias_dict.get("this"), None - name_dict: dict[str, object] = cast(dict[str, object], name_payload) - alias: str | None = ( - str(name_dict.get("name")) - if name_dict.get("quoted") is not True and name_dict.get("name") is not None - else None - ) - return alias_dict.get("this"), alias - - -def _payload_contains_unqualified_column(*, payload: object, column_name: str) -> bool: - if isinstance(payload, list | tuple): - return any( - _payload_contains_unqualified_column(payload=value, column_name=column_name) - for value in payload - ) - if not isinstance(payload, dict): - return False - payload_dict: dict[str, object] = cast(dict[str, object], payload) - column_payload: object = payload_dict.get("column") - if isinstance(column_payload, dict): - column_dict: dict[str, object] = cast(dict[str, object], column_payload) - name_payload: object = column_dict.get("name") - if isinstance(name_payload, dict): - name_dict: dict[str, object] = cast(dict[str, object], name_payload) - if ( - column_dict.get("table") is None - and name_dict.get("quoted") is not True - and str(name_dict.get("name") or "").lower() == column_name.lower() - ): - return True - return any( - key != _SELECT_KIND - and _payload_contains_unqualified_column(payload=value, column_name=column_name) - for key, value in payload_dict.items() ) def _optional_int(value: object) -> int | None: return value if isinstance(value, int) else None - - -def _diagnostic_position( - *, sql: str, message: str, line: int | None, column: int | None -) -> tuple[int | None, int | None]: - if line is not None and column is not None: - return line, column - match: re.Match[str] | None = re.search( - r"(?:Unknown column|Ambiguous column reference) '([^']+)'", message - ) - if match is None: - return line, column - identifier: str = match.group(1).rsplit(".", maxsplit=1)[-1] - occurrences: list[re.Match[str]] = list( - re.finditer(rf"(? str | None: - match: re.Match[str] | None = re.search( - r"(?:Unknown column|Ambiguous column reference) '([^']+)'", message - ) - if match is None: - return None - identifier: str = match.group(1).rsplit(".", maxsplit=1)[-1] - occurrence: re.Match[str] | None = re.search( - rf"(? latest[0]): - latest = matches[-1].start(), label - if latest is None: - return None - if latest[1] == _ORDER_BY_CONTEXT: - over_position: int = prefix.upper().rfind("OVER") - if over_position > latest[0] or ( - over_position >= 0 - and prefix[over_position:].count("(") > prefix[over_position:].count(")") - ): - return _WINDOW_ORDER_BY_CONTEXT - return latest[1] diff --git a/src/sqlbuild/compiler/sql_analysis/classes/binding_catalog.py b/src/sqlbuild/compiler/sql_analysis/classes/binding_catalog.py new file mode 100644 index 0000000000..0e0afc4255 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/classes/binding_catalog.py @@ -0,0 +1,108 @@ +"""Compile-lifetime ownership of the native binding catalog.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from copy import copy +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.constants import ( + CASE_SENSITIVE_BINDING_DIALECTS, + SQL_QUOTED_IDENTIFIER_DELIMITER, +) +from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest +from sqlbuild.compiler.sql_analysis.types import ( + NativeBindingRequest, + NativeCatalogModule, + NativeProjectCatalog, +) + + +class BindingCatalog: + def __init__( + self, + *, + dialect: str, + quoted_ignore_case: bool, + known_functions: tuple[str, ...], + known_types: tuple[str, ...], + relations: Mapping[str, Mapping[str, str]], + ) -> None: + self.schemas: dict[str, Mapping[str, str]] = dict(relations) + self.dialect: str = dialect + self.quoted_ignore_case: bool = quoted_ignore_case + self.analysis_shapes: dict[str, tuple[Mapping[str, str], Mapping[str, str]]] = {} + self.expression_shapes: dict[str, dict[str, str] | None] = {} + self.shared_analyses: dict[object, object] = {} + self.native: NativeProjectCatalog = cast(NativeCatalogModule, _native).ProjectCatalog( + { + "dialect": dialect, + "quoted_ignore_case": quoted_ignore_case, + "known_functions": known_functions, + "known_types": known_types, + "relations": relations, + } + ) + + def prepare_analysis( + self, + *, + types: Mapping[str, Mapping[str, str]], + nullability: Mapping[str, Mapping[str, str]], + ) -> None: + updates: dict[str, tuple[Mapping[str, str], Mapping[str, str]]] = {} + for name, columns in nullability.items(): + shape: tuple[Mapping[str, str], Mapping[str, str]] = (types.get(name, {}), columns) + if self.analysis_shapes.get(name) != shape: + updates[name] = shape + self.analysis_shapes[name] = shape + if updates: + self.native.update_analysis(updates) + + def with_relations(self, relations: Mapping[str, Mapping[str, str]]) -> BindingCatalog: + catalog: BindingCatalog = copy(self) + catalog.schemas = {**self.schemas, **relations} + catalog.analysis_shapes = dict(self.analysis_shapes) + catalog.expression_shapes = dict(self.expression_shapes) + catalog.native = self.native.with_relations(relations) + return catalog + + def inferred_schema( + self, *, sql: str, columns: dict[str, str], inputs: Mapping[str, Mapping[str, str]] + ) -> dict[str, str]: + if self.quoted_ignore_case or self.dialect not in CASE_SENSITIVE_BINDING_DIALECTS: + return columns + has_exact_input: bool = False + for shape in inputs.values(): + if any(SQL_QUOTED_IDENTIFIER_DELIMITER in name for name in shape): + has_exact_input = True + break + if SQL_QUOTED_IDENTIFIER_DELIMITER not in sql and not has_exact_input: + return columns + return self.native.inferred_schema(sql=sql, columns=columns, inputs=inputs) + + def prepare(self, requests: Sequence[SqlSchemaValidationRequest]) -> list[NativeBindingRequest]: + additions: dict[str, Mapping[str, str]] = {} + for request in requests: + for name, columns in request.schema.items(): + if columns and name not in self.schemas: + additions[name] = columns + self.schemas[name] = columns + if additions: + self.native.update_relations(additions) + prepared: list[NativeBindingRequest] = [] + for request in requests: + overrides: dict[str, Mapping[str, str]] = { + name: columns + for name, columns in request.schema.items() + if columns and self.schemas.get(name) != columns + } + prepared.append( + ( + request.sql, + [(name, bool(request.schema[name])) for name in sorted(request.schema)], + overrides, + ) + ) + return prepared diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 8bba4fab97..85b17433fc 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -164,3 +164,12 @@ POLYGLOT_ANALYSIS_TRANSFORM_DIRECT: str = "direct" POLYGLOT_ANALYSIS_TRANSFORM_STAR: str = "star" POLYGLOT_ANALYSIS_UNSAFE_TRANSFORMS: frozenset[str] = frozenset() +TYPE_CHECKED_DIALECTS: frozenset[str] = frozenset({"duckdb", "snowflake", "postgresql", "bigquery"}) +SQL_QUOTED_IDENTIFIER_DELIMITER: str = '"' +CASE_SENSITIVE_BINDING_DIALECTS: frozenset[str] = frozenset({"snowflake", "postgresql"}) +BINDING_SEVERITIES: frozenset[str] = frozenset({"error", "warning"}) +NATIVE_DIALECT_ALIASES: dict[str, str] = { + "postgres": "postgresql", + "motherduck": "duckdb", + "sqlserver": "tsql", +} diff --git a/src/sqlbuild/compiler/sql_analysis/main/_binding_catalog.py b/src/sqlbuild/compiler/sql_analysis/main/_binding_catalog.py new file mode 100644 index 0000000000..d6bcf39fba --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_binding_catalog.py @@ -0,0 +1,22 @@ +"""Create one native catalog for a compiled project's binding lifetime.""" + +from collections.abc import Mapping + +from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog + + +def create_binding_catalog( + *, + dialect: str, + quoted_ignore_case: bool, + known_functions: tuple[str, ...], + known_types: tuple[str, ...], + relations: Mapping[str, Mapping[str, str]], +) -> BindingCatalog: + return BindingCatalog( + dialect=dialect, + quoted_ignore_case=quoted_ignore_case, + known_functions=known_functions, + known_types=known_types, + relations=relations, + ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_identifier_case.py b/src/sqlbuild/compiler/sql_analysis/main/_identifier_case.py new file mode 100644 index 0000000000..556c9ed66c --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_identifier_case.py @@ -0,0 +1,10 @@ +"""Resolve the effective connection's quoted-identifier binding policy.""" + +from sqlbuild.compiler.sql_analysis._helpers.identifier_case import ( + ignores_quoted_case as _ignores_quoted_case, +) + + +def ignores_quoted_case(*, connection: dict[str, object], dialect: str | None) -> bool: + """Return whether quoted identifiers use the dialect's insensitive binding mode.""" + return _ignores_quoted_case(connection=connection, dialect=dialect) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py b/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py new file mode 100644 index 0000000000..d21014a46c --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis.py @@ -0,0 +1,23 @@ +"""Normalize analysis SQL using native offset-preserving transformations.""" + +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.types import NativePositionsModule + + +def normalize_analysis_sql( + *, + sql: str, + dialect: str | None, + stubs: dict[str, str] | None = None, + placeholders: dict[str, str] | None = None, +) -> str: + return cast(NativePositionsModule, _native).normalize_analysis_sql( + { + "sql": sql, + "dialect": dialect or "generic", + "stubs": stubs or {}, + "placeholders": placeholders or {}, + } + ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis_batch.py b/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis_batch.py new file mode 100644 index 0000000000..c6ebd32ad1 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_normalize_analysis_batch.py @@ -0,0 +1,21 @@ +"""Normalize a compile batch under one native GIL-release boundary.""" + +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.types import NativePositionsModule + + +def normalize_analysis_sqls( + *, + sqls: tuple[str, ...], + dialect: str | None, + placeholders: tuple[dict[str, str] | None, ...], +) -> list[str]: + requests: list[tuple[str, dict[str, str]]] = [ + (sql, defaults or {}) for sql, defaults in zip(sqls, placeholders, strict=True) + ] + return cast(NativePositionsModule, _native).normalize_analysis_sqls( + dialect=dialect or "generic", + requests=requests, + ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_normalize_for_polyglot.py b/src/sqlbuild/compiler/sql_analysis/main/_normalize_for_polyglot.py index 1a7261eacd..8252498b30 100644 --- a/src/sqlbuild/compiler/sql_analysis/main/_normalize_for_polyglot.py +++ b/src/sqlbuild/compiler/sql_analysis/main/_normalize_for_polyglot.py @@ -2,12 +2,15 @@ from __future__ import annotations -from sqlbuild.compiler.sql_analysis._helpers.polyglot_normalization import ( - normalize_sql_for_polyglot_impl, -) +from typing import cast + +import sqlbuild._native as _native +from sqlbuild.compiler.sql_analysis.types import NativePositionsModule def normalize_sql_for_polyglot(*, sql: str, dialect: str | None) -> str: """Return analysis-equivalent SQL for Polyglot's supported dialect surface.""" - return normalize_sql_for_polyglot_impl(sql=sql, dialect=dialect) + return cast(NativePositionsModule, _native).normalize_dialect_sql( + sql=sql, dialect=dialect or "generic" + ) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_prepare_schema_validation.py b/src/sqlbuild/compiler/sql_analysis/main/_prepare_schema_validation.py deleted file mode 100644 index b7cae89a6c..0000000000 --- a/src/sqlbuild/compiler/sql_analysis/main/_prepare_schema_validation.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Prepare shared native schema-validation inputs.""" - -from sqlbuild.compiler.sql_analysis._helpers.schema_validation import _request_payload -from sqlbuild.compiler.sql_analysis.models import SqlSchemaValidationRequest - - -def prepare_schema_validation(*, request: SqlSchemaValidationRequest) -> dict[str, object]: - """Prepare the schema-validation protocol for native compilation.""" - return _request_payload(request=request) diff --git a/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py b/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py new file mode 100644 index 0000000000..0165127490 --- /dev/null +++ b/src/sqlbuild/compiler/sql_analysis/main/_resolve_binding_column.py @@ -0,0 +1,28 @@ +"""Resolve schema column spellings using dialect identifier semantics.""" + +from collections.abc import Mapping + +from sqlbuild.compiler.sql_analysis.constants import ( + CASE_SENSITIVE_BINDING_DIALECTS, + NATIVE_DIALECT_ALIASES, + SNOWFLAKE_DIALECT_NAME, + SQL_QUOTED_IDENTIFIER_DELIMITER, +) + + +def resolve_binding_column( + *, name: str, columns: Mapping[str, str], dialect: str | None, ignore_quoted_case: bool = False +) -> str | None: + dialect = NATIVE_DIALECT_ALIASES.get(dialect or "generic", dialect) + + def identity(value: str) -> str: + quoted: bool = value.startswith(SQL_QUOTED_IDENTIFIER_DELIMITER) and value.endswith( + SQL_QUOTED_IDENTIFIER_DELIMITER + ) + raw: str = value[1:-1].replace('""', '"') if quoted else value + if quoted and dialect in CASE_SENSITIVE_BINDING_DIALECTS and not ignore_quoted_case: + return raw + return raw.upper() if dialect == SNOWFLAKE_DIALECT_NAME else raw.casefold() + + expected: str = identity(name) + return next((column for column in columns if identity(column) == expected), None) diff --git a/src/sqlbuild/compiler/sql_analysis/models.py b/src/sqlbuild/compiler/sql_analysis/models.py index 5eca31f593..c0f594b38e 100644 --- a/src/sqlbuild/compiler/sql_analysis/models.py +++ b/src/sqlbuild/compiler/sql_analysis/models.py @@ -3,12 +3,13 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Any @dataclass(frozen=True) class SqlBindingDiagnostic: - """One proven SQL binding failure returned by the native validator.""" + """One native binding error or non-blocking runtime-conversion warning.""" code: str message: str @@ -16,6 +17,7 @@ class SqlBindingDiagnostic: column: int | None = None start: int | None = None end: int | None = None + severity: str = "error" @dataclass(frozen=True) @@ -32,3 +34,7 @@ class SqlSchemaValidationRequest: sql: str dialect: str | None schema: Mapping[str, Mapping[str, str]] + known_functions: tuple[str, ...] = () + known_types: tuple[str, ...] = () + quoted_identifiers_ignore_case: bool = False + catalog: Any | None = field(default=None, repr=False, compare=False) diff --git a/src/sqlbuild/compiler/sql_analysis/types.py b/src/sqlbuild/compiler/sql_analysis/types.py index 59bc4a3e02..d30921b1f2 100644 --- a/src/sqlbuild/compiler/sql_analysis/types.py +++ b/src/sqlbuild/compiler/sql_analysis/types.py @@ -2,8 +2,59 @@ from __future__ import annotations +from collections.abc import Mapping from typing import Protocol +type NativeDiagnosticRow = tuple[str, str, int | None, int | None, int | None, int | None, str] +type NativeBindingRequest = tuple[str, list[tuple[str, bool]], dict[str, Mapping[str, str]]] + + +class NativeProjectCatalog(Protocol): + def inferred_schema( + self, *, sql: str, columns: Mapping[str, str], inputs: Mapping[str, Mapping[str, str]] + ) -> dict[str, str]: ... + def with_relations( + self, relations: Mapping[str, Mapping[str, str]] + ) -> NativeProjectCatalog: ... + def validation_payload(self, requests: list[NativeBindingRequest]) -> str: ... + def update_analysis( + self, relations: Mapping[str, tuple[Mapping[str, str], Mapping[str, str]]] + ) -> None: ... + def register_override(self, relations: Mapping[str, Mapping[str, str]]) -> int: ... + def analyze_compact(self, payload: bytes) -> bytes: ... + def update_relations(self, relations: Mapping[str, Mapping[str, str]]) -> None: ... + def binding_results( + self, requests: list[NativeBindingRequest] + ) -> list[list[NativeDiagnosticRow]]: ... + + +class NativeCatalogModule(Protocol): + def ProjectCatalog( + self, + request: dict[str, object], + ) -> NativeProjectCatalog: ... + + +class NativeBindingPositions(Protocol): + def position( + self, *, start: int | None, line: int | None, column: int | None, message: str + ) -> tuple[int | None, int | None]: ... + + +class NativePositionsModule(Protocol): + def normalize_analysis_sqls( + self, *, dialect: str, requests: list[tuple[str, dict[str, str]]] + ) -> list[str]: ... + def binding_diagnostics( + self, *, sql: str, dialect: str, rows: list[NativeDiagnosticRow] + ) -> list[NativeDiagnosticRow]: ... + def BindingPositions( + self, + request: dict[str, object], + ) -> NativeBindingPositions: ... + def normalize_analysis_sql(self, request: dict[str, object]) -> str: ... + def normalize_dialect_sql(self, *, sql: str, dialect: str) -> str: ... + class NativeValidationModule(Protocol): """Native schema-validation boundary.""" diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 51fd5091d0..f6f036dd21 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -1059,12 +1059,7 @@ def _execute_model_node( microbatch_event_store=microbatch_event_store, microbatch_event_store_resolver=microbatch_event_store_resolver, microbatch_scope=microbatch_scope, - microbatch_model_version_hash=( - microbatch_scope.virtual_model_version_hash - if microbatch_scope is not None - and microbatch_scope.virtual_model_version_hash is not None - else model_entry.fingerprint_version_hash - ), + microbatch_model_version_hash=model_entry.fingerprint_version_hash, microbatch_unaccounted_partition_policy=( model_entry.unaccounted_partition_policy or self._runtime.microbatch_unaccounted_partition_policy diff --git a/src/sqlbuild/executor/custom/models.py b/src/sqlbuild/executor/custom/models.py index 55e204f683..8b6111a2ed 100644 --- a/src/sqlbuild/executor/custom/models.py +++ b/src/sqlbuild/executor/custom/models.py @@ -83,63 +83,6 @@ def qualify_in_destination_schema(self, name: str) -> str: return self.qualify_name(name=name) -@dataclass(frozen=True) -class PrepareVersionContext: - """Context provided to optional custom prepare_version() functions.""" - - adapter: BaseAdapter - connection: Any - origin_relation: str - destination: str - destination_database: str | None - destination_schema: str | None - destination_name: str - config: dict[str, Any] - placeholders: dict[str, str] - run_id: str - environment: str - vars: dict[str, object] - unique_key: tuple[str, ...] - declared_columns: tuple[ColumnInfo, ...] - statement_recorder: StatementRecorder = field(default_factory=StatementRecorder) - - def execute_sql(self, sql: str) -> Any: - """Execute a SQL statement, recording it for runtime artifacts and verbose output.""" - return execute_sql_with_recording( - adapter=self.adapter, - connection=self.connection, - sql=sql, - statement_recorder=self.statement_recorder, - ) - - def log(self, message: str) -> None: - """Record a log message for verbose output.""" - self.statement_recorder.log(message) - - def qualify_name( - self, - *, - name: str, - database: str | None = None, - schema: str | None = None, - ) -> str: - """Return a fully-qualified relation name, preserving already-qualified input.""" - - return qualify_custom_relation( - adapter=self.adapter, - name=name, - destination_database=self.destination_database, - destination_schema=self.destination_schema, - database=database, - schema=schema, - ) - - def qualify_in_destination_schema(self, name: str) -> str: - """Return a relation name qualified into the destination database/schema.""" - - return self.qualify_name(name=name) - - @dataclass(frozen=True) class MaterializationResult: """Result returned by a custom materialize() function.""" diff --git a/src/sqlbuild/executor/functions/main/_execute.py b/src/sqlbuild/executor/functions/main/_execute.py index 600dcae888..2c07cd3bc8 100644 --- a/src/sqlbuild/executor/functions/main/_execute.py +++ b/src/sqlbuild/executor/functions/main/_execute.py @@ -9,12 +9,12 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import FunctionDefinition -from sqlbuild.compiler.compile.main.function_node_type import function_node_type from sqlbuild.compiler.compile.types import FunctionLanguage from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint from sqlbuild.compiler.planner.models import FunctionPlanEntry +from sqlbuild.compiler.resource_names.main.function_node_type import function_node_type from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.errors.contracts.main.error_code import error_code from sqlbuild.errors.contracts.main.error_help import error_help diff --git a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py index d0749b5dbd..074659d08c 100644 --- a/src/sqlbuild/executor/janitor/_helpers/archive_planning.py +++ b/src/sqlbuild/executor/janitor/_helpers/archive_planning.py @@ -45,7 +45,7 @@ def plan_janitor_archives( archive_retention_days: int, now: datetime, ) -> JanitorArchivePlanning: - """Plan direct-mode archives; virtual mode never archives or expires archives.""" + """Plan archives and archive expiry.""" if not direct_mode: return JanitorArchivePlanning( diff --git a/src/sqlbuild/executor/janitor/_helpers/classification.py b/src/sqlbuild/executor/janitor/_helpers/classification.py index eb2aa98a03..99413dc399 100644 --- a/src/sqlbuild/executor/janitor/_helpers/classification.py +++ b/src/sqlbuild/executor/janitor/_helpers/classification.py @@ -176,7 +176,7 @@ def classify_janitor_relations( retention_days: int, age_reader: JanitorRelationAgeReader, now: datetime, - direct_mode: bool = False, + direct_mode: bool = True, ) -> JanitorRelationClassification: """Split one schema's relations into delete candidates and skipped relations.""" @@ -264,7 +264,7 @@ def _relation_skip_reason( if relation_key in protected_relation_keys: return protection_reasons.get( relation_key, - "relation is referenced by a retained virtual checkpoint", + "relation is protected from cleanup", ) exclude_pattern: str | None = matching_exclude_pattern( key=relation_key, diff --git a/src/sqlbuild/executor/janitor/constants.py b/src/sqlbuild/executor/janitor/constants.py index 7fd9271740..506aa4c3d8 100644 --- a/src/sqlbuild/executor/janitor/constants.py +++ b/src/sqlbuild/executor/janitor/constants.py @@ -21,7 +21,6 @@ ) DIRECT_DEFAULT_RETENTION_DAYS: int = 14 -VIRTUAL_DEFAULT_RETENTION_DAYS: int = 30 ARCHIVE_NAME_PREFIX: str = "_SQB_ARCHIVE__" ARCHIVE_NAME_SEPARATOR: str = "__" diff --git a/src/sqlbuild/executor/janitor/main/execute.py b/src/sqlbuild/executor/janitor/main/execute.py index 58ad35c963..f12bb5b6a2 100644 --- a/src/sqlbuild/executor/janitor/main/execute.py +++ b/src/sqlbuild/executor/janitor/main/execute.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Callable from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -15,17 +14,11 @@ from sqlbuild.executor.janitor.models import ( JanitorArchiveCandidate, JanitorArchivedRelation, - JanitorCheckpointCandidate, JanitorDeleteCandidate, - JanitorDetachedVirtualEnvironmentCandidate, JanitorDirectStatePruneCandidate, JanitorExecutionResult, - JanitorExpiredLockCandidate, - JanitorExpiredVirtualEnvironmentCandidate, JanitorPlan, JanitorQueryDiffArtifactCandidate, - JanitorStateBackupCandidate, - JanitorVirtualStatePruneCandidate, ) from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -35,16 +28,6 @@ def execute_janitor_plan( plan: JanitorPlan, adapter: BaseAdapter, connection: Any, - delete_checkpoint: Callable[[JanitorCheckpointCandidate], None] | None = None, - delete_detached_virtual_environment: Callable[ - [JanitorDetachedVirtualEnvironmentCandidate], None - ] - | None = None, - delete_expired_virtual_environment: Callable[[JanitorExpiredVirtualEnvironmentCandidate], None] - | None = None, - delete_state_backup: Callable[[JanitorStateBackupCandidate], None] | None = None, - delete_expired_lock: Callable[[JanitorExpiredLockCandidate], None] | None = None, - prune_virtual_state: Callable[[JanitorVirtualStatePruneCandidate], object] | None = None, ) -> JanitorExecutionResult: """Delete all candidates in a janitor plan.""" @@ -57,12 +40,6 @@ def execute_janitor_plan( plan=plan, adapter=adapter, connection=connection, - delete_checkpoint=delete_checkpoint, - delete_detached_virtual_environment=delete_detached_virtual_environment, - delete_expired_virtual_environment=delete_expired_virtual_environment, - delete_state_backup=delete_state_backup, - delete_expired_lock=delete_expired_lock, - prune_virtual_state=prune_virtual_state, ) @@ -71,16 +48,6 @@ def _execute_janitor_plan( plan: JanitorPlan, adapter: BaseAdapter, connection: Any, - delete_checkpoint: Callable[[JanitorCheckpointCandidate], None] | None, - delete_detached_virtual_environment: Callable[ - [JanitorDetachedVirtualEnvironmentCandidate], None - ] - | None, - delete_expired_virtual_environment: Callable[[JanitorExpiredVirtualEnvironmentCandidate], None] - | None, - delete_state_backup: Callable[[JanitorStateBackupCandidate], None] | None, - delete_expired_lock: Callable[[JanitorExpiredLockCandidate], None] | None, - prune_virtual_state: Callable[[JanitorVirtualStatePruneCandidate], object] | None, ) -> JanitorExecutionResult: recorder: StatementRecorder = StatementRecorder() @@ -124,48 +91,18 @@ def _execute_janitor_plan( if_exists=True, statement_recorder=recorder, ) - deleted_checkpoints: tuple[JanitorCheckpointCandidate, ...] = apply_janitor_deletions( - candidates=plan.checkpoint_candidates, delete=delete_checkpoint - ) - deleted_detached_virtual_environments: tuple[ - JanitorDetachedVirtualEnvironmentCandidate, ... - ] = apply_janitor_deletions( - candidates=plan.detached_virtual_environment_candidates, - delete=delete_detached_virtual_environment, - ) - deleted_expired_virtual_environments: tuple[JanitorExpiredVirtualEnvironmentCandidate, ...] = ( - apply_janitor_deletions( - candidates=plan.expired_virtual_environment_candidates, - delete=delete_expired_virtual_environment, - ) - ) - deleted_state_backups: tuple[JanitorStateBackupCandidate, ...] = apply_janitor_deletions( - candidates=plan.state_backup_candidates, delete=delete_state_backup - ) - deleted_expired_locks: tuple[JanitorExpiredLockCandidate, ...] = apply_janitor_deletions( - candidates=plan.expired_lock_candidates, delete=delete_expired_lock - ) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = apply_janitor_deletions( candidates=plan.direct_state_prune_candidates, delete=lambda direct_state_candidate: adapter.execute( connection=connection, sql=direct_state_candidate.prune_sql ), ) - pruned_virtual_state: tuple[JanitorVirtualStatePruneCandidate, ...] = apply_janitor_deletions( - candidates=plan.virtual_state_prune_candidates, delete=prune_virtual_state - ) return JanitorExecutionResult( deleted=() if plan.direct_mode else plan.candidates, archived=archived, deleted_archives=deleted_archives, deleted_query_diff_artifacts=plan.query_diff_artifact_candidates, - deleted_checkpoints=deleted_checkpoints, - deleted_detached_virtual_environments=deleted_detached_virtual_environments, - deleted_expired_virtual_environments=deleted_expired_virtual_environments, - deleted_state_backups=deleted_state_backups, - deleted_expired_locks=deleted_expired_locks, pruned_direct_state=pruned_direct_state, - pruned_virtual_state=pruned_virtual_state, ) @@ -174,15 +111,4 @@ def _janitor_action_count(plan: JanitorPlan) -> int: relation_count += len(plan.query_diff_artifact_candidates) if plan.direct_mode: relation_count += len(plan.archive_candidates) + len(plan.archive_deletion_candidates) - return relation_count + sum( - len(candidates) - for candidates in ( - plan.checkpoint_candidates, - plan.detached_virtual_environment_candidates, - plan.expired_virtual_environment_candidates, - plan.state_backup_candidates, - plan.expired_lock_candidates, - plan.direct_state_prune_candidates, - plan.virtual_state_prune_candidates, - ) - ) + return relation_count + len(plan.direct_state_prune_candidates) diff --git a/src/sqlbuild/executor/janitor/main/plan.py b/src/sqlbuild/executor/janitor/main/plan.py index 45c88ec42f..5d744c433e 100644 --- a/src/sqlbuild/executor/janitor/main/plan.py +++ b/src/sqlbuild/executor/janitor/main/plan.py @@ -24,7 +24,6 @@ JanitorPlan, JanitorRelationScope, JanitorSchemaClassification, - JanitorStateCandidates, JanitorWarehouseFacts, ) from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -39,7 +38,6 @@ def build_janitor_plan( delete_tracked_only: bool = True, exclude_patterns: tuple[str, ...] = (), relation_scope: JanitorRelationScope | None = None, - state_candidates: JanitorStateCandidates | None = None, direct_settings: JanitorDirectModeSettings | None = None, ) -> JanitorPlan: """Build a desired-vs-warehouse cleanup plan for target schemas.""" @@ -47,9 +45,6 @@ def build_janitor_plan( scope: JanitorRelationScope = ( relation_scope if relation_scope is not None else JanitorRelationScope() ) - state: JanitorStateCandidates = ( - state_candidates if state_candidates is not None else JanitorStateCandidates() - ) direct: JanitorDirectModeSettings = direct_settings or JanitorDirectModeSettings() managed_target_schemas: set[tuple[str | None, str | None]] = collect_target_schemas(project) target_schemas: set[tuple[str | None, str | None]] = collect_scan_schemas( @@ -80,12 +75,6 @@ def build_janitor_plan( direct_mode=direct.enabled, archive_retention_days=direct.archive_retention_days, query_diff_artifact_candidates=query_diff_artifact_candidates, - checkpoint_candidates=state.checkpoint_candidates, - detached_virtual_environment_candidates=(state.detached_virtual_environment_candidates), - expired_virtual_environment_candidates=(state.expired_virtual_environment_candidates), - state_backup_candidates=state.state_backup_candidates, - expired_lock_candidates=state.expired_lock_candidates, - virtual_state_prune_candidates=state.virtual_state_prune_candidates, direct_state_prune_candidates=(), skipped_relations=query_diff_artifact_skipped, scanned_schema_count=len(query_artifact_schemas), @@ -149,12 +138,6 @@ def build_janitor_plan( archive_deletion_candidates=archives.archive_deletion_candidates, retained_archives=archives.retained_archives, query_diff_artifact_candidates=query_diff_artifact_candidates, - checkpoint_candidates=state.checkpoint_candidates, - detached_virtual_environment_candidates=state.detached_virtual_environment_candidates, - expired_virtual_environment_candidates=state.expired_virtual_environment_candidates, - state_backup_candidates=state.state_backup_candidates, - expired_lock_candidates=state.expired_lock_candidates, - virtual_state_prune_candidates=state.virtual_state_prune_candidates, direct_state_prune_candidates=direct_state_prune_candidates, skipped_relations=( *( diff --git a/src/sqlbuild/executor/janitor/main/resolve_retention_days.py b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py index 19e6372361..71a342c723 100644 --- a/src/sqlbuild/executor/janitor/main/resolve_retention_days.py +++ b/src/sqlbuild/executor/janitor/main/resolve_retention_days.py @@ -4,17 +4,14 @@ from sqlbuild.executor.janitor.constants import ( DIRECT_DEFAULT_RETENTION_DAYS, - VIRTUAL_DEFAULT_RETENTION_DAYS, ) -def resolve_janitor_retention_days( - *, override: int | None, configured: int | None, virtual_environments: bool -) -> int: +def resolve_janitor_retention_days(*, override: int | None, configured: int | None) -> int: """Resolve CLI override, explicit config, then the mode-specific default.""" if override is not None: return override if configured is not None: return configured - return VIRTUAL_DEFAULT_RETENTION_DAYS if virtual_environments else DIRECT_DEFAULT_RETENTION_DAYS + return DIRECT_DEFAULT_RETENTION_DAYS diff --git a/src/sqlbuild/executor/janitor/models.py b/src/sqlbuild/executor/janitor/models.py index 5601dfcc81..034fdf686f 100644 --- a/src/sqlbuild/executor/janitor/models.py +++ b/src/sqlbuild/executor/janitor/models.py @@ -88,49 +88,6 @@ class JanitorQueryDiffArtifactCandidate: key: JanitorRelationKey -@dataclass(frozen=True) -class JanitorCheckpointCandidate: - """One retained-history checkpoint eligible for pruning.""" - - checkpoint_id: str - virtual_environment_name: str - created_at: datetime | None - - -@dataclass(frozen=True) -class JanitorDetachedVirtualEnvironmentCandidate: - """One detached virtual environment eligible for state cleanup.""" - - virtual_environment_name: str - updated_at: datetime | None - - -@dataclass(frozen=True) -class JanitorExpiredVirtualEnvironmentCandidate: - """One non-active virtual environment eligible for TTL cleanup.""" - - virtual_environment_name: str - updated_at: datetime | None - - -@dataclass(frozen=True) -class JanitorStateBackupCandidate: - """One state migration backup eligible for pruning.""" - - backup_id: str - schema_name: str - created_at: datetime | None - - -@dataclass(frozen=True) -class JanitorExpiredLockCandidate: - """One expired state lock eligible for pruning.""" - - lock_key: str - owner_id: str - expires_at: datetime - - @dataclass(frozen=True) class JanitorDirectStatePruneCandidate: """One direct-mode state table eligible for history pruning.""" @@ -150,18 +107,6 @@ def display_name(self) -> str: return ".".join(parts) -@dataclass(frozen=True) -class JanitorVirtualStatePruneCandidate: - """One virtual-mode state table eligible for orphaned state pruning.""" - - schema: str - table_name: str - reason: str - - def display_name(self) -> str: - return f"{self.schema}.{self.table_name}" - - @dataclass(frozen=True) class JanitorSkippedRelation: """One stale relation skipped by a safety rule.""" @@ -253,22 +198,6 @@ class JanitorArchivePlanning: blocked_schemas: tuple[JanitorBlockedSchema, ...] -@dataclass(frozen=True) -class JanitorStateCandidates: - """Precomputed state-side cleanup candidates for one janitor plan.""" - - checkpoint_candidates: tuple[JanitorCheckpointCandidate, ...] = () - detached_virtual_environment_candidates: tuple[ - JanitorDetachedVirtualEnvironmentCandidate, ... - ] = () - expired_virtual_environment_candidates: tuple[ - JanitorExpiredVirtualEnvironmentCandidate, ... - ] = () - state_backup_candidates: tuple[JanitorStateBackupCandidate, ...] = () - expired_lock_candidates: tuple[JanitorExpiredLockCandidate, ...] = () - virtual_state_prune_candidates: tuple[JanitorVirtualStatePruneCandidate, ...] = () - - @dataclass(frozen=True) class JanitorRelationScope: """Scan scope and protection rules for one janitor plan.""" @@ -282,7 +211,7 @@ class JanitorRelationScope: class JanitorDirectModeSettings: """Direct-mode state-history settings.""" - enabled: bool = False + enabled: bool = True state_history_versions: int = 20 archive_retention_days: int = 14 @@ -293,7 +222,7 @@ class JanitorPlan: target_name: str | None retention_days: int - direct_mode: bool = False + direct_mode: bool = True archive_retention_days: int = 14 candidates: tuple[JanitorDeleteCandidate, ...] = field(default_factory=tuple) archive_candidates: tuple[JanitorArchiveCandidate, ...] = field(default_factory=tuple) @@ -302,21 +231,9 @@ class JanitorPlan: query_diff_artifact_candidates: tuple[JanitorQueryDiffArtifactCandidate, ...] = field( default_factory=tuple ) - checkpoint_candidates: tuple[JanitorCheckpointCandidate, ...] = field(default_factory=tuple) - detached_virtual_environment_candidates: tuple[ - JanitorDetachedVirtualEnvironmentCandidate, ... - ] = field(default_factory=tuple) - expired_virtual_environment_candidates: tuple[ - JanitorExpiredVirtualEnvironmentCandidate, ... - ] = field(default_factory=tuple) - state_backup_candidates: tuple[JanitorStateBackupCandidate, ...] = field(default_factory=tuple) - expired_lock_candidates: tuple[JanitorExpiredLockCandidate, ...] = field(default_factory=tuple) direct_state_prune_candidates: tuple[JanitorDirectStatePruneCandidate, ...] = field( default_factory=tuple ) - virtual_state_prune_candidates: tuple[JanitorVirtualStatePruneCandidate, ...] = field( - default_factory=tuple - ) skipped_relations: tuple[JanitorSkippedRelation, ...] = field(default_factory=tuple) skipped_schemas: tuple[JanitorSkippedSchema, ...] = field(default_factory=tuple) blocked_schemas: tuple[JanitorBlockedSchema, ...] = field(default_factory=tuple) @@ -335,16 +252,4 @@ class JanitorExecutionResult: deleted_query_diff_artifacts: tuple[JanitorQueryDiffArtifactCandidate, ...] = field( default_factory=tuple ) - deleted_checkpoints: tuple[JanitorCheckpointCandidate, ...] = field(default_factory=tuple) - deleted_detached_virtual_environments: tuple[ - JanitorDetachedVirtualEnvironmentCandidate, ... - ] = field(default_factory=tuple) - deleted_expired_virtual_environments: tuple[JanitorExpiredVirtualEnvironmentCandidate, ...] = ( - field(default_factory=tuple) - ) - deleted_state_backups: tuple[JanitorStateBackupCandidate, ...] = field(default_factory=tuple) - deleted_expired_locks: tuple[JanitorExpiredLockCandidate, ...] = field(default_factory=tuple) pruned_direct_state: tuple[JanitorDirectStatePruneCandidate, ...] = field(default_factory=tuple) - pruned_virtual_state: tuple[JanitorVirtualStatePruneCandidate, ...] = field( - default_factory=tuple - ) diff --git a/src/sqlbuild/executor/node_results/main/decode_json.py b/src/sqlbuild/executor/node_results/main/decode_json.py deleted file mode 100644 index 4a98ce0390..0000000000 --- a/src/sqlbuild/executor/node_results/main/decode_json.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public node result JSON decoding entrypoint.""" - -from __future__ import annotations - -from sqlbuild.executor.node_results._helpers.serialization import decode_json_b64 - - -def decode_node_result_json(*, value: str, label: str, node_name: str) -> object: - """Decode a persisted node result JSON value.""" - - return decode_json_b64(value=value, label=label, node_name=node_name) diff --git a/src/sqlbuild/executor/node_results/main/encode_json.py b/src/sqlbuild/executor/node_results/main/encode_json.py deleted file mode 100644 index 6ac71e81c3..0000000000 --- a/src/sqlbuild/executor/node_results/main/encode_json.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public node result JSON encoding entrypoint.""" - -from __future__ import annotations - -from sqlbuild.executor.node_results._helpers.serialization import encode_json_b64 - - -def encode_node_result_json(*, value: object, label: str, node_name: str) -> str: - """Encode a node result JSON value for persisted storage.""" - - return encode_json_b64(value=value, label=label, node_name=node_name) diff --git a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py index d507bc4999..fdf6a941b4 100644 --- a/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py +++ b/src/sqlbuild/executor/run/_helpers/materializations/snapshot.py @@ -20,6 +20,10 @@ ) from sqlbuild.adapter.type_system.main.normalize_type import normalize_type from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.compiler.planner.constants import ( + SNAPSHOT_DEFAULT_VALID_FROM_COLUMN, + SNAPSHOT_DEFAULT_VALID_TO_COLUMN, +) from sqlbuild.compiler.planner.models import ModelPlanEntry from sqlbuild.compiler.planner.types import ( HistoricalInput, @@ -71,8 +75,6 @@ ) from sqlbuild.spec.contracts.models import SnapshotsConfig -_DEFAULT_VALID_FROM_COLUMN: str = "valid_from" -_DEFAULT_VALID_TO_COLUMN: str = "valid_to" _SCHEMA_CHANGE_STRICTNESS: dict[SnapshotSchemaChangePolicy, int] = { SnapshotSchemaChangePolicy.APPEND_NEW_COLUMNS: 0, SnapshotSchemaChangePolicy.REQUIRE_CONFIRMATION: 1, @@ -1065,8 +1067,8 @@ def _require_updated_at(entry: ModelPlanEntry) -> str: def _valid_from_column(entry: ModelPlanEntry) -> str: - return entry.valid_from_column or _DEFAULT_VALID_FROM_COLUMN + return entry.valid_from_column or SNAPSHOT_DEFAULT_VALID_FROM_COLUMN def _valid_to_column(entry: ModelPlanEntry) -> str: - return entry.valid_to_column or _DEFAULT_VALID_TO_COLUMN + return entry.valid_to_column or SNAPSHOT_DEFAULT_VALID_TO_COLUMN diff --git a/src/sqlbuild/executor/testing/constants.py b/src/sqlbuild/executor/testing/constants.py index 26e967d345..1ef960c924 100644 --- a/src/sqlbuild/executor/testing/constants.py +++ b/src/sqlbuild/executor/testing/constants.py @@ -5,6 +5,7 @@ SQL_TEST_ASSERTION_FAILED_CODE: str = "T003" SQL_TEST_ENTRY_MISSING_CODE: str = "T004" SQL_TEST_EXPECTED_COLUMNS_CODE: str = "T005" +SQL_TEST_SIZE_BYTES: str = "bytes" SQL_TEST_NATIVE_RENDER_WORKERS: int = 4 SQL_TEST_IDENTIFIER_QUOTES: dict[str, str] = {'"': '"', "`": "`", "[": "]"} diff --git a/src/sqlbuild/executor/testing/main/_execute.py b/src/sqlbuild/executor/testing/main/_execute.py index 0847bb3c33..59a5d59020 100644 --- a/src/sqlbuild/executor/testing/main/_execute.py +++ b/src/sqlbuild/executor/testing/main/_execute.py @@ -5,6 +5,7 @@ from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.planner.models import SqlTestPlanEntry from sqlbuild.executor.testing._helpers.difference_samples import add_difference_samples from sqlbuild.executor.testing._helpers.expected_column_probe import ( @@ -49,16 +50,10 @@ def execute_sql_test( sql=comparison_sql, adapter=adapter, test_name=test_entry.name, + model_name=error_model_name, ) - except Exception: - error_message: str = ( - f"Combined unit test SQL for '{test_entry.name}' including '{error_model_name}' " - f"is {len(comparison_sql)} " - f"characters, which exceeds the recommended maximum of " - f"{adapter.recommended_max_sql_length()} for this adapter. This test is too " - "large for a single lightweight unit query. Consider splitting it into smaller " - "unit tests or moving it to a scenario test." - ) + except CompileInputError as error: + error_message: str = str(error) return SqlTestExecutionResult( test_name=test_entry.name, outcome=SqlTestOutcome.ERROR, diff --git a/src/sqlbuild/executor/testing/main/_sql_length.py b/src/sqlbuild/executor/testing/main/_sql_length.py index 22ede49534..3cb9b9c5e8 100644 --- a/src/sqlbuild/executor/testing/main/_sql_length.py +++ b/src/sqlbuild/executor/testing/main/_sql_length.py @@ -3,7 +3,9 @@ from __future__ import annotations from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.types import StatementSizeLimit from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.executor.testing.constants import SQL_TEST_SIZE_BYTES def validate_unit_test_sql_length( @@ -11,40 +13,18 @@ def validate_unit_test_sql_length( sql: str, adapter: BaseAdapter, test_name: str, + model_name: str, ) -> None: - limit: int | None = adapter.recommended_max_sql_length() - if limit is None: + size_limit: StatementSizeLimit | None = adapter.max_statement_size() + if size_limit is None: return - actual_length: int = len(sql) + limit, unit = size_limit + actual_length: int = len(sql.encode("utf-8")) if unit == SQL_TEST_SIZE_BYTES else len(sql) if actual_length <= limit: return raise CompileInputError( - _format_unit_test_sql_length_error( - sql=sql, - adapter=adapter, - test_name=test_name, - ) - ) - - -def _format_unit_test_sql_length_error( - *, - sql: str, - adapter: BaseAdapter, - test_name: str, -) -> str: - limit: int | None = adapter.recommended_max_sql_length() - actual_length: int = len(sql) - if limit is None: - return ( - f"Combined unit test SQL for '{test_name}' is {actual_length} characters, " - "which exceeds this adapter's configured recommendation. " - "This test is too large for a single lightweight unit query. " - "Consider splitting it into smaller unit tests or moving it to a scenario test." - ) - return ( - f"Combined unit test SQL for '{test_name}' is {actual_length} characters, " - f"which exceeds the recommended maximum of {limit} for this adapter. " - "This test is too large for a single lightweight unit query. " - "Consider splitting it into smaller unit tests or moving it to a scenario test." + f"Combined unit test SQL for '{test_name}' including '{model_name}' " + f"is {actual_length} {unit}, which exceeds the maximum of {limit} {unit} " + "for this adapter. Consider splitting it into smaller unit tests or moving it " + "to a scenario test." ) diff --git a/src/sqlbuild/integrations/dagster/_helpers/invocation.py b/src/sqlbuild/integrations/dagster/_helpers/invocation.py index 08432247af..5b9ca60432 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/invocation.py +++ b/src/sqlbuild/integrations/dagster/_helpers/invocation.py @@ -42,7 +42,6 @@ SOURCE_NODE_KIND, STANDALONE_CHECK_COMMANDS, SUCCESS_EXECUTION_STATUS, - VIRTUAL_ENV_FLAG, WARNING_CHECK_SEVERITY, ) @@ -267,7 +266,6 @@ def _with_event_output_args( or context is None or not args or args[0] not in LIVE_EVENT_COMMANDS - or VIRTUAL_ENV_FLAG in args or EVENT_OUTPUT_FLAG in args ): return args, None diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index 03b8ad1f02..d02a1fee47 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -51,7 +51,6 @@ STANDALONE_CHECK_COMMANDS: frozenset[str] = frozenset({"audit", CHECK_COMMAND, "test"}) EVENT_OUTPUT_FLAG: str = "--event-output" LIVE_EVENT_COMMANDS: frozenset[str] = ASSET_SELECTION_COMMANDS -VIRTUAL_ENV_FLAG: str = "--virtual-env" EXPLICIT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "-s", "--select-file"}) JSON_OUTPUT_FLAGS: frozenset[str] = frozenset({"--json", "--json-output"}) JSON_OUTPUT_FLAG: str = "--json-output" diff --git a/src/sqlbuild/integrations/dbt/_helpers/cli/mode.py b/src/sqlbuild/integrations/dbt/_helpers/cli/mode.py deleted file mode 100644 index b0a9e897d6..0000000000 --- a/src/sqlbuild/integrations/dbt/_helpers/cli/mode.py +++ /dev/null @@ -1,17 +0,0 @@ -"""dbt interop mode guard helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError - - -def enforce_dbt_interop_direct_mode(*, discovered_inputs: DiscoveredProjectInputs) -> None: - """Block dbt interop commands in virtual environment mode.""" - - if discovered_inputs.project_config.settings.virtual_environments: - raise DbtInteropConfigError( - "sqb dbt is not supported when virtual_environments = true", - code="C241", - help="Disable virtual_environments or run dbt and SQLBuild virtual builds separately.", - ) diff --git a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py index 24ec4a2f79..0a2aad61ec 100644 --- a/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py +++ b/src/sqlbuild/integrations/dbt/_helpers/pipeline/interop_prologue.py @@ -19,9 +19,6 @@ ) from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner from sqlbuild.integrations.dbt.exceptions import DbtInteropRuntimeError -from sqlbuild.integrations.dbt.main.cli._enforce_direct_mode import ( - enforce_dbt_interop_direct_mode, -) from sqlbuild.integrations.dbt.main.cli._parse_execution_args import parse_dbt_execution_args from sqlbuild.integrations.dbt.main.cli._resolve_executable import resolve_dbt_executable from sqlbuild.integrations.dbt.main.cli._route_interop_args import route_dbt_interop_args @@ -72,7 +69,6 @@ def resolve_dbt_execution_invocation( discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( project_dir=request.project_dir ) - enforce_dbt_interop_direct_mode(discovered_inputs=discovered_inputs) dbt_options: DbtCliOptions = resolve_dbt_plan_options( project_dir=request.project_dir, discovered_inputs=discovered_inputs, diff --git a/src/sqlbuild/integrations/dbt/main/cli/_enforce_direct_mode.py b/src/sqlbuild/integrations/dbt/main/cli/_enforce_direct_mode.py deleted file mode 100644 index 36b9076858..0000000000 --- a/src/sqlbuild/integrations/dbt/main/cli/_enforce_direct_mode.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Enforce the supported dbt interop mode.""" - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.cli.mode import ( - enforce_dbt_interop_direct_mode as _enforce, -) - - -def enforce_dbt_interop_direct_mode(*, discovered_inputs: DiscoveredProjectInputs) -> None: - """Reject dbt interop in virtual environment mode.""" - - _ = _enforce(discovered_inputs=discovered_inputs) diff --git a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py index cbf99a38b9..0a835ab1e6 100644 --- a/src/sqlbuild/integrations/dbt/main/pipeline/debug.py +++ b/src/sqlbuild/integrations/dbt/main/pipeline/debug.py @@ -8,9 +8,6 @@ from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.integrations.dbt.classes.dbt_runner import DbtRunner -from sqlbuild.integrations.dbt.main.cli._enforce_direct_mode import ( - enforce_dbt_interop_direct_mode, -) from sqlbuild.integrations.dbt.main.config._parse_config_overrides import ( parse_dbt_config_overrides, ) @@ -30,7 +27,6 @@ def debug_dbt_from_project( """Run dbt debug from SQLBuild project configuration.""" discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - enforce_dbt_interop_direct_mode(discovered_inputs=discovered_inputs) resolved: ResolvedDbtConfig = resolve_dbt_config( project_root=project_dir, config=discovered_inputs.project_config.dbt, diff --git a/src/sqlbuild/lint/_helpers/native_sql.py b/src/sqlbuild/lint/_helpers/native_sql.py index ab02e7f638..af631c937b 100644 --- a/src/sqlbuild/lint/_helpers/native_sql.py +++ b/src/sqlbuild/lint/_helpers/native_sql.py @@ -128,12 +128,17 @@ def run_native_sql_lint( if not isinstance(raw_diagnostics, list): raise NativeLintError("native lint response is missing a diagnostics list") for raw_diagnostic in raw_diagnostics: - violation: LintViolation | None = _authored_violation( - raw_diagnostic=raw_diagnostic, - body=body, - contents=contents_by_path[body.file_path], - dialect=config.dialect, - ) + try: + violation: LintViolation | None = _authored_violation( + raw_diagnostic=raw_diagnostic, + body=body, + contents=contents_by_path[body.file_path], + dialect=config.dialect, + ) + except NativeLintError as error: + violation = _parse_failure_violation( + error=error, body=body, contents=contents_by_path[body.file_path] + ) if violation is not None: violations_by_file.setdefault(body.file_path, []).append(violation) return {path: tuple(entries) for path, entries in violations_by_file.items()} @@ -167,10 +172,12 @@ def _native_responses( orjson.dumps(payloads, option=orjson.OPT_SORT_KEYS).decode() ) ) - except (TypeError, ValueError) as error: - raise NativeLintError(str(error)) from error + except (TypeError, ValueError, RuntimeError) as error: + return dict.fromkeys(keys, NativeLintError(str(error))) if not isinstance(decoded, list) or len(decoded) != len(keys): - raise NativeLintError("native lint engine returned an invalid batch response") + return dict.fromkeys( + keys, NativeLintError("native lint engine returned an invalid batch response") + ) results: tuple[_NativeResult, ...] = tuple(_native_result(response=value) for value in decoded) return dict(zip(keys, results, strict=True)) @@ -187,19 +194,19 @@ def _native_result(*, response: object) -> _NativeResult: result: dict[str, Any] = cast(dict[str, Any], decoded) if result.get("version") != _NATIVE_LINT_API_VERSION: return NativeLintError("native lint engine returned an unsupported response version") + if not isinstance(result.get("diagnostics"), list): + return NativeLintError("native lint response is missing a diagnostics list") return result def _parse_failure_violation( *, error: NativeLintError, body: LintBody, contents: str -) -> LintViolation | None: +) -> LintViolation: """Convert a per-body native parse rejection into an authored project fault.""" match: re.Match[str] | None = _PARSE_ERROR_POSITION_PATTERN.match(error.message) - if match is None: - return None - expanded_line: int = int(match.group("line")) - expanded_column: int = int(match.group("column")) + expanded_line: int = int(match.group("line")) if match else 1 + expanded_column: int = int(match.group("column")) if match else 1 expanded_lines: list[str] = body.lint_text.splitlines(keepends=True) expanded_offset: int = 0 if 1 <= expanded_line <= len(expanded_lines): diff --git a/src/sqlbuild/microbatches/classes/event_codec.py b/src/sqlbuild/microbatches/classes/event_codec.py index 58829c7534..9617ec5935 100644 --- a/src/sqlbuild/microbatches/classes/event_codec.py +++ b/src/sqlbuild/microbatches/classes/event_codec.py @@ -37,8 +37,6 @@ def values(event: MicrobatchEvent) -> tuple[object | None, ...]: event.scope.target_schema, event.scope.target_name, event.scope.physical_generation_id, - event.scope.virtual_environment_name, - event.scope.virtual_model_version_hash, event.origin_run_id, event.origin_run_started_at, event.execution_run_id, @@ -81,12 +79,6 @@ def from_row(row: tuple[Any, ...]) -> MicrobatchEvent: target_schema=MicrobatchEventCodec._optional_str(values["target_schema"]), target_name=str(values["target_name"]), physical_generation_id=str(values["physical_generation_id"]), - virtual_environment_name=MicrobatchEventCodec._optional_str( - values["virtual_environment_name"] - ), - virtual_model_version_hash=MicrobatchEventCodec._optional_str( - values["virtual_model_version_hash"] - ), ) return MicrobatchEvent( diff --git a/src/sqlbuild/microbatches/constants.py b/src/sqlbuild/microbatches/constants.py index ad507d4697..f302c98c0a 100644 --- a/src/sqlbuild/microbatches/constants.py +++ b/src/sqlbuild/microbatches/constants.py @@ -4,7 +4,6 @@ MICROBATCH_TABLE_NAME: str = "_sqlbuild_microbatches" DIRECT_MICROBATCH_SCOPE_KIND: str = "direct_logical" -VIRTUAL_MICROBATCH_SCOPE_KIND: str = "virtual_physical" MICROBATCH_GENERATION_COMMENT_PREFIX: str = "sqlbuild-generation:" MICROBATCH_REPLAY_GENERATION_PREFIX: str = "replay:" MICROBATCH_GENERATION_WILDCARD: str = "*" @@ -22,8 +21,6 @@ "target_schema", "target_name", "physical_generation_id", - "virtual_environment_name", - "virtual_model_version_hash", "origin_run_id", "origin_run_started_at", "execution_run_id", diff --git a/src/sqlbuild/microbatches/main/deterministic_event_id.py b/src/sqlbuild/microbatches/main/deterministic_event_id.py index aaac67c8a7..361f9cefeb 100644 --- a/src/sqlbuild/microbatches/main/deterministic_event_id.py +++ b/src/sqlbuild/microbatches/main/deterministic_event_id.py @@ -27,8 +27,6 @@ def deterministic_microbatch_event_id( scope.target_schema, scope.target_name, scope.physical_generation_id, - scope.virtual_environment_name, - scope.virtual_model_version_hash, record_type.value, partition_start, partition_end, diff --git a/src/sqlbuild/microbatches/models.py b/src/sqlbuild/microbatches/models.py index 578ad8abe3..7eb8e3800e 100644 --- a/src/sqlbuild/microbatches/models.py +++ b/src/sqlbuild/microbatches/models.py @@ -26,8 +26,6 @@ class MicrobatchScope: target_schema: str | None target_name: str physical_generation_id: str - virtual_environment_name: str | None = None - virtual_model_version_hash: str | None = None @dataclass(frozen=True) diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_host.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_host.py index 835a6ef631..988ce2e70c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_host.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_host.py @@ -97,6 +97,7 @@ def _decode_inputs(payload: dict[str, Any]) -> tuple[CompiledProject, RulesConfi def _finding_payload(finding: Finding) -> dict[str, object]: payload: dict[str, object] = asdict(finding) + payload.pop("affected_rules", None) payload["path"] = finding.path.as_posix() return payload diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 30dccae1a4..34933180c2 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -155,7 +155,7 @@ def evaluate_native( if not isinstance(raw_findings, list): raise RulesError("native rules engine returned invalid findings") return RulesResult( - findings=tuple(_decode_finding(value) for value in raw_findings), + findings=tuple(decode_rule_finding(value) for value in raw_findings), evaluated_models=int(payload.get("evaluated_models", 0)), cache_hits=max(0, int(payload.get("cache_hits", 0)) - retry_native_misses), cache_misses=int(payload.get("cache_misses", 0)) + retry_native_misses, @@ -194,7 +194,7 @@ def finalize_native_findings( raise RulesError(str(error)) from error if not isinstance(payload, list): raise RulesError("native rules engine returned invalid finalized findings") - return tuple(_decode_finding(value) for value in payload) + return tuple(decode_rule_finding(value) for value in payload) def load_native_config(project_dir: Path) -> dict[str, object]: @@ -312,12 +312,16 @@ def _model_payloads( for name in test.target_model_names: test_counts[name] = test_counts.get(name, 0) + 1 return [ - _model_payload( - model=model, - compiled_audit_count=audit_counts.get(model.name, 0), - targeting_test_count=test_counts.get(model.name, 0), - dialect=dialect, - include_type_proof=include_type_proof, + dict( + _model_payload( + model=model, + compiled_audit_count=audit_counts.get(model.name, 0), + targeting_test_count=test_counts.get(model.name, 0), + dialect=dialect, + include_type_proof=include_type_proof, + ), + sql_analysis_disabled=not project.settings.sql_analysis + or model.config.values.get("sql_analysis") is False, ) for model in project.models ] @@ -732,6 +736,7 @@ def _custom_host_payload( return None, None serializable_project: CompiledProject = replace( project, + binding_catalog=None, sql_expansions={}, loaded_macros={}, loader_functions=(), @@ -773,7 +778,7 @@ def _custom_host_payload( ) -def _decode_finding(value: object) -> Finding: +def decode_rule_finding(value: object) -> Finding: if not isinstance(value, dict): raise RulesError("native rules engine returned an invalid finding") payload: dict[str, object] = {str(key): item for key, item in value.items()} @@ -799,11 +804,13 @@ def _decode_finding(value: object) -> Finding: column=column, message=message, remediation=remediation, + unevaluated=payload.get("unevaluated") is True, ) def _finding_payload(finding: Finding) -> dict[str, object]: return { + "unevaluated": finding.unevaluated, "code": finding.code, "path": finding.path.as_posix(), "line": finding.line, diff --git a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py index 3cee460160..134d8c9227 100644 --- a/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py +++ b/src/sqlbuild/rule_engine/_helpers/guidance/presentation.py @@ -23,7 +23,14 @@ def format_result_text(*, result: RulesResult) -> str: f"[{finding.code}] {finding.message}\n" f" Remediation: {finding.remediation}" ) + if finding.affected_rules: + blocks.append(f" Affected Rules: {', '.join(finding.affected_rules)}") blocks.append(f"Found {len(result.findings)} Rules findings") + if result.unevaluated_resources: + blocks.append( + f"{result.evaluated_models} models evaluated, " + f"{result.unevaluated_resources} resources could not be evaluated" + ) return "\n".join(blocks) @@ -38,10 +45,16 @@ def format_result_json(*, result: RulesResult) -> str: "column": finding.column, "message": finding.message, "remediation": finding.remediation, + **( + {"affected_rules": list(finding.affected_rules)} + if finding.affected_rules + else {} + ), } ) payload: dict[str, object] = { "evaluated_models": result.evaluated_models, + "unevaluated_resources": result.unevaluated_resources, "cache_hits": result.cache_hits, "cache_misses": result.cache_misses, "finding_count": len(result.findings), diff --git a/src/sqlbuild/rule_engine/_helpers/run/findings.py b/src/sqlbuild/rule_engine/_helpers/run/findings.py new file mode 100644 index 0000000000..db350a1de3 --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/findings.py @@ -0,0 +1,34 @@ +"""Public evaluation faults grouped after per-Rule suppression policy.""" + +from dataclasses import replace +from pathlib import Path + +from sqlbuild.rule_engine.models import Finding + + +def group_unevaluated_findings(findings: tuple[Finding, ...]) -> tuple[Finding, ...]: + grouped: dict[tuple[Path, str], list[Finding]] = {} + result: list[Finding] = [] + for finding in findings: + if not finding.unevaluated: + result.append(finding) + continue + reason: str = finding.message.removeprefix( + f"Rule {finding.code} could not be evaluated" + ).lstrip(": ") + grouped.setdefault((finding.path, reason), []).append(finding) + for (_, reason), failures in grouped.items(): + affected_rules: tuple[str, ...] = tuple(sorted({failure.code for failure in failures})) + first: Finding = min(failures, key=lambda failure: (failure.line, failure.column)) + result.append( + replace( + first, + code="rules-unevaluated", + affected_rules=affected_rules, + message=( + f"{len(affected_rules)} selected Rules could not evaluate this resource: " + f"{reason}" + ), + ) + ) + return tuple(result) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index e9a3004feb..cab2fe127f 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -26,6 +26,7 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.lint.constants import TEMPLATE_INTERPOLATION_START +from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.main.build_expansion_context import build_expansion_context from sqlbuild.lint.main.collect_project_files import collect_project_files from sqlbuild.lint.main.run_lint import run_lint @@ -34,10 +35,12 @@ from sqlbuild.rule_engine._helpers.engine.config import resolve_rule_ignore_selectors from sqlbuild.rule_engine._helpers.engine.hermeticity import verify_custom_rules from sqlbuild.rule_engine._helpers.engine.native import ( + decode_rule_finding, evaluate_native, finalize_native_findings, native_catalogue, ) +from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config @@ -52,7 +55,7 @@ SqlExpansionReuse, ) -_SQL_RULE_CACHE_VERSION: str = "sql-rules-v2" +_SQL_RULE_CACHE_VERSION: str = "sql-rules-v3" _SQLBUILD_VERSION: str = version("sqlbuild") _SQL_RULE_SUPPRESSION_CODE: str = "SQBRSQL000" @@ -146,13 +149,21 @@ def evaluate_rules( ) findings: tuple[Finding, ...] = tuple( sorted( - finalized, + group_unevaluated_findings(finalized), key=lambda item: (item.path.as_posix(), item.line, item.column, item.code), ) ) return RulesRunResult( findings=findings, - evaluated_models=result.evaluated_models, + evaluated_models=max( + 0, + result.evaluated_models + - len( + {item.path for item in findings if item.unevaluated} + & {model.relative_path for model in selected_project.models} + ), + ), + unevaluated_resources=len({item.path for item in findings if item.unevaluated}), built_in_ms=sql_ms + result.built_in_ms, custom_ms=result.custom_ms, cache_hits=result.cache_hits + sql_result.cache_hits, @@ -345,7 +356,9 @@ def _run_sql_rules( if not codes: return _SqlRulesEvaluation(findings=(), cache_hits=0, cache_misses=0) models_by_path: dict[str, CompiledModel] = { - model.relative_path.as_posix(): model for model in project.models + model.relative_path.as_posix(): model + for model in project.models + if project.settings.sql_analysis and model.config.values.get("sql_analysis") is not False } bucket: dict[str, dict[str, object]] = ( _read_sql_rule_cache(project_dir) if cache_enabled else {} @@ -368,7 +381,9 @@ def _run_sql_rules( file_identities: dict[str, str] = {} project_files: dict[Path, str] | None = None if selected_model_paths is None: - model_paths: frozenset[Path] = frozenset(project_dir / path for path in models_by_path) + model_paths: frozenset[Path] = frozenset( + project_dir / model.relative_path for model in project.models + ) project_files = collect_project_files(project_dir=project_dir, selected_paths=None) file_path: Path contents: str @@ -418,11 +433,21 @@ def _run_sql_rules( else None ) selected_codes: frozenset[str] = frozenset((*codes, _SQL_RULE_SUPPRESSION_CODE)) - evaluated: tuple[Finding, ...] = tuple( - _lint_finding(violation=violation, project_dir=project_dir) - for violation in (() if result is None else result.violations) - if violation.code in selected_codes - ) + evaluated: list[Finding] = [] + for violation in () if result is None else result.violations: + if violation.code in selected_codes: + evaluated.append(_lint_finding(violation=violation, project_dir=project_dir)) + elif violation.code == NativeLintError.code: + finding: Finding = _lint_finding(violation=violation, project_dir=project_dir) + evaluated.extend( + replace( + finding, + code=code, + message=f"Rule {code} could not be evaluated: {violation.message}", + unevaluated=True, + ) + for code in codes + ) findings.extend(evaluated) written_identities: dict[str, str] = { **{path: identities[path] for path in misses}, @@ -560,42 +585,14 @@ def _cached_sql_findings( if not isinstance(values, list): return None try: - return tuple(_finding_from_cache_payload(value) for value in values) + return tuple(decode_rule_finding(value) for value in values) except (KeyError, RulesError, TypeError, ValueError): return None -def _finding_from_cache_payload(value: object) -> Finding: - if not isinstance(value, dict): - raise RulesError("cached SQL finding must be an object") - payload: dict[str, object] = {str(key): item for key, item in value.items()} - code: object = payload["code"] - path: object = payload["path"] - line: object = payload["line"] - column: object = payload["column"] - message: object = payload["message"] - remediation: object = payload["remediation"] - if ( - not isinstance(code, str) - or not isinstance(path, str) - or not isinstance(line, int) - or not isinstance(column, int) - or not isinstance(message, str) - or not isinstance(remediation, str) - ): - raise RulesError("cached SQL finding has invalid fields") - return Finding( - code=code, - path=Path(path), - line=line, - column=column, - message=message, - remediation=remediation, - ) - - def _finding_cache_payload(finding: Finding) -> dict[str, object]: return { + "unevaluated": finding.unevaluated, "code": finding.code, "path": finding.path.as_posix(), "line": finding.line, diff --git a/src/sqlbuild/rule_engine/models.py b/src/sqlbuild/rule_engine/models.py index f3651eef92..54d505a742 100644 --- a/src/sqlbuild/rule_engine/models.py +++ b/src/sqlbuild/rule_engine/models.py @@ -218,6 +218,8 @@ class Finding: column: int message: str remediation: str + unevaluated: bool = False + affected_rules: tuple[str, ...] = () @dataclass(frozen=True) @@ -301,6 +303,7 @@ class RulesConfig: class RulesResult: findings: tuple[Finding, ...] evaluated_models: int + unevaluated_resources: int = 0 cache_hits: int = 0 cache_misses: int = 0 built_in_ms: int = 0 @@ -339,6 +342,7 @@ class RulesRunResult: evaluated_models: int built_in_ms: int custom_ms: int + unevaluated_resources: int = 0 cache_hits: int = 0 cache_misses: int = 0 skipped_type_proof_rules: tuple[str, ...] = () diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index 980a7bb61e..eef94fef8d 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -180,7 +180,6 @@ RENAME_OPERATION_STRATEGY, "build_aside", "create_new", - "virtual", "append_new_columns", "sync_all_columns", "adapter", @@ -197,7 +196,6 @@ "sql_test", "staging_relation", "state_batch", - "virtual_environment", } ) OPERATION_SCOPES: frozenset[str] = frozenset( diff --git a/src/sqlbuild/spec/contracts/_helpers/project_config.py b/src/sqlbuild/spec/contracts/_helpers/project_config.py index ff160d20b1..bd620b96f9 100644 --- a/src/sqlbuild/spec/contracts/_helpers/project_config.py +++ b/src/sqlbuild/spec/contracts/_helpers/project_config.py @@ -49,6 +49,11 @@ def resolve_effective_scenario_config( project_limits: ScenarioSnapshotLimitsConfig = project_config.scenario.snapshot_limits local_limits: ScenarioSnapshotLimitsConfig = local_config.scenario.snapshot_limits return ScenarioConfig( + run_namespace=( + local_config.scenario.run_namespace + if local_config.scenario.run_namespace is not None + else project_config.scenario.run_namespace + ), local_type_overrides=local_type_overrides, snapshot_limits=ScenarioSnapshotLimitsConfig( max_rows_per_relation=_resolve_optional_int_override( diff --git a/src/sqlbuild/spec/contracts/_helpers/targets.py b/src/sqlbuild/spec/contracts/_helpers/targets.py index 1113741dfd..7be6cdf3fc 100644 --- a/src/sqlbuild/spec/contracts/_helpers/targets.py +++ b/src/sqlbuild/spec/contracts/_helpers/targets.py @@ -2,17 +2,14 @@ from __future__ import annotations -from sqlbuild.spec.contracts.constants import CHANGES_ONLY_SETTING_OVERRIDE_KEY from sqlbuild.spec.contracts.exceptions import SpecConfigError from sqlbuild.spec.contracts.models import ( ClonePolicy, ExecutionLimitsConfig, LocalClonePolicy, LocalConfig, - LocalStateConfig, LocalTargetConfig, ProjectConfig, - StateConfig, TargetConfig, ) @@ -80,11 +77,6 @@ def resolve_target_config( if local_target.defer_clone_from is not None else project_target.defer_clone_from ), - changes_only=( - local_target.changes_only - if local_target.changes_only is not None - else project_target.changes_only - ), compile_cache=( local_target.compile_cache if local_target.compile_cache is not None @@ -128,10 +120,6 @@ def resolve_target_config( project_clone=project_target.clone, local_clone=local_target.clone, ), - state=_merge_state_config( - project_state=project_target.state, - local_state=local_target.state, - ), ) return target_config @@ -163,37 +151,6 @@ def _merge_execution_limits( ) -def resolve_effective_changes_only( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - selected_target: str | None, - cli_changes_only: bool, -) -> bool: - """Resolve changes-only selection with CLI, target, and settings precedence.""" - - if cli_changes_only: - return True - target_name: str | None = resolve_target_name( - project_config=project_config, - local_config=local_config, - selected_target=selected_target, - ) - if target_name is not None: - target_config: TargetConfig = resolve_target_config( - project_config=project_config, - local_config=local_config, - target_name=target_name, - ) - if target_config.changes_only is not None: - return target_config.changes_only - return ( - local_config.settings.changes_only - if CHANGES_ONLY_SETTING_OVERRIDE_KEY in local_config.setting_overrides - else project_config.settings.changes_only - ) - - def _merge_clone_policy( *, project_clone: ClonePolicy, local_clone: LocalClonePolicy ) -> ClonePolicy: @@ -211,25 +168,3 @@ def _merge_clone_policy( else project_clone.allow_as_clone_destination ), ) - - -def _merge_state_config( - *, - project_state: StateConfig, - local_state: LocalStateConfig, -) -> StateConfig: - return StateConfig( - backend=local_state.backend if local_state.backend is not None else project_state.backend, - schema=local_state.schema if local_state.schema is not None else project_state.schema, - connection={**project_state.connection, **local_state.connection}, - allow_reset=( - local_state.allow_reset - if local_state.allow_reset is not None - else project_state.allow_reset - ), - unsuffixed_virtual_env=( - local_state.unsuffixed_virtual_env - if local_state.unsuffixed_virtual_env is not None - else project_state.unsuffixed_virtual_env - ), - ) diff --git a/src/sqlbuild/spec/contracts/constants.py b/src/sqlbuild/spec/contracts/constants.py index 35d6bd2b94..3214e623c5 100644 --- a/src/sqlbuild/spec/contracts/constants.py +++ b/src/sqlbuild/spec/contracts/constants.py @@ -7,10 +7,10 @@ LOADER_SCHEMA_TABLE_PART_COUNT: int = 2 LOADER_QUALIFIED_TABLE_PART_COUNT: int = 3 DEFAULT_SEED_CSV_SETTINGS: SeedCsvSettings = SeedCsvSettings() -CHANGES_ONLY_SETTING_OVERRIDE_KEY: str = "changes_only" CURSOR_POLICY_DISABLED: str = "disabled" ZERO_DAY_CURSOR_DURATION: str = "0d" EFFECTIVE_BATCH_SIZE_TOKEN: str = "effective" +SCENARIO_RUN_NAMESPACE_CONFIG_KEY: str = "run_namespace" TIME_TRAVEL_RETENTION_MATERIALIZATIONS: tuple[str, ...] = ( "table", "incremental", diff --git a/src/sqlbuild/spec/contracts/main/parse_scenario_run_namespace.py b/src/sqlbuild/spec/contracts/main/parse_scenario_run_namespace.py new file mode 100644 index 0000000000..422794b3ae --- /dev/null +++ b/src/sqlbuild/spec/contracts/main/parse_scenario_run_namespace.py @@ -0,0 +1,18 @@ +"""Validation for the optional scenario artifact run namespace.""" + +from __future__ import annotations + +import re + +from sqlbuild.spec.contracts.exceptions import SpecConfigError + + +def parse_scenario_run_namespace(value: object) -> str: + """Return a valid namespace or reject it without normalization.""" + + if not isinstance(value, str) or re.fullmatch(r"[A-Za-z0-9_.-]{1,128}", value) is None: + raise SpecConfigError( + "scenario.run_namespace must be 1–128 characters: " + "ASCII letters, digits, '-', '_' or '.'" + ) + return value diff --git a/src/sqlbuild/spec/contracts/main/resolve_effective_changes_only.py b/src/sqlbuild/spec/contracts/main/resolve_effective_changes_only.py deleted file mode 100644 index 5f7c12de07..0000000000 --- a/src/sqlbuild/spec/contracts/main/resolve_effective_changes_only.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public effective changes-only resolution operation.""" - -from sqlbuild.spec.contracts._helpers.targets import ( - resolve_effective_changes_only as _resolve_effective_changes_only, -) -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig - - -def resolve_effective_changes_only( - *, - project_config: ProjectConfig, - local_config: LocalConfig, - selected_target: str | None, - cli_changes_only: bool, -) -> bool: - """Resolve changes-only selection with CLI, target, and settings precedence.""" - - return _resolve_effective_changes_only( - project_config=project_config, - local_config=local_config, - selected_target=selected_target, - cli_changes_only=cli_changes_only, - ) diff --git a/src/sqlbuild/spec/contracts/models.py b/src/sqlbuild/spec/contracts/models.py index 8d92ed41af..fb38ff572e 100644 --- a/src/sqlbuild/spec/contracts/models.py +++ b/src/sqlbuild/spec/contracts/models.py @@ -91,17 +91,6 @@ class ResolvedTableType: declared: bool = False -@dataclass(frozen=True) -class StateConfig: - """Virtual mode state store configuration.""" - - backend: str | None = None - schema: str | None = None - connection: dict[str, object] = field(default_factory=dict) - allow_reset: bool = False - unsuffixed_virtual_env: str | None = None - - @dataclass(frozen=True) class ExecutionLimitsConfig: """Optional target-scoped build execution limits.""" @@ -112,17 +101,6 @@ class ExecutionLimitsConfig: remediation: str | None = None -@dataclass(frozen=True) -class LocalStateConfig: - """Local virtual environment mode state store overrides.""" - - backend: str | None = None - schema: str | None = None - connection: dict[str, object] = field(default_factory=dict) - allow_reset: bool | None = None - unsuffixed_virtual_env: str | None = None - - @dataclass(frozen=True) class TargetConfig: """One named target configuration.""" @@ -135,9 +113,7 @@ class TargetConfig: loader_schema: str | None = None defer_sources_to: str | None = None defer_clone_from: str | None = None - changes_only: bool | None = None clone: ClonePolicy = field(default_factory=ClonePolicy) - state: StateConfig = field(default_factory=StateConfig) compile_cache: bool | None = None time_travel_retention: AuthoredTimeTravelRetention | None = None time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] = field( @@ -162,9 +138,7 @@ class LocalTargetConfig: loader_schema: str | None = None defer_sources_to: str | None = None defer_clone_from: str | None = None - changes_only: bool | None = None clone: LocalClonePolicy = field(default_factory=LocalClonePolicy) - state: LocalStateConfig = field(default_factory=LocalStateConfig) compile_cache: bool | None = None time_travel_retention: AuthoredTimeTravelRetention | None = None time_travel_retention_by_materialization: dict[str, AuthoredTimeTravelRetention] | None = None @@ -184,8 +158,6 @@ class SettingsConfig: column_contract_mode: ColumnContractMode = ColumnContractMode.IMPLICIT concurrency: int = 1 auto_load_sources: bool = True - changes_only: bool = False - virtual_environments: bool = False microbatch_concurrency: bool = False microbatch_unaccounted_partition_policy: str = "synthesize" table_promotion_mode: str | None = None @@ -290,7 +262,6 @@ class DefaultsConfig: max_microbatches: int | None = None unaccounted_partition_policy: str | None = None replay_on_change: str | None = None - run_despite_unchanged: str | None = None row_diff_exclude_columns: tuple[str, ...] = field(default_factory=tuple) row_diff_tolerances: dict[str, object] = field(default_factory=dict) row_diff_sample_rows: int | None = None @@ -315,7 +286,6 @@ class JanitorConfig: enabled: bool = False retention_days: int | None = None archive_retention_days: int = 14 - max_checkpoints: int = 20 direct_state_history_versions: int = 20 delete_tracked_only: bool = True exclude_patterns: tuple[str, ...] = field(default_factory=tuple) @@ -345,6 +315,7 @@ class ScenarioSnapshotLimitsConfig: class ScenarioConfig: """Scenario command configuration.""" + run_namespace: str | None = None local_type_overrides: dict[str, dict[str, str]] = field(default_factory=dict) snapshot_limits: ScenarioSnapshotLimitsConfig = field( default_factory=ScenarioSnapshotLimitsConfig @@ -556,7 +527,7 @@ class SourceFreshnessAgePolicy: @dataclass(frozen=True) class SourceFreshnessConfig: - """Configured source freshness observation for virtual planning.""" + """Configured source freshness observation for planning.""" strategy: SourceFreshnessStrategy value_kind: SourceFreshnessValueKind | None = None diff --git a/src/sqlbuild/virtual/__init__.py b/src/sqlbuild/virtual/__init__.py deleted file mode 100644 index b2529e8578..0000000000 --- a/src/sqlbuild/virtual/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual environment mode runtime package.""" diff --git a/src/sqlbuild/virtual/diff/__init__.py b/src/sqlbuild/virtual/diff/__init__.py deleted file mode 100644 index 578917b9ea..0000000000 --- a/src/sqlbuild/virtual/diff/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual diff domain.""" diff --git a/src/sqlbuild/virtual/diff/_helpers/__init__.py b/src/sqlbuild/virtual/diff/_helpers/__init__.py deleted file mode 100644 index d49a60b736..0000000000 --- a/src/sqlbuild/virtual/diff/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual diff helper package.""" diff --git a/src/sqlbuild/virtual/diff/_helpers/diff.py b/src/sqlbuild/virtual/diff/_helpers/diff.py deleted file mode 100644 index cc5519b35a..0000000000 --- a/src/sqlbuild/virtual/diff/_helpers/diff.py +++ /dev/null @@ -1,292 +0,0 @@ -"""Virtual diff helper functions.""" - -from __future__ import annotations - -import time -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors -from sqlbuild.executor.diff.main.execute import execute_diff -from sqlbuild.executor.diff.models import DiffExecutionOptions, DiffExecutionResult -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.virtual.diff.models import VirtualDiffOptions, VirtualDiffState -from sqlbuild.virtual.executor.main._rewrite import rewrite_virtual_project_model_locations -from sqlbuild.virtual.planner.main._semantics import build_virtual_plan_semantics -from sqlbuild.virtual.planner.main._targets import build_virtual_destination_from_physical_relation -from sqlbuild.virtual.planner.models import VirtualPlanSemantics -from sqlbuild.virtual.state.main.environments._ref_model_versions import ( - read_ref_model_versions, -) -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - - -def resolve_virtual_diff_model_names( - *, graph: ProjectGraph, select: tuple[str, ...], exclude: tuple[str, ...] -) -> tuple[str, ...]: - """Resolve model names selected for virtual diff.""" - - if not select: - return tuple(model.name for model in graph.project.models) - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=select, - exclude=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - return tuple( - sorted(key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL) - ) - - -def read_physical_relations_for_refs( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, PhysicalRelationRecord]: - """Read tracked physical relations for VDE refs.""" - - relations: dict[str, PhysicalRelationRecord] = {} - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is not None: - relations[ref.model_name] = relation - return relations - - -def filter_models_with_changed_virtual_refs( - *, - selected_names: tuple[str, ...], - from_refs: tuple[VirtualEnvironmentModelRefRecord, ...], - to_refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> tuple[tuple[str, ...], tuple[str, ...]]: - """Split selected models into changed/missing refs and identical refs.""" - - from_hashes: dict[str, str] = {ref.model_name: ref.version_hash for ref in from_refs} - to_hashes: dict[str, str] = {ref.model_name: ref.version_hash for ref in to_refs} - changed_names: list[str] = [] - skipped_names: list[str] = [] - for name in selected_names: - from_hash: str | None = from_hashes.get(name) - to_hash: str | None = to_hashes.get(name) - if from_hash is not None and from_hash == to_hash: - skipped_names.append(name) - else: - changed_names.append(name) - return tuple(changed_names), tuple(skipped_names) - - -def rewrite_project_to_physical_relations( - *, adapter: BaseAdapter, project: CompiledProject, relations: dict[str, PhysicalRelationRecord] -) -> CompiledProject: - """Rewrite a project's model locations to tracked physical relations.""" - - targets: dict[str, CompiledRelationLocation] = { - model.name: build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=relations[model.name], - fallback_target=model.destination, - ) - for model in project.models - if model.name in relations - } - return rewrite_virtual_project_model_locations(project=project, rewritten_locations=targets) - - -def non_finalized_target_names( - environments: tuple[tuple[str, VirtualEnvironmentRecord | None], ...], -) -> tuple[str, ...]: - return tuple( - name - for name, environment in environments - if environment is None or environment.status != VirtualEnvironmentStatus.FINALIZED - ) - - -def is_working_environment(environment: VirtualEnvironmentRecord | None) -> bool: - return environment is None or environment.status != VirtualEnvironmentStatus.FINALIZED - - -def read_virtual_diff_state( - *, - backend: Any, - state_connection: Any, - schema: str, - graph: ProjectGraph, - from_virtual_environment_name: str, - to_virtual_environment_name: str, - require_finalized: bool, -) -> VirtualDiffState: - """Read refs, semantics, and physical relations for both diffed VDEs.""" - - from_environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - to_environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ) - from_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - ) - to_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ) - ) - if not from_refs: - raise PlannerInputError( - f"unknown virtual environment '{from_virtual_environment_name}'", - code="S011", - ) - if not to_refs: - raise PlannerInputError( - f"unknown virtual environment '{to_virtual_environment_name}'", - code="S011", - ) - from_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=graph, - bound_refs=from_refs, - bound_model_versions=read_ref_model_versions( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=from_refs, - ), - source_freshness_records=backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ), - ) - to_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=graph, - bound_refs=to_refs, - bound_model_versions=read_ref_model_versions( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=to_refs, - ), - source_freshness_records=backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ), - ) - if require_finalized: - non_finalized: tuple[str, ...] = non_finalized_target_names( - ( - (from_virtual_environment_name, from_environment), - (to_virtual_environment_name, to_environment), - ) - ) - if non_finalized: - raise PlannerInputError( - "whole-VDE virtual diff requires finalized VDEs; non-finalized VDEs: " - + ", ".join(non_finalized), - code="S012", - help="Re-run with --allow-partial-diff to inspect a working VDE.", - ) - return VirtualDiffState( - from_environment=from_environment, - to_environment=to_environment, - from_refs=from_refs, - to_refs=to_refs, - from_semantics=from_semantics, - to_semantics=to_semantics, - from_relations=read_physical_relations_for_refs( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=from_refs, - ), - to_relations=read_physical_relations_for_refs( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=to_refs, - ), - ) - - -def execute_virtual_diff_between_relations( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - left_project: CompiledProject, - right_project: CompiledProject, - compared_names: tuple[str, ...], - options: VirtualDiffOptions, - hooks: ConnectionHooks, -) -> DiffExecutionResult: - """Open a warehouse connection with progress callbacks and run the diff.""" - - started_at: float = time.perf_counter() - if hooks.on_connection_start is not None: - hooks.on_connection_start(1) - connection: Any - try: - connection = adapter.connect(connection_config) - except Exception: - if hooks.on_connection_error is not None: - hooks.on_connection_error(1, elapsed_seconds=time.perf_counter() - started_at) - raise - if hooks.on_connection_complete is not None: - hooks.on_connection_complete(1, elapsed_seconds=time.perf_counter() - started_at) - try: - return execute_diff( - adapter=adapter, - connection=connection, - left_project=left_project, - right_project=right_project, - selected_names=compared_names, - options=DiffExecutionOptions( - schema_only=options.schema_only, - bounded=options.bounded, - collect_samples=options.collect_samples, - max_column_examples=options.max_column_examples, - max_row_only_examples=options.max_row_only_examples, - max_models=options.max_models, - max_columns=options.max_columns, - sampling_override=options.sampling_override, - unique_key_override=options.unique_key_override, - unkeyed=options.unkeyed, - excluded_columns_override=options.excluded_columns_override, - tolerance_overrides=options.tolerance_overrides, - ), - ) - finally: - adapter.close(connection) diff --git a/src/sqlbuild/virtual/diff/main/__init__.py b/src/sqlbuild/virtual/diff/main/__init__.py deleted file mode 100644 index 1657c24a05..0000000000 --- a/src/sqlbuild/virtual/diff/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual diff public entrypoints.""" diff --git a/src/sqlbuild/virtual/diff/main/diff.py b/src/sqlbuild/virtual/diff/main/diff.py deleted file mode 100644 index 500137533c..0000000000 --- a/src/sqlbuild/virtual/diff/main/diff.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Virtual-mode diff entrypoint.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.executor.diff.models import DiffExecutionResult -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.virtual.diff._helpers.diff import ( - execute_virtual_diff_between_relations, - filter_models_with_changed_virtual_refs, - is_working_environment, - read_virtual_diff_state, - resolve_virtual_diff_model_names, - rewrite_project_to_physical_relations, -) -from sqlbuild.virtual.diff.models import VirtualDiffOptions, VirtualDiffState -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime - - -def run_virtual_diff( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - from_virtual_environment_name: str, - to_virtual_environment_name: str, - options: VirtualDiffOptions | None = None, - hooks: ConnectionHooks | None = None, -) -> tuple[ - DiffExecutionResult, - tuple[str, ...], - tuple[str, ...], - tuple[str, ...], - tuple[str, ...], - bool, - bool, -]: - """Run a diff between two VDEs in the active physical environment.""" - - resolved: VirtualDiffOptions = options if options is not None else VirtualDiffOptions() - resolved_hooks: ConnectionHooks = hooks if hooks is not None else ConnectionHooks() - on_progress: Callable[[str], None] | None = resolved_hooks.on_progress - compile_start: float = time.perf_counter() - if on_progress is not None: - on_progress("Compiling project...") - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - no_sql_validation=resolved.no_sql_validation, - cli_vars=resolved.cli_vars, - external_sql_reference_resolver=resolved.external_sql_reference_resolver, - ) - if on_progress is not None: - on_progress(f"Compiled project. ({time.perf_counter() - compile_start:.2f}s)") - selected_names: tuple[str, ...] = resolve_virtual_diff_model_names( - graph=graph, - select=resolved.select, - exclude=resolved.exclude, - ) - if not selected_names: - selected_names = tuple(model.name for model in graph.project.models) - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - state_connection: Any = backend.connect(config.connection) - try: - inspect_start: float = time.perf_counter() - if on_progress is not None: - on_progress("Inspecting virtual state...") - state: VirtualDiffState = read_virtual_diff_state( - backend=backend, - state_connection=state_connection, - schema=config.schema, - graph=graph, - from_virtual_environment_name=from_virtual_environment_name, - to_virtual_environment_name=to_virtual_environment_name, - require_finalized=not resolved.select and not resolved.allow_partial_diff, - ) - if on_progress is not None: - on_progress(f"Inspected virtual state. ({time.perf_counter() - inspect_start:.2f}s)") - finally: - backend.close(state_connection) - - compared_names: tuple[str, ...] - skipped_names: tuple[str, ...] - compared_names, skipped_names = filter_models_with_changed_virtual_refs( - selected_names=selected_names, - from_refs=state.from_refs, - to_refs=state.to_refs, - ) - missing: tuple[str, ...] = tuple( - name - for name in compared_names - if name not in state.from_relations or name not in state.to_relations - ) - if missing: - raise PlannerInputError( - "virtual diff selected models missing tracked physical relations: " - + ", ".join(missing), - code="S013", - ) - if not compared_names: - return ( - DiffExecutionResult(), - selected_names, - skipped_names, - state.from_semantics.stale_model_names, - state.to_semantics.stale_model_names, - is_working_environment(state.from_environment), - is_working_environment(state.to_environment), - ) - - left_project: CompiledProject = rewrite_project_to_physical_relations( - adapter=adapter, - project=graph.project, - relations=state.from_relations, - ) - right_project: CompiledProject = rewrite_project_to_physical_relations( - adapter=adapter, - project=graph.project, - relations=state.to_relations, - ) - result: DiffExecutionResult = execute_virtual_diff_between_relations( - adapter=adapter, - connection_config=connection_config, - left_project=left_project, - right_project=right_project, - compared_names=compared_names, - options=resolved, - hooks=resolved_hooks, - ) - return ( - result, - selected_names, - skipped_names, - state.from_semantics.stale_model_names, - state.to_semantics.stale_model_names, - is_working_environment(state.from_environment), - is_working_environment(state.to_environment), - ) diff --git a/src/sqlbuild/virtual/diff/models.py b/src/sqlbuild/virtual/diff/models.py deleted file mode 100644 index a8fc8974cc..0000000000 --- a/src/sqlbuild/virtual/diff/models.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Virtual diff domain models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from sqlbuild.adapter.contract.models import RowDiffTolerances -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.executor.diff.models import RowDiffSamplingOverride -from sqlbuild.virtual.planner.models import VirtualPlanSemantics -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRecord, -) - - -@dataclass(frozen=True) -class VirtualDiffOptions: - """Selection, comparison, and sampling options for one virtual diff run.""" - - no_sql_validation: bool = False - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - schema_only: bool = False - bounded: str | None = None - collect_samples: bool = False - max_column_examples: int = 20 - max_row_only_examples: int = 20 - max_models: int | None = None - max_columns: int | None = None - sampling_override: RowDiffSamplingOverride = field(default_factory=RowDiffSamplingOverride) - unique_key_override: tuple[str, ...] = () - unkeyed: bool = False - excluded_columns_override: tuple[str, ...] = () - tolerance_overrides: RowDiffTolerances | None = None - allow_partial_diff: bool = False - cli_vars: dict[str, object] | None = None - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - - -@dataclass(frozen=True) -class VirtualDiffState: - """Bound refs, semantics, and physical relations for both diffed VDEs.""" - - from_environment: VirtualEnvironmentRecord | None - to_environment: VirtualEnvironmentRecord | None - from_refs: tuple[VirtualEnvironmentModelRefRecord, ...] - to_refs: tuple[VirtualEnvironmentModelRefRecord, ...] - from_semantics: VirtualPlanSemantics - to_semantics: VirtualPlanSemantics - from_relations: dict[str, PhysicalRelationRecord] - to_relations: dict[str, PhysicalRelationRecord] diff --git a/src/sqlbuild/virtual/executor/__init__.py b/src/sqlbuild/virtual/executor/__init__.py deleted file mode 100644 index 2c8b8cec3b..0000000000 --- a/src/sqlbuild/virtual/executor/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual executor domain.""" diff --git a/src/sqlbuild/virtual/executor/_helpers/__init__.py b/src/sqlbuild/virtual/executor/_helpers/__init__.py deleted file mode 100644 index 3ba61aa5ab..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual executor helpers.""" diff --git a/src/sqlbuild/virtual/executor/_helpers/build.py b/src/sqlbuild/virtual/executor/_helpers/build.py deleted file mode 100644 index 2af08b9849..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/build.py +++ /dev/null @@ -1,2240 +0,0 @@ -"""Virtual-mode build entrypoint.""" - -from __future__ import annotations - -import hashlib -import json -import threading -import time -from collections.abc import Callable -from dataclasses import dataclass, replace -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -import sqlbuild.adapter.relations.main.resolve_qualified_name_parts as relation_parts -import sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name as rn -import sqlbuild.executor.build.types -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.compile.constants import MIGRATE_FROM_CONFIG_KEY -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.main.materializations import load_custom_materializations -from sqlbuild.compiler.pipeline.main.prepare_versions import ( - load_custom_prepare_version_functions, -) -from sqlbuild.compiler.pipeline.main.relation_targets import build_python_relation_targets -from sqlbuild.compiler.pipeline.models import CompilePipelineResult, ProjectGraph, PythonPlanEntry -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.changes.replay_on_change import resolve_replay_on_change -from sqlbuild.compiler.planner.main.execution.plan_entry import ( - build_plan_output_from_model_changes_phase, -) -from sqlbuild.compiler.planner.main.execution.warehouse_snapshot import ( - build_warehouse_snapshot_phase, -) -from sqlbuild.compiler.planner.main.selection.build_resources import expand_build_resource_selection -from sqlbuild.compiler.planner.models import ( - BackfillResult, - ChangeDetectionResult, - ModelChangesPlanInputs, - ModelPlanEntry, - PlannerScope, - PlannerWarehouseSnapshotResult, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - ChangeKind, - IncrementalMode, - PlanAction, - PlanReason, - WorkSelectionPolicy, -) -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) -from sqlbuild.compiler.python_nodes.types import PythonNodeStatus -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.executor.build.constants import INCREMENTAL_ACTIONS -from sqlbuild.executor.build.models import ( - BuildCallbacks, - BuildCustomizations, - BuildExecutionResult, - BuildInitialState, - BuildRuntimeParams, - SeedExecutionResult, -) -from sqlbuild.executor.build.types import BuildStatus, ExecutionStatus -from sqlbuild.executor.custom.models import ( - MaterializationContext, - MaterializationResult, - PrepareVersionContext, -) -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.pipeline.main.run import run_build_pipeline -from sqlbuild.executor.python_nodes.main.ingress import run_ingress_python_loader_nodes -from sqlbuild.executor.python_nodes.main.read_side import create_read_side_python_execution_tracker -from sqlbuild.executor.python_nodes.models import ( - IngressCallbacks, - PythonIngressLoaderExecutorResult, - PythonNodeExecutionResult, - PythonNodeRuntime, -) -from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder -from sqlbuild.microbatches.constants import VIRTUAL_MICROBATCH_SCOPE_KIND -from sqlbuild.microbatches.models import MicrobatchScope -from sqlbuild.microbatches.types import MicrobatchEventStore -from sqlbuild.python_nodes.models import SqlResourceRef -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import SnapshotsConfig -from sqlbuild.virtual.executor._helpers.environment_views import write_logical_vde_views -from sqlbuild.virtual.executor._helpers.functions import build_function_version_record -from sqlbuild.virtual.executor._helpers.rewrite import ( - build_rewritten_model_locations, - build_rewritten_seed_locations, - relation_type_for_model, - rewrite_project_function_locations, - rewrite_project_model_locations, - rewrite_project_seed_locations, -) -from sqlbuild.virtual.executor._helpers.seeding import ( - include_stale_upstream_seed_names, - resolve_virtual_seed_selection, - seed_virtual_physical_version, -) -from sqlbuild.virtual.executor.classes.microbatch_lease_manager import ( - VirtualMicrobatchLeaseManager, -) -from sqlbuild.virtual.executor.classes.node_result_store import VirtualNodeResultStore -from sqlbuild.virtual.executor.models import ( - VirtualBuildExecutionHooks, - VirtualBuildHooks, - VirtualBuildOptions, - VirtualBuildPipelineResult, - VirtualEnvironmentNames, -) -from sqlbuild.virtual.freshness.main._current_records import ( - build_current_virtual_source_freshness_records, -) -from sqlbuild.virtual.freshness.main._runtime_observation import ( - observe_virtual_environment_source_freshness, -) -from sqlbuild.virtual.freshness.main._runtime_persistence import ( - persist_virtual_environment_source_freshness, -) -from sqlbuild.virtual.freshness.models import SourceFreshnessRuntimeResult -from sqlbuild.virtual.planner.main._output import apply_virtual_plan_output -from sqlbuild.virtual.planner.main._python_identities import read_bound_virtual_python_identities -from sqlbuild.virtual.planner.main._python_plan_entries import build_virtual_python_plan_entries -from sqlbuild.virtual.planner.main._python_run_selection import build_virtual_python_run_selection -from sqlbuild.virtual.planner.main._selection import resolve_virtual_plan_model_selection -from sqlbuild.virtual.planner.main._semantics import ( - build_virtual_plan_semantics, -) -from sqlbuild.virtual.planner.main._targets import ( - build_virtual_destination_from_physical_relation, -) -from sqlbuild.virtual.planner.models import VirtualPlanSemantics -from sqlbuild.virtual.state.classes.microbatch_store import VirtualMicrobatchEventStore -from sqlbuild.virtual.state.main.checkpoints._build_finalized_checkpoint import ( - build_finalized_virtual_environment_checkpoint, -) -from sqlbuild.virtual.state.main.encoding._decode_state_text import decode_state_text -from sqlbuild.virtual.state.main.encoding._encode_state_text import encode_state_text -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.main.python_identities._python_node_identity_write import ( - try_record_virtual_python_node_identity, -) -from sqlbuild.virtual.state.models import ( - FinalizedVirtualEnvironmentCheckpoint, - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationAncestryRecord, - PhysicalRelationRecord, - SeedVersionRecord, - SourceFreshnessRecord, - StateBackendConfig, - StateLockLease, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - VirtualEnvironmentStatus, -) - - -@dataclass(frozen=True) -class _VirtualBuildRuntime: - """Shared inputs, state runtime, and naming context for one virtual build run.""" - - project_dir: Path - discovered_inputs: DiscoveredProjectInputs - adapter: BaseAdapter - connection_config: dict[str, object] - options: VirtualBuildOptions - hooks: VirtualBuildHooks - backend: Any - config: StateBackendConfig - names: VirtualEnvironmentNames - - -@dataclass(frozen=True) -class _VirtualBuildStateReads: - """Bound state, semantics, and resolved selection read for one virtual build run.""" - - bound_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] - previous_function_query_sqls: dict[str, str] - bound_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] - semantics: VirtualPlanSemantics - selected_model_names: tuple[str, ...] - selected_seed_names: tuple[str, ...] - desired_seed_version_hashes: dict[str, str] - available_seed_physical_relations: dict[str, PhysicalRelationRecord] - seed_load_names: tuple[str, ...] - effective_select: tuple[str, ...] - bound_physical_relations: dict[str, PhysicalRelationRecord] - - -@dataclass(frozen=True) -class _RewrittenVirtualProject: - """Version-rewritten project and deferred relations for one virtual build run.""" - - project: CompiledProject - deferred_relations: dict[str, RelationInfo] - - -@dataclass(frozen=True) -class _VirtualBuildPlan: - """Display plan and executor plan for one virtual build run.""" - - plan_output: PlanOutput - executor_plan_output: PlanOutput - - -@dataclass(frozen=True) -class _VirtualBuildResolution: - """Read-only virtual planning resolution shared by plan and build.""" - - runtime: _VirtualBuildRuntime - graph: ProjectGraph - reads: _VirtualBuildStateReads - rewritten: _RewrittenVirtualProject - plan: _VirtualBuildPlan - python_plan: _VirtualPythonPlan - compile_seconds: float - planning_seconds: float - - -@dataclass(frozen=True) -class _VirtualPythonPlan: - """Python-node graph, lifecycle, plan entries, and relation targets for one run.""" - - python_graph: PythonNodeGraph - lifecycle_plan: PythonSqlRunLifecyclePlan - plan_entries: tuple[PythonPlanEntry, ...] - relation_targets: dict[SqlResourceRef, str] - selected_python_node_names: frozenset[str] - - -@dataclass(frozen=True) -class _FunctionPersistOutcome: - """Final function version hashes and node types persisted for one run.""" - - final_function_hashes: dict[str, str] - function_ref_node_types: dict[str, str] - - -@dataclass(frozen=True) -class _SeedPersistOutcome: - """Final seed version hashes and physical relations persisted for one run.""" - - final_seed_hashes: dict[str, str] - final_seed_physical_relations: dict[str, PhysicalRelationRecord] - - -def resolve_virtual_plan( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - options: VirtualBuildOptions, - hooks: VirtualBuildHooks, -) -> CompilePipelineResult: - """Resolve the read-only virtual plan consumed by plan and build.""" - - resolution: _VirtualBuildResolution = _resolve_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=options, - hooks=hooks, - ) - return CompilePipelineResult( - project=resolution.rewritten.project, - plan_output=resolution.plan.plan_output, - python_node_names=resolution.python_plan.selected_python_node_names, - python_plan_entries=resolution.python_plan.plan_entries, - compile_seconds=resolution.compile_seconds, - planning_seconds=resolution.planning_seconds, - ) - - -def run_virtual_build( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - options: VirtualBuildOptions, - hooks: VirtualBuildHooks, -) -> VirtualBuildPipelineResult: - """Execute a virtual-mode build.""" - - resolution: _VirtualBuildResolution = _resolve_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=options, - hooks=hooks, - ) - runtime: _VirtualBuildRuntime = resolution.runtime - graph: ProjectGraph = resolution.graph - reads: _VirtualBuildStateReads = resolution.reads - rewritten: _RewrittenVirtualProject = resolution.rewritten - plan: _VirtualBuildPlan = resolution.plan - python_plan: _VirtualPythonPlan = resolution.python_plan - output: PlanOutput = plan.plan_output - entries: tuple[PythonPlanEntry, ...] = python_plan.plan_entries - exec_hooks: VirtualBuildExecutionHooks = ( - hooks.on_plan_ready( - project=rewritten.project, - plan_output=output, - python_plan_entries=entries, - ) - if hooks.on_plan_ready is not None - else VirtualBuildExecutionHooks() - ) - lease_manager: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=runtime.backend, - connection_config=runtime.config.connection, - warehouse_connection_config=runtime.connection_config, - schema=runtime.config.schema, - run_id=rewritten.project.run_id, - ) - try: - lease_manager.acquire( - entries=plan.executor_plan_output.model_entries, - expected_version_hashes=reads.semantics.expected_version_hashes, - ) - _register_virtual_microbatch_physical_relations( - runtime=runtime, - plan_output=plan.executor_plan_output, - expected_version_hashes=reads.semantics.expected_version_hashes, - ) - result, ingress_python_results, read_side_results = _execute_leased_virtual_build( - runtime=runtime, - graph=graph, - reads=reads, - rewritten=rewritten, - plan=plan, - python_plan=python_plan, - exec_hooks=exec_hooks, - microbatch_lease_check=lease_manager.assert_active, - model_version_leases=lease_manager.active_leases, - ) - finally: - lease_manager.close() - - return VirtualBuildPipelineResult( - project=rewritten.project, - display_plan_output=plan.plan_output, - execution_plan=plan.executor_plan_output, - execution_result=result, - virtual_environment_name=runtime.names.target_vde_name, - python_node_results=(*ingress_python_results, *read_side_results), - compile_seconds=resolution.compile_seconds, - planning_seconds=resolution.planning_seconds, - ) - - -def _register_virtual_microbatch_physical_relations( - *, - runtime: _VirtualBuildRuntime, - plan_output: PlanOutput, - expected_version_hashes: dict[str, str], -) -> None: - """Register leased microbatch targets before execution so failure-state GC can resolve them.""" - - entries: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in plan_output.model_entries - if entry.incremental_mode == IncrementalMode.MICROBATCH and entry.action != PlanAction.SKIP - ) - if not entries: - return - connection: Any = runtime.backend.connect(runtime.config.connection) - try: - for entry in entries: - version_hash: str = expected_version_hashes.get( - entry.name, - entry.fingerprint_version_hash or compute_query_hash(entry.fingerprint_query_sql), - ) - existing_relation: PhysicalRelationRecord | None = ( - runtime.backend.get_physical_relation( - connection=connection, - schema=runtime.config.schema, - model_name=entry.name, - version_hash=version_hash, - ) - ) - if existing_relation is not None: - continue - runtime.backend.upsert_physical_relation( - connection=connection, - schema=runtime.config.schema, - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=entry.name, - version_hash=version_hash, - database_name=entry.destination.database, - schema_name=entry.destination.schema or "", - relation_name=entry.destination.name, - relation_type="table", - ), - ) - finally: - runtime.backend.close(connection) - - -def _execute_leased_virtual_build( - *, - runtime: _VirtualBuildRuntime, - graph: ProjectGraph, - reads: _VirtualBuildStateReads, - rewritten: _RewrittenVirtualProject, - plan: _VirtualBuildPlan, - python_plan: _VirtualPythonPlan, - exec_hooks: VirtualBuildExecutionHooks, - microbatch_lease_check: Callable[[], None], - model_version_leases: tuple[StateLockLease, ...], -) -> tuple[ - BuildExecutionResult, - tuple[PythonNodeExecutionResult, ...], - tuple[PythonNodeExecutionResult, ...], -]: - """Execute and publish a virtual build while physical-version leases are held.""" - - with CostContext.scope( - run_id=rewritten.project.run_id, - resource_type="run", - resource_name=rewritten.project.effective_target_name or "virtual", - ledger_path=( - runtime.project_dir - / "target" - / "executions" - / rewritten.project.run_id - / "statements.jsonl" - ), - phase="virtual_ingress", - on_statement_complete=exec_hooks.on_statement_complete, - ): - ingress_result: PythonIngressLoaderExecutorResult | None = _run_ingress_python_nodes( - runtime=runtime, - python_plan=python_plan, - plan_output=plan.plan_output, - project=rewritten.project, - exec_hooks=exec_hooks, - ) - ingress_python_results: tuple[PythonNodeExecutionResult, ...] = ( - ingress_result.python_results if ingress_result is not None else () - ) - ingress_load_results: tuple[LoadExecutionResult, ...] = ( - ingress_result.load_results if ingress_result is not None else () - ) - ingress_failed: bool = any( - load_result.status == ExecutionStatus.FAILED for load_result in ingress_load_results - ) or any(result.status == PythonNodeStatus.FAILED for result in ingress_python_results) - if ingress_failed: - result: BuildExecutionResult = BuildExecutionResult( - status=BuildStatus.FAILED, load_results=ingress_load_results - ) - else: - result = _execute_virtual_build_plan( - runtime=runtime, - plan=plan, - project=rewritten.project, - python_plan=python_plan, - reads=reads, - exec_hooks=exec_hooks, - ingress_load_results=ingress_load_results, - microbatch_lease_check=microbatch_lease_check, - ) - if result.status != BuildStatus.SUCCESS: - return result, ingress_python_results, () - with CostContext.scope( - run_id=rewritten.project.run_id, - resource_type="run", - resource_name=rewritten.project.effective_target_name or "virtual", - ledger_path=( - runtime.project_dir - / "target" - / "executions" - / rewritten.project.run_id - / "statements.jsonl" - ), - phase="virtual_finalize", - on_statement_complete=exec_hooks.on_statement_complete, - ): - microbatch_lease_check() - _persist_successful_virtual_build( - runtime=runtime, - project=graph.project, - reads=reads, - plan_output=plan.executor_plan_output, - result=result, - model_version_leases=model_version_leases, - ) - read_side_results: tuple[PythonNodeExecutionResult, ...] = _run_read_side_python_nodes( - runtime=runtime, - python_plan=python_plan, - project=rewritten.project, - result=result, - ) - if any(python_result.status == PythonNodeStatus.FAILED for python_result in read_side_results): - result = replace(result, status=BuildStatus.FAILED) - return result, ingress_python_results, read_side_results - - -def _reject_model_migrations(*, project: CompiledProject) -> None: - declared: tuple[str, ...] = tuple( - model.name - for model in project.models - if model.config.values.get(MIGRATE_FROM_CONFIG_KEY) is not None - ) - if declared: - raise PlannerInputError( - "migrate_from is supported only in direct mode; virtual environments do not run " - f"model migrations (declared by: {', '.join(declared)})", - code="M105", - ) - - -def _resolve_virtual_build( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - options: VirtualBuildOptions, - hooks: VirtualBuildHooks, -) -> _VirtualBuildResolution: - compile_start: float = time.monotonic() - try: - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - selected_target=options.planning.selected_target, - no_sql_validation=options.planning.no_sql_validation, - no_cache=options.planning.no_cache, - cli_vars=options.planning.cli_vars, - external_sql_reference_resolver=options.planning.external_sql_reference_resolver, - on_progress=hooks.on_progress, - ) - finally: - compile_seconds: float = time.monotonic() - compile_start - timing_tracker: BuildPhaseTimingTracker | None = BuildPhaseTimingTracker.current() - if timing_tracker is not None: - timing_tracker.compile_seconds = compile_seconds - planning_start: float = time.monotonic() - try: - names: VirtualEnvironmentNames = _resolve_virtual_environment_names( - discovered_inputs=discovered_inputs, - options=options, - ) - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - runtime: _VirtualBuildRuntime = _VirtualBuildRuntime( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=options, - hooks=hooks, - backend=backend, - config=config, - names=names, - ) - reads: _VirtualBuildStateReads = _read_virtual_build_state(runtime=runtime, graph=graph) - rewritten: _RewrittenVirtualProject = _rewrite_virtual_project( - runtime=runtime, graph=graph, reads=reads - ) - plan: _VirtualBuildPlan = _plan_virtual_build( - runtime=runtime, - graph=graph, - project=rewritten.project, - reads=reads, - deferred_relations=rewritten.deferred_relations, - ) - python_plan: _VirtualPythonPlan = _prepare_virtual_python_execution( - runtime=runtime, - graph=graph, - project=rewritten.project, - plan_output=plan.plan_output, - selected_model_names=reads.selected_model_names, - ) - finally: - planning_seconds: float = time.monotonic() - planning_start - if timing_tracker is not None: - timing_tracker.planning_seconds = planning_seconds - return _VirtualBuildResolution( - runtime=runtime, - graph=graph, - reads=reads, - rewritten=rewritten, - plan=plan, - python_plan=python_plan, - compile_seconds=compile_seconds, - planning_seconds=planning_seconds, - ) - - -def _resolve_virtual_environment_names( - *, - discovered_inputs: DiscoveredProjectInputs, - options: VirtualBuildOptions, -) -> VirtualEnvironmentNames: - physical_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=options.planning.selected_target, - ) - unsuffixed_virtual_environment_name: str | None = None - if physical_target_name is not None: - unsuffixed_virtual_environment_name = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=physical_target_name, - ).state.unsuffixed_virtual_env - return VirtualEnvironmentNames( - target_vde_name=( - options.planning.virtual_environment_name or physical_target_name or "default" - ), - physical_target_name=physical_target_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - - -def _read_virtual_build_state( - *, - runtime: _VirtualBuildRuntime, - graph: ProjectGraph, -) -> _VirtualBuildStateReads: - backend: Any = runtime.backend - config: StateBackendConfig = runtime.config - options: VirtualBuildOptions = runtime.options - target_vde_name: str = runtime.names.target_vde_name - state_connection: Any = backend.connect(config.connection) - try: - bound_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = _read_or_initialize_refs( - backend=backend, - state_connection=state_connection, - config=config, - target_vde_name=target_vde_name, - baseline_vde_name=runtime.names.physical_target_name, - ) - bound_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( - backend.get_virtual_environment_function_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=target_vde_name, - ) - ) - previous_function_query_sqls: dict[str, str] = {} - function_ref: VirtualEnvironmentFunctionRefRecord - for function_ref in bound_function_refs: - function_version: FunctionVersionRecord | None = backend.get_function_version( - connection=state_connection, - schema=config.schema, - function_name=function_ref.function_name, - version_hash=function_ref.version_hash, - ) - if function_version is None: - continue - function_query_sql: str | None = decode_state_text(function_version.definition_text_b64) - if function_query_sql is not None: - previous_function_query_sqls[function_ref.function_name] = function_query_sql - bound_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=target_vde_name, - ) - ) - bound_model_versions: dict[str, ModelVersionRecord | None] = _read_bound_model_versions( - backend=backend, - state_connection=state_connection, - config=config, - bound_refs=bound_refs, - ) - source_freshness_records: tuple[SourceFreshnessRecord, ...] = ( - backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=config.schema, - virtual_environment_name=target_vde_name, - ) - ) - prebuild_source_connection: Any = runtime.adapter.connect(runtime.connection_config) - try: - current_source_freshness_records: tuple[SourceFreshnessRecord, ...] = ( - build_current_virtual_source_freshness_records( - adapter=runtime.adapter, - connection=prebuild_source_connection, - sources=tuple(source.source_entry for source in graph.project.sources), - virtual_environment_name=target_vde_name, - observed_at=datetime.now(UTC), - previous_records=source_freshness_records, - run_id=graph.project.run_id, - ) - ) - finally: - runtime.adapter.close(prebuild_source_connection) - semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=graph, - bound_refs=bound_refs, - bound_model_versions=bound_model_versions, - bound_seed_refs=bound_seed_refs, - source_freshness_records=current_source_freshness_records, - previous_source_freshness_records=source_freshness_records, - ) - work_selection_policy: WorkSelectionPolicy = ( - WorkSelectionPolicy.STALE_ONLY - if options.planning.changes_only - else WorkSelectionPolicy.ALL_SELECTED - ) - default_model_selection: tuple[str, ...] = ( - semantics.default_selection - if options.planning.changes_only - else tuple(sorted(model.name for model in graph.project.models)) - ) - selected_model_names: tuple[str, ...] - selected_seed_names: tuple[str, ...] = () - if options.seed_only: - selected_model_names = () - selected_seed_names = resolve_virtual_seed_selection( - graph=graph, - select=options.planning.select, - exclude=options.planning.exclude, - ) - else: - selected_model_names = resolve_virtual_plan_model_selection( - graph=graph, - select=options.planning.select, - exclude=options.planning.exclude, - default_selection=default_model_selection, - stale_model_names=semantics.stale_model_names, - include_stale_upstreams=options.planning.include_stale_upstreams, - work_selection_policy=work_selection_policy, - ) - selected_seed_names = ( - ( - semantics.stale_seed_names - if options.planning.changes_only - else tuple(sorted(seed.name for seed in graph.project.seeds)) - ) - if not options.planning.select and not options.planning.exclude - else () - ) - selected_seed_names = include_stale_upstream_seed_names( - graph=graph, - selected_model_names=selected_model_names, - selected_seed_names=selected_seed_names, - stale_seed_names=semantics.stale_seed_names, - ) - desired_seed_version_hashes: dict[str, str] = { - seed_name: semantics.expected_seed_version_hashes[seed_name] - for seed_name in selected_seed_names - if seed_name in semantics.expected_seed_version_hashes - } - available_seed_physical_relations: dict[str, PhysicalRelationRecord] = ( - _read_available_seed_physical_relations( - adapter=runtime.adapter, - connection_config=runtime.connection_config, - backend=backend, - state_connection=state_connection, - config=config, - desired_seed_version_hashes=desired_seed_version_hashes, - ) - ) - seed_load_names: tuple[str, ...] = selected_seed_names - if options.planning.changes_only: - seed_load_names = tuple( - seed_name - for seed_name in selected_seed_names - if seed_name not in available_seed_physical_relations - ) - effective_select: tuple[str, ...] = _build_virtual_planner_select( - graph=graph, - selected_model_names=selected_model_names, - selected_seed_names=selected_seed_names, - ) - bound_physical_relations: dict[str, PhysicalRelationRecord] = _read_bound_relations( - backend=backend, - state_connection=state_connection, - config=config, - bound_version_hashes=semantics.bound_version_hashes, - ) - finally: - backend.close(state_connection) - return _VirtualBuildStateReads( - bound_function_refs=bound_function_refs, - previous_function_query_sqls=previous_function_query_sqls, - bound_seed_refs=bound_seed_refs, - semantics=semantics, - selected_model_names=selected_model_names, - selected_seed_names=selected_seed_names, - desired_seed_version_hashes=desired_seed_version_hashes, - available_seed_physical_relations=available_seed_physical_relations, - seed_load_names=seed_load_names, - effective_select=effective_select, - bound_physical_relations=bound_physical_relations, - ) - - -def _rewrite_virtual_project( - *, - runtime: _VirtualBuildRuntime, - graph: ProjectGraph, - reads: _VirtualBuildStateReads, -) -> _RewrittenVirtualProject: - adapter: BaseAdapter = runtime.adapter - semantics: VirtualPlanSemantics = reads.semantics - selected_model_version_hashes: dict[str, str] = { - model_name: semantics.expected_version_hashes[model_name] - for model_name in reads.selected_model_names - if model_name in semantics.expected_version_hashes - } - seed_load_version_hashes: dict[str, str] = { - seed_name: reads.desired_seed_version_hashes[seed_name] - for seed_name in reads.seed_load_names - if seed_name in reads.desired_seed_version_hashes - } - rewritten_locations: dict[str, CompiledRelationLocation] = build_rewritten_model_locations( - project=graph.project, - adapter=adapter, - selected_model_version_hashes=selected_model_version_hashes, - bound_physical_relations=reads.bound_physical_relations, - ) - rewritten_project: CompiledProject = rewrite_project_model_locations( - project=graph.project, - rewritten_locations=rewritten_locations, - ) - rewritten_seed_locations: dict[str, CompiledRelationLocation] = build_rewritten_seed_locations( - project=graph.project, - adapter=adapter, - seed_load_version_hashes=seed_load_version_hashes, - available_physical_relations=reads.available_seed_physical_relations, - bound_seed_version_hashes=semantics.bound_seed_version_hashes, - virtual_environment_name=runtime.names.target_vde_name, - unsuffixed_virtual_environment_name=runtime.names.unsuffixed_virtual_environment_name, - ) - rewritten_project = rewrite_project_seed_locations( - project=rewritten_project, - rewritten_locations=rewritten_seed_locations, - ) - rewritten_project = rewrite_project_function_locations( - project=rewritten_project, - adapter=adapter, - virtual_environment_name=runtime.names.target_vde_name, - unsuffixed_virtual_environment_name=runtime.names.unsuffixed_virtual_environment_name, - ) - deferred_relations: dict[str, RelationInfo] = { - model_name: RelationInfo( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - relation_type=relation.relation_type, - ) - for model_name, relation in reads.bound_physical_relations.items() - } - return _RewrittenVirtualProject( - project=rewritten_project, - deferred_relations=deferred_relations, - ) - - -def _plan_virtual_build( - *, - runtime: _VirtualBuildRuntime, - graph: ProjectGraph, - project: CompiledProject, - reads: _VirtualBuildStateReads, - deferred_relations: dict[str, RelationInfo], -) -> _VirtualBuildPlan: - _reject_model_migrations(project=graph.project) - adapter: BaseAdapter = runtime.adapter - options: VirtualBuildOptions = runtime.options - hooks: VirtualBuildHooks = runtime.hooks - if reads.effective_select: - if hooks.on_connection_start is not None: - hooks.on_connection_start(1) - connection_start: float = time.monotonic() - try: - planning_connection: Any = adapter.connect(runtime.connection_config) - except Exception: - if hooks.on_connection_error is not None: - hooks.on_connection_error(1, elapsed_seconds=time.monotonic() - connection_start) - raise - if hooks.on_connection_complete is not None: - hooks.on_connection_complete(1, elapsed_seconds=time.monotonic() - connection_start) - try: - warehouse_result: PlannerWarehouseSnapshotResult = build_warehouse_snapshot_phase( - project=project, - adapter=adapter, - connection=planning_connection, - select=reads.effective_select, - exclude=(), - auto_load_sources=options.planning.auto_load_sources, - full_refresh=options.planning.full_refresh, - deferred_relations=deferred_relations, - on_progress=hooks.on_progress, - ) - plan_output: PlanOutput = build_plan_output_from_model_changes_phase( - project=project, - adapter=adapter, - connection=planning_connection, - scope=warehouse_result.scope, - snapshot=warehouse_result.snapshot, - model_changes=_build_virtual_model_changes( - project=project, - scope=warehouse_result.scope, - semantics=reads.semantics, - bound_physical_relations=reads.bound_physical_relations, - ), - inputs=ModelChangesPlanInputs( - cursor_overrides=options.planning.cursor_overrides, - full_refresh=options.planning.full_refresh, - reload_sources=options.planning.reload_sources, - project_config=runtime.discovered_inputs.project_config, - local_config=runtime.discovered_inputs.local_config, - defer_sources_to=options.planning.defer_sources_to, - seed_version_hashes=reads.semantics.expected_seed_version_hashes, - seed_metadata_jsons=reads.semantics.seed_identity_metadata_jsons, - seed_plan_reasons=reads.semantics.seed_plan_reasons, - max_microbatches=options.planning.max_microbatches, - plan_sql_tests=options.run_tests, - ), - ) - finally: - adapter.close(planning_connection) - else: - plan_output = PlanOutput( - execution_order=(), - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - selected_keys=frozenset(), - model_locations={ - model.name: ( - build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=reads.bound_physical_relations[model.name], - fallback_target=model.destination, - ) - if model.name in reads.bound_physical_relations - else model.destination - ) - for model in graph.project.models - }, - function_locations={ - function.name: function.destination for function in graph.project.functions - }, - seed_locations={seed.name: seed.destination for seed in graph.project.seeds}, - source_map={source.name: source.source_entry for source in graph.project.sources}, - ) - plan_output = apply_virtual_plan_output( - plan_output=plan_output, - target_name=runtime.names.target_vde_name, - semantics=reads.semantics, - selected_model_names=reads.selected_model_names, - previous_function_query_sqls=reads.previous_function_query_sqls, - ) - return _VirtualBuildPlan( - plan_output=plan_output, - executor_plan_output=_build_physical_seed_load_plan_output( - plan_output=plan_output, - seed_load_names=reads.seed_load_names, - ), - ) - - -def _prepare_virtual_python_execution( - *, - runtime: _VirtualBuildRuntime, - graph: ProjectGraph, - project: CompiledProject, - plan_output: PlanOutput, - selected_model_names: tuple[str, ...], -) -> _VirtualPythonPlan: - options: VirtualBuildOptions = runtime.options - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=runtime.discovered_inputs - ) - python_selection: PythonSqlRunSelection = build_virtual_python_run_selection( - discovered_inputs=runtime.discovered_inputs, - graph=graph, - plan_output=plan_output, - select=options.planning.select, - exclude=options.planning.exclude, - selected_model_names=selected_model_names, - include_python=options.planning.include_python, - ) - lifecycle_plan: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( - selection=python_selection, - python_graph=python_graph, - ) - previous_python_identities: dict[tuple[str, str], Fingerprint] = ( - read_bound_virtual_python_identities( - discovered_inputs=runtime.discovered_inputs, - project_dir=runtime.project_dir, - virtual_environment_name=options.planning.virtual_environment_name, - ) - ) - return _VirtualPythonPlan( - python_graph=python_graph, - lifecycle_plan=lifecycle_plan, - plan_entries=build_virtual_python_plan_entries( - discovered_inputs=runtime.discovered_inputs, - selection=python_selection, - previous_identities=previous_python_identities, - ), - relation_targets=build_python_relation_targets( - adapter=runtime.adapter, - project=project, - plan_output=plan_output, - required_refs=python_graph.selected_sql_refs( - selected_names=( - lifecycle_plan.ingress_python_node_names - | lifecycle_plan.read_side_python_node_names - ), - ), - ), - selected_python_node_names=python_selection.python_node_names, - ) - - -def _build_python_identity_recorder(runtime: _VirtualBuildRuntime) -> PythonIdentityRecorder: - def record_python_identity(*, identity: Any, _target_name: str | None) -> None: - state_connection: Any = runtime.backend.connect(runtime.config.connection) - try: - try_record_virtual_python_node_identity( - backend=runtime.backend, - state_connection=state_connection, - schema=runtime.config.schema, - virtual_environment_name=runtime.names.target_vde_name, - identity=identity, - ) - finally: - runtime.backend.close(state_connection) - - return record_python_identity - - -def _run_ingress_python_nodes( - *, - runtime: _VirtualBuildRuntime, - python_plan: _VirtualPythonPlan, - plan_output: PlanOutput, - project: CompiledProject, - exec_hooks: VirtualBuildExecutionHooks, -) -> PythonIngressLoaderExecutorResult | None: - if not python_plan.lifecycle_plan.ingress_python_node_names: - return None - adapter: BaseAdapter = runtime.adapter - options: VirtualBuildOptions = runtime.options - ingress_connection: Any = adapter.connect(runtime.connection_config) - try: - ingress_state_connection: Any = runtime.backend.connect(runtime.config.connection) - try: - ingress_result_store: VirtualNodeResultStore = VirtualNodeResultStore( - backend=runtime.backend, - state_connection=ingress_state_connection, - state_schema=runtime.config.schema, - virtual_environment_name=runtime.names.target_vde_name, - target_database=adapter.default_database(), - target_schema=adapter.default_schema(), - ) - return run_ingress_python_loader_nodes( - python_graph=python_plan.python_graph, - selected_python_names=python_plan.lifecycle_plan.ingress_python_node_names, - loader_functions=runtime.discovered_inputs.loader_functions, - source_map=plan_output.source_map, - runtime=PythonNodeRuntime( - adapter=adapter, - connection_config=runtime.connection_config, - connection=ingress_connection, - run_id=project.run_id, - target=runtime.names.target_vde_name, - vars=project.effective_vars, - is_reload=options.planning.reload_sources, - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - start_cursor_ts=options.start_cursor_ts, - end_cursor_ts=options.end_cursor_ts, - start_cursor_int=options.start_cursor_int, - end_cursor_int=options.end_cursor_int, - relation_targets=python_plan.relation_targets, - providers=options.providers, - result_store=ingress_result_store, - ), - callbacks=IngressCallbacks( - on_node_start=exec_hooks.on_node_start, - on_node_complete=exec_hooks.on_node_complete, - identity_recorder=_build_python_identity_recorder(runtime), - ), - ) - finally: - runtime.backend.close(ingress_state_connection) - finally: - adapter.close(ingress_connection) - - -def _execute_virtual_build_plan( - *, - runtime: _VirtualBuildRuntime, - plan: _VirtualBuildPlan, - project: CompiledProject, - python_plan: _VirtualPythonPlan, - reads: _VirtualBuildStateReads, - exec_hooks: VirtualBuildExecutionHooks, - ingress_load_results: tuple[LoadExecutionResult, ...], - microbatch_lease_check: Callable[[], None], -) -> BuildExecutionResult: - options: VirtualBuildOptions = runtime.options - microbatch_state_lock: threading.Lock = threading.Lock() - custom_materializations: dict[ - str, Callable[[MaterializationContext], MaterializationResult] - ] = load_custom_materializations(runtime.discovered_inputs.materialization_files) - prepare_version_functions: dict[str, Callable[[PrepareVersionContext], None]] = ( - load_custom_prepare_version_functions(runtime.discovered_inputs.materialization_files) - ) - result: BuildExecutionResult = run_build_pipeline( - plan=plan.executor_plan_output, - connection_config=runtime.connection_config, - adapter=runtime.adapter, - settings=project.settings, - runtime=BuildRuntimeParams( - snapshots=options.snapshots or SnapshotsConfig(), - allow_snapshot_schema_change=options.allow_snapshot_schema_change, - run_id=project.run_id, - runtime_dir=runtime.project_dir / "target", - run_tests=options.run_tests, - run_audits=options.run_audits, - fail_fast=options.fail_fast, - max_concurrency=( - options.concurrency - if options.concurrency is not None - else project.settings.concurrency - ), - loader_is_reload=options.planning.reload_sources, - start_cursor_ts=options.start_cursor_ts, - end_cursor_ts=options.end_cursor_ts, - start_cursor_int=options.start_cursor_int, - end_cursor_int=options.end_cursor_int, - query_change_tracking=False, - target=project.effective_target_name or "", - providers=options.providers, - microbatch_state_resolver=lambda entry, connection: _resolve_virtual_microbatch_state( - runtime=runtime, - entry=entry, - connection=connection, - model_version_hash=reads.semantics.expected_version_hashes.get(entry.name), - operation_lock=microbatch_state_lock, - ), - microbatch_location_state_resolver=( - lambda model_name, destination, version_hash, connection: ( - _resolve_virtual_microbatch_location_state( - runtime=runtime, - model_name=model_name, - destination=destination, - connection=connection, - model_version_hash=version_hash, - operation_lock=microbatch_state_lock, - ) - ) - ), - microbatch_lease_check=microbatch_lease_check, - ), - callbacks=BuildCallbacks( - on_node_start=exec_hooks.on_node_start, - on_node_complete=exec_hooks.on_node_complete, - on_sub_progress=exec_hooks.on_sub_progress, - on_scheduler_state=exec_hooks.on_scheduler_state, - on_statement_complete=exec_hooks.on_statement_complete, - before_model_materialize=_build_before_model_materialize( - runtime=runtime, - reads=reads, - run_id=project.run_id, - prepare_version_functions=prepare_version_functions, - ), - python_identity_recorder=_build_python_identity_recorder(runtime), - ), - customizations=BuildCustomizations( - custom_materializations=custom_materializations, - loader_functions=_sql_loader_functions_for_lifecycle_handoff( - discovered_inputs=runtime.discovered_inputs, - ingress_loader_names=python_plan.lifecycle_plan.ingress_loader_names, - ), - ), - initial_state=BuildInitialState( - precompleted_keys=frozenset( - _load_result_key(plan=plan.plan_output, result=load_result) - for load_result in ingress_load_results - ), - initial_load_results=ingress_load_results, - initial_failed_keys=frozenset( - _load_result_key(plan=plan.plan_output, result=load_result) - for load_result in ingress_load_results - if load_result.status != ExecutionStatus.SUCCESS - ), - ), - ) - if result.status == BuildStatus.SUCCESS and reads.available_seed_physical_relations: - result = replace( - result, - seed_results=result.seed_results - + tuple( - SeedExecutionResult(seed_name=seed_name, status=ExecutionStatus.SKIPPED) - for seed_name in sorted(reads.available_seed_physical_relations) - if seed_name not in reads.seed_load_names - ), - ) - return result - - -def _resolve_virtual_microbatch_state( - *, - runtime: _VirtualBuildRuntime, - entry: ModelPlanEntry, - connection: Any, - model_version_hash: str | None, - operation_lock: threading.Lock, -) -> tuple[MicrobatchEventStore, MicrobatchScope]: - version_hash: str = ( - model_version_hash - or entry.fingerprint_version_hash - or compute_query_hash(entry.fingerprint_query_sql) - ) - return _resolve_virtual_microbatch_location_state( - runtime=runtime, - model_name=entry.name, - destination=entry.destination, - connection=connection, - model_version_hash=version_hash, - operation_lock=operation_lock, - ) - - -def _resolve_virtual_microbatch_location_state( - *, - runtime: _VirtualBuildRuntime, - model_name: str, - destination: CompiledRelationLocation, - connection: Any, - model_version_hash: str | None, - operation_lock: threading.Lock, -) -> tuple[MicrobatchEventStore, MicrobatchScope]: - if model_version_hash is None: - raise ExecutorInputError(f"virtual microbatch model '{model_name}' has no version identity") - destination_identity: str = ".".join( - part - for part in ( - destination.database, - destination.schema, - destination.name, - ) - if part is not None - ) - state_realm: str = f"{runtime.config.backend.value}:{runtime.config.schema}" - warehouse_realm: str = hashlib.sha256( - json.dumps(runtime.connection_config, sort_keys=True, default=str).encode() - ).hexdigest() - scope_key: str = ( - f"{state_realm}:{warehouse_realm}:{model_name}:{model_version_hash}:{destination_identity}" - ) - relation_generation: str | None = runtime.adapter.physical_relation_generation( - connection=connection, - database=destination.database, - schema=destination.schema, - name=destination.name, - ) - physical_generation_id: str = f"{warehouse_realm}:{model_version_hash}:{destination_identity}" - if relation_generation is not None: - relation_generation_hash: str = hashlib.sha256(relation_generation.encode()).hexdigest() - physical_generation_id += f":{relation_generation_hash}" - return ( - VirtualMicrobatchEventStore( - backend=runtime.backend, - connection_config=runtime.config.connection, - schema=runtime.config.schema, - operation_lock=operation_lock, - ), - MicrobatchScope( - scope_kind=VIRTUAL_MICROBATCH_SCOPE_KIND, - scope_key=scope_key, - model_name=model_name, - target_database=destination.database, - target_schema=destination.schema, - target_name=destination.name, - physical_generation_id=physical_generation_id, - virtual_environment_name=runtime.names.target_vde_name, - virtual_model_version_hash=model_version_hash, - ), - ) - - -def _build_before_model_materialize( - *, - runtime: _VirtualBuildRuntime, - reads: _VirtualBuildStateReads, - run_id: str, - prepare_version_functions: dict[str, Callable[[PrepareVersionContext], None]], -) -> sqlbuild.executor.build.types.BeforeModelMaterializeCallback: - def before_model_materialize(*, entry: ModelPlanEntry, connection: Any) -> None: - if entry.action not in INCREMENTAL_ACTIONS and entry.action != PlanAction.CUSTOM: - return - parent_relation: PhysicalRelationRecord | None = reads.bound_physical_relations.get( - entry.name - ) - version_hash: str | None = reads.semantics.expected_version_hashes.get(entry.name) - if ( - parent_relation is None - or version_hash is None - or parent_relation.version_hash == version_hash - ): - return - state_connection: Any = runtime.backend.connect(runtime.config.connection) - try: - prepare_version: Callable[[PrepareVersionContext], None] | None = ( - prepare_version_functions.get(entry.custom_materialization_name or "") - if entry.action == PlanAction.CUSTOM - else None - ) - if prepare_version is not None: - _prepare_custom_virtual_version( - runtime=runtime, - connection=connection, - state_connection=state_connection, - entry=entry, - parent_relation=parent_relation, - version_hash=version_hash, - prepare_version=prepare_version, - run_id=run_id, - schema_prepared=True, - ) - else: - seed_virtual_physical_version( - adapter=runtime.adapter, - connection=connection, - backend=runtime.backend, - state_connection=state_connection, - state_schema=runtime.config.schema, - entry=entry, - parent_relation=parent_relation, - version_hash=version_hash, - schema_prepared=True, - ) - finally: - runtime.backend.close(state_connection) - - return before_model_materialize - - -def _run_read_side_python_nodes( - *, - runtime: _VirtualBuildRuntime, - python_plan: _VirtualPythonPlan, - project: CompiledProject, - result: BuildExecutionResult, -) -> tuple[PythonNodeExecutionResult, ...]: - if not python_plan.lifecycle_plan.read_side_python_node_names: - return () - adapter: BaseAdapter = runtime.adapter - options: VirtualBuildOptions = runtime.options - connection: Any = adapter.connect(runtime.connection_config) - try: - state_connection: Any = runtime.backend.connect(runtime.config.connection) - try: - result_store: VirtualNodeResultStore = VirtualNodeResultStore( - backend=runtime.backend, - state_connection=state_connection, - state_schema=runtime.config.schema, - virtual_environment_name=runtime.names.target_vde_name, - target_database=adapter.default_database(), - target_schema=adapter.default_schema(), - ) - tracker: Any = create_read_side_python_execution_tracker( - python_graph=python_plan.python_graph, - selected_python_names=python_plan.lifecycle_plan.read_side_python_node_names, - runtime=PythonNodeRuntime( - adapter=adapter, - connection_config=runtime.connection_config, - connection=connection, - run_id=project.run_id, - target=runtime.names.target_vde_name, - vars=project.effective_vars, - is_reload=options.planning.reload_sources, - default_database=adapter.default_database(), - default_schema=adapter.default_schema(), - relation_targets=python_plan.relation_targets, - start_cursor_ts=options.start_cursor_ts, - end_cursor_ts=options.end_cursor_ts, - start_cursor_int=options.start_cursor_int, - end_cursor_int=options.end_cursor_int, - providers=options.providers, - result_store=result_store, - ), - identity_recorder=_build_python_identity_recorder(runtime), - ) - load_result: LoadExecutionResult - for load_result in result.load_results: - tracker.record_sql_result(load_result) - model_result: Any - for model_result in result.model_results: - tracker.record_sql_result(model_result) - tracker.dispatch_ready_python_nodes() - tracker.finalize_unrun_python_nodes() - return tracker.results - finally: - runtime.backend.close(state_connection) - finally: - adapter.close(connection) - - -def _sql_loader_functions_for_lifecycle_handoff( - *, - discovered_inputs: DiscoveredProjectInputs, - ingress_loader_names: frozenset[str], -) -> tuple[Any, ...]: - loader_functions: frozenset[object] = frozenset( - loader.function for loader in discovered_inputs.loader_functions - ) - handoff_loaders: list[Any] = [] - for loader in discovered_inputs.loader_functions: - dependencies: list[object] = [] - for dependency in loader.depends_on: - if dependency in loader_functions or loader.name not in ingress_loader_names: - dependencies.append(dependency) - handoff_loaders.append(replace(loader, depends_on=tuple(dependencies))) - return tuple(handoff_loaders) - - -def _load_result_key(*, plan: PlanOutput, result: LoadExecutionResult) -> CompiledObjectKey: - for entry in plan.source_load_entries: - if entry.name == result.source_name: - return entry.key - raise PlannerInputError( - f"No source-load plan entry found for load result '{result.source_name}'" - ) - - -def _prepare_custom_virtual_version( - *, - runtime: _VirtualBuildRuntime, - connection: Any, - state_connection: Any, - entry: ModelPlanEntry, - parent_relation: PhysicalRelationRecord, - version_hash: str, - prepare_version: Callable[[PrepareVersionContext], None], - run_id: str, - schema_prepared: bool = False, -) -> None: - adapter: BaseAdapter = runtime.adapter - recorder: StatementRecorder = StatementRecorder() - if not schema_prepared: - adapter.ensure_schema( - connection=connection, - database=entry.destination.database, - schema=entry.destination.schema, - statement_recorder=recorder, - ) - destination: str = rn.resolve_relation_location_qualified_name( - adapter=adapter, location=entry.destination - ) - if adapter.relation_exists( - connection=connection, - database=entry.destination.database, - schema=entry.destination.schema, - name=entry.destination.name, - ): - adapter.drop( - connection=connection, - destination=destination, - if_exists=True, - statement_recorder=recorder, - ) - source: str = relation_parts.resolve_qualified_name_parts( - adapter=adapter, - database=parent_relation.database_name, - schema=parent_relation.schema_name, - name=parent_relation.relation_name, - ) - prepare_version( - PrepareVersionContext( - adapter=adapter, - connection=connection, - origin_relation=source, - destination=destination, - destination_database=entry.destination.database, - destination_schema=entry.destination.schema, - destination_name=entry.destination.name, - config=dict(entry.custom_config), - placeholders=dict(entry.custom_placeholders), - run_id=run_id, - environment=runtime.names.target_vde_name, - vars=runtime.options.planning.cli_vars or {}, - unique_key=entry.unique_key, - declared_columns=entry.declared_columns, - statement_recorder=recorder, - ) - ) - runtime.backend.upsert_physical_relation_ancestry( - connection=state_connection, - schema=runtime.config.schema, - record=PhysicalRelationAncestryRecord( - model_name=entry.name, - version_hash=version_hash, - parent_model_name=parent_relation.artifact_name, - parent_version_hash=parent_relation.version_hash, - seed_strategy="custom_prepare_version", - ), - ) - - -def _read_or_initialize_refs( - *, - backend: Any, - state_connection: Any, - config: StateBackendConfig, - target_vde_name: str, - baseline_vde_name: str | None, -) -> tuple[VirtualEnvironmentModelRefRecord, ...]: - environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=config.schema, - virtual_environment_name=target_vde_name, - ) - if environment is not None and environment.status == VirtualEnvironmentStatus.DETACHED: - raise PlannerInputError( - f"virtual environment '{target_vde_name}' is detached", - code="S028", - help="Run state adopt again or choose a non-detached virtual environment.", - ) - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=target_vde_name, - ) - if refs or baseline_vde_name is None or baseline_vde_name == target_vde_name: - return refs - baseline_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=baseline_vde_name, - ) - ) - return tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=target_vde_name, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in baseline_refs - ) - - -def _read_bound_model_versions( - *, - backend: Any, - state_connection: Any, - config: StateBackendConfig, - bound_refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, ModelVersionRecord | None]: - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in bound_refs - } - - -def _read_bound_relations( - *, - backend: Any, - state_connection: Any, - config: StateBackendConfig, - bound_version_hashes: dict[str, str], -) -> dict[str, PhysicalRelationRecord]: - relations: dict[str, PhysicalRelationRecord] = {} - for model_name, version_hash in bound_version_hashes.items(): - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=state_connection, - schema=config.schema, - model_name=model_name, - version_hash=version_hash, - ) - if relation is not None: - relations[model_name] = relation - return relations - - -def _read_available_seed_physical_relations( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - backend: Any, - state_connection: Any, - config: StateBackendConfig, - desired_seed_version_hashes: dict[str, str], -) -> dict[str, PhysicalRelationRecord]: - relations: dict[str, PhysicalRelationRecord] = {} - if not desired_seed_version_hashes: - return relations - records_by_seed: dict[str, PhysicalRelationRecord] = {} - seed_name: str - version_hash: str - for seed_name, version_hash in desired_seed_version_hashes.items(): - relation: PhysicalRelationRecord | None = backend.get_physical_relation_for_artifact( - connection=state_connection, - schema=config.schema, - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_name, - version_hash=version_hash, - ) - if relation is not None: - records_by_seed[seed_name] = relation - if not records_by_seed: - return relations - warehouse_connection: Any = adapter.connect(connection_config) - try: - relation_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=warehouse_connection, - locations=tuple( - (record.database_name, record.schema_name, record.relation_name) - for record in records_by_seed.values() - ), - ) - for seed_name, relation in records_by_seed.items(): - if relation_lookup.exists( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ): - relations[seed_name] = relation - finally: - adapter.close(warehouse_connection) - return relations - - -def _build_physical_seed_load_plan_output( - *, plan_output: PlanOutput, seed_load_names: tuple[str, ...] -) -> PlanOutput: - seed_load_name_set: frozenset[str] = frozenset(seed_load_names) - selected_load_seed_keys: frozenset[CompiledObjectKey] = frozenset( - entry.key for entry in plan_output.seed_entries if entry.name in seed_load_name_set - ) - selected_existing_seed_keys: frozenset[CompiledObjectKey] = frozenset( - entry.key for entry in plan_output.seed_entries if entry.name not in seed_load_name_set - ) - return replace( - plan_output, - execution_order=tuple( - key for key in plan_output.execution_order if key not in selected_existing_seed_keys - ), - seed_entries=tuple( - entry for entry in plan_output.seed_entries if entry.key in selected_load_seed_keys - ), - selected_keys=frozenset( - key for key in plan_output.selected_keys if key not in selected_existing_seed_keys - ), - ) - - -def _persist_successful_virtual_build( - *, - runtime: _VirtualBuildRuntime, - project: CompiledProject, - reads: _VirtualBuildStateReads, - plan_output: PlanOutput, - result: BuildExecutionResult, - model_version_leases: tuple[StateLockLease, ...], -) -> None: - semantics: VirtualPlanSemantics = reads.semantics - names: VirtualEnvironmentNames = runtime.names - final_version_hashes: dict[str, str] = dict(semantics.bound_version_hashes) - successful_model_names: frozenset[str] = frozenset( - model_result.model_name - for model_result in result.model_results - if model_result.status == ExecutionStatus.SUCCESS - ) - for entry in plan_output.model_entries: - if entry.name not in successful_model_names: - continue - final_version_hashes[entry.name] = semantics.expected_version_hashes[entry.name] - - state_connection: Any = runtime.backend.connect(runtime.config.connection) - try: - _observe_and_persist_source_freshness( - runtime=runtime, - state_connection=state_connection, - project=project, - load_results=result.load_results, - ) - _write_model_version_records( - runtime=runtime, - state_connection=state_connection, - project=project, - plan_output=plan_output, - semantics=semantics, - final_version_hashes=final_version_hashes, - ) - functions: _FunctionPersistOutcome = _persist_function_versions( - runtime=runtime, - state_connection=state_connection, - plan_output=plan_output, - bound_function_refs=reads.bound_function_refs, - ) - seeds: _SeedPersistOutcome = _persist_seed_versions( - runtime=runtime, - state_connection=state_connection, - plan_output=plan_output, - reads=reads, - seed_results=result.seed_results, - ) - stale_model_after_build: tuple[str, ...] = tuple( - model.name - for model in project.models - if final_version_hashes.get(model.name) - != semantics.expected_version_hashes.get(model.name) - ) - stale_seed_after_build: tuple[str, ...] = tuple( - seed.name - for seed in project.seeds - if seeds.final_seed_hashes.get(seed.name) - != semantics.expected_seed_version_hashes.get(seed.name) - ) - status: VirtualEnvironmentStatus = ( - VirtualEnvironmentStatus.FINALIZED - if not stale_model_after_build and not stale_seed_after_build - else VirtualEnvironmentStatus.ACTIVE - ) - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=names.target_vde_name, - status=status, - baseline_virtual_environment_name=( - names.physical_target_name - if names.physical_target_name != names.target_vde_name - else None - ), - ) - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=names.target_vde_name, - model_name=model_name, - version_hash=version_hash, - ) - for model_name, version_hash in sorted(final_version_hashes.items()) - ) - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = tuple( - VirtualEnvironmentFunctionRefRecord( - virtual_environment_name=names.target_vde_name, - node_type=functions.function_ref_node_types[function_name], - function_name=function_name, - version_hash=version_hash, - ) - for function_name, version_hash in sorted(functions.final_function_hashes.items()) - if function_name in functions.function_ref_node_types - ) - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = tuple( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=names.target_vde_name, - seed_name=seed_name, - version_hash=version_hash, - ) - for seed_name, version_hash in sorted(seeds.final_seed_hashes.items()) - ) - checkpoint: FinalizedVirtualEnvironmentCheckpoint | None = ( - build_finalized_virtual_environment_checkpoint( - virtual_environment_name=names.target_vde_name, - refs=refs, - function_refs=function_refs, - seed_refs=seed_refs, - ) - if status == VirtualEnvironmentStatus.FINALIZED - else None - ) - published: bool = ( - runtime.backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=state_connection, - schema=runtime.config.schema, - record=virtual_environment_record, - refs_by_node_type=_build_node_ref_groups( - refs=refs, seed_refs=seed_refs, function_refs=function_refs - ), - leases=model_version_leases, - checkpoint=checkpoint.checkpoint if checkpoint is not None else None, - checkpoint_refs=checkpoint.refs if checkpoint is not None else (), - checkpoint_function_refs=( - checkpoint.function_refs if checkpoint is not None else () - ), - checkpoint_seed_refs=checkpoint.seed_refs if checkpoint is not None else (), - ) - ) - if not published: - raise ExecutorInputError( - "virtual incremental physical-version lease ownership was lost before " - "model refs could be published" - ) - finally: - runtime.backend.close(state_connection) - - write_logical_vde_views( - project=project, - adapter=runtime.adapter, - connection_config=runtime.connection_config, - target_vde_name=names.target_vde_name, - unsuffixed_virtual_environment_name=names.unsuffixed_virtual_environment_name, - plan_output=plan_output, - final_version_hashes=final_version_hashes, - final_seed_physical_relations=seeds.final_seed_physical_relations, - ) - - -def _observe_and_persist_source_freshness( - *, - runtime: _VirtualBuildRuntime, - state_connection: Any, - project: CompiledProject, - load_results: tuple[LoadExecutionResult, ...], -) -> None: - previous_source_freshness_records: tuple[SourceFreshnessRecord, ...] = ( - runtime.backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=runtime.config.schema, - virtual_environment_name=runtime.names.target_vde_name, - ) - ) - source_observation_connection: Any = runtime.adapter.connect(runtime.connection_config) - try: - source_freshness_result: SourceFreshnessRuntimeResult = ( - observe_virtual_environment_source_freshness( - adapter=runtime.adapter, - connection=source_observation_connection, - sources=tuple(source.source_entry for source in project.sources), - virtual_environment_name=runtime.names.target_vde_name, - observed_at=datetime.now(UTC), - run_id=project.run_id, - load_results=load_results, - previous_records=previous_source_freshness_records, - ) - ) - finally: - runtime.adapter.close(source_observation_connection) - persist_virtual_environment_source_freshness( - backend=runtime.backend, - state_connection=state_connection, - schema=runtime.config.schema, - virtual_environment_name=runtime.names.target_vde_name, - result=source_freshness_result, - ) - - -def _write_model_version_records( - *, - runtime: _VirtualBuildRuntime, - state_connection: Any, - project: CompiledProject, - plan_output: PlanOutput, - semantics: VirtualPlanSemantics, - final_version_hashes: dict[str, str], -) -> None: - model_entries_by_name: dict[str, Any] = { - entry.name: entry for entry in plan_output.model_entries - } - model: CompiledModel - for model in project.models: - version_hash: str | None = final_version_hashes.get(model.name) - if version_hash is None: - continue - entry: Any | None = model_entries_by_name.get(model.name) - existing_model_version: ModelVersionRecord | None = runtime.backend.get_model_version( - connection=state_connection, - schema=runtime.config.schema, - model_name=model.name, - version_hash=version_hash, - ) - if existing_model_version is None: - metadata_json: str = semantics.expected_metadata_jsons.get(model.name, "{}") - runtime.backend.upsert_model_version( - connection=state_connection, - schema=runtime.config.schema, - record=ModelVersionRecord( - model_name=model.name, - version_hash=version_hash, - definition_identity_hash=semantics.expected_local_hashes.get( - model.name, version_hash - ), - identity_metadata_hash=hashlib.sha256( - metadata_json.encode("utf-8") - ).hexdigest(), - status=ModelVersionStatus.READY, - definition_text_b64=encode_state_text(model.query_sql), - identity_metadata_json_b64=encode_state_text(metadata_json), - compiled_sql_b64=( - encode_state_text(entry.resolved_sql) if entry is not None else None - ), - ), - ) - target: CompiledRelationLocation | None = entry.destination if entry is not None else None - if target is not None: - existing_physical_relation: PhysicalRelationRecord | None = ( - runtime.backend.get_physical_relation( - connection=state_connection, - schema=runtime.config.schema, - model_name=model.name, - version_hash=version_hash, - ) - ) - if existing_physical_relation is None: - runtime.backend.upsert_physical_relation( - connection=state_connection, - schema=runtime.config.schema, - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=model.name, - version_hash=version_hash, - database_name=target.database, - schema_name=target.schema or "", - relation_name=target.name, - relation_type=relation_type_for_model( - str(model.config.values.get("materialized", "table")) - ), - ), - ) - - -def _persist_function_versions( - *, - runtime: _VirtualBuildRuntime, - state_connection: Any, - plan_output: PlanOutput, - bound_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], -) -> _FunctionPersistOutcome: - final_function_hashes: dict[str, str] = { - ref.function_name: ref.version_hash for ref in bound_function_refs - } - function_ref_node_types: dict[str, str] = { - ref.function_name: ref.node_type for ref in bound_function_refs - } - function_entry: Any - for function_entry in plan_output.function_entries: - function_version: FunctionVersionRecord = build_function_version_record(function_entry) - final_function_hashes[function_entry.name] = function_version.version_hash - function_ref_node_types[function_entry.name] = str(function_entry.key.resource_type) - existing_function_version: FunctionVersionRecord | None = ( - runtime.backend.get_function_version( - connection=state_connection, - schema=runtime.config.schema, - function_name=function_version.function_name, - version_hash=function_version.version_hash, - ) - ) - if existing_function_version is None: - runtime.backend.upsert_function_version( - connection=state_connection, - schema=runtime.config.schema, - record=function_version, - ) - return _FunctionPersistOutcome( - final_function_hashes=final_function_hashes, - function_ref_node_types=function_ref_node_types, - ) - - -def _persist_seed_versions( - *, - runtime: _VirtualBuildRuntime, - state_connection: Any, - plan_output: PlanOutput, - reads: _VirtualBuildStateReads, - seed_results: tuple[SeedExecutionResult, ...], -) -> _SeedPersistOutcome: - semantics: VirtualPlanSemantics = reads.semantics - final_seed_hashes: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in reads.bound_seed_refs - } - final_seed_physical_relations: dict[str, PhysicalRelationRecord] = dict( - reads.available_seed_physical_relations - ) - for seed_name, relation in reads.available_seed_physical_relations.items(): - final_seed_hashes[seed_name] = relation.version_hash - successful_seed_names: frozenset[str] = frozenset( - seed_result.seed_name - for seed_result in seed_results - if seed_result.status == ExecutionStatus.SUCCESS - ) - for seed_name in successful_seed_names: - version_hash: str | None = semantics.expected_seed_version_hashes.get(seed_name) - if version_hash is None: - continue - target: CompiledRelationLocation | None = plan_output.seed_locations.get(seed_name) - metadata_json: str = semantics.seed_identity_metadata_jsons.get(seed_name, "{}") - existing_seed_version: SeedVersionRecord | None = runtime.backend.get_seed_version( - connection=state_connection, - schema=runtime.config.schema, - seed_name=seed_name, - version_hash=version_hash, - ) - if existing_seed_version is None: - runtime.backend.upsert_seed_version( - connection=state_connection, - schema=runtime.config.schema, - record=SeedVersionRecord( - seed_name=seed_name, - version_hash=version_hash, - identity_metadata_hash=hashlib.sha256( - metadata_json.encode("utf-8") - ).hexdigest(), - identity_metadata_json_b64=encode_state_text(metadata_json), - status=ModelVersionStatus.READY, - ), - ) - if target is not None: - existing_seed_physical_relation: PhysicalRelationRecord | None = ( - runtime.backend.get_physical_relation_for_artifact( - connection=state_connection, - schema=runtime.config.schema, - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_name, - version_hash=version_hash, - ) - ) - if existing_seed_physical_relation is None: - seed_physical_relation: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_name, - version_hash=version_hash, - database_name=target.database, - schema_name=target.schema or "", - relation_name=target.name, - relation_type="table", - ) - runtime.backend.upsert_physical_relation( - connection=state_connection, - schema=runtime.config.schema, - record=seed_physical_relation, - ) - final_seed_physical_relations[seed_name] = seed_physical_relation - else: - final_seed_physical_relations[seed_name] = existing_seed_physical_relation - final_seed_hashes[seed_name] = version_hash - for seed_name, version_hash in final_seed_hashes.items(): - if seed_name in final_seed_physical_relations: - continue - seed_physical_relation = runtime.backend.get_physical_relation_for_artifact( - connection=state_connection, - schema=runtime.config.schema, - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_name, - version_hash=version_hash, - ) - if seed_physical_relation is not None: - final_seed_physical_relations[seed_name] = seed_physical_relation - return _SeedPersistOutcome( - final_seed_hashes=final_seed_hashes, - final_seed_physical_relations=final_seed_physical_relations, - ) - - -def _build_node_ref_groups( - *, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...], - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], -) -> dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]]: - return { - "model": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="model", - node_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - ), - "seed": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="seed", - node_name=ref.seed_name, - version_hash=ref.version_hash, - ) - for ref in seed_refs - ), - "udf": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in function_refs - if ref.node_type == CompiledResourceType.UDF - ), - "table_fn": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in function_refs - if ref.node_type == CompiledResourceType.TABLE_FN - ), - } - - -def _build_virtual_model_changes( - *, - project: CompiledProject, - scope: PlannerScope, - semantics: VirtualPlanSemantics, - bound_physical_relations: dict[str, PhysicalRelationRecord], -) -> dict[str, ChangeDetectionResult]: - changes: dict[str, ChangeDetectionResult] = {} - model: CompiledModel - for model in project.models: - if model.key not in scope.selected_keys: - continue - changes[model.name] = _build_virtual_model_change( - model=model, - semantics=semantics, - bound_physical_relations=bound_physical_relations, - ) - return changes - - -def _build_virtual_planner_select( - *, - graph: ProjectGraph, - selected_model_names: tuple[str, ...], - selected_seed_names: tuple[str, ...] = (), -) -> tuple[str, ...]: - selected_model_keys: frozenset[CompiledObjectKey] = frozenset( - key - for model_name in selected_model_names - if (key := graph.all_keys.get(model_name)) is not None - ) - selected_seed_keys: frozenset[CompiledObjectKey] = frozenset( - key - for seed_name in selected_seed_names - if (key := graph.all_keys.get(seed_name)) is not None - ) - expanded_keys: frozenset[CompiledObjectKey] = expand_build_resource_selection( - selected_keys=selected_model_keys | selected_seed_keys, - upstream=graph.upstream_deps, - downstream=graph.downstream_deps, - include_upstream_functions=True, - include_upstream_seeds=False, - include_downstream_functions=True, - ) - return tuple(sorted(key.name for key in expanded_keys)) - - -def _build_virtual_model_change( - *, - model: CompiledModel, - semantics: VirtualPlanSemantics, - bound_physical_relations: dict[str, PhysicalRelationRecord], -) -> ChangeDetectionResult: - metadata_json: str = semantics.expected_metadata_jsons.get(model.name, "{}") - previous_metadata_json: str | None = semantics.bound_metadata_jsons.get(model.name) - bound_physical_relation: PhysicalRelationRecord | None = bound_physical_relations.get( - model.name - ) - previous_version_hash: str | None = ( - bound_physical_relation.version_hash if bound_physical_relation is not None else None - ) - if bound_physical_relation is None: - return ChangeDetectionResult( - model_name=model.name, - change_kind=ChangeKind.FIRST_RUN, - fingerprint_metadata_json=metadata_json, - previous_metadata_json=previous_metadata_json, - backfill=BackfillResult(action=BackfillAction.FULL), - ) - root_reason: PlanReason | None = semantics.stale_root_reasons.get(model.name) - if root_reason == PlanReason.CONFIG_CHANGED: - return ChangeDetectionResult( - model_name=model.name, - change_kind=ChangeKind.CONFIG_CHANGED, - config_changed=True, - fingerprint_metadata_json=metadata_json, - previous_metadata_json=previous_metadata_json, - previous_version_hash=previous_version_hash, - ) - if root_reason in (PlanReason.QUERY_CHANGED, PlanReason.FUNCTION_CHANGED): - raw_replay_policy: object | None = model.config.values.get("replay_on_change") - replay_policy: str | None = ( - raw_replay_policy if isinstance(raw_replay_policy, str) else None - ) - return ChangeDetectionResult( - model_name=model.name, - change_kind=ChangeKind.QUERY_CHANGED, - query_changed=True, - fingerprint_metadata_json=metadata_json, - previous_metadata_json=previous_metadata_json, - previous_version_hash=previous_version_hash, - backfill=resolve_replay_on_change(replay_on_change=replay_policy), - ) - return ChangeDetectionResult( - model_name=model.name, - change_kind=ChangeKind.NO_CHANGE, - fingerprint_metadata_json=metadata_json, - previous_metadata_json=previous_metadata_json, - previous_version_hash=previous_version_hash, - ) diff --git a/src/sqlbuild/virtual/executor/_helpers/clone.py b/src/sqlbuild/virtual/executor/_helpers/clone.py deleted file mode 100644 index f9843dfc7d..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/clone.py +++ /dev/null @@ -1,828 +0,0 @@ -"""Virtual clone helper operations.""" - -from __future__ import annotations - -import hashlib -import uuid -from collections.abc import Callable -from datetime import timedelta -from functools import partial -from pathlib import Path -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompiledRelationLocation, - CompiledSeed, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.clone import run_clone_pipeline -from sqlbuild.compiler.pipeline.main.project_graph import ( - build_project_graph_from_compiled_project, -) -from sqlbuild.compiler.pipeline.models import ( - ClonePipelineConnection, - ClonePipelineResult, - ProjectGraph, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ModelPlanEntry, SeedPlanEntry -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.runtime.observability.classes.resource_attempt_lifecycle import ( - ResourceAttemptLifecycle, -) -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.virtual.executor._helpers.rewrite import ( - build_physical_destination, - build_physical_seed_destination, - relation_type_for_model, -) -from sqlbuild.virtual.executor.models import ( - CloneOriginLookup, - CloneProjectContext, - CloneVersions, - VirtualCloneItemResult, -) -from sqlbuild.virtual.planner.main._semantics import build_virtual_plan_semantics -from sqlbuild.virtual.planner.models import VirtualPlanSemantics -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.main.encoding._encode_state_text import encode_state_text -from sqlbuild.virtual.state.main.locks._model_version_lock import acquire_model_version_lease -from sqlbuild.virtual.state.main.locks._release_lock import release_state_lease -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - PhysicalRelationRecord, - SeedVersionRecord, - StateLockLease, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import ModelVersionStatus, PhysicalArtifactType - - -def build_clone_graph_from_project(project: CompiledProject) -> ProjectGraph: - return build_project_graph_from_compiled_project(project=project) - - -def build_workspace_model_versions( - *, - project: CompiledProject, - model_entries: tuple[ModelPlanEntry, ...], - model_names: tuple[str, ...], - version_hashes: dict[str, str], - local_hashes: dict[str, str], - metadata_jsons: dict[str, str], -) -> dict[str, ModelVersionRecord]: - models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} - model_entries_by_name: dict[str, ModelPlanEntry] = { - entry.name: entry for entry in model_entries - } - records: dict[str, ModelVersionRecord] = {} - for name in model_names: - model: CompiledModel = models_by_name[name] - entry: ModelPlanEntry | None = model_entries_by_name.get(name) - metadata_json: str = metadata_jsons.get(name, "{}") - records[name] = ModelVersionRecord( - model_name=name, - version_hash=version_hashes[name], - definition_identity_hash=local_hashes.get(name, version_hashes[name]), - identity_metadata_hash=hashlib.sha256(metadata_json.encode("utf-8")).hexdigest(), - status=ModelVersionStatus.READY, - definition_text_b64=encode_state_text(model.query_sql), - identity_metadata_json_b64=encode_state_text(metadata_json), - compiled_sql_b64=encode_state_text(entry.resolved_sql) if entry is not None else None, - ) - return records - - -def build_workspace_seed_versions( - *, - project: CompiledProject, - seed_entries: tuple[SeedPlanEntry, ...], - seed_names: tuple[str, ...], - version_hashes: dict[str, str], - metadata_jsons: dict[str, str], -) -> dict[str, SeedVersionRecord]: - seeds_by_name: dict[str, CompiledSeed] = {seed.name: seed for seed in project.seeds} - seed_entries_by_name: dict[str, SeedPlanEntry] = {entry.name: entry for entry in seed_entries} - records: dict[str, SeedVersionRecord] = {} - for name in seed_names: - if name not in seeds_by_name or name not in seed_entries_by_name: - continue - metadata_json: str = metadata_jsons.get(name, "{}") - records[name] = SeedVersionRecord( - seed_name=name, - version_hash=version_hashes[name], - identity_metadata_hash=hashlib.sha256(metadata_json.encode("utf-8")).hexdigest(), - identity_metadata_json_b64=encode_state_text(metadata_json), - status=ModelVersionStatus.READY, - ) - return records - - -def hydrate_relation( - *, - adapter: BaseAdapter, - destination_connection: Any, - origin_location: CompiledRelationLocation, - destination_location: CompiledRelationLocation, -) -> str: - with OperationLifecycle(operation_kind="clone", operation_name="clone_relation_transfer"): - adapter.ensure_schema( - connection=destination_connection, - database=destination_location.database, - schema=destination_location.schema or "", - statement_recorder=StatementRecorder(), - ) - adapter.durable_clone( - connection=destination_connection, - origin=resolve_relation_location_qualified_name( - adapter=adapter, location=origin_location - ), - destination=resolve_relation_location_qualified_name( - adapter=adapter, location=destination_location - ), - origin_is_transient=_location_is_transient( - adapter=adapter, connection=destination_connection, location=origin_location - ), - statement_recorder=StatementRecorder(), - ) - return "hydrated" - - -def hydrate_and_register_relation( - *, - adapter: BaseAdapter, - destination_connection: Any, - origin_location: CompiledRelationLocation, - destination_location: CompiledRelationLocation, - resource_kind: str, - resource_name: str, - run_id: str, - register: Callable[[], None], -) -> str: - """Hydrate fresh physical state and keep its attempt active through registration.""" - - if adapter.relation_exists( - connection=destination_connection, - database=destination_location.database, - schema=destination_location.schema, - name=destination_location.name, - ): - register() - return "reused" - with ResourceAttemptLifecycle( - resource_id=f"{resource_kind}:{resource_name}", - resource_kind=resource_kind, - resource_name=resource_name, - run_id=run_id, - ): - action: str = hydrate_relation( - adapter=adapter, - destination_connection=destination_connection, - origin_location=origin_location, - destination_location=destination_location, - ) - register() - return action - - -def _location_is_transient( - *, adapter: BaseAdapter, connection: Any, location: CompiledRelationLocation -) -> bool: - """Return whether the origin warehouse relation is transient, defaulting to False.""" - - if location.schema is None: - return False - relations: tuple[Any, ...] = adapter.list_relations( - connection=connection, - database=location.database, - schemas=(location.schema,), - names=(location.name,), - ) - target_name: str = location.name.lower() - for relation in relations: - if relation.name == target_name: - return bool(relation.is_transient) - return False - - -def acquire_model_lease( - *, - backend: Any, - config_schema: str, - config_connection: dict[str, object], - model_name: str, - version_hash: str, -) -> StateLockLease | None: - connection: Any = backend.connect(config_connection) - try: - return acquire_model_version_lease( - backend=backend, - connection=connection, - schema=config_schema, - model_name=model_name, - version_hash=version_hash, - owner_id=f"clone:{uuid.uuid4()}", - ttl=timedelta(minutes=10), - ) - finally: - backend.close(connection) - - -def release_model_lease( - *, backend: Any, config_schema: str, config_connection: dict[str, object], lease: StateLockLease -) -> None: - connection: Any = backend.connect(config_connection) - try: - release_state_lease( - backend=backend, connection=connection, schema=config_schema, lease=lease - ) - finally: - backend.close(connection) - - -def register_hydrated_relation( - *, - backend: Any, - config_schema: str, - config_connection: dict[str, object], - model_version: ModelVersionRecord, - model: CompiledModel, - destination: CompiledRelationLocation, -) -> None: - connection: Any = backend.connect(config_connection) - try: - if ( - backend.get_model_version( - connection=connection, - schema=config_schema, - model_name=model_version.model_name, - version_hash=model_version.version_hash, - ) - is None - ): - backend.upsert_model_version( - connection=connection, schema=config_schema, record=model_version - ) - if ( - backend.get_physical_relation( - connection=connection, - schema=config_schema, - model_name=model.name, - version_hash=model_version.version_hash, - ) - is None - ): - backend.upsert_physical_relation( - connection=connection, - schema=config_schema, - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=model.name, - version_hash=model_version.version_hash, - database_name=destination.database, - schema_name=destination.schema or "", - relation_name=destination.name, - relation_type=relation_type_for_model( - MaterializationType( - model.config.values.get("materialized", MaterializationType.TABLE) - ) - ), - ), - ) - finally: - backend.close(connection) - - -def register_hydrated_seed_relation( - *, - backend: Any, - config_schema: str, - config_connection: dict[str, object], - seed_version: SeedVersionRecord, - destination: CompiledRelationLocation, -) -> None: - connection: Any = backend.connect(config_connection) - try: - if ( - backend.get_seed_version( - connection=connection, - schema=config_schema, - seed_name=seed_version.seed_name, - version_hash=seed_version.version_hash, - ) - is None - ): - backend.upsert_seed_version( - connection=connection, schema=config_schema, record=seed_version - ) - if ( - backend.get_physical_relation_for_artifact( - connection=connection, - schema=config_schema, - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_version.seed_name, - version_hash=seed_version.version_hash, - ) - is None - ): - backend.upsert_physical_relation( - connection=connection, - schema=config_schema, - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_version.seed_name, - version_hash=seed_version.version_hash, - database_name=destination.database, - schema_name=destination.schema or "", - relation_name=destination.name, - relation_type="table", - ), - ) - finally: - backend.close(connection) - - -def compile_clone_pipeline( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - destination_connection: ClonePipelineConnection, - no_sql_validation: bool, - select: tuple[str, ...], - exclude: tuple[str, ...], - cli_vars: dict[str, object] | None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, -) -> ClonePipelineResult: - """Compile origin and destination projects for one clone run.""" - - return run_clone_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - no_sql_validation=no_sql_validation, - select=select, - exclude=exclude, - cli_vars=cli_vars, - destination_connection=destination_connection, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - - -def build_clone_project_context(clone_pipeline: ClonePipelineResult) -> CloneProjectContext: - """Build the destination graph and node lookups from the clone pipeline result.""" - - return CloneProjectContext( - destination_graph=build_clone_graph_from_project( - project=clone_pipeline.destination_project - ), - model_names=tuple(entry.name for entry in clone_pipeline.destination_model_entries), - seed_names=tuple(entry.name for entry in clone_pipeline.destination_seed_entries), - destination_models_by_name={ - model.name: model for model in clone_pipeline.destination_project.models - }, - origin_models_by_name={model.name: model for model in clone_pipeline.origin_project.models}, - destination_seeds_by_name={ - seed.name: seed for seed in clone_pipeline.destination_project.seeds - }, - origin_seeds_by_name={seed.name: seed for seed in clone_pipeline.origin_project.seeds}, - ) - - -def resolve_clone_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - clone_pipeline: ClonePipelineResult, - context: CloneProjectContext, - virtual_environment_name: str | None, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - origin_target_name: str, -) -> CloneVersions: - """Resolve model and seed version records to hydrate.""" - - if virtual_environment_name is None: - origin_schema: str | None = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=origin_target_name, - ).state.schema - if origin_schema is None: - raise StateBackendConfigError( - f"Target '{origin_target_name}' state config must define schema" - ) - return _resolve_workspace_clone_versions( - clone_pipeline=clone_pipeline, - context=context, - backend=backend, - state_connection=state_connection, - schema=origin_schema, - origin_virtual_environment_name=origin_target_name, - ) - return _read_virtual_environment_clone_versions( - backend=backend, - state_connection=state_connection, - schema=schema, - context=context, - virtual_environment_name=virtual_environment_name, - ) - - -def build_clone_origin_lookup( - *, - adapter: BaseAdapter, - destination_connection: Any, - context: CloneProjectContext, - versions: CloneVersions, -) -> CloneOriginLookup: - """Build origin lookup locations and the relation existence lookup.""" - - model_locations: dict[str, CompiledRelationLocation] = { - model_name: build_physical_destination( - adapter=adapter, - target=context.origin_models_by_name[model_name].destination, - model_name=model_name, - version_hash=versions.version_hashes[model_name], - ) - for model_name in context.model_names - } - seed_locations: dict[str, CompiledRelationLocation] = { - seed_name: build_physical_seed_destination( - adapter=adapter, - target=context.origin_seeds_by_name[seed_name].destination, - seed_name=seed_name, - version_hash=versions.seed_versions[seed_name].version_hash, - ) - for seed_name in context.seed_names - } - with OperationLifecycle( - operation_kind="clone", operation_name="clone_relation_inspection" - ) as inspection: - lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=destination_connection, - locations=tuple( - (location.database, location.schema, location.name) - for location in (*model_locations.values(), *seed_locations.values()) - ), - ) - inspection.completed(metadata={"item_count": len(model_locations) + len(seed_locations)}) - return CloneOriginLookup( - model_locations=model_locations, - seed_locations=seed_locations, - lookup=lookup, - ) - - -def hydrate_clone_model_relations( - *, - backend: Any, - adapter: BaseAdapter, - destination_connection: Any, - config_schema: str, - config_connection: Any, - context: CloneProjectContext, - versions: CloneVersions, - origin_lookup: CloneOriginLookup, - skip_locked: bool, - run_id: str, -) -> tuple[VirtualCloneItemResult, ...]: - """Hydrate selected model relations from origin warehouse artifacts.""" - - results: list[VirtualCloneItemResult] = [] - for model_name in context.model_names: - destination_model: CompiledModel = context.destination_models_by_name[model_name] - origin_model: CompiledModel = context.origin_models_by_name[model_name] - version_hash: str = versions.version_hashes[model_name] - origin_location: CompiledRelationLocation = build_physical_destination( - adapter=adapter, - target=origin_model.destination, - model_name=model_name, - version_hash=version_hash, - ) - lookup_location: CompiledRelationLocation = origin_lookup.model_locations[model_name] - destination_location: CompiledRelationLocation = build_physical_destination( - adapter=adapter, - target=destination_model.destination, - model_name=model_name, - version_hash=version_hash, - ) - if not origin_lookup.lookup.exists( - database=lookup_location.database, - schema=lookup_location.schema, - name=lookup_location.name, - ): - results.append( - VirtualCloneItemResult( - PhysicalArtifactType.MODEL, model_name, version_hash, "missing" - ) - ) - continue - lease: StateLockLease | None = acquire_model_lease( - backend=backend, - config_schema=config_schema, - config_connection=config_connection, - model_name=model_name, - version_hash=version_hash, - ) - if lease is None: - if skip_locked: - results.append( - VirtualCloneItemResult( - PhysicalArtifactType.MODEL, - model_name, - version_hash, - "skipped_locked", - ) - ) - continue - raise PlannerInputError( - f"model version '{model_name}:{version_hash}' is locked", - code="S022", - help="Re-run with --skip-locked to hydrate other model versions.", - ) - try: - action: str = hydrate_and_register_relation( - adapter=adapter, - destination_connection=destination_connection, - origin_location=origin_location, - destination_location=destination_location, - resource_kind="model", - resource_name=model_name, - run_id=run_id, - register=partial( - register_hydrated_relation, - backend=backend, - config_schema=config_schema, - config_connection=config_connection, - model_version=versions.model_versions[model_name], - model=destination_model, - destination=destination_location, - ), - ) - results.append( - VirtualCloneItemResult(PhysicalArtifactType.MODEL, model_name, version_hash, action) - ) - finally: - release_model_lease( - backend=backend, - config_schema=config_schema, - config_connection=config_connection, - lease=lease, - ) - return tuple(results) - - -def hydrate_clone_seed_relations( - *, - backend: Any, - adapter: BaseAdapter, - destination_connection: Any, - config_schema: str, - config_connection: Any, - context: CloneProjectContext, - versions: CloneVersions, - origin_lookup: CloneOriginLookup, - run_id: str, -) -> tuple[VirtualCloneItemResult, ...]: - """Hydrate selected seed relations from origin warehouse artifacts.""" - - results: list[VirtualCloneItemResult] = [] - for seed_name in context.seed_names: - destination_seed: CompiledSeed = context.destination_seeds_by_name[seed_name] - origin_seed: CompiledSeed = context.origin_seeds_by_name[seed_name] - seed_version: SeedVersionRecord = versions.seed_versions[seed_name] - seed_version_hash: str = seed_version.version_hash - origin_location: CompiledRelationLocation = build_physical_seed_destination( - adapter=adapter, - target=origin_seed.destination, - seed_name=seed_name, - version_hash=seed_version_hash, - ) - lookup_location: CompiledRelationLocation = origin_lookup.seed_locations[seed_name] - destination_location: CompiledRelationLocation = build_physical_seed_destination( - adapter=adapter, - target=destination_seed.destination, - seed_name=seed_name, - version_hash=seed_version_hash, - ) - if not origin_lookup.lookup.exists( - database=lookup_location.database, - schema=lookup_location.schema, - name=lookup_location.name, - ): - results.append( - VirtualCloneItemResult( - PhysicalArtifactType.SEED, - seed_name, - seed_version_hash, - "missing", - ) - ) - continue - action: str = hydrate_and_register_relation( - adapter=adapter, - destination_connection=destination_connection, - origin_location=origin_location, - destination_location=destination_location, - resource_kind="seed", - resource_name=seed_name, - run_id=run_id, - register=partial( - register_hydrated_seed_relation, - backend=backend, - config_schema=config_schema, - config_connection=config_connection, - seed_version=seed_version, - destination=destination_location, - ), - ) - results.append( - VirtualCloneItemResult( - PhysicalArtifactType.SEED, - seed_name, - seed_version_hash, - action, - ) - ) - return tuple(results) - - -def _resolve_workspace_clone_versions( - *, - clone_pipeline: ClonePipelineResult, - context: CloneProjectContext, - backend: Any, - state_connection: Any, - schema: str, - origin_virtual_environment_name: str, -) -> CloneVersions: - semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=context.destination_graph, - bound_refs=(), - bound_model_versions={}, - ) - version_hashes: dict[str, str] = dict(semantics.expected_version_hashes) - model_versions: dict[str, ModelVersionRecord] = build_workspace_model_versions( - project=clone_pipeline.destination_project, - model_entries=clone_pipeline.destination_model_entries, - model_names=context.model_names, - version_hashes=version_hashes, - local_hashes=semantics.expected_local_hashes, - metadata_jsons=semantics.expected_metadata_jsons, - ) - source_dependent_names: tuple[str, ...] = tuple( - name - for name in context.model_names - if name in semantics.source_freshness_incomplete_model_names - ) - if source_dependent_names: - origin_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=origin_virtual_environment_name, - ) - ) - origin_hashes: dict[str, str] = {ref.model_name: ref.version_hash for ref in origin_refs} - missing_origin_refs: tuple[str, ...] = tuple( - name for name in source_dependent_names if name not in origin_hashes - ) - if missing_origin_refs: - raise PlannerInputError( - "origin virtual state is required for source-dependent workspace clone models: " - + ", ".join(missing_origin_refs), - code="S021", - ) - for name in source_dependent_names: - version_hash: str = origin_hashes[name] - version: ModelVersionRecord | None = backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=name, - version_hash=version_hash, - ) - if version is None: - raise PlannerInputError( - "origin virtual state has a source-dependent ref without model version: " - + name, - code="S021", - ) - version_hashes[name] = version_hash - model_versions[name] = version - seed_versions: dict[str, SeedVersionRecord] = build_workspace_seed_versions( - project=clone_pipeline.destination_project, - seed_entries=clone_pipeline.destination_seed_entries, - seed_names=context.seed_names, - version_hashes=semantics.expected_seed_version_hashes, - metadata_jsons=semantics.seed_identity_metadata_jsons, - ) - return CloneVersions( - mode=( - "workspace fingerprints + origin VDE refs" - if source_dependent_names - else "workspace fingerprints" - ), - version_hashes=version_hashes, - model_versions=model_versions, - seed_versions=seed_versions, - origin_state_used=bool(source_dependent_names), - ) - - -def _read_virtual_environment_clone_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - context: CloneProjectContext, - virtual_environment_name: str, -) -> CloneVersions: - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - if not refs: - raise PlannerInputError( - f"unknown destination virtual environment '{virtual_environment_name}'", - code="S019", - ) - ref_hashes: dict[str, str] = {ref.model_name: ref.version_hash for ref in refs} - missing_refs: tuple[str, ...] = tuple( - name for name in context.model_names if name not in ref_hashes - ) - if missing_refs: - raise PlannerInputError( - "destination virtual environment is missing selected refs: " + ", ".join(missing_refs), - code="S020", - ) - model_versions: dict[str, ModelVersionRecord] = {} - for name in context.model_names: - version_hash: str = ref_hashes[name] - record: ModelVersionRecord | None = backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=name, - version_hash=version_hash, - ) - if record is None: - raise PlannerInputError( - "destination virtual environment has a ref without model version state: " + name, - code="S021", - ) - model_versions[name] = record - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - ) - seed_hashes: dict[str, str] = {ref.seed_name: ref.version_hash for ref in seed_refs} - missing_seed_refs: tuple[str, ...] = tuple( - name for name in context.seed_names if name not in seed_hashes - ) - if missing_seed_refs: - raise PlannerInputError( - "destination virtual environment is missing selected seed refs: " - + ", ".join(missing_seed_refs), - code="S020", - ) - seed_versions: dict[str, SeedVersionRecord] = {} - for name in context.seed_names: - seed_version_hash: str = seed_hashes[name] - seed_record: SeedVersionRecord | None = backend.get_seed_version( - connection=state_connection, - schema=schema, - seed_name=name, - version_hash=seed_version_hash, - ) - if seed_record is None: - raise PlannerInputError( - "destination virtual environment has a ref without seed version state: " + name, - code="S021", - ) - seed_versions[name] = seed_record - return CloneVersions( - mode="destination VDE refs", - version_hashes=ref_hashes, - model_versions=model_versions, - seed_versions=seed_versions, - ) diff --git a/src/sqlbuild/virtual/executor/_helpers/environment_views.py b/src/sqlbuild/virtual/executor/_helpers/environment_views.py deleted file mode 100644 index a53f357b3e..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/environment_views.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Shared VDE view refresh and function publish phase for executor runs.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from typing import Any - -import sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name as rn -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -from sqlbuild.virtual.executor._helpers.rewrite import build_virtual_destination -from sqlbuild.virtual.executor._helpers.rollback import publish_function_versions -from sqlbuild.virtual.executor.models import VirtualEnvironmentPhysicalRelations -from sqlbuild.virtual.planner.main._targets import ( - build_virtual_destination_from_physical_relation, -) -from sqlbuild.virtual.state.models import FunctionVersionRecord, PhysicalRelationRecord - - -def write_vde_views( - *, - project: CompiledProject, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - model_physical_locations: dict[str, CompiledRelationLocation], - seed_physical_locations: dict[str, CompiledRelationLocation], - on_connection_start: Callable[[int], None] | None = None, - on_connection_complete: ConnectionElapsedCallback | None = None, - on_connection_error: ConnectionElapsedCallback | None = None, -) -> None: - """Create or replace logical VDE views from resolved physical locations.""" - - started_at: float = time.perf_counter() - if on_connection_start is not None: - on_connection_start(1) - connection: Any - try: - connection = adapter.connect(connection_config) - except Exception: - if on_connection_error is not None: - on_connection_error(1, elapsed_seconds=time.perf_counter() - started_at) - raise - if on_connection_complete is not None: - on_connection_complete(1, elapsed_seconds=time.perf_counter() - started_at) - recorder: StatementRecorder = StatementRecorder() - view_entries: list[tuple[CompiledRelationLocation, CompiledRelationLocation]] = [] - for model in project.models: - model_location: CompiledRelationLocation | None = model_physical_locations.get(model.name) - if model_location is not None: - view_entries.append((model.destination, model_location)) - for seed in project.seeds: - seed_location: CompiledRelationLocation | None = seed_physical_locations.get(seed.name) - if seed_location is not None: - view_entries.append((seed.destination, seed_location)) - try: - for destination, physical_target in view_entries: - virtual_target: CompiledRelationLocation = build_virtual_destination( - adapter=adapter, - target=destination, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - adapter.ensure_schema( - connection=connection, - database=virtual_target.database, - schema=virtual_target.schema, - statement_recorder=recorder, - ) - adapter.create_view_as( - connection=connection, - destination=rn.resolve_relation_location_qualified_name( - adapter=adapter, location=virtual_target - ), - sql=( - "SELECT * FROM " - + rn.resolve_relation_location_qualified_name( - adapter=adapter, location=physical_target - ) - ), - statement_recorder=recorder, - ) - finally: - adapter.close(connection) - - -def write_vde_views_from_records( - *, - project: CompiledProject, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - physical_relations: dict[str, PhysicalRelationRecord], - seed_physical_relations: dict[str, PhysicalRelationRecord], - on_connection_start: Callable[[int], None] | None = None, - on_connection_complete: ConnectionElapsedCallback | None = None, - on_connection_error: ConnectionElapsedCallback | None = None, -) -> None: - """Create or replace logical VDE views from tracked physical relation records.""" - - model_locations: dict[str, CompiledRelationLocation] = {} - for model in project.models: - model_relation: PhysicalRelationRecord | None = physical_relations.get(model.name) - if model_relation is None: - continue - model_locations[model.name] = build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=model_relation, - fallback_target=model.destination, - ) - seed_locations: dict[str, CompiledRelationLocation] = {} - for seed in project.seeds: - seed_relation: PhysicalRelationRecord | None = seed_physical_relations.get(seed.name) - if seed_relation is None: - continue - seed_locations[seed.name] = build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=seed_relation, - fallback_target=seed.destination, - ) - write_vde_views( - project=project, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_physical_locations=model_locations, - seed_physical_locations=seed_locations, - on_connection_start=on_connection_start, - on_connection_complete=on_connection_complete, - on_connection_error=on_connection_error, - ) - - -def write_logical_vde_views( - *, - project: CompiledProject, - adapter: BaseAdapter, - connection_config: dict[str, object], - target_vde_name: str, - unsuffixed_virtual_environment_name: str | None, - plan_output: PlanOutput, - final_version_hashes: dict[str, str], - final_seed_physical_relations: dict[str, PhysicalRelationRecord], -) -> None: - """Write logical model and seed views for finalized physical versions.""" - - model_locations: dict[str, CompiledRelationLocation] = {} - for model in project.models: - if model.name not in final_version_hashes: - continue - model_locations[model.name] = plan_output.model_locations.get(model.name, model.destination) - seed_locations: dict[str, CompiledRelationLocation] = {} - for seed in project.seeds: - relation: PhysicalRelationRecord | None = final_seed_physical_relations.get(seed.name) - if relation is None: - continue - seed_locations[seed.name] = build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=relation, - fallback_target=seed.destination, - ) - write_vde_views( - project=project, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=target_vde_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_physical_locations=model_locations, - seed_physical_locations=seed_locations, - ) - - -def write_virtual_environment_views( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - relations: VirtualEnvironmentPhysicalRelations, - function_versions: dict[str, FunctionVersionRecord], - hooks: ConnectionHooks, -) -> None: - """Refresh target VDE views and publish the given function versions.""" - - refresh_start: float = time.perf_counter() - if hooks.on_progress is not None: - hooks.on_progress("Refreshing target VDE views...") - write_vde_views_from_records( - project=graph.project, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - physical_relations=relations.model_relations, - seed_physical_relations=relations.seed_relations, - on_connection_start=hooks.on_connection_start, - on_connection_complete=hooks.on_connection_complete, - on_connection_error=hooks.on_connection_error, - ) - if function_versions: - publish_function_versions( - adapter=adapter, - connection_config=connection_config, - graph=graph, - virtual_environment_name=virtual_environment_name, - function_versions=function_versions, - ) - if hooks.on_progress is not None: - hooks.on_progress( - f"Refreshed target VDE views. ({time.perf_counter() - refresh_start:.2f}s)" - ) diff --git a/src/sqlbuild/virtual/executor/_helpers/functions.py b/src/sqlbuild/virtual/executor/_helpers/functions.py deleted file mode 100644 index 22436b788b..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/functions.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Virtual function version helpers.""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import asdict -from typing import Any - -from sqlbuild.compiler.compile.models import FunctionArgument, FunctionReturnColumn -from sqlbuild.compiler.compile.types import FunctionLanguage -from sqlbuild.compiler.planner.models import FunctionPlanEntry -from sqlbuild.virtual.state.main.encoding._decode_state_text import decode_state_text -from sqlbuild.virtual.state.main.encoding._encode_state_text import encode_state_text -from sqlbuild.virtual.state.models import FunctionVersionRecord -from sqlbuild.virtual.state.types import ModelVersionStatus - - -def build_function_version_record(function_entry: FunctionPlanEntry) -> FunctionVersionRecord: - """Build a state row for one planned function definition.""" - - fingerprint_sql: str = function_entry.fingerprint_query_sql - return FunctionVersionRecord( - function_name=function_entry.name, - version_hash=function_version_hash(fingerprint_sql), - language=function_entry.language.value, - returns=function_entry.returns, - arguments_json_b64=encode_state_text(_json_dataclasses(function_entry.arguments)), - return_columns_json_b64=encode_state_text(_json_dataclasses(function_entry.return_columns)), - packages_json_b64=encode_state_text(_json_values(function_entry.packages)), - runtime_version=function_entry.runtime_version, - entry_point=function_entry.entry_point, - body_sql_b64=encode_state_text(function_entry.body_sql), - definition_text_b64=encode_state_text(fingerprint_sql), - status=ModelVersionStatus.READY, - ) - - -def function_version_hash(fingerprint_sql: str) -> str: - """Return the function version hash for fingerprint SQL.""" - - return hashlib.sha256(fingerprint_sql.encode("utf-8")).hexdigest() - - -def decode_function_arguments(record: FunctionVersionRecord) -> tuple[FunctionArgument, ...]: - """Decode persisted function arguments.""" - - values: list[dict[str, object]] = _decode_json_list(record.arguments_json_b64) - return tuple( - FunctionArgument(name=str(value["name"]), type=str(value["type"])) for value in values - ) - - -def decode_function_return_columns( - record: FunctionVersionRecord, -) -> tuple[FunctionReturnColumn, ...]: - """Decode persisted table-function return columns.""" - - values: list[dict[str, object]] = _decode_json_list(record.return_columns_json_b64) - return tuple( - FunctionReturnColumn(name=str(value["name"]), type=str(value["type"])) for value in values - ) - - -def decode_function_packages(record: FunctionVersionRecord) -> tuple[str, ...]: - """Decode persisted function packages.""" - - values: list[object] = _decode_json_list(record.packages_json_b64) - return tuple(str(value) for value in values) - - -def decode_function_body_sql(record: FunctionVersionRecord) -> str: - """Decode persisted function body SQL.""" - - return decode_state_text(record.body_sql_b64) or "" - - -def decode_function_language(record: FunctionVersionRecord) -> FunctionLanguage: - """Decode persisted function language.""" - - return FunctionLanguage(record.language) - - -def _json_dataclasses(values: tuple[FunctionArgument | FunctionReturnColumn | object, ...]) -> str: - encoded: list[dict[str, str]] = [] - for value in values: - if isinstance(value, FunctionArgument | FunctionReturnColumn): - encoded.append(asdict(value)) - return _json_values(tuple(encoded)) - - -def _json_values(values: tuple[object, ...]) -> str: - return json.dumps(values, sort_keys=True, separators=(",", ":")) - - -def _decode_json_list(value_b64: str) -> list[Any]: - decoded: str | None = decode_state_text(value_b64) - if decoded is None: - return [] - value: object = json.loads(decoded) - if not isinstance(value, list): - return [] - return value diff --git a/src/sqlbuild/virtual/executor/_helpers/project_context.py b/src/sqlbuild/virtual/executor/_helpers/project_context.py deleted file mode 100644 index 23a1961fbc..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/project_context.py +++ /dev/null @@ -1,63 +0,0 @@ -"""Shared project graph and target-context phase for virtual executor runs.""" - -from __future__ import annotations - -import time -from collections.abc import Callable - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.executor.models import VirtualProjectContext - - -def resolve_virtual_project_context( - *, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - no_sql_validation: bool, - cli_vars: dict[str, object] | None, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - on_progress: Callable[[str], None] | None, -) -> VirtualProjectContext: - """Compile the project graph and resolve active target VDE naming.""" - - compile_start: float = time.perf_counter() - if on_progress is not None: - on_progress("Compiling project...") - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - no_sql_validation=no_sql_validation, - cli_vars=cli_vars, - external_sql_reference_resolver=external_sql_reference_resolver, - ) - if on_progress is not None: - on_progress(f"Compiled project. ({time.perf_counter() - compile_start:.2f}s)") - return VirtualProjectContext( - graph=graph, - unsuffixed_virtual_environment_name=_resolve_unsuffixed_virtual_environment_name( - discovered_inputs - ), - ) - - -def _resolve_unsuffixed_virtual_environment_name( - discovered_inputs: DiscoveredProjectInputs, -) -> str | None: - active_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if active_target_name is None: - return None - return resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=active_target_name, - ).state.unsuffixed_virtual_env diff --git a/src/sqlbuild/virtual/executor/_helpers/promote.py b/src/sqlbuild/virtual/executor/_helpers/promote.py deleted file mode 100644 index ef23e8b9d9..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/promote.py +++ /dev/null @@ -1,563 +0,0 @@ -"""Virtual promote helper functions.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import WorkSelectionPolicy -from sqlbuild.virtual.executor._helpers.seeding import read_seed_physical_relations -from sqlbuild.virtual.executor.models import ( - PromoteEnvironmentState, - PromoteRefUpdate, - PromoteResolution, - PromoteSelection, - PromoteSemantics, - VirtualEnvironmentPhysicalRelations, -) -from sqlbuild.virtual.planner.main._selection import resolve_virtual_plan_model_selection -from sqlbuild.virtual.planner.main._semantics import build_virtual_plan_semantics -from sqlbuild.virtual.planner.main._upstreams import build_virtual_stale_required_upstream_closure -from sqlbuild.virtual.planner.models import VirtualPlanSemantics -from sqlbuild.virtual.state.main.checkpoints._checkpoints import ( - create_finalized_virtual_environment_checkpoint, -) -from sqlbuild.virtual.state.main.environments._ref_model_versions import ( - read_ref_model_versions, -) -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationRecord, - SourceFreshnessRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - - -def read_promote_environment_state( - *, - backend: Any, - state_connection: Any, - schema: str, - from_virtual_environment_name: str, - to_virtual_environment_name: str, -) -> PromoteEnvironmentState: - """Read bound refs and environment records for one promote run.""" - - source_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - ) - target_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ) - ) - source_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( - backend.get_virtual_environment_function_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - ) - from_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - ) - to_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ) - ) - if not source_refs: - raise PlannerInputError( - f"unknown source virtual environment '{from_virtual_environment_name}'", - code="S011", - ) - source_environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - if ( - source_environment is not None - and source_environment.status == VirtualEnvironmentStatus.DETACHED - ): - raise PlannerInputError( - f"source virtual environment '{from_virtual_environment_name}' is detached", - code="S028", - ) - target_environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ) - if ( - target_environment is not None - and target_environment.status == VirtualEnvironmentStatus.DETACHED - ): - raise PlannerInputError( - f"target virtual environment '{to_virtual_environment_name}' is detached", - code="S028", - ) - source_freshness_records: tuple[SourceFreshnessRecord, ...] = ( - backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=schema, - virtual_environment_name=from_virtual_environment_name, - ) - ) - target_freshness_records: tuple[SourceFreshnessRecord, ...] = ( - backend.get_virtual_environment_source_freshness( - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - ) - ) - return PromoteEnvironmentState( - source_refs=source_refs, - target_refs=target_refs, - source_function_refs=source_function_refs, - from_seed_refs=from_seed_refs, - to_seed_refs=to_seed_refs, - source_environment=source_environment, - target_environment=target_environment, - source_freshness_records=source_freshness_records, - target_freshness_records=target_freshness_records, - ) - - -def build_promote_semantics( - *, - backend: Any, - state_connection: Any, - schema: str, - graph: ProjectGraph, - environment_state: PromoteEnvironmentState, -) -> PromoteSemantics: - """Build source and target virtual plan semantics from bound refs.""" - - source_versions: dict[str, ModelVersionRecord | None] = read_ref_model_versions( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=environment_state.source_refs, - ) - target_versions: dict[str, ModelVersionRecord | None] = read_ref_model_versions( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=environment_state.target_refs, - ) - source_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=graph, - bound_refs=environment_state.source_refs, - bound_model_versions=source_versions, - bound_seed_refs=environment_state.from_seed_refs, - source_freshness_records=environment_state.source_freshness_records, - ) - target_semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=graph, - bound_refs=environment_state.target_refs, - bound_model_versions=target_versions, - bound_seed_refs=environment_state.to_seed_refs, - source_freshness_records=environment_state.target_freshness_records, - ) - return PromoteSemantics(source=source_semantics, target=target_semantics) - - -def resolve_promote_selection( - *, - graph: ProjectGraph, - environment_state: PromoteEnvironmentState, - source_semantics: VirtualPlanSemantics, - select: tuple[str, ...], - exclude: tuple[str, ...], - include_stale_upstreams: bool, -) -> PromoteSelection: - """Resolve and validate the promote model and seed selection scope.""" - - selected_model_names: tuple[str, ...] = resolve_virtual_plan_model_selection( - graph=graph, - select=select, - exclude=exclude, - default_selection=tuple(model.name for model in graph.project.models), - stale_model_names=source_semantics.stale_model_names, - include_stale_upstreams=include_stale_upstreams, - work_selection_policy=WorkSelectionPolicy.ALL_SELECTED, - ) - if not select: - selected_model_names = tuple(model.name for model in graph.project.models) - if ( - environment_state.source_environment is None - or environment_state.source_environment.status != VirtualEnvironmentStatus.FINALIZED - ): - raise PlannerInputError( - "whole-VDE promotion requires a finalized source virtual environment", - code="S018", - help="Use --select for a coherent partial promotion from a working source VDE.", - ) - selected_seed_names: tuple[str, ...] = selected_upstream_seed_names( - graph=graph, - selected_model_names=selected_model_names, - all_seed_names=tuple(seed.name for seed in graph.project.seeds), - include_all=not select, - ) - source_ref_map: dict[str, str] = { - ref.model_name: ref.version_hash for ref in environment_state.source_refs - } - from_seed_ref_map: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in environment_state.from_seed_refs - } - missing_source_refs: tuple[str, ...] = tuple( - model_name for model_name in selected_model_names if model_name not in source_ref_map - ) - if missing_source_refs: - raise PlannerInputError( - "source virtual environment is missing selected refs: " - + ", ".join(missing_source_refs), - code="S015", - ) - missing_from_seed_refs: tuple[str, ...] = tuple( - seed_name for seed_name in selected_seed_names if seed_name not in from_seed_ref_map - ) - if missing_from_seed_refs: - raise PlannerInputError( - "source virtual environment is missing selected seed refs: " - + ", ".join(missing_from_seed_refs), - code="S015", - ) - return PromoteSelection( - selected_model_names=selected_model_names, - selected_seed_names=selected_seed_names, - source_ref_map=source_ref_map, - from_seed_ref_map=from_seed_ref_map, - ) - - -def resolve_promote_final_refs( - *, - graph: ProjectGraph, - environment_state: PromoteEnvironmentState, - selection: PromoteSelection, - target_semantics: VirtualPlanSemantics, - select: tuple[str, ...], - include_stale_upstreams: bool, - allow_partial_promotion: bool, -) -> PromoteResolution: - """Resolve final ref hashes, staleness, and target status for the promotion.""" - - selected_model_names: tuple[str, ...] = selection.selected_model_names - selected_seed_names: tuple[str, ...] = selection.selected_seed_names - final_version_hashes: dict[str, str] = { - ref.model_name: ref.version_hash for ref in environment_state.target_refs - } - final_seed_hashes: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in environment_state.to_seed_refs - } - for model_name in selected_model_names: - final_version_hashes[model_name] = selection.source_ref_map[model_name] - for seed_name in selected_seed_names: - final_seed_hashes[seed_name] = selection.from_seed_ref_map[seed_name] - stale_after: tuple[str, ...] = _stale_models_after_promotion( - graph=graph, - final_version_hashes=final_version_hashes, - expected_version_hashes=target_semantics.expected_version_hashes, - ) - if not select: - stale_after = () - stale_upstreams: tuple[str, ...] = build_virtual_stale_required_upstream_closure( - graph=graph, - selected_model_names=selected_model_names, - stale_model_names=stale_after, - ) - if stale_upstreams and not include_stale_upstreams: - raise PlannerInputError( - "selected promotion scope is missing stale required upstream models: " - + ", ".join(stale_upstreams), - code="S016", - help="Re-run with --include-stale-upstreams to add required upstream refs.", - ) - if stale_upstreams: - selected_model_names = tuple(sorted({*selected_model_names, *stale_upstreams})) - selected_seed_names = selected_upstream_seed_names( - graph=graph, - selected_model_names=selected_model_names, - all_seed_names=tuple(seed.name for seed in graph.project.seeds), - include_all=False, - ) - for model_name in stale_upstreams: - final_version_hashes[model_name] = selection.source_ref_map[model_name] - for seed_name in selected_seed_names: - if seed_name not in selection.from_seed_ref_map: - raise PlannerInputError( - "source virtual environment is missing selected seed refs: " + seed_name, - code="S015", - ) - final_seed_hashes[seed_name] = selection.from_seed_ref_map[seed_name] - stale_after = _stale_models_after_promotion( - graph=graph, - final_version_hashes=final_version_hashes, - expected_version_hashes=target_semantics.expected_version_hashes, - ) - if stale_after and not allow_partial_promotion: - raise PlannerInputError( - "promotion would leave target virtual environment working; remaining stale models: " - + ", ".join(stale_after), - code="S017", - help="Re-run with --allow-partial-promotion to accept a working target VDE.", - ) - status: VirtualEnvironmentStatus = ( - VirtualEnvironmentStatus.FINALIZED if not stale_after else VirtualEnvironmentStatus.ACTIVE - ) - return PromoteResolution( - selected_model_names=selected_model_names, - selected_seed_names=selected_seed_names, - final_version_hashes=final_version_hashes, - final_seed_hashes=final_seed_hashes, - stale_after=stale_after, - status=status, - ) - - -def build_promote_ref_update( - *, - backend: Any, - state_connection: Any, - schema: str, - from_virtual_environment_name: str, - to_virtual_environment_name: str, - resolution: PromoteResolution, - source_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...], - select: tuple[str, ...], -) -> PromoteRefUpdate: - """Build the target environment record and replacement ref groups.""" - - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=to_virtual_environment_name, - status=resolution.status, - baseline_virtual_environment_name=from_virtual_environment_name, - ) - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=to_virtual_environment_name, - model_name=model_name, - version_hash=version_hash, - ) - for model_name, version_hash in sorted(resolution.final_version_hashes.items()) - ) - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = tuple( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=to_virtual_environment_name, - seed_name=seed_name, - version_hash=version_hash, - ) - for seed_name, version_hash in sorted(resolution.final_seed_hashes.items()) - ) - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = () - function_versions: dict[str, FunctionVersionRecord] = {} - if not select: - function_refs = tuple( - VirtualEnvironmentFunctionRefRecord( - virtual_environment_name=to_virtual_environment_name, - node_type=ref.node_type, - function_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in source_function_refs - ) - for ref in function_refs: - function_version: FunctionVersionRecord | None = backend.get_function_version( - connection=state_connection, - schema=schema, - function_name=ref.function_name, - version_hash=ref.version_hash, - ) - if function_version is not None: - function_versions[ref.function_name] = function_version - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] = { - "model": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="model", - node_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - ), - "seed": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="seed", - node_name=ref.seed_name, - version_hash=ref.version_hash, - ) - for ref in seed_refs - ), - } - if not select: - refs_by_node_type["udf"] = tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in function_refs - if ref.node_type == CompiledResourceType.UDF - ) - refs_by_node_type["table_fn"] = tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in function_refs - if ref.node_type == CompiledResourceType.TABLE_FN - ) - return PromoteRefUpdate( - virtual_environment_record=virtual_environment_record, - refs=refs, - seed_refs=seed_refs, - function_refs=function_refs, - function_versions=function_versions, - refs_by_node_type=refs_by_node_type, - ) - - -def write_promote_environment_update( - *, - backend: Any, - state_connection: Any, - schema: str, - to_virtual_environment_name: str, - update: PromoteRefUpdate, -) -> None: - """Persist the promoted environment record, ref groups, and checkpoint.""" - - backend.upsert_virtual_environment_and_replace_node_ref_groups( - connection=state_connection, - schema=schema, - record=update.virtual_environment_record, - refs_by_node_type=update.refs_by_node_type, - ) - finalized: bool = update.virtual_environment_record.status == VirtualEnvironmentStatus.FINALIZED - if finalized and update.refs: - create_finalized_virtual_environment_checkpoint( - backend=backend, - connection=state_connection, - schema=schema, - virtual_environment_name=to_virtual_environment_name, - refs=update.refs, - function_refs=update.function_refs, - seed_refs=update.seed_refs, - ) - - -def read_promote_physical_relations( - *, - backend: Any, - state_connection: Any, - schema: str, - update: PromoteRefUpdate, -) -> VirtualEnvironmentPhysicalRelations: - """Read tracked physical relations backing the promoted refs.""" - - model_relations: dict[str, PhysicalRelationRecord] = _read_physical_relations( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=update.refs, - ) - seed_relations: dict[str, PhysicalRelationRecord] = read_seed_physical_relations( - backend=backend, - state_connection=state_connection, - schema=schema, - seed_version_hashes={ref.seed_name: ref.version_hash for ref in update.seed_refs}, - ) - return VirtualEnvironmentPhysicalRelations( - model_relations=model_relations, - seed_relations=seed_relations, - ) - - -def _stale_models_after_promotion( - *, - graph: ProjectGraph, - final_version_hashes: dict[str, str], - expected_version_hashes: dict[str, str], -) -> tuple[str, ...]: - return tuple( - model.name - for model in graph.project.models - if final_version_hashes.get(model.name) != expected_version_hashes.get(model.name) - ) - - -def _read_physical_relations( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, PhysicalRelationRecord]: - relations: dict[str, PhysicalRelationRecord] = {} - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is not None: - relations[ref.model_name] = relation - return relations - - -def selected_upstream_seed_names( - *, - graph: ProjectGraph, - selected_model_names: tuple[str, ...], - all_seed_names: tuple[str, ...], - include_all: bool, -) -> tuple[str, ...]: - """Resolve seed names reachable upstream from the selected models.""" - - if include_all: - return all_seed_names - selected: set[str] = set() - pending: list[CompiledObjectKey] = [ - model.key for model in graph.project.models if model.name in selected_model_names - ] - seen: set[CompiledObjectKey] = set() - while pending: - key: CompiledObjectKey = pending.pop() - if key in seen: - continue - seen.add(key) - upstream_key: CompiledObjectKey - for upstream_key in graph.upstream_deps.get(key, ()): - if upstream_key.resource_type == CompiledResourceType.SEED: - selected.add(upstream_key.name) - continue - pending.append(upstream_key) - return tuple(sorted(selected)) diff --git a/src/sqlbuild/virtual/executor/_helpers/rewrite.py b/src/sqlbuild/virtual/executor/_helpers/rewrite.py deleted file mode 100644 index cbe58c3129..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/rewrite.py +++ /dev/null @@ -1,247 +0,0 @@ -"""Virtual build target rewrite helpers.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( - resolve_qualified_name_parts, -) -from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, - CompiledProject, - CompiledRelationLocation, - CompiledSeed, -) -from sqlbuild.compiler.planner.types import MaterializationType -from sqlbuild.virtual.planner.main._targets import ( - build_virtual_destination_from_physical_relation, -) -from sqlbuild.virtual.state.models import PhysicalRelationRecord - - -def build_physical_destination( - *, - adapter: BaseAdapter, - target: CompiledRelationLocation, - model_name: str, - version_hash: str, -) -> CompiledRelationLocation: - """Build the physical version target for a virtual-mode model build.""" - - physical_schema: str | None = ( - f"{target.schema}__sqb_physical" if target.schema is not None else None - ) - physical_name: str = f"{model_name}__v_{version_hash[:8]}" - return CompiledRelationLocation( - database=target.database, - schema=physical_schema, - name=physical_name, - qualified_name=resolve_qualified_name_parts( - adapter=adapter, - database=target.database, - schema=physical_schema, - name=physical_name, - ), - logical_schema=target.logical_schema, - logical_database=target.logical_database, - ) - - -def build_physical_seed_destination( - *, - adapter: BaseAdapter, - target: CompiledRelationLocation, - seed_name: str, - version_hash: str, -) -> CompiledRelationLocation: - """Build the physical version target for a virtual-mode seed load.""" - - physical_schema: str | None = ( - f"{target.schema}__sqb_physical" if target.schema is not None else None - ) - physical_name: str = f"{seed_name}__v_{version_hash[:8]}" - return CompiledRelationLocation( - database=target.database, - schema=physical_schema, - name=physical_name, - qualified_name=resolve_qualified_name_parts( - adapter=adapter, - database=target.database, - schema=physical_schema, - name=physical_name, - ), - logical_schema=target.logical_schema, - logical_database=target.logical_database, - ) - - -def build_virtual_destination( - *, - adapter: BaseAdapter, - target: CompiledRelationLocation, - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None = None, -) -> CompiledRelationLocation: - """Build the logical VDE view target for a model.""" - - virtual_schema: str | None - if target.schema is None: - virtual_schema = None - elif unsuffixed_virtual_environment_name == virtual_environment_name: - virtual_schema = target.schema - else: - virtual_schema = f"{target.schema}__{virtual_environment_name}" - return CompiledRelationLocation( - database=target.database, - schema=virtual_schema, - name=target.name, - qualified_name=resolve_qualified_name_parts( - adapter=adapter, - database=target.database, - schema=virtual_schema, - name=target.name, - ), - logical_schema=target.logical_schema, - logical_database=target.logical_database, - ) - - -def build_rewritten_seed_locations( - *, - project: CompiledProject, - adapter: BaseAdapter, - seed_load_version_hashes: dict[str, str], - available_physical_relations: dict[str, PhysicalRelationRecord], - bound_seed_version_hashes: dict[str, str], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, -) -> dict[str, CompiledRelationLocation]: - """Resolve physical and virtual seed targets used during a virtual build.""" - - rewritten: dict[str, CompiledRelationLocation] = {} - for seed in project.seeds: - version_hash: str | None = seed_load_version_hashes.get(seed.name) - available_relation: PhysicalRelationRecord | None = available_physical_relations.get( - seed.name - ) - if version_hash is not None: - rewritten[seed.name] = build_physical_seed_destination( - adapter=adapter, - target=seed.destination, - seed_name=seed.name, - version_hash=version_hash, - ) - elif available_relation is not None: - rewritten[seed.name] = build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=available_relation, - fallback_target=seed.destination, - ) - elif seed.name in bound_seed_version_hashes: - rewritten[seed.name] = build_virtual_destination( - adapter=adapter, - target=seed.destination, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - return rewritten - - -def rewrite_project_model_locations( - *, - project: CompiledProject, - rewritten_locations: dict[str, CompiledRelationLocation], -) -> CompiledProject: - """Return a compiled project with selected model locations replaced.""" - - rewritten_models: tuple[CompiledModel, ...] = tuple( - replace(model, destination=rewritten_locations.get(model.name, model.destination)) - for model in project.models - ) - return replace(project, models=rewritten_models) - - -def rewrite_project_seed_locations( - *, - project: CompiledProject, - rewritten_locations: dict[str, CompiledRelationLocation], -) -> CompiledProject: - """Return a compiled project with selected seed locations replaced.""" - - rewritten_seeds: tuple[CompiledSeed, ...] = tuple( - replace(seed, destination=rewritten_locations.get(seed.name, seed.destination)) - for seed in project.seeds - ) - return replace(project, seeds=rewritten_seeds) - - -def rewrite_project_function_locations( - *, - project: CompiledProject, - adapter: BaseAdapter, - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None = None, -) -> CompiledProject: - """Return a compiled project with functions published into the VDE schema.""" - - rewritten_functions: tuple[CompiledFunction, ...] = tuple( - replace( - function, - destination=build_virtual_destination( - adapter=adapter, - target=function.destination, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ), - fingerprint_destination=build_virtual_destination( - adapter=adapter, - target=function.fingerprint_destination, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ), - ) - for function in project.functions - ) - return replace(project, functions=rewritten_functions) - - -def relation_type_for_model(materialized: str | None) -> str: - """Return the persisted physical relation type for a model materialization.""" - - if materialized == MaterializationType.VIEW: - return "view" - return "table" - - -def build_rewritten_model_locations( - *, - project: CompiledProject, - adapter: BaseAdapter, - selected_model_version_hashes: dict[str, str], - bound_physical_relations: dict[str, PhysicalRelationRecord], -) -> dict[str, CompiledRelationLocation]: - """Build rewritten model locations for virtual build execution.""" - - rewritten_locations: dict[str, CompiledRelationLocation] = {} - model: CompiledModel - for model in project.models: - selected_version_hash: str | None = selected_model_version_hashes.get(model.name) - if selected_version_hash is not None: - rewritten_locations[model.name] = build_physical_destination( - adapter=adapter, - target=model.destination, - model_name=model.name, - version_hash=selected_version_hash, - ) - continue - bound_relation: PhysicalRelationRecord | None = bound_physical_relations.get(model.name) - if bound_relation is not None: - rewritten_locations[model.name] = build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=bound_relation, - fallback_target=model.destination, - ) - return rewritten_locations diff --git a/src/sqlbuild/virtual/executor/_helpers/rollback.py b/src/sqlbuild/virtual/executor/_helpers/rollback.py deleted file mode 100644 index 198331aa0c..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/rollback.py +++ /dev/null @@ -1,643 +0,0 @@ -"""Virtual rollback helper functions.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import FunctionDefinition, RelationLookup -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.virtual.executor._helpers.functions import ( - decode_function_arguments, - decode_function_body_sql, - decode_function_language, - decode_function_packages, - decode_function_return_columns, -) -from sqlbuild.virtual.executor._helpers.promote import selected_upstream_seed_names -from sqlbuild.virtual.executor._helpers.rewrite import build_virtual_destination -from sqlbuild.virtual.executor._helpers.seeding import read_seed_physical_relations -from sqlbuild.virtual.executor.models import ( - RollbackCheckpointState, - RollbackRefUpdate, - RollbackResolution, - VirtualEnvironmentPhysicalRelations, -) -from sqlbuild.virtual.planner.main._selection import resolve_virtual_plan_model_selection -from sqlbuild.virtual.planner.main._targets import build_virtual_destination_from_physical_relation -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - PhysicalRelationRecord, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - - -def resolve_target_checkpoint( - *, - backend: Any, - state_connection: Any, - schema: str, - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...], - current_ref_map: dict[str, str], - checkpoint_id: str | None, -) -> tuple[ - VirtualEnvironmentCheckpointRecord | None, - tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], -]: - checkpoint: VirtualEnvironmentCheckpointRecord - for checkpoint in checkpoints: - if checkpoint_id is not None and checkpoint.checkpoint_id != checkpoint_id: - continue - checkpoint_model_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_model_refs( - connection=state_connection, - schema=schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - checkpoint_ref_map: dict[str, str] = { - ref.model_name: ref.version_hash for ref in checkpoint_model_refs - } - if checkpoint_id is not None or checkpoint_ref_map != current_ref_map: - return checkpoint, checkpoint_model_refs - if checkpoint_id is not None: - raise PlannerInputError(f"unknown checkpoint '{checkpoint_id}'", code="S026") - return None, () - - -def resolve_selected_model_names( - *, - graph: ProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], - all_model_names: tuple[str, ...], - target_checkpoint_model_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], -) -> tuple[str, ...]: - if select: - return resolve_virtual_plan_model_selection( - graph=graph, - select=select, - exclude=exclude, - default_selection=all_model_names, - stale_model_names=(), - ) - excluded: set[str] = set( - resolve_virtual_plan_model_selection( - graph=graph, - select=exclude, - exclude=(), - default_selection=(), - stale_model_names=(), - ) - if exclude - else () - ) - return tuple( - ref.model_name for ref in target_checkpoint_model_refs if ref.model_name not in excluded - ) - - -def stale_after_rollback( - *, - graph: ProjectGraph, - final_version_hashes: dict[str, str], - expected_version_hashes: dict[str, str], -) -> tuple[str, ...]: - return tuple( - model.name - for model in graph.project.models - if final_version_hashes.get(model.name) != expected_version_hashes.get(model.name) - ) - - -def guard_partial_rollback_scope( - *, - graph: ProjectGraph, - selected_model_names: tuple[str, ...], - stale_after: tuple[str, ...], - checkpoint_ref_map: dict[str, str], - final_version_hashes: dict[str, str], - include_stale_upstreams: bool, -) -> tuple[tuple[str, ...], dict[str, str]]: - checkpoint_mismatched_models: tuple[str, ...] = tuple( - model_name - for model_name, checkpoint_hash in checkpoint_ref_map.items() - if final_version_hashes.get(model_name) != checkpoint_hash - ) - required_upstreams: tuple[str, ...] = resolve_virtual_plan_model_selection( - graph=graph, - select=selected_model_names, - exclude=(), - default_selection=selected_model_names, - stale_model_names=checkpoint_mismatched_models, - include_stale_upstreams=True, - ) - stale_upstream_set: set[str] = set(required_upstreams) - set(selected_model_names) - if stale_upstream_set and not include_stale_upstreams: - raise PlannerInputError( - "selected rollback scope is missing stale required upstream models: " - + ", ".join(sorted(stale_upstream_set)), - code="S028", - help="Re-run with --include-stale-upstreams to add required upstream refs.", - ) - for model_name in stale_upstream_set: - if model_name not in checkpoint_ref_map: - raise PlannerInputError( - "checkpoint is missing required upstream refs: " + model_name, - code="S025", - ) - final_version_hashes[model_name] = checkpoint_ref_map[model_name] - return tuple(sorted({*selected_model_names, *stale_upstream_set})), final_version_hashes - - -def read_physical_relations( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], -) -> dict[str, PhysicalRelationRecord]: - relations: dict[str, PhysicalRelationRecord] = {} - ref: VirtualEnvironmentCheckpointModelRefRecord - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is None: - raise PlannerInputError( - f"checkpoint references missing physical relation for model '{ref.model_name}'", - code="S022", - ) - relations[ref.model_name] = relation - return relations - - -def validate_physical_relations_exist( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - models_by_name: dict[str, CompiledModel], - physical_relations: dict[str, PhysicalRelationRecord], -) -> None: - targets_by_model: dict[str, CompiledRelationLocation] = {} - model_name: str - relation: PhysicalRelationRecord - for model_name, relation in physical_relations.items(): - model: CompiledModel | None = models_by_name.get(model_name) - if model is None: - raise PlannerInputError( - f"checkpoint references unknown model '{model_name}'", - code="S023", - ) - targets_by_model[model_name] = build_virtual_destination_from_physical_relation( - adapter=adapter, - relation=relation, - fallback_target=model.destination, - ) - connection: Any = adapter.connect(connection_config) - try: - relation_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - (target.database, target.schema, target.name) - for target in targets_by_model.values() - ), - ) - for model_name, target in targets_by_model.items(): - if not relation_lookup.exists( - database=target.database, schema=target.schema, name=target.name - ): - raise PlannerInputError( - f"checkpoint references missing warehouse relation for model '{model_name}'", - code="S024", - ) - finally: - adapter.close(connection) - - -def read_function_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...], -) -> dict[str, FunctionVersionRecord]: - versions: dict[str, FunctionVersionRecord] = {} - ref: VirtualEnvironmentCheckpointFunctionRefRecord - for ref in refs: - record: FunctionVersionRecord | None = backend.get_function_version( - connection=state_connection, - schema=schema, - function_name=ref.function_name, - version_hash=ref.version_hash, - ) - if record is None: - raise PlannerInputError( - f"checkpoint references missing function version for '{ref.function_name}'", - code="S029", - ) - versions[ref.function_name] = record - return versions - - -def publish_function_versions( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - graph: ProjectGraph, - virtual_environment_name: str, - function_versions: dict[str, FunctionVersionRecord], -) -> None: - functions_by_name: dict[str, CompiledFunction] = { - function.name: function for function in graph.project.functions - } - connection: Any = adapter.connect(connection_config) - recorder: StatementRecorder = StatementRecorder() - try: - for function_name, record in function_versions.items(): - function: CompiledFunction | None = functions_by_name.get(function_name) - if function is None: - continue - target: CompiledRelationLocation = build_virtual_destination( - adapter=adapter, - target=function.destination, - virtual_environment_name=virtual_environment_name, - ) - adapter.ensure_schema( - connection=connection, - database=target.database, - schema=target.schema, - statement_recorder=recorder, - ) - if target.qualified_name is None: - continue - adapter.create_function( - connection=connection, - definition=FunctionDefinition( - destination=target.qualified_name, - arguments=decode_function_arguments(record), - returns=record.returns, - body_sql=decode_function_body_sql(record), - return_columns=decode_function_return_columns(record), - language=decode_function_language(record), - runtime_version=record.runtime_version, - entry_point=record.entry_point, - packages=decode_function_packages(record), - source_file_path=None, - ), - statement_recorder=recorder, - ) - finally: - adapter.close(connection) - - -def read_rollback_checkpoint_state( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, - checkpoint_id: str | None, -) -> RollbackCheckpointState: - """Read current refs and resolve the rollback target checkpoint.""" - - environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - if environment is not None and environment.status == VirtualEnvironmentStatus.DETACHED: - raise PlannerInputError( - f"virtual environment '{virtual_environment_name}' is detached", - code="S028", - ) - current_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - ) - if not current_refs: - raise PlannerInputError( - f"unknown virtual environment '{virtual_environment_name}'", - code="S020", - ) - current_ref_map: dict[str, str] = {ref.model_name: ref.version_hash for ref in current_refs} - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - backend.list_virtual_environment_checkpoints( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - ) - target_checkpoint, target_checkpoint_model_refs = resolve_target_checkpoint( - backend=backend, - state_connection=state_connection, - schema=schema, - checkpoints=checkpoints, - current_ref_map=current_ref_map, - checkpoint_id=checkpoint_id, - ) - if target_checkpoint is None: - raise PlannerInputError( - "no previous finalized checkpoint is available for rollback", - code="S021", - ) - target_checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_function_refs( - connection=state_connection, - schema=schema, - checkpoint_id=target_checkpoint.checkpoint_id, - ) - ) - target_checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_seed_refs( - connection=state_connection, - schema=schema, - checkpoint_id=target_checkpoint.checkpoint_id, - ) - ) - return RollbackCheckpointState( - current_ref_map=current_ref_map, - target_checkpoint=target_checkpoint, - checkpoint_model_refs=target_checkpoint_model_refs, - checkpoint_function_refs=target_checkpoint_function_refs, - checkpoint_seed_refs=target_checkpoint_seed_refs, - ) - - -def resolve_rollback_final_refs( - *, - backend: Any, - state_connection: Any, - schema: str, - graph: ProjectGraph, - virtual_environment_name: str, - checkpoint_state: RollbackCheckpointState, - select: tuple[str, ...], - exclude: tuple[str, ...], - include_stale_upstreams: bool, - allow_partial_rollback: bool, -) -> RollbackResolution: - """Resolve final ref hashes, scope, and target status for the rollback.""" - - selected_model_names: tuple[str, ...] = resolve_selected_model_names( - graph=graph, - select=select, - exclude=exclude, - all_model_names=tuple(model.name for model in graph.project.models), - target_checkpoint_model_refs=checkpoint_state.checkpoint_model_refs, - ) - checkpoint_ref_map: dict[str, str] = { - ref.model_name: ref.version_hash for ref in checkpoint_state.checkpoint_model_refs - } - checkpoint_seed_ref_map: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in checkpoint_state.checkpoint_seed_refs - } - missing_checkpoint_model_refs: tuple[str, ...] = tuple( - model_name for model_name in selected_model_names if model_name not in checkpoint_ref_map - ) - if missing_checkpoint_model_refs: - raise PlannerInputError( - "checkpoint is missing selected refs: " + ", ".join(missing_checkpoint_model_refs), - code="S025", - ) - final_version_hashes: dict[str, str] = dict(checkpoint_state.current_ref_map) - for model_name in selected_model_names: - final_version_hashes[model_name] = checkpoint_ref_map[model_name] - current_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - ) - final_seed_hashes: dict[str, str] = { - ref.seed_name: ref.version_hash for ref in current_seed_refs - } - selected_seed_names: tuple[str, ...] = selected_upstream_seed_names( - graph=graph, - selected_model_names=selected_model_names, - all_seed_names=tuple(seed.name for seed in graph.project.seeds), - include_all=not bool(select or exclude), - ) - missing_checkpoint_seed_refs: tuple[str, ...] = tuple( - seed_name for seed_name in selected_seed_names if seed_name not in checkpoint_seed_ref_map - ) - if missing_checkpoint_seed_refs: - raise PlannerInputError( - "checkpoint is missing selected seed refs: " + ", ".join(missing_checkpoint_seed_refs), - code="S025", - ) - for seed_name in selected_seed_names: - final_seed_hashes[seed_name] = checkpoint_seed_ref_map[seed_name] - stale_after: tuple[str, ...] = stale_after_rollback( - graph=graph, - final_version_hashes=final_version_hashes, - expected_version_hashes=checkpoint_ref_map, - ) - is_partial_scope: bool = bool(select or exclude) - if is_partial_scope: - selected_model_names, final_version_hashes = guard_partial_rollback_scope( - graph=graph, - selected_model_names=selected_model_names, - stale_after=stale_after, - checkpoint_ref_map=checkpoint_ref_map, - final_version_hashes=final_version_hashes, - include_stale_upstreams=include_stale_upstreams, - ) - stale_after = stale_after_rollback( - graph=graph, - final_version_hashes=final_version_hashes, - expected_version_hashes=checkpoint_ref_map, - ) - if stale_after and not allow_partial_rollback: - raise PlannerInputError( - "rollback would leave target virtual environment working; " - "remaining stale models: " + ", ".join(stale_after), - code="S027", - help="Re-run with --allow-partial-rollback to accept a working target VDE.", - ) - status: VirtualEnvironmentStatus = ( - VirtualEnvironmentStatus.FINALIZED - if not is_partial_scope or not stale_after - else VirtualEnvironmentStatus.ACTIVE - ) - rolled_back_model_names: tuple[str, ...] = tuple( - sorted( - model_name - for model_name, version_hash in checkpoint_state.current_ref_map.items() - if final_version_hashes.get(model_name) != version_hash - ) - ) - return RollbackResolution( - final_version_hashes=final_version_hashes, - final_seed_hashes=final_seed_hashes, - is_partial_scope=is_partial_scope, - status=status, - rolled_back_model_names=rolled_back_model_names, - ) - - -def read_rollback_physical_relations( - *, - backend: Any, - state_connection: Any, - schema: str, - checkpoint_id: str, - resolution: RollbackResolution, -) -> VirtualEnvironmentPhysicalRelations: - """Read tracked physical relations backing the rollback target refs.""" - - model_relations: dict[str, PhysicalRelationRecord] = read_physical_relations( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=tuple( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id=checkpoint_id, - model_name=model_name, - version_hash=version_hash, - ) - for model_name, version_hash in sorted(resolution.final_version_hashes.items()) - ), - ) - seed_relations: dict[str, PhysicalRelationRecord] = read_seed_physical_relations( - backend=backend, - state_connection=state_connection, - schema=schema, - seed_version_hashes=resolution.final_seed_hashes, - ) - missing_seed_names: tuple[str, ...] = tuple( - sorted(set(resolution.final_seed_hashes) - set(seed_relations)) - ) - if missing_seed_names: - raise PlannerInputError( - f"checkpoint references missing physical relation for seed '{missing_seed_names[0]}'", - code="S022", - ) - return VirtualEnvironmentPhysicalRelations( - model_relations=model_relations, - seed_relations=seed_relations, - ) - - -def build_rollback_ref_update( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, - resolution: RollbackResolution, - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...], -) -> RollbackRefUpdate: - """Build the environment record and replacement ref groups for the rollback.""" - - target_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=virtual_environment_name, - model_name=model_name, - version_hash=version_hash, - ) - for model_name, version_hash in sorted(resolution.final_version_hashes.items()) - ) - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=virtual_environment_name, - status=resolution.status, - ) - destination_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = tuple( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=virtual_environment_name, - seed_name=seed_name, - version_hash=version_hash, - ) - for seed_name, version_hash in sorted(resolution.final_seed_hashes.items()) - ) - function_versions: dict[str, FunctionVersionRecord] = read_function_versions( - backend=backend, - state_connection=state_connection, - schema=schema, - refs=checkpoint_function_refs, - ) - target_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = tuple( - VirtualEnvironmentFunctionRefRecord( - virtual_environment_name=virtual_environment_name, - node_type=("table_fn" if decode_function_return_columns(function_version) else "udf"), - function_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in checkpoint_function_refs - if (function_version := function_versions.get(ref.function_name)) is not None - ) - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] = { - "model": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="model", - node_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in target_refs - ), - "seed": tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="seed", - node_name=ref.seed_name, - version_hash=ref.version_hash, - ) - for ref in destination_seed_refs - ), - } - if not resolution.is_partial_scope: - refs_by_node_type["udf"] = tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in target_function_refs - if ref.node_type == CompiledResourceType.UDF - ) - refs_by_node_type["table_fn"] = tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in target_function_refs - if ref.node_type == CompiledResourceType.TABLE_FN - ) - return RollbackRefUpdate( - virtual_environment_record=virtual_environment_record, - refs=target_refs, - seed_refs=destination_seed_refs, - function_refs=target_function_refs, - function_versions=function_versions, - refs_by_node_type=refs_by_node_type, - ) diff --git a/src/sqlbuild/virtual/executor/_helpers/seeding.py b/src/sqlbuild/virtual/executor/_helpers/seeding.py deleted file mode 100644 index 23c73b5a89..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/seeding.py +++ /dev/null @@ -1,259 +0,0 @@ -"""Virtual physical-version seeding helpers.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( - resolve_qualified_name_parts, -) -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import IncrementalStrategy, PlanAction -from sqlbuild.cursor_algebra.main.sentinel_to_token import sentinel_to_token -from sqlbuild.executor.build.constants import INCREMENTAL_ACTIONS -from sqlbuild.virtual.state.models import PhysicalRelationAncestryRecord, PhysicalRelationRecord -from sqlbuild.virtual.state.types import PhysicalArtifactType - - -def read_seed_physical_relations( - *, - backend: Any, - state_connection: Any, - schema: str, - seed_version_hashes: dict[str, str], -) -> dict[str, PhysicalRelationRecord]: - """Read available physical relations for seed version hashes.""" - - relations: dict[str, PhysicalRelationRecord] = {} - seed_name: str - version_hash: str - for seed_name, version_hash in seed_version_hashes.items(): - relation: PhysicalRelationRecord | None = backend.get_physical_relation_for_artifact( - connection=state_connection, - schema=schema, - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_name, - version_hash=version_hash, - ) - if relation is not None: - relations[seed_name] = relation - return relations - - -def seed_virtual_physical_version( - *, - adapter: BaseAdapter, - connection: Any, - backend: Any, - state_connection: Any, - state_schema: str, - entry: ModelPlanEntry, - parent_relation: PhysicalRelationRecord | None, - version_hash: str | None, - schema_prepared: bool = False, -) -> None: - """Seed one incremental physical version target before DML execution.""" - - recorder: StatementRecorder = StatementRecorder() - if not _should_seed_physical_version(entry=entry): - return - if parent_relation is None or version_hash is None: - return - if parent_relation.version_hash == version_hash: - return - - if not schema_prepared: - adapter.ensure_schema( - connection=connection, - database=entry.destination.database, - schema=entry.destination.schema, - statement_recorder=recorder, - ) - target: str = resolve_relation_location_qualified_name( - adapter=adapter, location=entry.destination - ) - if adapter.relation_exists( - connection=connection, - database=entry.destination.database, - schema=entry.destination.schema, - name=entry.destination.name, - ): - return - source: str = resolve_qualified_name_parts( - adapter=adapter, - database=parent_relation.database_name, - schema=parent_relation.schema_name, - name=parent_relation.relation_name, - ) - seed_strategy: str = _seed_physical_relation( - adapter=adapter, - connection=connection, - origin=source, - destination=target, - entry=entry, - origin_is_transient=_origin_is_transient( - adapter=adapter, - connection=connection, - database=parent_relation.database_name, - schema=parent_relation.schema_name, - name=parent_relation.relation_name, - ), - statement_recorder=recorder, - ) - backend.upsert_physical_relation_ancestry( - connection=state_connection, - schema=state_schema, - record=PhysicalRelationAncestryRecord( - model_name=entry.name, - version_hash=version_hash, - parent_model_name=parent_relation.artifact_name, - parent_version_hash=parent_relation.version_hash, - seed_strategy=seed_strategy, - ), - ) - - -def _seed_physical_relation( - *, - adapter: BaseAdapter, - connection: Any, - origin: str, - destination: str, - entry: ModelPlanEntry, - origin_is_transient: bool, - statement_recorder: StatementRecorder, -) -> str: - if _requires_append_bounded_seed(entry=entry): - cursor_start: str = _cursor_start_for_append_seed(entry=entry) - adapter.create_table_as( - connection=connection, - destination=destination, - sql=adapter.render_seed_select_before_cursor( - origin=origin, - cursor_column=entry.cursor_column or "", - cursor_end_exclusive=cursor_start, - cursor_type=entry.cursor_type, - ), - statement_recorder=statement_recorder, - ) - return "bounded_append_copy" - - if adapter.supports_durable_clone(): - adapter.durable_clone( - connection=connection, - origin=origin, - destination=destination, - origin_is_transient=origin_is_transient, - statement_recorder=statement_recorder, - ) - return "durable_clone" - - adapter.create_table_as( - connection=connection, - destination=destination, - sql=f"SELECT * FROM {origin}", - statement_recorder=statement_recorder, - ) - return "copy" - - -def _origin_is_transient( - *, - adapter: BaseAdapter, - connection: Any, - database: str | None, - schema: str | None, - name: str, -) -> bool: - """Return whether the origin warehouse relation is transient, defaulting to False.""" - - if schema is None: - return False - relations: tuple[Any, ...] = adapter.list_relations( - connection=connection, database=database, schemas=(schema,), names=(name,) - ) - target_name: str = name.lower() - for relation in relations: - if relation.name == target_name: - return bool(relation.is_transient) - return False - - -def _should_seed_physical_version(*, entry: ModelPlanEntry) -> bool: - return entry.action in INCREMENTAL_ACTIONS or entry.action == PlanAction.CUSTOM - - -def _requires_append_bounded_seed(*, entry: ModelPlanEntry) -> bool: - return ( - entry.incremental_strategy == IncrementalStrategy.APPEND and entry.cursor_bounds is not None - ) - - -def _cursor_start_for_append_seed(*, entry: ModelPlanEntry) -> str: - if entry.cursor_column is None or entry.cursor_bounds is None: - raise PlannerInputError( - f"bounded append seeding for '{entry.name}' requires cursor bounds and cursor_column" - ) - return sentinel_to_token(sentinel=entry.cursor_bounds.start) - - -def resolve_virtual_seed_selection( - *, - graph: ProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> tuple[str, ...]: - """Resolve the seed names matched by the requested selectors.""" - - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=select, - exclude=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - return tuple( - sorted(key.name for key in selected_keys if key.resource_type == CompiledResourceType.SEED) - ) - - -def include_stale_upstream_seed_names( - *, - graph: ProjectGraph, - selected_model_names: tuple[str, ...], - selected_seed_names: tuple[str, ...], - stale_seed_names: tuple[str, ...], -) -> tuple[str, ...]: - """Add stale seeds feeding the selected models to the selected seed names.""" - - selected: set[str] = set(selected_seed_names) - stale_seed_name_set: frozenset[str] = frozenset(stale_seed_names) - pending: list[CompiledObjectKey] = [ - model.key for model in graph.project.models if model.name in selected_model_names - ] - seen: set[CompiledObjectKey] = set() - while pending: - key: CompiledObjectKey = pending.pop() - if key in seen: - continue - seen.add(key) - upstream_key: CompiledObjectKey - for upstream_key in graph.upstream_deps.get(key, ()): - if upstream_key.resource_type == CompiledResourceType.SEED: - if upstream_key.name in stale_seed_name_set: - selected.add(upstream_key.name) - continue - pending.append(upstream_key) - return tuple(sorted(selected)) diff --git a/src/sqlbuild/virtual/executor/_helpers/state_operations.py b/src/sqlbuild/virtual/executor/_helpers/state_operations.py deleted file mode 100644 index 7d1f7848a4..0000000000 --- a/src/sqlbuild/virtual/executor/_helpers/state_operations.py +++ /dev/null @@ -1,99 +0,0 @@ -"""State operation and lease bookkeeping phases for virtual executor runs.""" - -from __future__ import annotations - -import uuid -from datetime import timedelta -from typing import Any - -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.virtual.executor.models import StateOperationHandle -from sqlbuild.virtual.state.main.environments._record_operation import record_state_operation -from sqlbuild.virtual.state.main.locks._locks import acquire_virtual_environment_lease -from sqlbuild.virtual.state.models import StateLockLease -from sqlbuild.virtual.state.types import StateOperationStatus, StateOperationType - - -def create_state_operation_handle(operation_type: StateOperationType) -> StateOperationHandle: - """Create a unique handle identifying one executor state operation.""" - - return StateOperationHandle( - operation_id=f"{operation_type.value}:{uuid.uuid4()}", - operation_type=operation_type, - ) - - -def write_state_operation_started( - *, - backend: Any, - state_connection: Any, - schema: str, - handle: StateOperationHandle, - virtual_environment_name: str, - message: str, -) -> None: - """Record the RUNNING start event for one executor state operation.""" - - record_state_operation( - backend=backend, - connection=state_connection, - schema=schema, - operation_id=handle.operation_id, - operation_type=handle.operation_type, - status=StateOperationStatus.RUNNING, - action="start", - virtual_environment_name=virtual_environment_name, - message=message, - ) - - -def write_state_operation_result( - *, - backend: Any, - state_connection: Any, - schema: str, - handle: StateOperationHandle, - status: StateOperationStatus, - message: str, -) -> None: - """Record the finish event and final status for one executor state operation.""" - - record_state_operation( - backend=backend, - connection=state_connection, - schema=schema, - operation_id=handle.operation_id, - operation_type=None, - status=status, - action="finish", - virtual_environment_name=None, - message=message, - ) - - -def acquire_virtual_environment_lease_or_raise( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, - owner_prefix: str, - locked_error_code: str, - ttl: timedelta = timedelta(minutes=10), -) -> StateLockLease: - """Acquire the environment lease or raise a structured locked error.""" - - lease: StateLockLease | None = acquire_virtual_environment_lease( - backend=backend, - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - owner_id=f"{owner_prefix}:{uuid.uuid4()}", - ttl=ttl, - ) - if lease is None: - raise PlannerInputError( - f"virtual environment '{virtual_environment_name}' is locked", - code=locked_error_code, - ) - return lease diff --git a/src/sqlbuild/virtual/executor/classes/__init__.py b/src/sqlbuild/virtual/executor/classes/__init__.py deleted file mode 100644 index ba22347590..0000000000 --- a/src/sqlbuild/virtual/executor/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual executor classes.""" diff --git a/src/sqlbuild/virtual/executor/classes/microbatch_lease_manager.py b/src/sqlbuild/virtual/executor/classes/microbatch_lease_manager.py deleted file mode 100644 index 11cbb935ba..0000000000 --- a/src/sqlbuild/virtual/executor/classes/microbatch_lease_manager.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Renewable physical-version lease manager for virtual microbatch mutation.""" - -from __future__ import annotations - -import hashlib -import json -import threading -from datetime import datetime, timedelta -from typing import Any - -from sqlbuild.compiler.fingerprints.main.compute_query_hash import compute_query_hash -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import IncrementalMode, MaterializationType, PlanAction -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.main.locks._model_version_lock import acquire_model_version_lease -from sqlbuild.virtual.state.main.locks._release_lock import release_state_lease -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - StateLockLease, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRetentionRecord, -) -from sqlbuild.virtual.state.types import PhysicalArtifactType - -_LEASE_TTL: timedelta = timedelta(minutes=5) -_LEASE_RENEW_INTERVAL_SECONDS: float = 60.0 - - -class VirtualMicrobatchLeaseManager: - """Protect shared incremental versions and lease microbatch mutations.""" - - def __init__( - self, - *, - backend: StateBackend, - connection_config: dict[str, object], - warehouse_connection_config: dict[str, object], - schema: str, - run_id: str, - ) -> None: - self._backend = backend - self._connection_config = connection_config - self._schema = schema - self._owner_id = f"build:{run_id}" - self._warehouse_realm = hashlib.sha256( - json.dumps(warehouse_connection_config, sort_keys=True, default=str).encode() - ).hexdigest() - self._leases: tuple[StateLockLease, ...] = () - self._stop = threading.Event() - self._lost = threading.Event() - self._thread: threading.Thread | None = None - - def acquire( - self, - *, - entries: tuple[ModelPlanEntry, ...], - expected_version_hashes: dict[str, str], - ) -> None: - incremental_entries: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in entries - if entry.materialization_type == MaterializationType.INCREMENTAL - and entry.action != PlanAction.SKIP - ) - lease_entries: tuple[ModelPlanEntry, ...] = tuple( - entry - for entry in incremental_entries - if entry.incremental_mode == IncrementalMode.MICROBATCH - or entry.action == PlanAction.CREATE_TABLE - ) - if not lease_entries: - return - connection: Any = self._backend.connect(self._connection_config) - leases: list[StateLockLease] = [] - try: - entries_by_version: dict[tuple[str, str], ModelPlanEntry] = {} - for entry in lease_entries: - version_hash: str = expected_version_hashes.get( - entry.name, - entry.fingerprint_version_hash - or compute_query_hash(entry.fingerprint_query_sql), - ) - entries_by_version[(entry.name, version_hash)] = entry - entry_key: tuple[str, str] - for entry_key in sorted(entries_by_version): - entry: ModelPlanEntry = entries_by_version[entry_key] - version_hash = entry_key[1] - lease: StateLockLease | None = acquire_model_version_lease( - backend=self._backend, - connection=connection, - schema=self._schema, - model_name=f"{self._warehouse_realm}:{entry.name}", - version_hash=version_hash, - owner_id=self._owner_id, - ttl=_LEASE_TTL, - ) - if lease is None: - raise PlannerInputError( - "virtual incremental physical version is already being mutated: " - f"{entry.name}@{version_hash}" - ) - leases.append(lease) - for entry_key in sorted(entries_by_version): - self._reject_shared_full_refresh( - connection=connection, - entry=entries_by_version[entry_key], - version_hash=entry_key[1], - ) - except BaseException: - self._release(connection=connection, leases=tuple(leases)) - raise - finally: - self._backend.close(connection) - self._leases = tuple(leases) - if not leases: - return - self._thread = threading.Thread( - target=self._renew_until_stopped, - name=f"sqlbuild-lease-{self._owner_id}", - daemon=True, - ) - self._thread.start() - - def assert_active(self) -> None: - """Fence target DML after a renewal failure or lost owner row.""" - - if self._lost.is_set(): - raise ExecutorInputError( - "virtual incremental physical-version lease was lost; refusing further target DML" - ) - - @property - def active_leases(self) -> tuple[StateLockLease, ...]: - """Return the leases that must still be owned when model refs are published.""" - - return self._leases - - def close(self) -> None: - self._stop.set() - if self._thread is not None: - self._thread.join(timeout=5.0) - if not self._leases: - return - connection: Any = self._backend.connect(self._connection_config) - try: - self._release(connection=connection, leases=self._leases) - finally: - self._backend.close(connection) - - def _renew_until_stopped(self) -> None: - while not self._stop.wait(_LEASE_RENEW_INTERVAL_SECONDS): - connection: Any = self._backend.connect(self._connection_config) - try: - expires_at: datetime = datetime.now() + _LEASE_TTL - for lease in self._leases: - if not self._backend.renew_lock( - connection=connection, - schema=self._schema, - lock_key=lease.lock_key, - owner_id=lease.owner_id, - expires_at=expires_at, - ): - self._lost.set() - return - except Exception: - self._lost.set() - return - finally: - self._backend.close(connection) - - def _release(self, *, connection: Any, leases: tuple[StateLockLease, ...]) -> None: - for lease in reversed(leases): - release_state_lease( - backend=self._backend, - connection=connection, - schema=self._schema, - lease=lease, - ) - - def _reject_shared_full_refresh( - self, *, connection: Any, entry: ModelPlanEntry, version_hash: str - ) -> None: - existing: PhysicalRelationRecord | None = self._backend.get_physical_relation_for_artifact( - connection=connection, - schema=self._schema, - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=entry.name, - version_hash=version_hash, - ) - if ( - entry.action == PlanAction.CREATE_TABLE - and existing is not None - and self._model_version_is_referenced( - connection=connection, - model_name=entry.name, - version_hash=version_hash, - ) - ): - raise PlannerInputError( - "virtual incremental full refresh cannot replace a shared physical version: " - f"{entry.name}@{version_hash}; change the model definition to create a new " - "immutable version" - ) - - def _model_version_is_referenced( - self, *, connection: Any, model_name: str, version_hash: str - ) -> bool: - environments: tuple[VirtualEnvironmentRetentionRecord, ...] = ( - self._backend.list_virtual_environments( - connection=connection, - schema=self._schema, - ) - ) - for environment in environments: - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - self._backend.get_virtual_environment_model_refs( - connection=connection, - schema=self._schema, - virtual_environment_name=environment.virtual_environment_name, - ) - ) - if any( - ref.model_name == model_name and ref.version_hash == version_hash for ref in refs - ): - return True - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - self._backend.list_virtual_environment_checkpoints( - connection=connection, - schema=self._schema, - virtual_environment_name=environment.virtual_environment_name, - ) - ) - for checkpoint in checkpoints: - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - self._backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, - schema=self._schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - if any( - ref.model_name == model_name and ref.version_hash == version_hash - for ref in checkpoint_refs - ): - return True - return False diff --git a/src/sqlbuild/virtual/executor/classes/node_result_store.py b/src/sqlbuild/virtual/executor/classes/node_result_store.py deleted file mode 100644 index cdca2c3e2c..0000000000 --- a/src/sqlbuild/virtual/executor/classes/node_result_store.py +++ /dev/null @@ -1,69 +0,0 @@ -"""VDE state-backed runtime node result store.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.executor.node_results.classes.node_result_store import NodeResultStore -from sqlbuild.executor.node_results.models import ( - NodeResultEnvelope, - NodeResultQuery, - NodeResultRecord, -) -from sqlbuild.virtual.state.classes.state_backend import StateBackend - - -class VirtualNodeResultStore(NodeResultStore): - """State-backed node result store for virtual-mode execution.""" - - def __init__( - self, - *, - backend: StateBackend, - state_connection: Any, - state_schema: str, - virtual_environment_name: str, - target_database: str | None, - target_schema: str | None, - target_name: str | None = None, - ) -> None: - self.backend: StateBackend = backend - self.state_connection: Any = state_connection - self.state_schema: str = state_schema - self.virtual_environment_name: str = virtual_environment_name - super().__init__( - database=target_database, - schema=target_schema, - target_name=target_name, - ) - - def write(self, record: NodeResultRecord) -> None: - scoped_record: NodeResultRecord = NodeResultRecord( - node_type=record.node_type, - node_name=record.node_name, - target_database=self.database, - target_schema=self.schema, - target_name=self.target_name, - run_id=record.run_id, - status=record.status, - payload=record.payload, - metadata=record.metadata, - error_message=record.error_message, - materialized=record.materialized, - ts=record.ts, - ) - self.backend.insert_node_result( - connection=self.state_connection, - schema=self.state_schema, - virtual_environment_name=self.virtual_environment_name, - record=scoped_record, - ) - self._cache_record(record=scoped_record) - - def _read_persisted(self, *, query: NodeResultQuery) -> tuple[NodeResultEnvelope, ...]: - return self.backend.read_node_results( - connection=self.state_connection, - schema=self.state_schema, - virtual_environment_name=self.virtual_environment_name, - query=query, - ) diff --git a/src/sqlbuild/virtual/executor/constants.py b/src/sqlbuild/virtual/executor/constants.py deleted file mode 100644 index 4c2bd7c2f2..0000000000 --- a/src/sqlbuild/virtual/executor/constants.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Virtual executor constants.""" - -from __future__ import annotations - -DUCKDB_MEMORY_DATABASE: str = ":memory:" -VIRTUAL_CLONE_HYDRATED_ACTION: str = "hydrated" -VIRTUAL_CLONE_REUSED_ACTION: str = "reused" -VIRTUAL_CLONE_MISSING_ACTION: str = "missing" -VIRTUAL_CLONE_SKIPPED_LOCKED_ACTION: str = "skipped_locked" -VIRTUAL_CLONE_FOUND_ACTIONS: tuple[str, ...] = ( - VIRTUAL_CLONE_HYDRATED_ACTION, - VIRTUAL_CLONE_REUSED_ACTION, -) diff --git a/src/sqlbuild/virtual/executor/main/__init__.py b/src/sqlbuild/virtual/executor/main/__init__.py deleted file mode 100644 index bd4472f29b..0000000000 --- a/src/sqlbuild/virtual/executor/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual executor entrypoints.""" diff --git a/src/sqlbuild/virtual/executor/main/_logical_target.py b/src/sqlbuild/virtual/executor/main/_logical_target.py deleted file mode 100644 index 0095659e95..0000000000 --- a/src/sqlbuild/virtual/executor/main/_logical_target.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public virtual logical target helper.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.virtual.executor._helpers.rewrite import build_virtual_destination - - -def build_virtual_logical_destination( - *, - adapter: BaseAdapter, - target: CompiledRelationLocation, - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None = None, -) -> CompiledRelationLocation: - """Build the logical virtual target for a model or function.""" - - return build_virtual_destination( - adapter=adapter, - target=target, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) diff --git a/src/sqlbuild/virtual/executor/main/_physical_target.py b/src/sqlbuild/virtual/executor/main/_physical_target.py deleted file mode 100644 index d33b328004..0000000000 --- a/src/sqlbuild/virtual/executor/main/_physical_target.py +++ /dev/null @@ -1,24 +0,0 @@ -"""Public virtual physical target helper.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.virtual.executor._helpers.rewrite import build_physical_destination - - -def build_virtual_physical_destination( - *, - adapter: BaseAdapter, - target: CompiledRelationLocation, - model_name: str, - version_hash: str, -) -> CompiledRelationLocation: - """Build the physical virtual-version target for a model.""" - - return build_physical_destination( - adapter=adapter, - target=target, - model_name=model_name, - version_hash=version_hash, - ) diff --git a/src/sqlbuild/virtual/executor/main/_relation_type.py b/src/sqlbuild/virtual/executor/main/_relation_type.py deleted file mode 100644 index ee6d01046c..0000000000 --- a/src/sqlbuild/virtual/executor/main/_relation_type.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public virtual relation type helpers.""" - -from __future__ import annotations - -from sqlbuild.virtual.executor._helpers.rewrite import relation_type_for_model - - -def resolve_model_relation_type(materialized: str | None) -> str: - """Return the persisted physical relation type for a model materialization.""" - - return relation_type_for_model(materialized) diff --git a/src/sqlbuild/virtual/executor/main/_resolve_plan.py b/src/sqlbuild/virtual/executor/main/_resolve_plan.py deleted file mode 100644 index 7ac40f3406..0000000000 --- a/src/sqlbuild/virtual/executor/main/_resolve_plan.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Shared virtual plan resolution public entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.virtual.executor._helpers.build import resolve_virtual_plan as _resolve_virtual_plan -from sqlbuild.virtual.executor.models import VirtualBuildHooks, VirtualBuildOptions -from sqlbuild.virtual.planner.models import VirtualPlanOptions - - -def resolve_virtual_plan( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - options: VirtualPlanOptions, - hooks: ConnectionHooks, -) -> CompilePipelineResult: - """Resolve a build-grade virtual plan without executing or persisting it.""" - - return _resolve_virtual_plan( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=VirtualBuildOptions( - planning=options, - snapshots=discovered_inputs.project_config.snapshots, - ), - hooks=VirtualBuildHooks( - on_progress=hooks.on_progress, - on_connection_start=hooks.on_connection_start, - on_connection_complete=hooks.on_connection_complete, - on_connection_error=hooks.on_connection_error, - ), - ) diff --git a/src/sqlbuild/virtual/executor/main/_rewrite.py b/src/sqlbuild/virtual/executor/main/_rewrite.py deleted file mode 100644 index 6a9c962da0..0000000000 --- a/src/sqlbuild/virtual/executor/main/_rewrite.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public virtual project rewrite helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation -from sqlbuild.virtual.executor._helpers.rewrite import rewrite_project_model_locations - - -def rewrite_virtual_project_model_locations( - *, project: CompiledProject, rewritten_locations: dict[str, CompiledRelationLocation] -) -> CompiledProject: - """Rewrite model locations for a virtual execution consumer.""" - - return rewrite_project_model_locations(project=project, rewritten_locations=rewritten_locations) diff --git a/src/sqlbuild/virtual/executor/main/_views.py b/src/sqlbuild/virtual/executor/main/_views.py deleted file mode 100644 index 9012ff8b06..0000000000 --- a/src/sqlbuild/virtual/executor/main/_views.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Public helpers for logical VDE view refreshes.""" - -from __future__ import annotations - -from collections.abc import Callable - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -from sqlbuild.virtual.executor._helpers.environment_views import write_vde_views_from_records -from sqlbuild.virtual.state.models import PhysicalRelationRecord - - -def refresh_logical_vde_views( - *, - project: CompiledProject, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None = None, - physical_relations: dict[str, PhysicalRelationRecord], - seed_physical_relations: dict[str, PhysicalRelationRecord] | None = None, - on_connection_start: Callable[[int], None] | None = None, - on_connection_complete: ConnectionElapsedCallback | None = None, - on_connection_error: ConnectionElapsedCallback | None = None, -) -> None: - """Create or replace logical VDE views from tracked physical relations.""" - - write_vde_views_from_records( - project=project, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - physical_relations=physical_relations, - seed_physical_relations=seed_physical_relations or {}, - on_connection_start=on_connection_start, - on_connection_complete=on_connection_complete, - on_connection_error=on_connection_error, - ) diff --git a/src/sqlbuild/virtual/executor/main/build.py b/src/sqlbuild/virtual/executor/main/build.py deleted file mode 100644 index 2ef301d4fa..0000000000 --- a/src/sqlbuild/virtual/executor/main/build.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Virtual build public entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.executor._helpers.build import run_virtual_build as _run_virtual_build -from sqlbuild.virtual.executor.models import ( - VirtualBuildHooks, - VirtualBuildOptions, - VirtualBuildPipelineResult, -) - - -def run_virtual_build( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - options: VirtualBuildOptions | None = None, - hooks: VirtualBuildHooks | None = None, -) -> VirtualBuildPipelineResult: - """Execute a virtual-mode build.""" - - return _run_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=options if options is not None else VirtualBuildOptions(), - hooks=hooks if hooks is not None else VirtualBuildHooks(), - ) diff --git a/src/sqlbuild/virtual/executor/main/clone.py b/src/sqlbuild/virtual/executor/main/clone.py deleted file mode 100644 index 11e4683830..0000000000 --- a/src/sqlbuild/virtual/executor/main/clone.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Virtual clone public entrypoint.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ClonePipelineConnection, ClonePipelineResult -from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle -from sqlbuild.runtime.observability.main.run_scope import run_scope -from sqlbuild.virtual.executor._helpers.clone import ( - build_clone_origin_lookup, - build_clone_project_context, - compile_clone_pipeline, - hydrate_clone_model_relations, - hydrate_clone_seed_relations, - resolve_clone_versions, -) -from sqlbuild.virtual.executor.models import ( - CloneOptions, - CloneOriginLookup, - CloneProjectContext, - CloneVersions, - VirtualCloneItemResult, - VirtualCloneResult, -) -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime - - -def run_virtual_clone( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - origin_target_name: str, - destination_target_name: str, - destination_connection_config: dict[str, object], - options: CloneOptions | None = None, -) -> VirtualCloneResult: - """Hydrate target physical versions from matching source warehouse artifacts.""" - - resolved_options: CloneOptions = options if options is not None else CloneOptions() - with OperationLifecycle(operation_kind="clone", operation_name="clone_target_connection"): - destination_connection: Any = adapter.connect(destination_connection_config) - try: - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - selected_target=destination_target_name, - ) - with OperationLifecycle(operation_kind="clone", operation_name="clone_state_connection"): - state_connection: Any = backend.connect(config.connection) - try: - clone_pipeline: ClonePipelineResult = compile_clone_pipeline( - discovered_inputs=discovered_inputs, - adapter=adapter, - origin_target_name=origin_target_name, - destination_target_name=destination_target_name, - destination_connection=ClonePipelineConnection( - config=destination_connection_config, - handle=destination_connection, - ), - no_sql_validation=resolved_options.no_sql_validation, - select=resolved_options.select, - exclude=resolved_options.exclude, - cli_vars=resolved_options.cli_vars, - external_sql_reference_resolver=resolved_options.external_sql_reference_resolver, - ) - with run_scope(clone_pipeline.destination_project.run_id): - with OperationLifecycle( - operation_kind="clone", operation_name="clone_execution" - ) as lifecycle: - context: CloneProjectContext = build_clone_project_context(clone_pipeline) - with OperationLifecycle( - operation_kind="clone", operation_name="clone_state_inspection" - ): - versions: CloneVersions = resolve_clone_versions( - backend=backend, - state_connection=state_connection, - schema=config.schema, - clone_pipeline=clone_pipeline, - context=context, - virtual_environment_name=resolved_options.virtual_environment_name, - discovered_inputs=discovered_inputs, - project_dir=project_dir, - origin_target_name=origin_target_name, - ) - origin_lookup: CloneOriginLookup = build_clone_origin_lookup( - adapter=adapter, - destination_connection=destination_connection, - context=context, - versions=versions, - ) - model_items: tuple[VirtualCloneItemResult, ...] = hydrate_clone_model_relations( - backend=backend, - adapter=adapter, - destination_connection=destination_connection, - config_schema=config.schema, - config_connection=config.connection, - context=context, - versions=versions, - origin_lookup=origin_lookup, - skip_locked=resolved_options.skip_locked, - run_id=clone_pipeline.destination_project.run_id, - ) - seed_items: tuple[VirtualCloneItemResult, ...] = hydrate_clone_seed_relations( - backend=backend, - adapter=adapter, - destination_connection=destination_connection, - config_schema=config.schema, - config_connection=config.connection, - context=context, - versions=versions, - origin_lookup=origin_lookup, - run_id=clone_pipeline.destination_project.run_id, - ) - result: VirtualCloneResult = VirtualCloneResult( - mode=versions.mode, - origin_environment=origin_target_name, - destination_environment=destination_target_name, - destination_virtual_environment=resolved_options.virtual_environment_name, - origin_state_used=versions.origin_state_used, - item_results=(*model_items, *seed_items), - ) - if result.missing_count != 0: - lifecycle.failed(error_code="clone_execution_failed") - return result - finally: - backend.close(state_connection) - finally: - adapter.close(destination_connection) diff --git a/src/sqlbuild/virtual/executor/main/promote.py b/src/sqlbuild/virtual/executor/main/promote.py deleted file mode 100644 index e460471f34..0000000000 --- a/src/sqlbuild/virtual/executor/main/promote.py +++ /dev/null @@ -1,222 +0,0 @@ -"""Virtual promote public entrypoint.""" - -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.runtime.observability.classes.operation_lifecycle import ( - OperationAttributes, - OperationLifecycle, -) -from sqlbuild.runtime.observability.main.canonicalize_operation_adapter import ( - canonicalize_operation_adapter, -) -from sqlbuild.virtual.executor._helpers.environment_views import write_virtual_environment_views -from sqlbuild.virtual.executor._helpers.project_context import resolve_virtual_project_context -from sqlbuild.virtual.executor._helpers.promote import ( - build_promote_ref_update, - build_promote_semantics, - read_promote_environment_state, - read_promote_physical_relations, - resolve_promote_final_refs, - resolve_promote_selection, - write_promote_environment_update, -) -from sqlbuild.virtual.executor._helpers.state_operations import ( - acquire_virtual_environment_lease_or_raise, - create_state_operation_handle, - write_state_operation_result, - write_state_operation_started, -) -from sqlbuild.virtual.executor.models import ( - PromoteEnvironmentState, - PromoteOptions, - PromoteRefUpdate, - PromoteResolution, - PromoteSelection, - PromoteSemantics, - StateOperationHandle, - VirtualEnvironmentPhysicalRelations, - VirtualProjectContext, -) -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.main.locks._release_lock import release_state_lease -from sqlbuild.virtual.state.models import StateLockLease -from sqlbuild.virtual.state.types import StateOperationStatus, StateOperationType - - -def run_virtual_promote( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - from_virtual_environment_name: str, - to_virtual_environment_name: str, - options: PromoteOptions, - hooks: ConnectionHooks, -) -> tuple[str, tuple[str, ...], tuple[str, ...]]: - """Promote refs from one VDE to another and refresh target views.""" - - on_progress: Callable[[str], None] | None = hooks.on_progress - context: VirtualProjectContext = resolve_virtual_project_context( - discovered_inputs=discovered_inputs, - adapter=adapter, - no_sql_validation=options.no_sql_validation, - cli_vars=options.cli_vars, - external_sql_reference_resolver=options.external_sql_reference_resolver, - on_progress=on_progress, - ) - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - handle: StateOperationHandle = create_state_operation_handle(StateOperationType.PROMOTE) - with OperationLifecycle( - operation_kind="warehouse", - operation_name="relation_promotion", - operation_id=handle.operation_id, - attributes=OperationAttributes( - phase="reconcile", - strategy="virtual", - adapter=canonicalize_operation_adapter(adapter.adapter_name), - target_kind="virtual_environment", - ), - ) as lifecycle: - state_connection: Any | None = None - lease: StateLockLease | None = None - state_started = False - try: - state_connection = backend.connect(config.connection) - write_state_operation_started( - backend=backend, - state_connection=state_connection, - schema=config.schema, - handle=handle, - virtual_environment_name=to_virtual_environment_name, - message=( - f"promote from {from_virtual_environment_name} to {to_virtual_environment_name}" - ), - ) - state_started = True - lease = acquire_virtual_environment_lease_or_raise( - backend=backend, - state_connection=state_connection, - schema=config.schema, - virtual_environment_name=to_virtual_environment_name, - owner_prefix="promote", - locked_error_code="S014", - ) - environment_state: PromoteEnvironmentState = read_promote_environment_state( - backend=backend, - state_connection=state_connection, - schema=config.schema, - from_virtual_environment_name=from_virtual_environment_name, - to_virtual_environment_name=to_virtual_environment_name, - ) - semantics: PromoteSemantics = build_promote_semantics( - backend=backend, - state_connection=state_connection, - schema=config.schema, - graph=context.graph, - environment_state=environment_state, - ) - selection: PromoteSelection = resolve_promote_selection( - graph=context.graph, - environment_state=environment_state, - source_semantics=semantics.source, - select=options.select, - exclude=options.exclude, - include_stale_upstreams=options.include_stale_upstreams, - ) - resolution: PromoteResolution = resolve_promote_final_refs( - graph=context.graph, - environment_state=environment_state, - selection=selection, - target_semantics=semantics.target, - select=options.select, - include_stale_upstreams=options.include_stale_upstreams, - allow_partial_promotion=options.allow_partial_promotion, - ) - update: PromoteRefUpdate = build_promote_ref_update( - backend=backend, - state_connection=state_connection, - schema=config.schema, - from_virtual_environment_name=from_virtual_environment_name, - to_virtual_environment_name=to_virtual_environment_name, - resolution=resolution, - source_function_refs=environment_state.source_function_refs, - select=options.select, - ) - write_promote_environment_update( - backend=backend, - state_connection=state_connection, - schema=config.schema, - to_virtual_environment_name=to_virtual_environment_name, - update=update, - ) - relations: VirtualEnvironmentPhysicalRelations = read_promote_physical_relations( - backend=backend, - state_connection=state_connection, - schema=config.schema, - update=update, - ) - write_virtual_environment_views( - graph=context.graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=to_virtual_environment_name, - unsuffixed_virtual_environment_name=context.unsuffixed_virtual_environment_name, - relations=relations, - function_versions=update.function_versions, - hooks=hooks, - ) - write_state_operation_result( - backend=backend, - state_connection=state_connection, - schema=config.schema, - handle=handle, - status=StateOperationStatus.SUCCEEDED, - message=f"promoted {resolution.promoted_model_count} models", - ) - except Exception as error: - if state_started: - write_state_operation_result( - backend=backend, - state_connection=state_connection, - schema=config.schema, - handle=handle, - status=StateOperationStatus.FAILED, - message=f"{error}", - ) - raise - finally: - _ = _close_promote_runtime( - backend=backend, - state_connection=state_connection, - schema=config.schema, - lease=lease, - ) - lifecycle.completed(metadata={"changed_count": resolution.promoted_model_count}) - return resolution.status.value, resolution.selected_model_names, resolution.stale_after - - -def _close_promote_runtime( - *, backend: Any, state_connection: Any | None, schema: str, lease: StateLockLease | None -) -> None: - """Release promotion-owned state resources before the canonical terminal.""" - - if lease is not None: - _ = release_state_lease( - backend=backend, - connection=state_connection, - schema=schema, - lease=lease, - ) - if state_connection is not None: - backend.close(state_connection) diff --git a/src/sqlbuild/virtual/executor/main/rollback.py b/src/sqlbuild/virtual/executor/main/rollback.py deleted file mode 100644 index 028530fb82..0000000000 --- a/src/sqlbuild/virtual/executor/main/rollback.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Virtual rollback public entrypoint.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.virtual.executor._helpers.environment_views import write_virtual_environment_views -from sqlbuild.virtual.executor._helpers.project_context import resolve_virtual_project_context -from sqlbuild.virtual.executor._helpers.rollback import ( - build_rollback_ref_update, - read_rollback_checkpoint_state, - read_rollback_physical_relations, - resolve_rollback_final_refs, - validate_physical_relations_exist, -) -from sqlbuild.virtual.executor._helpers.state_operations import ( - acquire_virtual_environment_lease_or_raise, -) -from sqlbuild.virtual.executor.models import ( - RollbackCheckpointState, - RollbackOptions, - RollbackRefUpdate, - RollbackResolution, - VirtualEnvironmentPhysicalRelations, - VirtualProjectContext, -) -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.main.locks._release_lock import release_state_lease -from sqlbuild.virtual.state.models import StateLockLease -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - - -def run_virtual_rollback( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - options: RollbackOptions, - hooks: ConnectionHooks, -) -> tuple[str, tuple[str, ...], VirtualEnvironmentStatus]: - """Rollback a VDE to the previous finalized checkpoint.""" - - on_progress: Callable[[str], None] | None = hooks.on_progress - context: VirtualProjectContext = resolve_virtual_project_context( - discovered_inputs=discovered_inputs, - adapter=adapter, - no_sql_validation=options.no_sql_validation, - cli_vars=options.cli_vars, - external_sql_reference_resolver=options.external_sql_reference_resolver, - on_progress=on_progress, - ) - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - state_connection: Any = backend.connect(config.connection) - lease: StateLockLease | None = None - try: - inspect_start: float = time.perf_counter() - if on_progress is not None: - on_progress("Inspecting virtual state...") - lease = acquire_virtual_environment_lease_or_raise( - backend=backend, - state_connection=state_connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - owner_prefix="rollback", - locked_error_code="S019", - ) - checkpoint_state: RollbackCheckpointState = read_rollback_checkpoint_state( - backend=backend, - state_connection=state_connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - checkpoint_id=options.checkpoint_id, - ) - resolution: RollbackResolution = resolve_rollback_final_refs( - backend=backend, - state_connection=state_connection, - schema=config.schema, - graph=context.graph, - virtual_environment_name=virtual_environment_name, - checkpoint_state=checkpoint_state, - select=options.select, - exclude=options.exclude, - include_stale_upstreams=options.include_stale_upstreams, - allow_partial_rollback=options.allow_partial_rollback, - ) - relations: VirtualEnvironmentPhysicalRelations = read_rollback_physical_relations( - backend=backend, - state_connection=state_connection, - schema=config.schema, - checkpoint_id=checkpoint_state.target_checkpoint.checkpoint_id, - resolution=resolution, - ) - validate_physical_relations_exist( - adapter=adapter, - connection_config=connection_config, - models_by_name={model.name: model for model in context.graph.project.models}, - physical_relations=relations.model_relations, - ) - update: RollbackRefUpdate = build_rollback_ref_update( - backend=backend, - state_connection=state_connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - resolution=resolution, - checkpoint_function_refs=checkpoint_state.checkpoint_function_refs, - ) - backend.upsert_virtual_environment_and_replace_node_ref_groups( - connection=state_connection, - schema=config.schema, - record=update.virtual_environment_record, - refs_by_node_type=update.refs_by_node_type, - ) - if on_progress is not None: - on_progress(f"Inspected virtual state. ({time.perf_counter() - inspect_start:.2f}s)") - finally: - if lease is not None: - _ = release_state_lease( - backend=backend, - connection=state_connection, - schema=config.schema, - lease=lease, - ) - backend.close(state_connection) - write_virtual_environment_views( - graph=context.graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=context.unsuffixed_virtual_environment_name, - relations=relations, - function_versions=(update.function_versions if not resolution.is_partial_scope else {}), - hooks=hooks, - ) - return ( - checkpoint_state.target_checkpoint.checkpoint_id, - resolution.rolled_back_model_names, - resolution.status, - ) diff --git a/src/sqlbuild/virtual/executor/main/virtual_target.py b/src/sqlbuild/virtual/executor/main/virtual_target.py deleted file mode 100644 index 5c805c58f0..0000000000 --- a/src/sqlbuild/virtual/executor/main/virtual_target.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Public virtual target construction entrypoint.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.virtual.executor._helpers.rewrite import ( - build_virtual_destination as _build_virtual_destination, -) - - -def build_virtual_destination( - *, - adapter: BaseAdapter, - target: CompiledRelationLocation, - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None = None, -) -> CompiledRelationLocation: - """Build the logical VDE view target for a model.""" - - return _build_virtual_destination( - adapter=adapter, - target=target, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) diff --git a/src/sqlbuild/virtual/executor/models.py b/src/sqlbuild/virtual/executor/models.py deleted file mode 100644 index 9f88f4e69a..0000000000 --- a/src/sqlbuild/virtual/executor/models.py +++ /dev/null @@ -1,362 +0,0 @@ -"""Virtual executor result models.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass, field -from datetime import datetime - -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledProject, - CompiledRelationLocation, - CompiledSeed, -) -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver -from sqlbuild.cost.models import StatementExecutionTelemetry -from sqlbuild.executor.build.models import BuildExecutionResult, SchedulerState -from sqlbuild.executor.python_nodes.models import PythonNodeExecutionResult -from sqlbuild.provider.main.runtime import ProviderContainer -from sqlbuild.runtime.contracts.types import ( - ConnectionElapsedCallback, - NodeStartCallback, -) -from sqlbuild.spec.contracts.models import SnapshotsConfig -from sqlbuild.virtual.executor.constants import ( - VIRTUAL_CLONE_FOUND_ACTIONS, - VIRTUAL_CLONE_HYDRATED_ACTION, - VIRTUAL_CLONE_MISSING_ACTION, - VIRTUAL_CLONE_REUSED_ACTION, - VIRTUAL_CLONE_SKIPPED_LOCKED_ACTION, -) -from sqlbuild.virtual.executor.types import VirtualPlanReadyCallback -from sqlbuild.virtual.planner.models import VirtualPlanOptions, VirtualPlanSemantics -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationRecord, - SeedVersionRecord, - SourceFreshnessRecord, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import ( - PhysicalArtifactType, - StateOperationType, - VirtualEnvironmentStatus, -) - - -@dataclass(frozen=True) -class VirtualBuildExecutionHooks: - """Callbacks to use once a virtual build plan is ready.""" - - on_node_start: NodeStartCallback | None = None - on_node_complete: Callable[[object], None] | None = None - on_sub_progress: Callable[[str], None] | None = None - on_scheduler_state: Callable[[SchedulerState], None] | None = None - on_statement_complete: Callable[[StatementExecutionTelemetry], None] | None = None - - -@dataclass(frozen=True) -class VirtualBuildOptions: - """Execution options composing the shared virtual planning contract.""" - - planning: VirtualPlanOptions = field(default_factory=VirtualPlanOptions) - seed_only: bool = False - fail_fast: bool = False - allow_snapshot_schema_change: bool = False - concurrency: int | None = None - run_tests: bool = True - run_audits: bool = True - snapshots: SnapshotsConfig | None = None - start_cursor_ts: datetime | None = None - end_cursor_ts: datetime | None = None - start_cursor_int: int | None = None - end_cursor_int: int | None = None - providers: ProviderContainer | None = None - - -@dataclass(frozen=True) -class VirtualBuildHooks: - """Plan-ready, progress, and connection callbacks for one virtual build run.""" - - on_plan_ready: VirtualPlanReadyCallback | None = None - on_progress: Callable[[str], None] | None = None - on_connection_start: Callable[[int], None] | None = None - on_connection_complete: ConnectionElapsedCallback | None = None - on_connection_error: ConnectionElapsedCallback | None = None - - -@dataclass(frozen=True) -class VirtualEnvironmentNames: - """Resolved target VDE, physical target, and unsuffixed naming context.""" - - target_vde_name: str - physical_target_name: str | None = None - unsuffixed_virtual_environment_name: str | None = None - - -@dataclass(frozen=True) -class PromoteOptions: - """Selection and policy options for one virtual promote run.""" - - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - allow_partial_promotion: bool = False - include_stale_upstreams: bool = False - no_sql_validation: bool = False - cli_vars: dict[str, object] | None = None - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - - -@dataclass(frozen=True) -class RollbackOptions: - """Selection and policy options for one virtual rollback run.""" - - checkpoint_id: str | None = None - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - allow_partial_rollback: bool = False - include_stale_upstreams: bool = False - no_sql_validation: bool = False - cli_vars: dict[str, object] | None = None - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - - -@dataclass(frozen=True) -class CloneOptions: - """Selection and policy options for one virtual clone run.""" - - virtual_environment_name: str | None = None - skip_locked: bool = False - no_sql_validation: bool = False - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - - -@dataclass(frozen=True) -class VirtualBuildPipelineResult: - """Result returned by the virtual build pipeline.""" - - project: CompiledProject - display_plan_output: PlanOutput - execution_plan: PlanOutput - execution_result: BuildExecutionResult - virtual_environment_name: str - python_node_results: tuple[PythonNodeExecutionResult, ...] = field(default_factory=tuple) - compile_seconds: float | None = None - planning_seconds: float | None = None - - -@dataclass(frozen=True) -class VirtualCloneItemResult: - """One virtual clone hydration result.""" - - artifact_type: PhysicalArtifactType - artifact_name: str - version_hash: str - action: str - message: str | None = None - - -@dataclass(frozen=True) -class VirtualCloneResult: - """Result returned by virtual physical-version hydration.""" - - mode: str - origin_environment: str - destination_environment: str - destination_virtual_environment: str | None = None - origin_state_used: bool = False - item_results: tuple[VirtualCloneItemResult, ...] = field(default_factory=tuple) - - @property - def selected_count(self) -> int: - return len(self.item_results) - - @property - def found_count(self) -> int: - return sum(1 for item in self.item_results if item.action in VIRTUAL_CLONE_FOUND_ACTIONS) - - @property - def hydrated_count(self) -> int: - return sum(1 for item in self.item_results if item.action == VIRTUAL_CLONE_HYDRATED_ACTION) - - @property - def reused_count(self) -> int: - return sum(1 for item in self.item_results if item.action == VIRTUAL_CLONE_REUSED_ACTION) - - @property - def missing_count(self) -> int: - return sum(1 for item in self.item_results if item.action == VIRTUAL_CLONE_MISSING_ACTION) - - @property - def skipped_locked_count(self) -> int: - return sum( - 1 for item in self.item_results if item.action == VIRTUAL_CLONE_SKIPPED_LOCKED_ACTION - ) - - -@dataclass(frozen=True) -class StateOperationHandle: - """Identity of one in-flight executor state operation.""" - - operation_id: str - operation_type: StateOperationType - - -@dataclass(frozen=True) -class VirtualProjectContext: - """Compiled project graph plus active target VDE naming context.""" - - graph: ProjectGraph - unsuffixed_virtual_environment_name: str | None - - -@dataclass(frozen=True) -class PromoteEnvironmentState: - """Bound refs and environment records read for one promote run.""" - - source_refs: tuple[VirtualEnvironmentModelRefRecord, ...] - target_refs: tuple[VirtualEnvironmentModelRefRecord, ...] - source_function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] - from_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] - to_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] - source_environment: VirtualEnvironmentRecord | None - target_environment: VirtualEnvironmentRecord | None - source_freshness_records: tuple[SourceFreshnessRecord, ...] - target_freshness_records: tuple[SourceFreshnessRecord, ...] - - -@dataclass(frozen=True) -class PromoteSemantics: - """Source and target virtual plan semantics for one promote run.""" - - source: VirtualPlanSemantics - target: VirtualPlanSemantics - - -@dataclass(frozen=True) -class PromoteSelection: - """Validated promote selection scope with source ref lookups.""" - - selected_model_names: tuple[str, ...] - selected_seed_names: tuple[str, ...] - source_ref_map: dict[str, str] - from_seed_ref_map: dict[str, str] - - -@dataclass(frozen=True) -class PromoteResolution: - """Final ref hashes, staleness, and target status for one promote run.""" - - selected_model_names: tuple[str, ...] - selected_seed_names: tuple[str, ...] - final_version_hashes: dict[str, str] - final_seed_hashes: dict[str, str] - stale_after: tuple[str, ...] - status: VirtualEnvironmentStatus - - @property - def promoted_model_count(self) -> int: - return len(self.selected_model_names) - - -@dataclass(frozen=True) -class PromoteRefUpdate: - """Target environment record and replacement ref groups to persist.""" - - virtual_environment_record: VirtualEnvironmentRecord - refs: tuple[VirtualEnvironmentModelRefRecord, ...] - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] - function_versions: dict[str, FunctionVersionRecord] - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] - - -@dataclass(frozen=True) -class VirtualEnvironmentPhysicalRelations: - """Tracked physical relations backing an environment's model and seed refs.""" - - model_relations: dict[str, PhysicalRelationRecord] - seed_relations: dict[str, PhysicalRelationRecord] - - -@dataclass(frozen=True) -class RollbackCheckpointState: - """Current ref map and resolved rollback checkpoint refs.""" - - current_ref_map: dict[str, str] - target_checkpoint: VirtualEnvironmentCheckpointRecord - checkpoint_model_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] - - -@dataclass(frozen=True) -class RollbackResolution: - """Final ref hashes, scope, and target status for one rollback run.""" - - final_version_hashes: dict[str, str] - final_seed_hashes: dict[str, str] - is_partial_scope: bool - status: VirtualEnvironmentStatus - rolled_back_model_names: tuple[str, ...] - - -@dataclass(frozen=True) -class CloneProjectContext: - """Destination graph and origin/destination node lookups for one clone run.""" - - destination_graph: ProjectGraph - model_names: tuple[str, ...] - seed_names: tuple[str, ...] - destination_models_by_name: dict[str, CompiledModel] - origin_models_by_name: dict[str, CompiledModel] - destination_seeds_by_name: dict[str, CompiledSeed] - origin_seeds_by_name: dict[str, CompiledSeed] - - -@dataclass(frozen=True) -class CloneVersions: - """Resolved model and seed versions to hydrate for one clone run.""" - - mode: str - version_hashes: dict[str, str] - model_versions: dict[str, ModelVersionRecord] - seed_versions: dict[str, SeedVersionRecord] - origin_state_used: bool = False - - -@dataclass(frozen=True) -class CloneOriginLookup: - """Origin lookup locations, alias, and existence lookup for one clone run.""" - - model_locations: dict[str, CompiledRelationLocation] - seed_locations: dict[str, CompiledRelationLocation] - lookup: RelationLookup - - -@dataclass(frozen=True) -class RollbackRefUpdate: - """Environment record and replacement ref groups for one rollback run.""" - - virtual_environment_record: VirtualEnvironmentRecord - refs: tuple[VirtualEnvironmentModelRefRecord, ...] - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] - function_versions: dict[str, FunctionVersionRecord] - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] diff --git a/src/sqlbuild/virtual/executor/types.py b/src/sqlbuild/virtual/executor/types.py deleted file mode 100644 index e5bfa0081a..0000000000 --- a/src/sqlbuild/virtual/executor/types.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Virtual executor callback types.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Protocol - -if TYPE_CHECKING: - from sqlbuild.compiler.compile.models import CompiledProject - from sqlbuild.compiler.pipeline.models import PythonPlanEntry - from sqlbuild.compiler.planner.models import PlanOutput - from sqlbuild.virtual.executor.models import VirtualBuildExecutionHooks - - -class VirtualPlanReadyCallback(Protocol): - def __call__( - self, - *, - project: CompiledProject, - plan_output: PlanOutput, - python_plan_entries: tuple[PythonPlanEntry, ...], - ) -> VirtualBuildExecutionHooks: ... diff --git a/src/sqlbuild/virtual/freshness/__init__.py b/src/sqlbuild/virtual/freshness/__init__.py deleted file mode 100644 index 62048a18e0..0000000000 --- a/src/sqlbuild/virtual/freshness/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual source freshness helpers.""" diff --git a/src/sqlbuild/virtual/freshness/_helpers/__init__.py b/src/sqlbuild/virtual/freshness/_helpers/__init__.py deleted file mode 100644 index b84fc49f0b..0000000000 --- a/src/sqlbuild/virtual/freshness/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Source freshness helper modules.""" diff --git a/src/sqlbuild/virtual/freshness/_helpers/runtime.py b/src/sqlbuild/virtual/freshness/_helpers/runtime.py deleted file mode 100644 index 0e837fc6bb..0000000000 --- a/src/sqlbuild/virtual/freshness/_helpers/runtime.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Virtual source freshness runtime observation helpers.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.adapter.contract.exceptions import AdapterUserError -from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.record_equivalence import ( - source_freshness_records_equivalent, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind -from sqlbuild.virtual.freshness.main._observation import observe_configured_source_freshness -from sqlbuild.virtual.freshness.main._state_record import source_freshness_record_from_observation -from sqlbuild.virtual.freshness.models import SourceFreshnessRuntimeResult -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -def observe_virtual_environment_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - virtual_environment_name: str, - observed_at: datetime, - run_id: str | None = None, - load_results: tuple[Any, ...] = (), - previous_records: tuple[SourceFreshnessRecord, ...] = (), -) -> SourceFreshnessRuntimeResult: - """Observe current source freshness records without using them for skip decisions.""" - - records: list[SourceFreshnessRecord] = [] - unknown_source_names: list[str] = [] - preserved_source_names: list[str] = [] - generated_source_names: list[str] = [] - load_result_by_source: dict[str, Any] = {result.source_name: result for result in load_results} - previous_record_by_source: dict[str, SourceFreshnessRecord] = { - record.source_name: record for record in previous_records - } - - source: SourceEntry - for source in sources: - load_result: Any | None = load_result_by_source.get(source.name) - if _is_soft_skipped_load(load_result): - previous_record: SourceFreshnessRecord | None = previous_record_by_source.get( - source.name - ) - if previous_record is None: - unknown_source_names.append(source.name) - continue - records.append(previous_record) - preserved_source_names.append(source.name) - continue - - if source.managed: - if load_result is None or load_result.status != ExecutionStatus.SUCCESS: - unknown_source_names.append(source.name) - continue - managed_record: SourceFreshnessRecord | None = _managed_loader_freshness_record( - adapter=adapter, - connection=connection, - source=source, - virtual_environment_name=virtual_environment_name, - observed_at=observed_at, - run_id=run_id, - ) - if managed_record is None: - unknown_source_names.append(source.name) - continue - records.append( - _record_with_lag_tolerance_applied( - source=source, - current_record=managed_record, - previous_record=previous_record_by_source.get(source.name), - ) - ) - if source.freshness is None: - generated_source_names.append(source.name) - continue - - observation: SourceFreshnessObservation | None = _observe_unmanaged_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - if observation is None: - unknown_source_names.append(source.name) - continue - current_record: SourceFreshnessRecord = source_freshness_record_from_observation( - observation=observation, - virtual_environment_name=virtual_environment_name, - ) - records.append( - _record_with_lag_tolerance_applied( - source=source, - current_record=current_record, - previous_record=previous_record_by_source.get(source.name), - ) - ) - - return SourceFreshnessRuntimeResult( - records=tuple(sorted(records, key=lambda record: record.source_name)), - unknown_source_names=tuple(sorted(unknown_source_names)), - preserved_source_names=tuple(sorted(preserved_source_names)), - generated_source_names=tuple(sorted(generated_source_names)), - ) - - -def persist_virtual_environment_source_freshness( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, - result: SourceFreshnessRuntimeResult, -) -> None: - """Persist the latest observed freshness records for a virtual environment.""" - - backend.replace_virtual_environment_source_freshness( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - records=result.records, - ) - - -def build_current_virtual_source_freshness_records( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - virtual_environment_name: str, - observed_at: datetime, - previous_records: tuple[SourceFreshnessRecord, ...], - run_id: str | None = None, -) -> tuple[SourceFreshnessRecord, ...]: - """Return source records for pre-loader virtual freshness comparisons.""" - - unmanaged_sources: tuple[SourceEntry, ...] = tuple( - source for source in sources if not source.managed - ) - current_unmanaged_result: SourceFreshnessRuntimeResult = ( - observe_virtual_environment_source_freshness( - adapter=adapter, - connection=connection, - sources=unmanaged_sources, - virtual_environment_name=virtual_environment_name, - observed_at=observed_at, - ) - ) - managed_sources: tuple[SourceEntry, ...] = tuple(source for source in sources if source.managed) - managed_source_names: frozenset[str] = frozenset(source.name for source in managed_sources) - sources_by_name: dict[str, SourceEntry] = {source.name: source for source in unmanaged_sources} - previous_records_by_source: dict[str, SourceFreshnessRecord] = { - record.source_name: record for record in previous_records - } - previous_managed_records: tuple[SourceFreshnessRecord, ...] = tuple( - record for record in previous_records if record.source_name in managed_source_names - ) - previous_managed_source_names: frozenset[str] = frozenset( - record.source_name for record in previous_managed_records - ) - generated_managed_records: tuple[SourceFreshnessRecord, ...] = () - if run_id is not None: - generated_managed_records = tuple( - _generated_managed_loader_record( - source=source, - virtual_environment_name=virtual_environment_name, - observed_at=observed_at, - run_id=run_id, - ) - for source in managed_sources - if source.freshness is None and source.name not in previous_managed_source_names - ) - current_unmanaged_records: tuple[SourceFreshnessRecord, ...] = tuple( - _record_with_lag_tolerance_applied( - source=sources_by_name[record.source_name], - current_record=record, - previous_record=previous_records_by_source.get(record.source_name), - ) - for record in current_unmanaged_result.records - ) - return tuple( - sorted( - (*previous_managed_records, *generated_managed_records, *current_unmanaged_records), - key=lambda record: record.source_name, - ) - ) - - -def _record_with_lag_tolerance_applied( - *, - source: SourceEntry, - current_record: SourceFreshnessRecord, - previous_record: SourceFreshnessRecord | None, -) -> SourceFreshnessRecord: - if previous_record is None or source.freshness is None: - return current_record - if source_freshness_records_equivalent( - previous_record=previous_record, - current_record=current_record, - lag_tolerance=source.freshness.lag_tolerance, - ): - return previous_record - return current_record - - -def _generated_managed_loader_record( - *, - source: SourceEntry, - virtual_environment_name: str, - observed_at: datetime, - run_id: str, -) -> SourceFreshnessRecord: - return SourceFreshnessRecord( - virtual_environment_name=virtual_environment_name, - source_name=source.name, - strategy="loader", - value_kind=SourceFreshnessValueKind.STRING.value, - data_version=run_id, - data_version_hash=source_freshness_data_version_hash( - source_name=source.name, - strategy="loader", - value_kind=SourceFreshnessValueKind.STRING, - data_version=run_id, - ), - observed_at=observed_at, - ) - - -def _observe_unmanaged_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation | None: - try: - if source.freshness is not None: - return observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - if ( - not source.managed - and source.expression is None - and source.table is not None - and adapter.supports_table_freshness_metadata() - ): - return observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=_source_with_adapter_freshness(source), - observed_at=observed_at, - ) - except AdapterUserError: - return None - return None - - -def _managed_loader_freshness_record( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - virtual_environment_name: str, - observed_at: datetime, - run_id: str | None, -) -> SourceFreshnessRecord | None: - if source.freshness is not None: - try: - observation: SourceFreshnessObservation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) - except AdapterUserError: - return None - return source_freshness_record_from_observation( - observation=observation, - virtual_environment_name=virtual_environment_name, - ) - if run_id is None: - return None - return _generated_managed_loader_record( - source=source, - virtual_environment_name=virtual_environment_name, - observed_at=observed_at, - run_id=run_id, - ) - - -def _is_soft_skipped_load(load_result: Any | None) -> bool: - return ( - load_result is not None - and load_result.status == ExecutionStatus.SKIPPED - and load_result.skip_mode == SkipMode.SOFT - ) - - -def _source_with_adapter_freshness(source: SourceEntry) -> SourceEntry: - return SourceEntry( - name=source.name, - database=source.database, - schema=source.schema, - table=source.table, - loader=source.loader, - managed=source.managed, - integration_loader=source.integration_loader, - freshness=SourceFreshnessConfig(strategy=SourceFreshnessStrategy.ADAPTER), - write_strategy=source.write_strategy, - load_batch_size=source.load_batch_size, - cursor_column=source.cursor_column, - unique_key=source.unique_key, - expression=source.expression, - description=source.description, - type_enforcement=source.type_enforcement, - contract=source.contract, - meta=source.meta, - columns=source.columns, - audits=source.audits, - ) diff --git a/src/sqlbuild/virtual/freshness/_helpers/state.py b/src/sqlbuild/virtual/freshness/_helpers/state.py deleted file mode 100644 index d4d7cce4bc..0000000000 --- a/src/sqlbuild/virtual/freshness/_helpers/state.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Virtual source freshness state-record helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness.main.data_version_hash import ( - source_freshness_data_version_hash, -) -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -def source_freshness_record_from_observation( - *, observation: SourceFreshnessObservation, virtual_environment_name: str -) -> SourceFreshnessRecord: - """Build a persisted virtual state record from an observed source freshness value.""" - - normalized_data_version: str = normalize_source_freshness_data_version( - value=observation.data_version, - value_kind=observation.value_kind, - ) - return SourceFreshnessRecord( - virtual_environment_name=virtual_environment_name, - source_name=observation.source_name, - strategy=observation.strategy.value, - value_kind=observation.value_kind.value, - data_version=normalized_data_version, - data_version_hash=source_freshness_data_version_hash( - source_name=observation.source_name, - strategy=observation.strategy, - value_kind=observation.value_kind, - data_version=normalized_data_version, - ), - observed_at=observation.observed_at, - ) diff --git a/src/sqlbuild/virtual/freshness/main/__init__.py b/src/sqlbuild/virtual/freshness/main/__init__.py deleted file mode 100644 index 958e2533c5..0000000000 --- a/src/sqlbuild/virtual/freshness/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Source freshness runtime entrypoints.""" diff --git a/src/sqlbuild/virtual/freshness/main/_current_records.py b/src/sqlbuild/virtual/freshness/main/_current_records.py deleted file mode 100644 index eab32afcc4..0000000000 --- a/src/sqlbuild/virtual/freshness/main/_current_records.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Public source freshness current-records entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.virtual.freshness._helpers.runtime import ( - build_current_virtual_source_freshness_records as _build_current_records, -) -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -def build_current_virtual_source_freshness_records( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - virtual_environment_name: str, - observed_at: datetime, - previous_records: tuple[SourceFreshnessRecord, ...], - run_id: str | None = None, -) -> tuple[SourceFreshnessRecord, ...]: - """Return source records for pre-loader virtual freshness comparisons.""" - - return _build_current_records( - adapter=adapter, - connection=connection, - sources=sources, - virtual_environment_name=virtual_environment_name, - observed_at=observed_at, - previous_records=previous_records, - run_id=run_id, - ) diff --git a/src/sqlbuild/virtual/freshness/main/_observation.py b/src/sqlbuild/virtual/freshness/main/_observation.py deleted file mode 100644 index 0133669454..0000000000 --- a/src/sqlbuild/virtual/freshness/main/_observation.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public source freshness observation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.compiler.source_freshness.main.observation import ( - observe_configured_source_freshness as _observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.spec.contracts.models import SourceEntry - - -def observe_configured_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - source: SourceEntry, - observed_at: datetime, -) -> SourceFreshnessObservation: - """Observe one configured source freshness value.""" - - return _observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=observed_at, - ) diff --git a/src/sqlbuild/virtual/freshness/main/_runtime_observation.py b/src/sqlbuild/virtual/freshness/main/_runtime_observation.py deleted file mode 100644 index c9d13c7834..0000000000 --- a/src/sqlbuild/virtual/freshness/main/_runtime_observation.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Public source freshness runtime observation entrypoint.""" - -from __future__ import annotations - -from datetime import datetime -from typing import Any - -from sqlbuild.adapter.contract.classes.strict_adapter import StrictAdapter -from sqlbuild.spec.contracts.models import SourceEntry -from sqlbuild.virtual.freshness._helpers.runtime import ( - observe_virtual_environment_source_freshness as _observe_virtual_environment_source_freshness, -) -from sqlbuild.virtual.freshness.models import SourceFreshnessRuntimeResult -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -def observe_virtual_environment_source_freshness( - *, - adapter: StrictAdapter, - connection: Any, - sources: tuple[SourceEntry, ...], - virtual_environment_name: str, - observed_at: datetime, - run_id: str | None = None, - load_results: tuple[Any, ...] = (), - previous_records: tuple[SourceFreshnessRecord, ...] = (), -) -> SourceFreshnessRuntimeResult: - """Observe current source freshness records without using them for skip decisions.""" - - return _observe_virtual_environment_source_freshness( - adapter=adapter, - connection=connection, - sources=sources, - virtual_environment_name=virtual_environment_name, - observed_at=observed_at, - run_id=run_id, - load_results=load_results, - previous_records=previous_records, - ) diff --git a/src/sqlbuild/virtual/freshness/main/_runtime_persistence.py b/src/sqlbuild/virtual/freshness/main/_runtime_persistence.py deleted file mode 100644 index a27de14e7a..0000000000 --- a/src/sqlbuild/virtual/freshness/main/_runtime_persistence.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Public source freshness runtime persistence entrypoint.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.virtual.freshness._helpers.runtime import ( - persist_virtual_environment_source_freshness as _persist_virtual_environment_source_freshness, -) -from sqlbuild.virtual.freshness.models import SourceFreshnessRuntimeResult - - -def persist_virtual_environment_source_freshness( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, - result: SourceFreshnessRuntimeResult, -) -> None: - """Persist the latest observed freshness records for a virtual environment.""" - - _ = _persist_virtual_environment_source_freshness( - backend=backend, - state_connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - result=result, - ) diff --git a/src/sqlbuild/virtual/freshness/main/_state_record.py b/src/sqlbuild/virtual/freshness/main/_state_record.py deleted file mode 100644 index 6bb3d3ec53..0000000000 --- a/src/sqlbuild/virtual/freshness/main/_state_record.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public source freshness state-record entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.virtual.freshness._helpers.state import ( - source_freshness_record_from_observation as _source_freshness_record_from_observation, -) -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -def source_freshness_record_from_observation( - *, observation: SourceFreshnessObservation, virtual_environment_name: str -) -> SourceFreshnessRecord: - """Build a persisted state record from an observed source freshness value.""" - - return _source_freshness_record_from_observation( - observation=observation, - virtual_environment_name=virtual_environment_name, - ) diff --git a/src/sqlbuild/virtual/freshness/models.py b/src/sqlbuild/virtual/freshness/models.py deleted file mode 100644 index 8d3c206145..0000000000 --- a/src/sqlbuild/virtual/freshness/models.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Source freshness observation models for virtual environments.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessObservation as SourceFreshnessObservation, -) -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -@dataclass(frozen=True) -class SourceFreshnessRuntimeResult: - """Result of observing source freshness for one virtual environment.""" - - records: tuple[SourceFreshnessRecord, ...] - unknown_source_names: tuple[str, ...] = () - preserved_source_names: tuple[str, ...] = () - generated_source_names: tuple[str, ...] = () diff --git a/src/sqlbuild/virtual/planner/__init__.py b/src/sqlbuild/virtual/planner/__init__.py deleted file mode 100644 index 08e930671b..0000000000 --- a/src/sqlbuild/virtual/planner/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual planner domain.""" diff --git a/src/sqlbuild/virtual/planner/_helpers/__init__.py b/src/sqlbuild/virtual/planner/_helpers/__init__.py deleted file mode 100644 index 10f858bae5..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual planner helpers.""" diff --git a/src/sqlbuild/virtual/planner/_helpers/output.py b/src/sqlbuild/virtual/planner/_helpers/output.py deleted file mode 100644 index eeb92e8091..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/output.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Virtual planner output post-processing helpers.""" - -from __future__ import annotations - -from dataclasses import replace - -from sqlbuild.compiler.planner.models import ( - CascadeResult, - FunctionPlanEntry, - ModelPlanEntry, - PlanOutput, - RunDespiteUnchangedDecision, - RunDespiteUnchangedPlanningResult, - SeedPlanEntry, -) -from sqlbuild.compiler.planner.types import BackfillAction, PlanReason - - -def rewrite_virtual_plan_entries( - *, - plan_output: PlanOutput, - stale_root_reasons: dict[str, PlanReason], - stale_root_causes: dict[str, str], - stale_root_cause_reasons: dict[str, PlanReason] | None = None, - previous_query_sqls: dict[str, str] | None = None, - current_metadata_jsons: dict[str, str] | None = None, - previous_metadata_jsons: dict[str, str] | None = None, - previous_function_query_sqls: dict[str, str] | None = None, - run_despite_unchanged: RunDespiteUnchangedPlanningResult | None = None, - seed_plan_reasons: dict[str, PlanReason] | None = None, -) -> PlanOutput: - """Rewrite direct planner entries with virtual-specific reasons and causes.""" - - rewritten_entries: list[ModelPlanEntry] = [] - cause_reasons: dict[str, PlanReason] = stale_root_cause_reasons or stale_root_reasons - entry: ModelPlanEntry - for entry in plan_output.model_entries: - run_decision: RunDespiteUnchangedDecision | None = ( - run_despite_unchanged.decisions.get(entry.name) - if run_despite_unchanged is not None - else None - ) - if entry.name in stale_root_reasons: - rewritten_entries.append( - replace( - entry, - reason=stale_root_reasons[entry.name], - config_changed=( - entry.config_changed - or stale_root_reasons[entry.name] == PlanReason.CONFIG_CHANGED - ), - cascade=None, - run_despite_unchanged=run_decision, - previous_query_sql=(previous_query_sqls or {}).get( - entry.name, - entry.previous_query_sql, - ), - fingerprint_metadata_json=(current_metadata_jsons or {}).get(entry.name), - previous_metadata_json=(previous_metadata_jsons or {}).get(entry.name), - ) - ) - continue - root_cause: str | None = stale_root_causes.get(entry.name) - if root_cause is not None: - rewritten_entries.append( - replace( - entry, - reason=PlanReason.NO_CHANGE, - cascade=CascadeResult( - effective_action=BackfillAction.FORWARD_ONLY, - effective_duration=None, - root_cause=root_cause, - root_reason=cause_reasons[root_cause], - causes=(), - ), - ) - ) - continue - rewritten_entries.append(entry) - rewritten_function_entries: list[FunctionPlanEntry] = [] - function_entry: FunctionPlanEntry - for function_entry in plan_output.function_entries: - previous_function_query_sql: str | None = (previous_function_query_sqls or {}).get( - function_entry.name, - function_entry.previous_query_sql, - ) - if previous_function_query_sql is not None: - rewritten_function_entries.append( - replace( - function_entry, - previous_query_sql=previous_function_query_sql, - reason=( - PlanReason.QUERY_CHANGED - if previous_function_query_sql != function_entry.fingerprint_query_sql - else function_entry.reason - ), - ) - ) - continue - rewritten_function_entries.append(function_entry) - rewritten_seed_entries: list[SeedPlanEntry] = [] - seed_entry: SeedPlanEntry - for seed_entry in plan_output.seed_entries: - seed_reason: PlanReason | None = (seed_plan_reasons or {}).get(seed_entry.name) - if seed_reason is None: - rewritten_seed_entries.append(seed_entry) - continue - rewritten_seed_entries.append(replace(seed_entry, reason=seed_reason)) - return replace( - plan_output, - model_entries=tuple(rewritten_entries), - function_entries=tuple(rewritten_function_entries), - seed_entries=tuple(rewritten_seed_entries), - ) - - -def with_virtual_metadata( - *, - plan_output: PlanOutput, - target_name: str | None, - stale_model_names: tuple[str, ...], - stale_root_names: tuple[str, ...], - remaining_stale_model_names: tuple[str, ...] = (), - source_freshness_observed_source_names: tuple[str, ...] = (), - source_freshness_unchanged_source_names: tuple[str, ...] = (), - source_freshness_incomplete_source_names: tuple[str, ...] = (), - source_freshness_incomplete_model_names: tuple[str, ...] = (), -) -> PlanOutput: - """Attach virtual-specific metadata to a plan output.""" - - metadata: dict[str, object] = dict(plan_output.metadata) - metadata.update( - { - "virtual_mode": True, - "virtual_environment_name": target_name, - "virtual_environment_status": "finalized" if not stale_model_names else "working", - "virtual_stale_model_names": stale_model_names, - "virtual_stale_root_names": stale_root_names, - "virtual_remaining_stale_model_names": remaining_stale_model_names, - "virtual_source_freshness_observed_source_names": ( - source_freshness_observed_source_names - ), - "virtual_source_freshness_unchanged_source_names": ( - source_freshness_unchanged_source_names - ), - "virtual_source_freshness_incomplete_source_names": ( - source_freshness_incomplete_source_names - ), - "virtual_source_freshness_incomplete_model_names": ( - source_freshness_incomplete_model_names - ), - } - ) - return replace(plan_output, metadata=metadata) diff --git a/src/sqlbuild/virtual/planner/_helpers/planning.py b/src/sqlbuild/virtual/planner/_helpers/planning.py deleted file mode 100644 index 27f96153ea..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/planning.py +++ /dev/null @@ -1,591 +0,0 @@ -"""Pure virtual planning helpers.""" - -from __future__ import annotations - -import json -from typing import Any - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.main.identity.seed_identity import build_seed_identity -from sqlbuild.compiler.planner.main.identity.version_identity_function_hashes import ( - build_function_local_hashes as build_shared_function_local_hashes, -) -from sqlbuild.compiler.planner.main.identity.version_identity_local_hash import ( - build_model_local_identity_hash, -) -from sqlbuild.compiler.planner.main.identity.version_identity_model_metadata import ( - build_model_version_identity_metadata_json, -) -from sqlbuild.compiler.planner.main.identity.version_identity_stale_model_names import ( - build_version_identity_stale_model_names, -) -from sqlbuild.compiler.planner.main.identity.version_identity_version_hash import ( - build_model_version_identity_hash, -) -from sqlbuild.compiler.planner.main.selection.model_downstream_closure import ( - build_downstream_model_names, -) -from sqlbuild.compiler.planner.main.selection.model_upstream_closure import ( - build_upstream_model_names, -) -from sqlbuild.compiler.planner.main.selection.selection import resolve_project_selectors -from sqlbuild.compiler.planner.types import PlanReason, WorkSelectionPolicy -from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - SourceFreshnessRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) - - -def build_expected_local_hashes( - *, - graph: ProjectGraph, -) -> dict[str, str]: - """Derive local-only semantic hashes for functions and models.""" - - function_local_hashes: dict[str, str] = build_function_local_hashes(graph=graph) - expected_local_hashes: dict[str, str] = dict(function_local_hashes) - model_metadata_jsons: dict[str, str] = build_model_fingerprint_metadata_jsons( - graph=graph, - function_local_hashes=function_local_hashes, - ) - models_by_name: dict[str, Any] = {model.name: model for model in graph.project.models} - key: Any - for key in _topologically_order_keys(graph): - if key.resource_type != CompiledResourceType.MODEL: - continue - model: Any | None = models_by_name.get(key.name) - if model is None: - continue - expected_local_hashes[model.name] = build_model_local_identity_hash( - query_sql=model.query_sql, - metadata_json=model_metadata_jsons[model.name], - ) - return expected_local_hashes - - -def build_function_local_hashes(*, graph: ProjectGraph) -> dict[str, str]: - """Derive local-only semantic hashes for functions.""" - - return build_shared_function_local_hashes(functions=graph.project.functions) - - -def build_expected_seed_version_hashes(*, graph: ProjectGraph) -> dict[str, str]: - """Derive expected seed version hashes from current compiled seed identities.""" - - version_hashes: dict[str, str] = {} - for seed in graph.project.seeds: - seed_version_hash, _metadata_json = build_seed_identity(seed) - version_hashes[seed.name] = seed_version_hash - return version_hashes - - -def build_seed_identity_metadata_jsons(*, graph: ProjectGraph) -> dict[str, str]: - """Build deterministic seed identity metadata JSON by seed name.""" - - metadata_jsons: dict[str, str] = {} - for seed in graph.project.seeds: - _seed_version_hash, metadata_json = build_seed_identity(seed) - metadata_jsons[seed.name] = metadata_json - return metadata_jsons - - -def build_model_fingerprint_metadata_jsons( - *, - graph: ProjectGraph, - function_local_hashes: dict[str, str] | None = None, -) -> dict[str, str]: - """Build deterministic non-query model fingerprint metadata JSON by model.""" - - function_hashes: dict[str, str] = function_local_hashes or build_function_local_hashes( - graph=graph - ) - result: dict[str, str] = {} - hook_version_hashes: dict[str, str] = { - name: identity.version_hash - for name, identity in build_hook_identities(graph.project.hook_functions).items() - } - models_by_name: dict[str, Any] = {model.name: model for model in graph.project.models} - key: Any - for key in _topologically_order_keys(graph): - if key.resource_type != CompiledResourceType.MODEL: - continue - model: Any | None = models_by_name.get(key.name) - if model is None: - continue - result[model.name] = build_model_version_identity_metadata_json( - model=model, - function_local_hashes=function_hashes, - hook_version_hashes=hook_version_hashes, - ) - return result - - -def build_expected_version_hashes( - *, - graph: ProjectGraph, - expected_local_hashes: dict[str, str], - source_version_hashes: dict[str, str] | None = None, - seed_version_hashes: dict[str, str] | None = None, -) -> dict[str, str]: - """Derive expected model version hashes from current code and upstream hashes.""" - - source_hashes: dict[str, str] = source_version_hashes or {} - expected_hashes: dict[str, str] = { - function.name: expected_local_hashes[function.name] for function in graph.project.functions - } - expected_hashes.update(seed_version_hashes or build_expected_seed_version_hashes(graph=graph)) - models_by_name: dict[str, Any] = {model.name: model for model in graph.project.models} - - key: Any - for key in _topologically_order_keys(graph): - if key.resource_type != CompiledResourceType.MODEL: - continue - model: Any | None = models_by_name.get(key.name) - if model is None: - continue - expected_hashes[model.name] = build_model_version_identity_hash( - local_hash=expected_local_hashes[model.name], - upstream_deps=model.deps, - upstream_version_hashes=expected_hashes, - source_version_hashes=source_hashes, - ) - return expected_hashes - - -def build_source_freshness_incomplete_model_names( - *, - graph: ProjectGraph, - source_version_hashes: dict[str, str], -) -> tuple[str, ...]: - """Return models whose upstream source freshness proof is incomplete.""" - - incomplete_source_keys: tuple[Any, ...] = tuple( - source.key for source in graph.project.sources if source.name not in source_version_hashes - ) - incomplete_model_names: frozenset[str] = frozenset( - key.name - for key in transitive_closure_many( - starts=incomplete_source_keys, edges=graph.downstream_deps, include_starts=False - ) - if key.resource_type == CompiledResourceType.MODEL - ) - return tuple(sorted(incomplete_model_names)) - - -def build_bound_version_hashes( - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, str]: - """Index bound version hashes by model name from VDE refs.""" - - return {ref.model_name: ref.version_hash for ref in refs} - - -def build_bound_seed_version_hashes( - refs: tuple[VirtualEnvironmentSeedRefRecord, ...], -) -> dict[str, str]: - """Index bound version hashes by seed name from VDE seed refs.""" - - return {ref.seed_name: ref.version_hash for ref in refs} - - -def build_stale_seed_names( - *, - seed_names: tuple[str, ...], - expected_seed_version_hashes: dict[str, str], - bound_seed_version_hashes: dict[str, str], -) -> tuple[str, ...]: - """Return seed names whose bound and expected version hashes differ.""" - - return tuple( - sorted( - seed_name - for seed_name in seed_names - if bound_seed_version_hashes.get(seed_name) - != expected_seed_version_hashes.get(seed_name) - ) - ) - - -def build_seed_plan_reasons( - *, - seed_names: tuple[str, ...], - expected_seed_version_hashes: dict[str, str], - bound_seed_version_hashes: dict[str, str], -) -> dict[str, PlanReason]: - """Classify VDE seed plan reasons from typed seed refs.""" - - reasons: dict[str, PlanReason] = {} - seed_name: str - for seed_name in seed_names: - bound_version_hash: str | None = bound_seed_version_hashes.get(seed_name) - expected_version_hash: str | None = expected_seed_version_hashes.get(seed_name) - if bound_version_hash is None: - reasons[seed_name] = PlanReason.FIRST_RUN - elif expected_version_hash is not None and bound_version_hash != expected_version_hash: - reasons[seed_name] = PlanReason.CONFIG_CHANGED - else: - reasons[seed_name] = PlanReason.NO_CHANGE - return reasons - - -def build_source_version_hashes( - records: tuple[SourceFreshnessRecord, ...], -) -> dict[str, str]: - """Index source freshness input hashes by source name.""" - - return {record.source_name: record.data_version_hash for record in records} - - -def build_source_freshness_unchanged_source_names( - *, - previous_records: tuple[SourceFreshnessRecord, ...], - current_records: tuple[SourceFreshnessRecord, ...], -) -> tuple[str, ...]: - """Return source names whose current freshness record matches previous state.""" - - previous_by_source: dict[str, SourceFreshnessRecord] = { - record.source_name: record for record in previous_records - } - return tuple( - sorted( - record.source_name - for record in current_records - if previous_by_source.get(record.source_name) is not None - and previous_by_source[record.source_name].data_version_hash == record.data_version_hash - ) - ) - - -def build_stale_model_names( - *, - model_names: tuple[str, ...], - expected_version_hashes: dict[str, str], - bound_version_hashes: dict[str, str], - source_freshness_incomplete_model_names: tuple[str, ...] = (), -) -> tuple[str, ...]: - """Return model names whose bound and expected hashes differ.""" - - return build_version_identity_stale_model_names( - model_names=model_names, - expected_version_hashes=expected_version_hashes, - built_version_hashes=bound_version_hashes, - forced_stale_model_names=source_freshness_incomplete_model_names, - ) - - -def build_bound_local_hashes( - model_versions: dict[str, ModelVersionRecord | None], -) -> dict[str, str]: - """Index bound local semantic hashes by model name from model-version records.""" - - return { - model_name: model_version.definition_identity_hash - for model_name, model_version in model_versions.items() - if model_version is not None - } - - -def build_stale_root_reasons( - *, - stale_model_names: tuple[str, ...], - expected_local_hashes: dict[str, str], - bound_version_hashes: dict[str, str], - bound_local_hashes: dict[str, str], - current_query_sqls: dict[str, str] | None = None, - bound_previous_query_sqls: dict[str, str] | None = None, - expected_metadata_jsons: dict[str, str] | None = None, - bound_metadata_jsons: dict[str, str] | None = None, -) -> dict[str, PlanReason]: - """Classify stale root models by first-run vs local semantic change.""" - - stale_root_reasons: dict[str, PlanReason] = {} - stale_model_name: str - for stale_model_name in stale_model_names: - bound_version_hash: str | None = bound_version_hashes.get(stale_model_name) - if bound_version_hash is None: - stale_root_reasons[stale_model_name] = PlanReason.FIRST_RUN - continue - expected_local_hash: str | None = expected_local_hashes.get(stale_model_name) - bound_local_hash: str | None = bound_local_hashes.get(stale_model_name) - if expected_local_hash != bound_local_hash: - current_query_sql: str | None = (current_query_sqls or {}).get(stale_model_name) - previous_query_sql: str | None = (bound_previous_query_sqls or {}).get(stale_model_name) - if previous_query_sql is not None and current_query_sql != previous_query_sql: - stale_root_reasons[stale_model_name] = PlanReason.QUERY_CHANGED - continue - expected_metadata_json: str | None = (expected_metadata_jsons or {}).get( - stale_model_name - ) - bound_metadata_json: str | None = (bound_metadata_jsons or {}).get(stale_model_name) - if _metadata_config_payload(expected_metadata_json) != _metadata_config_payload( - bound_metadata_json - ): - stale_root_reasons[stale_model_name] = PlanReason.CONFIG_CHANGED - continue - stale_root_reasons[stale_model_name] = PlanReason.QUERY_CHANGED - return stale_root_reasons - - -def _metadata_config_payload(metadata_json: str | None) -> object: - if metadata_json is None: - return None - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return metadata_json - if not isinstance(payload, dict): - return None - return payload.get("config", {}) - - -def _metadata_function_hashes(metadata_json: str | None) -> dict[str, str]: - if metadata_json is None: - return {} - try: - payload: object = json.loads(metadata_json) - except json.JSONDecodeError: - return {} - if not isinstance(payload, dict): - return {} - raw_hashes: object = payload.get("local_function_hashes", {}) - if not isinstance(raw_hashes, dict): - return {} - return {str(name): str(value) for name, value in raw_hashes.items()} - - -def build_stale_root_causes( - *, - stale_model_names: tuple[str, ...], - stale_root_reasons: dict[str, PlanReason], - graph: ProjectGraph, - stale_root_source_causes: dict[str, str] | None = None, -) -> dict[str, str]: - """Assign each non-root stale model to one stale root cause.""" - - causes: dict[str, str] = {} - root_source_causes: dict[str, str] = stale_root_source_causes or {} - stale_root_name: str - for stale_root_name in sorted(stale_root_reasons): - start_key: Any | None = graph.all_keys.get(stale_root_name) - if start_key is None: - continue - stack: list[Any] = [start_key] - visited: set[Any] = set() - while stack: - current: Any = stack.pop() - if current in visited: - continue - visited.add(current) - if ( - current.resource_type == CompiledResourceType.MODEL - and current.name in stale_model_names - and current.name not in stale_root_reasons - and current.name not in causes - ): - causes[current.name] = root_source_causes.get(stale_root_name, stale_root_name) - downstream_key: Any - for downstream_key in graph.downstream_deps.get(current, ()): # pragma: no branch - stack.append(downstream_key) - return causes - - -def build_stale_root_source_causes( - *, - stale_root_reasons: dict[str, PlanReason], - expected_metadata_jsons: dict[str, str], - bound_metadata_jsons: dict[str, str], -) -> dict[str, str]: - """Map stale root models to changed function names when function metadata is the root.""" - - result: dict[str, str] = {} - model_name: str - for model_name, reason in stale_root_reasons.items(): - if reason != PlanReason.QUERY_CHANGED: - continue - expected_hashes: dict[str, str] = _metadata_function_hashes( - expected_metadata_jsons.get(model_name) - ) - bound_hashes: dict[str, str] = _metadata_function_hashes( - bound_metadata_jsons.get(model_name) - ) - changed_function_names: tuple[str, ...] = tuple( - sorted( - function_name - for function_name, expected_hash in expected_hashes.items() - if bound_hashes.get(function_name) != expected_hash - ) - ) - if len(changed_function_names) == 1: - result[model_name] = changed_function_names[0] - return result - - -def build_stale_root_cause_reasons( - *, - stale_root_reasons: dict[str, PlanReason], - stale_root_source_causes: dict[str, str], -) -> dict[str, PlanReason]: - """Map display cause names to display cause reasons.""" - - result: dict[str, PlanReason] = {} - for root_name, root_reason in stale_root_reasons.items(): - cause_name: str = stale_root_source_causes.get(root_name, root_name) - cause_reason: PlanReason = root_reason - if cause_name != root_name and root_reason == PlanReason.QUERY_CHANGED: - cause_reason = PlanReason.FUNCTION_CHANGED - result[cause_name] = cause_reason - return result - - -def build_default_virtual_selection( - *, - stale_model_names: tuple[str, ...], - graph: ProjectGraph, -) -> tuple[str, ...]: - """Return stale models plus their downstream model closure.""" - - start_keys: tuple[CompiledObjectKey, ...] = tuple( - key - for model_name in stale_model_names - if (key := graph.all_keys.get(model_name)) is not None - ) - return tuple( - sorted( - build_downstream_model_names( - start_keys=start_keys, - downstream_deps=graph.downstream_deps, - ) - ) - ) - - -def resolve_virtual_model_selection( - *, - graph: ProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], - default_selection: tuple[str, ...], - stale_model_names: tuple[str, ...], - include_stale_upstreams: bool = False, - work_selection_policy: WorkSelectionPolicy = WorkSelectionPolicy.ALL_SELECTED, -) -> tuple[str, ...]: - """Resolve and guard the effective virtual model selection.""" - - selected_model_names: tuple[str, ...] = ( - _resolve_selected_model_names(graph=graph, select=select, exclude=exclude) - if select - else _apply_exclude_to_model_names( - graph=graph, - model_names=default_selection, - exclude=exclude, - ) - ) - if work_selection_policy == WorkSelectionPolicy.STALE_ONLY: - default_set: set[str] = set(default_selection) - selected_model_names = tuple( - model_name for model_name in selected_model_names if model_name in default_set - ) - stale_upstream_names: tuple[str, ...] = build_stale_required_upstream_closure( - graph=graph, - selected_model_names=selected_model_names, - stale_model_names=stale_model_names, - ) - if stale_upstream_names and not include_stale_upstreams: - stale_list: str = ", ".join(stale_upstream_names) - raise PlannerInputError( - f"selected virtual scope is missing stale required upstream models: {stale_list}", - code="S010", - help=( - "Re-run with --include-stale-upstreams to add the minimal required " - "upstream closure." - ), - ) - if include_stale_upstreams: - selected_model_names = tuple(sorted({*selected_model_names, *stale_upstream_names})) - return selected_model_names - - -def build_stale_required_upstream_closure( - *, - graph: ProjectGraph, - selected_model_names: tuple[str, ...], - stale_model_names: tuple[str, ...], -) -> tuple[str, ...]: - """Return stale model ancestors required by the selected models.""" - - selected: set[str] = set(selected_model_names) - stale: set[str] = set(stale_model_names) - start_keys: tuple[CompiledObjectKey, ...] = tuple( - key - for model_name in selected_model_names - if (key := graph.all_keys.get(model_name)) is not None - ) - upstream_start_keys: list[CompiledObjectKey] = [] - for key in start_keys: - for upstream_key in graph.upstream_deps.get(key, ()): - upstream_start_keys.append(upstream_key) - required: frozenset[str] = build_upstream_model_names( - start_keys=tuple(upstream_start_keys), - upstream_deps=graph.upstream_deps, - ) - return tuple(sorted(model_name for model_name in required if model_name in stale - selected)) - - -def _resolve_selected_model_names( - *, - graph: ProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], -) -> tuple[str, ...]: - selected_keys: frozenset[CompiledObjectKey] = resolve_project_selectors( - select=select, - exclude=exclude, - all_keys=graph.all_keys, - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - tag_index=graph.tag_index, - path_index=graph.path_index, - ) - return tuple( - sorted(key.name for key in selected_keys if key.resource_type == CompiledResourceType.MODEL) - ) - - -def _apply_exclude_to_model_names( - *, - graph: ProjectGraph, - model_names: tuple[str, ...], - exclude: tuple[str, ...], -) -> tuple[str, ...]: - if not model_names or not exclude: - return model_names - excluded: set[str] = set(_resolve_selected_model_names(graph=graph, select=exclude, exclude=())) - return tuple(model_name for model_name in model_names if model_name not in excluded) - - -def _topologically_order_keys(graph: ProjectGraph) -> tuple[Any, ...]: - indegree: dict[Any, int] = {key: len(deps) for key, deps in graph.upstream_deps.items()} - ready: list[Any] = sorted( - (key for key, count in indegree.items() if count == 0), - key=lambda obj: (obj.resource_type, obj.name), - ) - ordered: list[Any] = [] - while ready: - current: Any = ready.pop(0) - ordered.append(current) - downstream_key: Any - for downstream_key in graph.downstream_deps.get(current, ()): # pragma: no branch - if downstream_key not in indegree: - continue - indegree[downstream_key] -= 1 - if indegree[downstream_key] == 0: - ready.append(downstream_key) - ready.sort(key=lambda obj: (obj.resource_type, obj.name)) - return tuple(ordered) diff --git a/src/sqlbuild/virtual/planner/_helpers/python_node_closure.py b/src/sqlbuild/virtual/planner/_helpers/python_node_closure.py deleted file mode 100644 index 8de231ee47..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/python_node_closure.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Virtual Python-node closure helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.graph.main.transitive_closure_many import transitive_closure_many -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.models import DiscoveredPythonNode, PythonNodeGraph -from sqlbuild.compiler.python_nodes.types import PythonNodeKind - - -def planned_source_loader_python_names( - *, plan_output: PlanOutput, python_graph: PythonNodeGraph -) -> frozenset[str]: - """Return source-loader Python names required by the current virtual plan.""" - - loader_names: frozenset[str] = frozenset( - entry.loader - for entry in plan_output.source_load_entries - if entry.loader in python_graph.nodes_by_name - and python_graph.nodes_by_name[entry.loader].kind == PythonNodeKind.LOADER - ) - return loader_names | python_upstream_closure( - selected_python_names=loader_names, - python_graph=python_graph, - ) - - -def sql_attached_python_names( - *, selected_sql_names: frozenset[str], python_graph: PythonNodeGraph -) -> frozenset[str]: - """Return read-side Python names attached to selected SQL resources.""" - - mutable_direct_names: set[str] = set() - for node in python_graph.nodes: - if node.kind not in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - continue - if any(sql_dep.name in selected_sql_names for sql_dep in node.sql_deps): - mutable_direct_names.add(node.name) - direct_names: frozenset[str] = frozenset(mutable_direct_names) - downstream_names: frozenset[str] = python_downstream_closure( - selected_python_names=direct_names, - python_graph=python_graph, - ) - return direct_names | downstream_names - - -def python_upstream_closure( - *, selected_python_names: frozenset[str], python_graph: PythonNodeGraph -) -> frozenset[str]: - """Return all Python upstreams for selected Python nodes.""" - - return transitive_closure_many( - starts=selected_python_names, edges=python_graph.upstream_deps, include_starts=False - ) - - -def python_downstream_closure( - *, selected_python_names: frozenset[str], python_graph: PythonNodeGraph -) -> frozenset[str]: - """Return task/asset downstreams for selected Python nodes.""" - - names: set[str] = set() - pending: list[str] = [] - for node_name in selected_python_names: - pending.extend(python_graph.downstream_deps.get(node_name, ())) - while pending: - current: str = pending.pop(0) - if current in names: - continue - node: DiscoveredPythonNode = python_graph.nodes_by_name[current] - if node.kind in {PythonNodeKind.TASK, PythonNodeKind.ASSET}: - names.add(current) - pending.extend(python_graph.downstream_deps.get(current, ())) - return frozenset(names) diff --git a/src/sqlbuild/virtual/planner/_helpers/run_despite_unchanged.py b/src/sqlbuild/virtual/planner/_helpers/run_despite_unchanged.py deleted file mode 100644 index 963891033e..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/run_despite_unchanged.py +++ /dev/null @@ -1,85 +0,0 @@ -"""Virtual planner bridge for run_despite_unchanged planning.""" - -from __future__ import annotations - -from datetime import UTC, datetime - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.main.changes.run_despite_unchanged import ( - build_run_despite_unchanged_planning_result, -) -from sqlbuild.compiler.planner.models import PlannerScope, RunDespiteUnchangedPlanningResult -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, -) -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord as DirectSourceFreshnessRecord, -) -from sqlbuild.virtual.state.models import SourceFreshnessRecord - - -def build_virtual_run_despite_unchanged_planning_result( - *, - graph: ProjectGraph, - source_freshness_records: tuple[SourceFreshnessRecord, ...], - already_stale_model_names: frozenset[str], -) -> RunDespiteUnchangedPlanningResult: - """Return VDE models selected by run_despite_unchanged semantics.""" - - source_freshness: DirectSourceFreshnessPlanningResult = _direct_source_freshness_result( - source_freshness_records - ) - return build_run_despite_unchanged_planning_result( - scope=_planner_scope_from_graph(graph), - source_freshness=source_freshness, - already_stale_model_names=already_stale_model_names, - now=( - source_freshness.observed_records[0].observed_at - if source_freshness.observed_records - else datetime.now(UTC) - ), - ) - - -def _planner_scope_from_graph(graph: ProjectGraph) -> PlannerScope: - return PlannerScope( - upstream_deps=graph.upstream_deps, - downstream_deps=graph.downstream_deps, - all_keys=graph.all_keys, - models_by_name={model.name: model for model in graph.project.models}, - selected_keys=frozenset(graph.all_keys.values()), - execution_order=tuple(graph.upstream_deps), - ) - - -def _direct_source_freshness_result( - records: tuple[SourceFreshnessRecord, ...], -) -> DirectSourceFreshnessPlanningResult: - observed_records: tuple[DirectSourceFreshnessRecord, ...] = tuple( - DirectSourceFreshnessRecord( - source_name=record.source_name, - target_database=None, - target_schema=None, - target_name=None, - run_id=record.virtual_environment_name, - strategy=record.strategy, - value_kind=record.value_kind, - data_version=record.data_version, - data_version_hash=record.data_version_hash, - observed_at=record.observed_at, - ) - for record in records - ) - return DirectSourceFreshnessPlanningResult( - observed_records=observed_records, - unchanged_identities=frozenset( - SourceFreshnessIdentity( - source_name=record.source_name, - target_database=None, - target_schema=None, - target_name=None, - ) - for record in records - ), - ) diff --git a/src/sqlbuild/virtual/planner/_helpers/semantics_facts.py b/src/sqlbuild/virtual/planner/_helpers/semantics_facts.py deleted file mode 100644 index 0c2039bfa4..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/semantics_facts.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Expected, bound, and staleness fact builders for virtual plan semantics.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import RunDespiteUnchangedPlanningResult -from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.virtual.planner._helpers.planning import ( - build_bound_local_hashes, - build_bound_seed_version_hashes, - build_bound_version_hashes, - build_default_virtual_selection, - build_expected_local_hashes, - build_expected_seed_version_hashes, - build_expected_version_hashes, - build_model_fingerprint_metadata_jsons, - build_seed_identity_metadata_jsons, - build_seed_plan_reasons, - build_source_freshness_incomplete_model_names, - build_source_version_hashes, - build_stale_model_names, - build_stale_root_cause_reasons, - build_stale_root_causes, - build_stale_root_reasons, - build_stale_root_source_causes, - build_stale_seed_names, -) -from sqlbuild.virtual.planner._helpers.run_despite_unchanged import ( - build_virtual_run_despite_unchanged_planning_result, -) -from sqlbuild.virtual.planner._helpers.state_metadata import ( - decode_model_version_metadata_jsons, - decode_model_version_query_sqls, -) -from sqlbuild.virtual.planner.models import ( - BoundIdentityFacts, - ExpectedIdentityFacts, - StalenessFacts, -) -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - SourceFreshnessRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) - - -def build_expected_identity_facts( - *, - graph: ProjectGraph, - source_freshness_records: tuple[SourceFreshnessRecord, ...], -) -> ExpectedIdentityFacts: - """Derive expected hashes and source-freshness coverage from the graph.""" - - expected_local_hashes: dict[str, str] = build_expected_local_hashes(graph=graph) - expected_seed_version_hashes: dict[str, str] = build_expected_seed_version_hashes(graph=graph) - source_version_hashes: dict[str, str] = build_source_version_hashes(source_freshness_records) - return ExpectedIdentityFacts( - local_hashes=expected_local_hashes, - metadata_jsons=build_model_fingerprint_metadata_jsons(graph=graph), - seed_version_hashes=expected_seed_version_hashes, - seed_identity_metadata_jsons=build_seed_identity_metadata_jsons(graph=graph), - version_hashes=build_expected_version_hashes( - graph=graph, - expected_local_hashes=expected_local_hashes, - source_version_hashes=source_version_hashes, - seed_version_hashes=expected_seed_version_hashes, - ), - source_version_hashes=source_version_hashes, - source_freshness_observed_source_names=tuple(sorted(source_version_hashes)), - source_freshness_incomplete_source_names=tuple( - sorted( - source.name - for source in graph.project.sources - if source.name not in source_version_hashes - ) - ), - source_freshness_incomplete_model_names=build_source_freshness_incomplete_model_names( - graph=graph, - source_version_hashes=source_version_hashes, - ), - ) - - -def build_bound_identity_facts( - *, - bound_refs: tuple[VirtualEnvironmentModelRefRecord, ...], - bound_model_versions: dict[str, ModelVersionRecord | None], - bound_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...], -) -> BoundIdentityFacts: - """Decode bound hashes and previous definitions from persisted state.""" - - return BoundIdentityFacts( - version_hashes=build_bound_version_hashes(bound_refs), - seed_version_hashes=build_bound_seed_version_hashes(bound_seed_refs), - local_hashes=build_bound_local_hashes(bound_model_versions), - previous_query_sqls=decode_model_version_query_sqls(bound_model_versions), - metadata_jsons=decode_model_version_metadata_jsons(bound_model_versions), - ) - - -def build_staleness_facts( - *, - graph: ProjectGraph, - expected: ExpectedIdentityFacts, - bound: BoundIdentityFacts, - source_freshness_records: tuple[SourceFreshnessRecord, ...], -) -> StalenessFacts: - """Classify stale models and seeds with their root reasons and causes.""" - - stale_seed_names: tuple[str, ...] = build_stale_seed_names( - seed_names=tuple(seed.name for seed in graph.project.seeds), - expected_seed_version_hashes=expected.seed_version_hashes, - bound_seed_version_hashes=bound.seed_version_hashes, - ) - seed_plan_reasons: dict[str, PlanReason] = build_seed_plan_reasons( - seed_names=tuple(seed.name for seed in graph.project.seeds), - expected_seed_version_hashes=expected.seed_version_hashes, - bound_seed_version_hashes=bound.seed_version_hashes, - ) - identity_stale_model_names: tuple[str, ...] = build_stale_model_names( - model_names=tuple(model.name for model in graph.project.models), - expected_version_hashes=expected.version_hashes, - bound_version_hashes=bound.version_hashes, - source_freshness_incomplete_model_names=expected.source_freshness_incomplete_model_names, - ) - run_despite_unchanged: RunDespiteUnchangedPlanningResult = ( - build_virtual_run_despite_unchanged_planning_result( - graph=graph, - source_freshness_records=source_freshness_records, - already_stale_model_names=frozenset(identity_stale_model_names), - ) - ) - stale_model_names: tuple[str, ...] = tuple( - sorted(set(identity_stale_model_names) | set(run_despite_unchanged.stale_model_names)) - ) - stale_root_reasons: dict[str, PlanReason] = build_stale_root_reasons( - stale_model_names=identity_stale_model_names, - expected_local_hashes=expected.local_hashes, - bound_version_hashes=bound.version_hashes, - bound_local_hashes=bound.local_hashes, - current_query_sqls={model.name: model.query_sql for model in graph.project.models}, - bound_previous_query_sqls=bound.previous_query_sqls, - expected_metadata_jsons=expected.metadata_jsons, - bound_metadata_jsons=bound.metadata_jsons, - ) - stale_root_reasons = { - **stale_root_reasons, - **{ - model_name: PlanReason.RUN_DESPITE_UNCHANGED - for model_name in run_despite_unchanged.root_model_names - }, - } - stale_root_source_causes: dict[str, str] = build_stale_root_source_causes( - stale_root_reasons=stale_root_reasons, - expected_metadata_jsons=expected.metadata_jsons, - bound_metadata_jsons=bound.metadata_jsons, - ) - return StalenessFacts( - stale_seed_names=stale_seed_names, - seed_plan_reasons=seed_plan_reasons, - stale_model_names=stale_model_names, - default_selection=build_default_virtual_selection( - stale_model_names=stale_model_names, - graph=graph, - ), - run_despite_unchanged=run_despite_unchanged, - stale_root_reasons=stale_root_reasons, - stale_root_causes=build_stale_root_causes( - stale_model_names=stale_model_names, - stale_root_reasons=stale_root_reasons, - graph=graph, - stale_root_source_causes=stale_root_source_causes, - ), - stale_root_cause_reasons=build_stale_root_cause_reasons( - stale_root_reasons=stale_root_reasons, - stale_root_source_causes=stale_root_source_causes, - ), - ) diff --git a/src/sqlbuild/virtual/planner/_helpers/state_metadata.py b/src/sqlbuild/virtual/planner/_helpers/state_metadata.py deleted file mode 100644 index c88945b3a8..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/state_metadata.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Helpers for decoding virtual model-version metadata.""" - -from __future__ import annotations - -from sqlbuild.virtual.state.main.encoding._decode_state_text import decode_state_text -from sqlbuild.virtual.state.models import ModelVersionRecord - - -def decode_model_version_query_sqls( - model_versions: dict[str, ModelVersionRecord | None], -) -> dict[str, str]: - """Decode persisted model-version fingerprint SQL by model name.""" - - result: dict[str, str] = {} - for model_name, model_version in model_versions.items(): - if model_version is None: - continue - query_sql: str | None = decode_state_text(model_version.definition_text_b64) - if query_sql is None: - continue - result[model_name] = query_sql - return result - - -def decode_model_version_metadata_jsons( - model_versions: dict[str, ModelVersionRecord | None], -) -> dict[str, str]: - """Decode persisted model-version fingerprint metadata JSON by model name.""" - - result: dict[str, str] = {} - for model_name, model_version in model_versions.items(): - if model_version is None: - continue - metadata_json: str | None = decode_state_text(model_version.identity_metadata_json_b64) - if metadata_json is None: - continue - result[model_name] = metadata_json - return result diff --git a/src/sqlbuild/virtual/planner/_helpers/targets.py b/src/sqlbuild/virtual/planner/_helpers/targets.py deleted file mode 100644 index 517b48fe05..0000000000 --- a/src/sqlbuild/virtual/planner/_helpers/targets.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Virtual planner target helpers.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.virtual.state.models import PhysicalRelationRecord - - -def build_destination_from_physical_relation( - *, - adapter: BaseAdapter, - relation: PhysicalRelationRecord, - fallback_target: CompiledRelationLocation, -) -> CompiledRelationLocation: - """Rebuild a compiled relation location from a stored physical relation record.""" - - return CompiledRelationLocation( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - qualified_name=adapter.render_qualified_name( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ), - logical_schema=fallback_target.logical_schema, - logical_database=fallback_target.logical_database, - ) diff --git a/src/sqlbuild/virtual/planner/main/__init__.py b/src/sqlbuild/virtual/planner/main/__init__.py deleted file mode 100644 index a41823d1af..0000000000 --- a/src/sqlbuild/virtual/planner/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual planner main entrypoints.""" diff --git a/src/sqlbuild/virtual/planner/main/_output.py b/src/sqlbuild/virtual/planner/main/_output.py deleted file mode 100644 index 153a5cc421..0000000000 --- a/src/sqlbuild/virtual/planner/main/_output.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Public virtual planner output entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.virtual.planner._helpers.output import ( - rewrite_virtual_plan_entries, - with_virtual_metadata, -) -from sqlbuild.virtual.planner.models import VirtualPlanSemantics - - -def apply_virtual_plan_output( - *, - plan_output: PlanOutput, - target_name: str, - semantics: VirtualPlanSemantics, - selected_model_names: tuple[str, ...] = (), - previous_function_query_sqls: dict[str, str] | None = None, -) -> PlanOutput: - """Rewrite plan entries and attach virtual environment metadata.""" - - rewritten: PlanOutput = rewrite_virtual_plan_entries( - plan_output=plan_output, - stale_root_reasons=semantics.stale_root_reasons, - stale_root_causes=semantics.stale_root_causes, - stale_root_cause_reasons=semantics.stale_root_cause_reasons, - previous_query_sqls=semantics.bound_previous_query_sqls, - current_metadata_jsons=semantics.expected_metadata_jsons, - previous_metadata_jsons=semantics.bound_metadata_jsons, - previous_function_query_sqls=previous_function_query_sqls, - run_despite_unchanged=semantics.run_despite_unchanged, - seed_plan_reasons=semantics.seed_plan_reasons, - ) - return with_virtual_metadata( - plan_output=rewritten, - target_name=target_name, - stale_model_names=semantics.stale_model_names, - stale_root_names=tuple(sorted(semantics.stale_root_reasons)), - remaining_stale_model_names=tuple( - sorted(set(semantics.stale_model_names) - set(selected_model_names)) - ), - source_freshness_observed_source_names=(semantics.source_freshness_observed_source_names), - source_freshness_unchanged_source_names=(semantics.source_freshness_unchanged_source_names), - source_freshness_incomplete_source_names=( - semantics.source_freshness_incomplete_source_names - ), - source_freshness_incomplete_model_names=(semantics.source_freshness_incomplete_model_names), - ) diff --git a/src/sqlbuild/virtual/planner/main/_python_identities.py b/src/sqlbuild/virtual/planner/main/_python_identities.py deleted file mode 100644 index 32c3fbd6e4..0000000000 --- a/src/sqlbuild/virtual/planner/main/_python_identities.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Virtual planner entrypoint for Python identity state.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.main.python_identities._python_node_identity_read import ( - read_virtual_python_identity_fingerprints, -) - - -def read_bound_virtual_python_identities( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - virtual_environment_name: str | None, -) -> dict[tuple[str, str], Fingerprint]: - """Read Python identity rows bound to the active virtual environment.""" - - target_name: str | None = virtual_environment_name or resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if target_name is None: - return {} - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - state_connection: Any = backend.connect(config.connection) - try: - return read_virtual_python_identity_fingerprints( - backend=backend, - state_connection=state_connection, - schema=config.schema, - virtual_environment_name=target_name, - ) - finally: - backend.close(state_connection) diff --git a/src/sqlbuild/virtual/planner/main/_python_plan_entries.py b/src/sqlbuild/virtual/planner/main/_python_plan_entries.py deleted file mode 100644 index 53f83d2c20..0000000000 --- a/src/sqlbuild/virtual/planner/main/_python_plan_entries.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Virtual-mode Python node plan-entry entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.pipeline.main.python_plan_entries import build_python_plan_entries -from sqlbuild.compiler.pipeline.models import PythonPlanEntry -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_lifecycle import build_python_sql_run_lifecycle -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunLifecyclePlan, - PythonSqlRunSelection, -) - - -def build_virtual_python_plan_entries( - *, - discovered_inputs: DiscoveredProjectInputs, - selection: PythonSqlRunSelection, - previous_identities: dict[tuple[str, str], Fingerprint] | None = None, -) -> tuple[PythonPlanEntry, ...]: - """Return display entries for a virtual Python selection.""" - - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - lifecycle_plan: PythonSqlRunLifecyclePlan = build_python_sql_run_lifecycle( - selection=selection, - python_graph=python_graph, - ) - return build_python_plan_entries( - lifecycle_plan=lifecycle_plan, - python_graph=python_graph, - previous_identities=previous_identities, - ) diff --git a/src/sqlbuild/virtual/planner/main/_python_run_selection.py b/src/sqlbuild/virtual/planner/main/_python_run_selection.py deleted file mode 100644 index 6c9c30529f..0000000000 --- a/src/sqlbuild/virtual/planner/main/_python_run_selection.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Virtual-mode Python node run-selection entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.compile.models import CompiledObjectKey -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.compiler.python_nodes.main.graph import build_discovered_python_node_graph -from sqlbuild.compiler.python_nodes.main.run_selection_graph import ( - resolve_python_sql_run_selection_from_graph, -) -from sqlbuild.compiler.python_nodes.main.selection_validation import validate_python_sql_selection -from sqlbuild.compiler.python_nodes.models import ( - PythonNodeGraph, - PythonSqlRunSelection, - PythonSqlSelection, -) -from sqlbuild.virtual.planner._helpers.python_node_closure import ( - planned_source_loader_python_names, - python_upstream_closure, - sql_attached_python_names, -) - - -def build_virtual_python_run_selection( - *, - discovered_inputs: DiscoveredProjectInputs, - graph: ProjectGraph, - plan_output: PlanOutput, - select: tuple[str, ...], - exclude: tuple[str, ...], - selected_model_names: tuple[str, ...], - include_python: bool, -) -> PythonSqlRunSelection: - """Return selected virtual Python nodes plus SQL keys used for lifecycle planning.""" - - python_graph: PythonNodeGraph = build_discovered_python_node_graph( - discovered_inputs=discovered_inputs - ) - selected_python_names: frozenset[str] = planned_source_loader_python_names( - plan_output=plan_output, python_graph=python_graph - ) - validation_sql_keys: frozenset[CompiledObjectKey] = frozenset(plan_output.selected_keys) - if include_python: - raw_select: tuple[str, ...] = select or selected_model_names - if raw_select: - run_selection: PythonSqlRunSelection = resolve_python_sql_run_selection_from_graph( - select=raw_select, - exclude=exclude, - project_graph=graph, - python_graph=python_graph, - ) - selected_python_names = selected_python_names | run_selection.python_node_names - validation_sql_keys = validation_sql_keys | run_selection.sql_keys - selected_python_names = selected_python_names | sql_attached_python_names( - selected_sql_names=frozenset(key.name for key in plan_output.selected_keys), - python_graph=python_graph, - ) - selected_python_names = selected_python_names | python_upstream_closure( - selected_python_names=selected_python_names, - python_graph=python_graph, - ) - validate_python_sql_selection( - selection=PythonSqlSelection( - sql_keys=validation_sql_keys, - python_node_names=selected_python_names, - ), - project_graph=graph, - python_graph=python_graph, - ) - return PythonSqlRunSelection( - sql_keys=frozenset(plan_output.selected_keys), - python_node_names=selected_python_names, - ) diff --git a/src/sqlbuild/virtual/planner/main/_selection.py b/src/sqlbuild/virtual/planner/main/_selection.py deleted file mode 100644 index c450f279d8..0000000000 --- a/src/sqlbuild/virtual/planner/main/_selection.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public virtual selection entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.types import WorkSelectionPolicy -from sqlbuild.virtual.planner._helpers.planning import resolve_virtual_model_selection - - -def resolve_virtual_plan_model_selection( - *, - graph: ProjectGraph, - select: tuple[str, ...], - exclude: tuple[str, ...], - default_selection: tuple[str, ...], - stale_model_names: tuple[str, ...], - include_stale_upstreams: bool = False, - work_selection_policy: WorkSelectionPolicy = WorkSelectionPolicy.ALL_SELECTED, -) -> tuple[str, ...]: - """Resolve the coherent virtual model selection for plan/build.""" - - return resolve_virtual_model_selection( - graph=graph, - select=select, - exclude=exclude, - default_selection=default_selection, - stale_model_names=stale_model_names, - include_stale_upstreams=include_stale_upstreams, - work_selection_policy=work_selection_policy, - ) diff --git a/src/sqlbuild/virtual/planner/main/_semantics.py b/src/sqlbuild/virtual/planner/main/_semantics.py deleted file mode 100644 index c7a188f22a..0000000000 --- a/src/sqlbuild/virtual/planner/main/_semantics.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Public virtual planning semantics entrypoint.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.virtual.planner._helpers.planning import ( - build_source_freshness_unchanged_source_names, -) -from sqlbuild.virtual.planner._helpers.semantics_facts import ( - build_bound_identity_facts, - build_expected_identity_facts, - build_staleness_facts, -) -from sqlbuild.virtual.planner.models import ( - BoundIdentityFacts, - ExpectedIdentityFacts, - StalenessFacts, - VirtualPlanSemantics, -) -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - SourceFreshnessRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) - - -def build_virtual_plan_semantics( - *, - graph: ProjectGraph, - bound_refs: tuple[VirtualEnvironmentModelRefRecord, ...], - bound_model_versions: dict[str, ModelVersionRecord | None], - bound_seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = (), - source_freshness_records: tuple[SourceFreshnessRecord, ...] = (), - previous_source_freshness_records: tuple[SourceFreshnessRecord, ...] = (), -) -> VirtualPlanSemantics: - """Derive expected hashes, bound hashes, stale models, and stale roots.""" - - expected: ExpectedIdentityFacts = build_expected_identity_facts( - graph=graph, - source_freshness_records=source_freshness_records, - ) - bound: BoundIdentityFacts = build_bound_identity_facts( - bound_refs=bound_refs, - bound_model_versions=bound_model_versions, - bound_seed_refs=bound_seed_refs, - ) - staleness: StalenessFacts = build_staleness_facts( - graph=graph, - expected=expected, - bound=bound, - source_freshness_records=source_freshness_records, - ) - return VirtualPlanSemantics( - expected_local_hashes=expected.local_hashes, - expected_metadata_jsons=expected.metadata_jsons, - expected_version_hashes=expected.version_hashes, - expected_seed_version_hashes=expected.seed_version_hashes, - seed_identity_metadata_jsons=expected.seed_identity_metadata_jsons, - bound_version_hashes=bound.version_hashes, - bound_seed_version_hashes=bound.seed_version_hashes, - bound_local_hashes=bound.local_hashes, - bound_previous_query_sqls=bound.previous_query_sqls, - bound_metadata_jsons=bound.metadata_jsons, - source_freshness_observed_source_names=expected.source_freshness_observed_source_names, - source_freshness_unchanged_source_names=( - build_source_freshness_unchanged_source_names( - previous_records=previous_source_freshness_records, - current_records=source_freshness_records, - ) - ), - source_freshness_incomplete_source_names=expected.source_freshness_incomplete_source_names, - source_freshness_incomplete_model_names=expected.source_freshness_incomplete_model_names, - stale_seed_names=staleness.stale_seed_names, - seed_plan_reasons=staleness.seed_plan_reasons, - stale_model_names=staleness.stale_model_names, - default_selection=staleness.default_selection, - run_despite_unchanged=staleness.run_despite_unchanged, - stale_root_reasons=staleness.stale_root_reasons, - stale_root_causes=staleness.stale_root_causes, - stale_root_cause_reasons=staleness.stale_root_cause_reasons, - ) diff --git a/src/sqlbuild/virtual/planner/main/_targets.py b/src/sqlbuild/virtual/planner/main/_targets.py deleted file mode 100644 index a720258254..0000000000 --- a/src/sqlbuild/virtual/planner/main/_targets.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public virtual target helpers.""" - -from __future__ import annotations - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.virtual.planner._helpers.targets import build_destination_from_physical_relation -from sqlbuild.virtual.state.models import PhysicalRelationRecord - - -def build_virtual_destination_from_physical_relation( - *, - adapter: BaseAdapter, - relation: PhysicalRelationRecord, - fallback_target: CompiledRelationLocation, -) -> CompiledRelationLocation: - """Build a compiled relation location from a tracked physical relation.""" - - return build_destination_from_physical_relation( - adapter=adapter, - relation=relation, - fallback_target=fallback_target, - ) diff --git a/src/sqlbuild/virtual/planner/main/_upstreams.py b/src/sqlbuild/virtual/planner/main/_upstreams.py deleted file mode 100644 index 912abc99ae..0000000000 --- a/src/sqlbuild/virtual/planner/main/_upstreams.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public virtual upstream closure helpers.""" - -from __future__ import annotations - -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.virtual.planner._helpers.planning import build_stale_required_upstream_closure - - -def build_virtual_stale_required_upstream_closure( - *, - graph: ProjectGraph, - selected_model_names: tuple[str, ...], - stale_model_names: tuple[str, ...], -) -> tuple[str, ...]: - """Return stale required upstreams for a selected virtual model scope.""" - - return build_stale_required_upstream_closure( - graph=graph, - selected_model_names=selected_model_names, - stale_model_names=stale_model_names, - ) diff --git a/src/sqlbuild/virtual/planner/main/plan.py b/src/sqlbuild/virtual/planner/main/plan.py deleted file mode 100644 index 20bc9bfc9e..0000000000 --- a/src/sqlbuild/virtual/planner/main/plan.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Virtual-mode planning entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.models import CompilePipelineResult -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.runtime.contracts.models import ConnectionHooks -from sqlbuild.virtual.executor.main._resolve_plan import resolve_virtual_plan -from sqlbuild.virtual.planner.models import VirtualPlanOptions - - -def run_virtual_plan_pipeline( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object] | None = None, - options: VirtualPlanOptions | None = None, - hooks: ConnectionHooks | None = None, -) -> CompilePipelineResult: - """Run the read-only build-grade virtual planning pipeline.""" - - if connection_config is None: - raise PlannerInputError("virtual planning requires explicit connection_config") - return resolve_virtual_plan( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=options if options is not None else VirtualPlanOptions(), - hooks=hooks if hooks is not None else ConnectionHooks(), - ) diff --git a/src/sqlbuild/virtual/planner/models.py b/src/sqlbuild/virtual/planner/models.py deleted file mode 100644 index 042d18a954..0000000000 --- a/src/sqlbuild/virtual/planner/models.py +++ /dev/null @@ -1,105 +0,0 @@ -"""Virtual planner models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - -from sqlbuild.compiler.planner.models import CursorOverrides, RunDespiteUnchangedPlanningResult -from sqlbuild.compiler.planner.types import PlanReason -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -@dataclass(frozen=True) -class VirtualPlanOptions: - """Selection, deferral, and cursor options for one virtual planning run.""" - - selected_target: str | None = None - no_sql_validation: bool = False - defer_sources_to: str | None = None - source_deferral_enabled: bool = True - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cursor_overrides: CursorOverrides | None = None - full_refresh: bool = False - virtual_environment_name: str | None = None - include_stale_upstreams: bool = False - changes_only: bool = False - auto_load_sources: bool = False - reload_sources: bool = False - include_python: bool = True - cli_vars: dict[str, object] | None = None - external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None - no_cache: bool = False - max_microbatches: int | None = None - - -@dataclass(frozen=True) -class ExpectedIdentityFacts: - """Expected identity hashes and source-freshness coverage from the graph.""" - - local_hashes: dict[str, str] = field(default_factory=dict) - metadata_jsons: dict[str, str] = field(default_factory=dict) - seed_version_hashes: dict[str, str] = field(default_factory=dict) - seed_identity_metadata_jsons: dict[str, str] = field(default_factory=dict) - version_hashes: dict[str, str] = field(default_factory=dict) - source_version_hashes: dict[str, str] = field(default_factory=dict) - source_freshness_observed_source_names: tuple[str, ...] = () - source_freshness_incomplete_source_names: tuple[str, ...] = () - source_freshness_incomplete_model_names: tuple[str, ...] = () - - -@dataclass(frozen=True) -class BoundIdentityFacts: - """Bound identity hashes decoded from persisted virtual state.""" - - version_hashes: dict[str, str] = field(default_factory=dict) - seed_version_hashes: dict[str, str] = field(default_factory=dict) - local_hashes: dict[str, str] = field(default_factory=dict) - previous_query_sqls: dict[str, str] = field(default_factory=dict) - metadata_jsons: dict[str, str] = field(default_factory=dict) - - -@dataclass(frozen=True) -class StalenessFacts: - """Stale models/seeds with their root reasons and causes.""" - - stale_seed_names: tuple[str, ...] = () - seed_plan_reasons: dict[str, PlanReason] = field(default_factory=dict) - stale_model_names: tuple[str, ...] = () - default_selection: tuple[str, ...] = () - run_despite_unchanged: RunDespiteUnchangedPlanningResult = field( - default_factory=RunDespiteUnchangedPlanningResult - ) - stale_root_reasons: dict[str, PlanReason] = field(default_factory=dict) - stale_root_causes: dict[str, str] = field(default_factory=dict) - stale_root_cause_reasons: dict[str, PlanReason] = field(default_factory=dict) - - -@dataclass(frozen=True) -class VirtualPlanSemantics: - """Derived virtual planning semantics for one VDE.""" - - expected_local_hashes: dict[str, str] = field(default_factory=dict) - expected_metadata_jsons: dict[str, str] = field(default_factory=dict) - expected_version_hashes: dict[str, str] = field(default_factory=dict) - expected_seed_version_hashes: dict[str, str] = field(default_factory=dict) - seed_identity_metadata_jsons: dict[str, str] = field(default_factory=dict) - bound_version_hashes: dict[str, str] = field(default_factory=dict) - bound_seed_version_hashes: dict[str, str] = field(default_factory=dict) - bound_local_hashes: dict[str, str] = field(default_factory=dict) - bound_previous_query_sqls: dict[str, str] = field(default_factory=dict) - bound_metadata_jsons: dict[str, str] = field(default_factory=dict) - source_freshness_observed_source_names: tuple[str, ...] = () - source_freshness_unchanged_source_names: tuple[str, ...] = () - source_freshness_incomplete_source_names: tuple[str, ...] = () - source_freshness_incomplete_model_names: tuple[str, ...] = () - stale_seed_names: tuple[str, ...] = () - seed_plan_reasons: dict[str, PlanReason] = field(default_factory=dict) - stale_model_names: tuple[str, ...] = () - default_selection: tuple[str, ...] = () - run_despite_unchanged: RunDespiteUnchangedPlanningResult = field( - default_factory=RunDespiteUnchangedPlanningResult - ) - stale_root_reasons: dict[str, PlanReason] = field(default_factory=dict) - stale_root_causes: dict[str, str] = field(default_factory=dict) - stale_root_cause_reasons: dict[str, PlanReason] = field(default_factory=dict) diff --git a/src/sqlbuild/virtual/reconcile/__init__.py b/src/sqlbuild/virtual/reconcile/__init__.py deleted file mode 100644 index f005319f68..0000000000 --- a/src/sqlbuild/virtual/reconcile/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual reconcile domain.""" diff --git a/src/sqlbuild/virtual/reconcile/_helpers/reconcile.py b/src/sqlbuild/virtual/reconcile/_helpers/reconcile.py deleted file mode 100644 index 2955bbfb1d..0000000000 --- a/src/sqlbuild/virtual/reconcile/_helpers/reconcile.py +++ /dev/null @@ -1,648 +0,0 @@ -"""Virtual reconcile helper logic.""" - -from __future__ import annotations - -from datetime import timedelta -from pathlib import Path -from typing import Any -from uuid import uuid4 - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.models import RelationInfo, RelationLookup -from sqlbuild.adapter.contract.types import RelationType -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.adapter.type_system.main.normalize_relation_type import normalize_relation_type -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.executor.main._views import refresh_logical_vde_views -from sqlbuild.virtual.reconcile.constants import RECONCILE_REPAIR_VIEW_COMMAND -from sqlbuild.virtual.state.main.environments.runtime import build_state_runtime -from sqlbuild.virtual.state.main.locks._locks import acquire_virtual_environment_lease -from sqlbuild.virtual.state.main.locks._release_lock import release_state_lease -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - ReconcileEventRecord, - StateLockLease, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import PhysicalArtifactType, ReconcileAction, StateOperationStatus - - -def run_virtual_reconcile( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str | None, - command: str | None, - model_name: str | None, - seed_name: str | None, - physical_relation_name: str | None, -) -> str: - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - ) - resolved_virtual_environment_name: str | None = virtual_environment_name or resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - active_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - unsuffixed_virtual_environment_name: str | None = None - if active_target_name is not None: - unsuffixed_virtual_environment_name = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=active_target_name, - ).state.unsuffixed_virtual_env - if resolved_virtual_environment_name is None: - raise PlannerInputError( - "reconcile requires --virtual-env or a default environment", - code="C247", - ) - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - state_connection: Any = backend.connect(config.connection) - lease: StateLockLease | None = None - try: - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=resolved_virtual_environment_name, - ) - ) - ref_map: dict[str, str] = {ref.model_name: ref.version_hash for ref in refs} - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=resolved_virtual_environment_name, - ) - ) - seed_ref_map: dict[str, str] = {ref.seed_name: ref.version_hash for ref in seed_refs} - physical_map: dict[str, PhysicalRelationRecord] = {} - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=state_connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is not None: - physical_map[ref.model_name] = relation - seed_physical_map: dict[str, PhysicalRelationRecord] = {} - for ref in seed_refs: - relation = backend.get_physical_relation_for_artifact( - connection=state_connection, - schema=config.schema, - artifact_type=PhysicalArtifactType.SEED, - artifact_name=ref.seed_name, - version_hash=ref.version_hash, - ) - if relation is not None: - seed_physical_map[ref.seed_name] = relation - - if command == RECONCILE_REPAIR_VIEW_COMMAND: - if (model_name is None) == (seed_name is None): - raise PlannerInputError( - "reconcile repair-view requires exactly one of --model or --seed", - code="C248", - ) - lease = acquire_virtual_environment_lease( - backend=backend, - connection=state_connection, - schema=config.schema, - virtual_environment_name=resolved_virtual_environment_name, - owner_id=f"reconcile:{uuid4()}", - ttl=timedelta(minutes=10), - ) - if lease is None: - raise PlannerInputError( - f"virtual environment '{resolved_virtual_environment_name}' is locked", - code="S014", - ) - if seed_name is not None: - repair_seed_view( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=resolved_virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - seed_name=seed_name, - seed_physical_map=seed_physical_map, - ) - _record_reconcile_event( - backend=backend, - state_connection=state_connection, - schema=config.schema, - action=ReconcileAction.REPAIR_VIEW, - message=f"repaired seed view for {seed_name}", - ) - return ( - "Repair\n" - f" seed {seed_name}\n" - f" VDE {resolved_virtual_environment_name}\n" - " action recreate logical seed view from state\n" - " result repaired" - ) - if model_name is None: - raise PlannerInputError( - "reconcile repair-view requires exactly one of --model or --seed", - code="C248", - ) - repair_view( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=resolved_virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_name=model_name, - physical_map=physical_map, - ) - _record_reconcile_event( - backend=backend, - state_connection=state_connection, - schema=config.schema, - action=ReconcileAction.REPAIR_VIEW, - message=f"repaired view for {model_name}", - ) - return ( - "Repair\n" - f" model {model_name}\n" - f" VDE {resolved_virtual_environment_name}\n" - " action recreate logical view from state\n" - " result repaired" - ) - - if command == ReconcileAction.ATTACH: - if model_name is None or physical_relation_name is None: - raise PlannerInputError( - "reconcile attach requires --model and --physical-relation", - code="C249", - ) - lease = acquire_virtual_environment_lease( - backend=backend, - connection=state_connection, - schema=config.schema, - virtual_environment_name=resolved_virtual_environment_name, - owner_id=f"reconcile:{uuid4()}", - ttl=timedelta(minutes=10), - ) - if lease is None: - raise PlannerInputError( - f"virtual environment '{resolved_virtual_environment_name}' is locked", - code="S014", - ) - selected_relation: PhysicalRelationRecord = resolve_attach_relation( - adapter=adapter, - backend=backend, - state_connection=state_connection, - schema=config.schema, - model_name=model_name, - physical_relation_name=physical_relation_name, - ) - validate_logical_target_repairable( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=resolved_virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_name=model_name, - ) - backend.replace_virtual_environment_model_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=resolved_virtual_environment_name, - refs=build_attached_refs( - existing_refs=refs, - virtual_environment_name=resolved_virtual_environment_name, - model_name=model_name, - version_hash=selected_relation.version_hash, - ), - ) - repair_view( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=resolved_virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_name=model_name, - physical_map={model_name: selected_relation}, - ) - _record_reconcile_event( - backend=backend, - state_connection=state_connection, - schema=config.schema, - action=ReconcileAction.ATTACH, - message=f"attached {model_name} to {physical_relation_name}", - ) - return ( - "Attach\n" - f" model {model_name}\n" - f" VDE {resolved_virtual_environment_name}\n" - f" physical {physical_relation_name}\n" - " result attached" - ) - - _record_reconcile_event( - backend=backend, - state_connection=state_connection, - schema=config.schema, - action=ReconcileAction.REPORT, - message=f"report for {resolved_virtual_environment_name}", - ) - return build_reconcile_report( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=resolved_virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_name=model_name, - ref_map=ref_map, - physical_map=physical_map, - seed_ref_map=seed_ref_map, - seed_physical_map=seed_physical_map, - ) - finally: - if lease is not None: - release_state_lease( - backend=backend, - connection=state_connection, - schema=config.schema, - lease=lease, - ) - backend.close(state_connection) - - -def build_reconcile_report( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - model_name: str | None, - ref_map: dict[str, str], - physical_map: dict[str, PhysicalRelationRecord], - seed_ref_map: dict[str, str], - seed_physical_map: dict[str, PhysicalRelationRecord], -) -> str: - target_names: tuple[str, ...] = ( - (model_name,) - if model_name is not None - else tuple(model.name for model in graph.project.models) - ) - relation_types: dict[str, str] = list_virtual_relation_types( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - physical_relation_lookup: RelationLookup = _build_physical_relation_lookup( - adapter=adapter, - connection_config=connection_config, - relations=(*physical_map.values(), *seed_physical_map.values()), - ) - issues: list[str] = [] - for model in graph.project.models: - if model.name not in target_names: - continue - if model.name not in ref_map: - issues.append(f"missing ref: {model.name}") - continue - relation: PhysicalRelationRecord | None = physical_map.get(model.name) - if relation is None: - issues.append(f"missing tracked physical relation: {model.name}") - continue - if not physical_relation_lookup.exists( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ): - issues.append(f"missing physical relation: {model.name}") - relation_type: str | None = relation_types.get(model.name) - if relation_type is None: - issues.append(f"missing logical target: {model.name}") - elif normalize_relation_type(relation_type) == RelationType.TABLE: - issues.append(f"logical target is table: {model.name}") - if model_name is None: - for seed in graph.project.seeds: - if seed.name not in seed_ref_map: - issues.append(f"missing seed ref: {seed.name}") - continue - relation = seed_physical_map.get(seed.name) - if relation is None: - issues.append(f"missing tracked physical seed relation: {seed.name}") - continue - if not physical_relation_lookup.exists( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ): - issues.append(f"missing physical seed relation: {seed.name}") - relation_type = relation_types.get(seed.name) - if relation_type is None: - issues.append(f"missing logical seed target: {seed.name}") - elif normalize_relation_type(relation_type) == RelationType.TABLE: - issues.append(f"logical seed target is table: {seed.name}") - if not issues: - return f"Reconcile report for {virtual_environment_name}: no issues." - return "Reconcile report for " + virtual_environment_name + ":\n- " + "\n- ".join(issues) - - -def repair_view( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - model_name: str, - physical_map: dict[str, PhysicalRelationRecord], -) -> None: - validate_logical_target_repairable( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - model_name=model_name, - ) - relation: PhysicalRelationRecord | None = physical_map.get(model_name) - if relation is None: - raise PlannerInputError( - f"missing tracked physical relation for '{model_name}'", code="C251" - ) - if not physical_relation_exists( - adapter=adapter, connection_config=connection_config, relation=relation - ): - raise PlannerInputError(f"missing physical relation for '{model_name}'", code="C252") - refresh_logical_vde_views( - project=graph.project, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - physical_relations={model_name: relation}, - ) - - -def repair_seed_view( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - seed_name: str, - seed_physical_map: dict[str, PhysicalRelationRecord], -) -> None: - validate_seed_logical_target_repairable( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - seed_name=seed_name, - ) - relation: PhysicalRelationRecord | None = seed_physical_map.get(seed_name) - if relation is None: - raise PlannerInputError( - f"missing tracked physical relation for seed '{seed_name}'", code="C251" - ) - if not physical_relation_exists( - adapter=adapter, connection_config=connection_config, relation=relation - ): - raise PlannerInputError(f"missing physical relation for seed '{seed_name}'", code="C252") - refresh_logical_vde_views( - project=graph.project, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - physical_relations={}, - seed_physical_relations={seed_name: relation}, - ) - - -def validate_logical_target_repairable( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - model_name: str, -) -> None: - relation_types: dict[str, str] = list_virtual_relation_types( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - relation_type: str | None = relation_types.get(model_name) - if relation_type is not None and normalize_relation_type(relation_type) == RelationType.TABLE: - raise PlannerInputError( - f"logical target for '{model_name}' is a table; repair-view will not overwrite it", - code="C250", - ) - - -def validate_seed_logical_target_repairable( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, - seed_name: str, -) -> None: - relation_types: dict[str, str] = list_virtual_relation_types( - graph=graph, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - unsuffixed_virtual_environment_name=unsuffixed_virtual_environment_name, - ) - relation_type: str | None = relation_types.get(seed_name) - if relation_type is not None and normalize_relation_type(relation_type) == RelationType.TABLE: - raise PlannerInputError( - f"logical seed target for '{seed_name}' is a table; repair-view will not overwrite it", - code="C250", - ) - - -def resolve_attach_relation( - *, - adapter: BaseAdapter, - backend: Any, - state_connection: Any, - schema: str, - model_name: str, - physical_relation_name: str, -) -> PhysicalRelationRecord: - candidates: tuple[PhysicalRelationRecord, ...] = backend.list_physical_relations_for_model( - connection=state_connection, - schema=schema, - model_name=model_name, - ) - for candidate in candidates: - rendered: str = resolve_relation_location_qualified_name( - adapter=adapter, location=fallback_target(candidate) - ) - if adapter.relation_names_match(left=rendered, right=physical_relation_name): - return candidate - raise PlannerInputError( - ( - f"physical relation '{physical_relation_name}' is not a tracked relation " - f"for '{model_name}'" - ), - code="C253", - ) - - -def build_attached_refs( - *, - existing_refs: tuple[VirtualEnvironmentModelRefRecord, ...], - virtual_environment_name: str, - model_name: str, - version_hash: str, -) -> tuple[VirtualEnvironmentModelRefRecord, ...]: - ref_map: dict[str, str] = {ref.model_name: ref.version_hash for ref in existing_refs} - ref_map[model_name] = version_hash - return tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=virtual_environment_name, - model_name=name, - version_hash=hash_value, - ) - for name, hash_value in sorted(ref_map.items()) - ) - - -def list_virtual_relation_types( - *, - graph: ProjectGraph, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str, - unsuffixed_virtual_environment_name: str | None, -) -> dict[str, str]: - def virtual_schema_for(schema: str | None) -> str | None: - if schema is None: - return None - if unsuffixed_virtual_environment_name == virtual_environment_name: - return schema - return f"{schema}__{virtual_environment_name}" - - artifacts: tuple[tuple[str, str | None, str | None, str], ...] = tuple( - ( - model.name, - model.destination.database, - virtual_schema_for(model.destination.schema), - model.destination.name, - ) - for model in graph.project.models - ) + tuple( - ( - seed.name, - seed.destination.database, - virtual_schema_for(seed.destination.schema), - seed.destination.name, - ) - for seed in graph.project.seeds - ) - connection: Any = adapter.connect(connection_config) - try: - lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple((database, schema, name) for _, database, schema, name in artifacts), - ) - result: dict[str, str] = {} - for artifact_name, database, schema, name in artifacts: - relation: RelationInfo | None = lookup.get(database=database, schema=schema, name=name) - if relation is not None: - result[artifact_name] = relation.relation_type - return result - finally: - adapter.close(connection) - - -def _build_physical_relation_lookup( - *, - adapter: BaseAdapter, - connection_config: dict[str, object], - relations: tuple[PhysicalRelationRecord, ...], -) -> RelationLookup: - connection: Any = adapter.connect(connection_config) - try: - return build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - (relation.database_name, relation.schema_name, relation.relation_name) - for relation in relations - ), - ) - finally: - adapter.close(connection) - - -def physical_relation_exists( - *, adapter: BaseAdapter, connection_config: dict[str, object], relation: PhysicalRelationRecord -) -> bool: - connection: Any = adapter.connect(connection_config) - try: - return adapter.relation_exists( - connection=connection, - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - ) - finally: - adapter.close(connection) - - -def fallback_target(relation: PhysicalRelationRecord) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - qualified_name=relation.relation_name, - ) - - -def _record_reconcile_event( - *, backend: Any, state_connection: Any, schema: str, action: ReconcileAction, message: str -) -> None: - backend.create_reconcile_event( - connection=state_connection, - schema=schema, - record=ReconcileEventRecord( - event_id=uuid4().hex, - action=action, - status=StateOperationStatus.SUCCEEDED, - message=message, - ), - ) diff --git a/src/sqlbuild/virtual/reconcile/constants.py b/src/sqlbuild/virtual/reconcile/constants.py deleted file mode 100644 index cc1db7035a..0000000000 --- a/src/sqlbuild/virtual/reconcile/constants.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Virtual reconcile constants.""" - -from __future__ import annotations - -RECONCILE_REPAIR_VIEW_COMMAND: str = "repair-view" diff --git a/src/sqlbuild/virtual/reconcile/main/__init__.py b/src/sqlbuild/virtual/reconcile/main/__init__.py deleted file mode 100644 index 36b8b4df5c..0000000000 --- a/src/sqlbuild/virtual/reconcile/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual reconcile public entrypoints.""" diff --git a/src/sqlbuild/virtual/reconcile/main/reconcile.py b/src/sqlbuild/virtual/reconcile/main/reconcile.py deleted file mode 100644 index fa91b99e9a..0000000000 --- a/src/sqlbuild/virtual/reconcile/main/reconcile.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Virtual reconcile public entrypoint.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.reconcile._helpers.reconcile import ( - run_virtual_reconcile as _run_virtual_reconcile, -) - - -def run_virtual_reconcile( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - adapter: BaseAdapter, - connection_config: dict[str, object], - virtual_environment_name: str | None, - command: str | None, - model_name: str | None, - seed_name: str | None, - physical_relation_name: str | None, -) -> str: - """Inspect or repair virtual reconcile state.""" - - return _run_virtual_reconcile( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - virtual_environment_name=virtual_environment_name, - command=command, - model_name=model_name, - seed_name=seed_name, - physical_relation_name=physical_relation_name, - ) diff --git a/src/sqlbuild/virtual/state/__init__.py b/src/sqlbuild/virtual/state/__init__.py deleted file mode 100644 index 83579b9fcb..0000000000 --- a/src/sqlbuild/virtual/state/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual mode state store package.""" diff --git a/src/sqlbuild/virtual/state/_helpers/__init__.py b/src/sqlbuild/virtual/state/_helpers/__init__.py deleted file mode 100644 index 9bfab89907..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""State backend helper package.""" diff --git a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/__init__.py b/src/sqlbuild/virtual/state/_helpers/state_lifecycle/__init__.py deleted file mode 100644 index fbb3f52c7a..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state lifecycle helpers.""" diff --git a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/adopt.py b/src/sqlbuild/virtual/state/_helpers/state_lifecycle/adopt.py deleted file mode 100644 index 084763303d..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/adopt.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Virtual state adopt helper logic.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationLookup -from sqlbuild.adapter.contract.types import RelationType -from sqlbuild.adapter.relations.main.relation_lookup import build_relation_lookup -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.executor.main._logical_target import build_virtual_logical_destination -from sqlbuild.virtual.executor.main._physical_target import build_virtual_physical_destination -from sqlbuild.virtual.executor.main._relation_type import resolve_model_relation_type -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.main.environments._record_operation import record_state_operation -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - PhysicalRelationRecord, - StateBackendConfig, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - StateOperationStatus, - StateOperationType, - VirtualEnvironmentStatus, -) - - -def adopt_into_virtual_state( - *, - discovered_inputs: DiscoveredProjectInputs, - config: StateBackendConfig, - backend: StateBackend, - state_connection: Any, - adapter: BaseAdapter, - connection: Any, - allow_copy: bool, -) -> str: - active_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if active_target_name is None: - raise PlannerInputError("state adopt requires an active target", code="C259") - operation_id: str = f"adopt:{active_target_name}" - record_state_operation( - backend=backend, - connection=state_connection, - schema=config.schema, - operation_id=operation_id, - operation_type=StateOperationType.ADOPT, - status=StateOperationStatus.RUNNING, - action="start", - virtual_environment_name=active_target_name, - message="starting adopt", - ) - try: - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - ) - refs: list[VirtualEnvironmentModelRefRecord] = [] - recorder: StatementRecorder = StatementRecorder() - relation_lookup: RelationLookup = build_relation_lookup( - adapter=adapter, - connection=connection, - locations=tuple( - ( - model.destination.database, - model.destination.schema, - model.destination.name, - ) - for model in graph.project.models - ), - ) - for model in graph.project.models: - if not relation_lookup.exists( - database=model.destination.database, - schema=model.destination.schema, - name=model.destination.name, - ): - continue - version_hash: str = model.name - physical_target: CompiledRelationLocation = build_virtual_physical_destination( - adapter=adapter, - target=model.destination, - model_name=model.name, - version_hash=version_hash, - ) - adapter.ensure_schema( - connection=connection, - database=physical_target.database, - schema=physical_target.schema, - statement_recorder=recorder, - ) - model_relation_type: str = resolve_model_relation_type( - str(model.config.values.get("materialized", "table")) - ) - adapter.move_or_copy_relation( - connection=connection, - origin=resolve_relation_location_qualified_name( - adapter=adapter, location=model.destination - ), - destination=resolve_relation_location_qualified_name( - adapter=adapter, location=physical_target - ), - remove_origin=model_relation_type != RelationType.VIEW, - allow_copy_fallback=allow_copy, - statement_recorder=recorder, - ) - if model_relation_type == RelationType.VIEW: - adapter.drop_view( - connection=connection, - destination=resolve_relation_location_qualified_name( - adapter=adapter, location=model.destination - ), - statement_recorder=recorder, - ) - virtual_target: CompiledRelationLocation = build_virtual_logical_destination( - adapter=adapter, - target=model.destination, - virtual_environment_name=active_target_name, - unsuffixed_virtual_environment_name=active_target_name, - ) - adapter.create_view_as( - connection=connection, - destination=resolve_relation_location_qualified_name( - adapter=adapter, location=virtual_target - ), - sql=( - "SELECT * FROM " - + resolve_relation_location_qualified_name( - adapter=adapter, location=physical_target - ) - ), - statement_recorder=recorder, - ) - backend.upsert_model_version( - connection=state_connection, - schema=config.schema, - record=ModelVersionRecord( - model_name=model.name, - version_hash=version_hash, - definition_identity_hash=version_hash, - identity_metadata_hash=version_hash, - status=ModelVersionStatus.READY, - ), - ) - backend.upsert_physical_relation( - connection=state_connection, - schema=config.schema, - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=model.name, - version_hash=version_hash, - database_name=physical_target.database, - schema_name=physical_target.schema or "", - relation_name=physical_target.name, - relation_type="table", - ), - ) - refs.append( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=active_target_name, - model_name=model.name, - version_hash=version_hash, - ) - ) - backend.upsert_virtual_environment( - connection=state_connection, - schema=config.schema, - record=VirtualEnvironmentRecord( - virtual_environment_name=active_target_name, - status=VirtualEnvironmentStatus.FINALIZED, - ), - ) - backend.replace_virtual_environment_model_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=active_target_name, - refs=tuple(refs), - ) - record_state_operation( - backend=backend, - connection=state_connection, - schema=config.schema, - operation_id=operation_id, - operation_type=None, - status=StateOperationStatus.SUCCEEDED, - action="finish", - virtual_environment_name=None, - message=f"adopted {len(refs)} models", - ) - return f"Adopted {len(refs)} models into virtual environment {active_target_name}." - except BaseException as error: - record_state_operation( - backend=backend, - connection=state_connection, - schema=config.schema, - operation_id=operation_id, - operation_type=None, - status=StateOperationStatus.FAILED, - action="fail", - virtual_environment_name=None, - message=str(error), - ) - raise diff --git a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/detach.py b/src/sqlbuild/virtual/state/_helpers/state_lifecycle/detach.py deleted file mode 100644 index 4ffbaa5043..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/detach.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Virtual state detach helper logic.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.types import RelationType -from sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name import ( - resolve_relation_location_qualified_name, -) -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.virtual.executor.main._logical_target import build_virtual_logical_destination -from sqlbuild.virtual.executor.main._relation_type import resolve_model_relation_type -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.main.environments._record_operation import record_state_operation -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - StateBackendConfig, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import ( - StateOperationStatus, - StateOperationType, - VirtualEnvironmentStatus, -) - - -def detach_from_virtual_state( - *, - discovered_inputs: DiscoveredProjectInputs, - config: StateBackendConfig, - backend: StateBackend, - state_connection: Any, - adapter: BaseAdapter, - connection: Any, - allow_copy: bool, -) -> str: - active_target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=None, - ) - if active_target_name is None: - raise PlannerInputError("state detach requires an active target", code="C260") - environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=state_connection, - schema=config.schema, - virtual_environment_name=active_target_name, - ) - if environment is None or environment.status != VirtualEnvironmentStatus.FINALIZED: - raise PlannerInputError( - "state detach requires a finalized virtual environment", - code="C261", - ) - operation_id: str = f"detach:{active_target_name}" - record_state_operation( - backend=backend, - connection=state_connection, - schema=config.schema, - operation_id=operation_id, - operation_type=StateOperationType.DETACH, - status=StateOperationStatus.RUNNING, - action="start", - virtual_environment_name=active_target_name, - message="starting detach", - ) - try: - graph: ProjectGraph = build_project_graph( - discovered_inputs=discovered_inputs, - adapter=adapter, - ) - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=state_connection, - schema=config.schema, - virtual_environment_name=active_target_name, - ) - ) - ref_map: dict[str, str] = {ref.model_name: ref.version_hash for ref in refs} - recorder: StatementRecorder = StatementRecorder() - detached_count: int = 0 - for model in graph.project.models: - version_hash: str | None = ref_map.get(model.name) - if version_hash is None: - continue - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=state_connection, - schema=config.schema, - model_name=model.name, - version_hash=version_hash, - ) - if relation is None: - continue - adapter.drop_view( - connection=connection, - destination=resolve_relation_location_qualified_name( - adapter=adapter, - location=build_virtual_logical_destination( - adapter=adapter, - target=model.destination, - virtual_environment_name=active_target_name, - unsuffixed_virtual_environment_name=active_target_name, - ), - ), - statement_recorder=recorder, - ) - model_relation_type: str = resolve_model_relation_type( - str(model.config.values.get("materialized", "table")) - ) - if model_relation_type == RelationType.VIEW: - adapter.create_view_as( - connection=connection, - destination=resolve_relation_location_qualified_name( - adapter=adapter, location=model.destination - ), - sql=model.query_sql, - statement_recorder=recorder, - ) - detached_count += 1 - continue - physical_target: CompiledRelationLocation = CompiledRelationLocation( - database=relation.database_name, - schema=relation.schema_name, - name=relation.relation_name, - qualified_name=relation.relation_name, - ) - adapter.move_or_copy_relation( - connection=connection, - origin=resolve_relation_location_qualified_name( - adapter=adapter, location=physical_target - ), - destination=resolve_relation_location_qualified_name( - adapter=adapter, location=model.destination - ), - remove_origin=False, - allow_copy_fallback=allow_copy, - statement_recorder=recorder, - ) - detached_count += 1 - backend.upsert_virtual_environment( - connection=state_connection, - schema=config.schema, - record=VirtualEnvironmentRecord( - virtual_environment_name=active_target_name, - status=VirtualEnvironmentStatus.DETACHED, - baseline_virtual_environment_name=environment.baseline_virtual_environment_name, - finalized_at=environment.finalized_at, - ), - ) - record_state_operation( - backend=backend, - connection=state_connection, - schema=config.schema, - operation_id=operation_id, - operation_type=None, - status=StateOperationStatus.SUCCEEDED, - action="finish", - virtual_environment_name=None, - message=f"detached {detached_count} models", - ) - return f"Detached {detached_count} models from virtual environment {active_target_name}." - except BaseException as error: - record_state_operation( - backend=backend, - connection=state_connection, - schema=config.schema, - operation_id=operation_id, - operation_type=None, - status=StateOperationStatus.FAILED, - action="fail", - virtual_environment_name=None, - message=str(error), - ) - raise diff --git a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/detached_retention.py b/src/sqlbuild/virtual/state/_helpers/state_lifecycle/detached_retention.py deleted file mode 100644 index 9048d440d5..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/detached_retention.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Detached virtual environment retention helpers.""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta - -from sqlbuild.virtual.state.models import ( - DetachedVirtualEnvironmentInspection, - PhysicalRelationRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRetentionRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - - -def build_detached_environment_inspection( - *, - environments: tuple[VirtualEnvironmentRetentionRecord, ...], - refs_by_environment: dict[str, tuple[VirtualEnvironmentModelRefRecord, ...]], - physical_relations_by_ref: dict[tuple[str, str], PhysicalRelationRecord], - retention_days: int, - now: datetime, -) -> DetachedVirtualEnvironmentInspection: - """Classify detached cleanup candidates and retained physical relation refs.""" - - cleanup_names: set[str] = { - environment.virtual_environment_name - for environment in environments - if _eligible_for_cleanup( - environment=environment, - retention_days=retention_days, - now=now, - ) - } - cleanup_relations: dict[tuple[str | None, str, str], PhysicalRelationRecord] = {} - retained_relations: dict[tuple[str | None, str, str], PhysicalRelationRecord] = {} - environment: VirtualEnvironmentRetentionRecord - for environment in environments: - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = refs_by_environment.get( - environment.virtual_environment_name, - (), - ) - ref: VirtualEnvironmentModelRefRecord - for ref in refs: - relation: PhysicalRelationRecord | None = physical_relations_by_ref.get( - (ref.model_name, ref.version_hash) - ) - if relation is None: - continue - relation_key: tuple[str | None, str, str] = ( - relation.database_name, - relation.schema_name, - relation.relation_name, - ) - if environment.virtual_environment_name in cleanup_names: - cleanup_relations[relation_key] = relation - continue - retained_relations[relation_key] = relation - return DetachedVirtualEnvironmentInspection( - cleanup_virtual_environments=tuple( - environment - for environment in environments - if environment.virtual_environment_name in cleanup_names - ), - cleanup_physical_relations=tuple( - cleanup_relations[key] - for key in sorted(cleanup_relations, key=lambda item: (item[0] or "", item[1], item[2])) - ), - retained_physical_relations=tuple( - retained_relations[key] - for key in sorted( - retained_relations, key=lambda item: (item[0] or "", item[1], item[2]) - ) - ), - ) - - -def _eligible_for_cleanup( - *, environment: VirtualEnvironmentRetentionRecord, retention_days: int, now: datetime -) -> bool: - if environment.status != VirtualEnvironmentStatus.DETACHED: - return False - if retention_days == 0: - return True - if environment.updated_at is None: - return False - return _aware(environment.updated_at) <= _aware(now) - timedelta(days=retention_days) - - -def _aware(value: datetime) -> datetime: - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value diff --git a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/lifecycle_output.py b/src/sqlbuild/virtual/state/_helpers/state_lifecycle/lifecycle_output.py deleted file mode 100644 index 48007abed0..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/lifecycle_output.py +++ /dev/null @@ -1,73 +0,0 @@ -"""CLI summary rendering for state lifecycle commands.""" - -from __future__ import annotations - -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.virtual.state.constants import STATE_TABLE_COLUMNS -from sqlbuild.virtual.state.models import StateBackendConfig - - -def format_state_lifecycle_summary( - *, - title: str, - config: StateBackendConfig, - use_color: bool, - backup_id: str | None = None, -) -> str: - """Render the post-command state store summary block.""" - - style: CliStyle = CliStyle(use_color=use_color) - rendered_title: str = style.success_strong(title) - state_label: str = style.success("State store:") - tables_label: str = style.success("Tables:") - lines: list[str] = ["", rendered_title, "", state_label] - lines.append(_summary_row(label="backend", value=config.backend.value, use_color=use_color)) - lines.append(_summary_row(label="schema", value=config.schema, use_color=use_color)) - database: object | None = config.connection.get("database") - if database is not None: - lines.append(_summary_row(label="database", value=str(database), use_color=use_color)) - if backup_id is not None: - lines.append(_summary_row(label="backup", value=backup_id, use_color=use_color)) - lines.append("") - lines.append(tables_label) - lines.append( - _summary_row( - label="created/validated", - value=str(len(STATE_TABLE_COLUMNS)), - use_color=use_color, - ) - ) - lines.append( - _summary_row( - label="current state", - value=( - "model_versions, function_versions, physical_relations, " - "physical_relation_ancestry, virtual_environments, " - "virtual_environment_node_refs, locks" - ), - use_color=use_color, - emphasize_value=False, - ) - ) - lines.append( - _summary_row( - label="history", - value=( - "virtual_environment_checkpoints, virtual_environment_checkpoint_model_refs, " - "virtual_environment_checkpoint_function_refs, plan_runs, " - "virtual_environment_model_ref_events, microbatch_events, reconcile_events, " - "state_migration_events" - ), - use_color=use_color, - emphasize_value=False, - ) - ) - lines.append("") - return "\n".join(lines) - - -def _summary_row(*, label: str, value: str, use_color: bool, emphasize_value: bool = True) -> str: - style: CliStyle = CliStyle(use_color=use_color) - rendered_label: str = style.muted(f"{label}:") - rendered_value: str = style.object_name(value) if emphasize_value else value - return f" {rendered_label:<24} {rendered_value}" diff --git a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/microbatch_replay_retention.py b/src/sqlbuild/virtual/state/_helpers/state_lifecycle/microbatch_replay_retention.py deleted file mode 100644 index 5d017913d3..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_lifecycle/microbatch_replay_retention.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Derive physical-version retention roots from virtual microbatch replay history.""" - -from __future__ import annotations - -from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage -from sqlbuild.microbatches.main.project_replay import project_replay_requirement -from sqlbuild.microbatches.models import ( - MicrobatchCoverageProjection, - MicrobatchEvent, - MicrobatchInterval, - ReplayRequirementProjection, -) -from sqlbuild.microbatches.types import MicrobatchRecordType, ReplayRequirementState -from sqlbuild.virtual.state.models import MicrobatchReplayRoot - - -def active_microbatch_replay_roots( - *, events: tuple[MicrobatchEvent, ...] -) -> tuple[MicrobatchReplayRoot, ...]: - """Return model versions whose latest replay requirement is incomplete.""" - - latest_requirements: dict[tuple[str, str], MicrobatchEvent] = {} - for event in events: - if ( - event.record_type != MicrobatchRecordType.REPLAY_REQUIREMENT - or event.required_model_version_hash is None - ): - continue - identity: tuple[str, str] = ( - event.scope.model_name, - event.scope.physical_generation_id.partition(":")[0], - ) - current: MicrobatchEvent | None = latest_requirements.get(identity) - if current is None or (current.created_at, current.event_id) < ( - event.created_at, - event.event_id, - ): - latest_requirements[identity] = event - - roots: list[MicrobatchReplayRoot] = [] - for requirement in latest_requirements.values(): - required_version_hash: str | None = requirement.required_model_version_hash - if required_version_hash is None: - continue - scope_events: tuple[MicrobatchEvent, ...] = tuple( - event - for event in events - if event.scope.scope_kind == requirement.scope.scope_kind - and event.scope.scope_key == requirement.scope.scope_key - and event.scope.physical_generation_id == requirement.scope.physical_generation_id - ) - expected: tuple[MicrobatchInterval, ...] = ( - MicrobatchInterval(start=requirement.run_start, end=requirement.run_end), - ) - coverage: MicrobatchCoverageProjection = project_microbatch_coverage( - events=scope_events, - expected_intervals=expected, - cursor_type=requirement.cursor_type, - ) - projection: ReplayRequirementProjection = project_replay_requirement( - requirement=requirement, - current_model_version_hash=required_version_hash, - expected_intervals=expected, - coverage=coverage, - cursor_type=requirement.cursor_type, - ) - if projection.state == ReplayRequirementState.INCOMPLETE: - roots.append( - MicrobatchReplayRoot( - model_name=requirement.scope.model_name, - version_hash=required_version_hash, - previous_version_hash=requirement.previous_model_version_hash, - ) - ) - return tuple(sorted(roots, key=lambda root: (root.model_name, root.version_hash))) diff --git a/src/sqlbuild/virtual/state/_helpers/state_runtime/__init__.py b/src/sqlbuild/virtual/state/_helpers/state_runtime/__init__.py deleted file mode 100644 index b2b8d4b705..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_runtime/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state runtime helpers.""" diff --git a/src/sqlbuild/virtual/state/_helpers/state_runtime/backend.py b/src/sqlbuild/virtual/state/_helpers/state_runtime/backend.py deleted file mode 100644 index a76e4bd2a1..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_runtime/backend.py +++ /dev/null @@ -1,20 +0,0 @@ -"""State backend construction helpers.""" - -from __future__ import annotations - -from sqlbuild.virtual.state.classes.duckdb import DuckDbStateBackend -from sqlbuild.virtual.state.classes.postgres import PostgresStateBackend -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.types import StateBackendName - - -def build_state_backend(backend_name: StateBackendName) -> StateBackend: - """Build the configured virtual state backend implementation.""" - - match backend_name: - case StateBackendName.DUCKDB: - return DuckDbStateBackend() - case StateBackendName.POSTGRES: - return PostgresStateBackend() - raise StateBackendConfigError(f"Unsupported state backend: {backend_name}") diff --git a/src/sqlbuild/virtual/state/_helpers/state_runtime/config.py b/src/sqlbuild/virtual/state/_helpers/state_runtime/config.py deleted file mode 100644 index ff3ce67838..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_runtime/config.py +++ /dev/null @@ -1,77 +0,0 @@ -"""State backend config resolution helpers.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config -from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name -from sqlbuild.spec.contracts.models import TargetConfig -from sqlbuild.virtual.state.constants import DUCKDB_MEMORY_DATABASE -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import StateBackendConfig -from sqlbuild.virtual.state.types import StateBackendName - - -def resolve_state_backend_config( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - selected_target: str | None = None, -) -> StateBackendConfig: - """Resolve state backend config from the active physical target.""" - - if not discovered_inputs.project_config.settings.virtual_environments: - raise StateBackendConfigError("State commands require virtual_environments = true") - - target_name: str | None = resolve_target_name( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - selected_target=selected_target, - ) - if target_name is None: - raise StateBackendConfigError( - "State commands require an active target with [targets..state]" - ) - target_config: TargetConfig = resolve_target_config( - project_config=discovered_inputs.project_config, - local_config=discovered_inputs.local_config, - target_name=target_name, - ) - backend_name: str | None = target_config.state.backend - if backend_name is None: - raise StateBackendConfigError(f"Target '{target_name}' does not configure a state backend") - schema: str | None = target_config.state.schema - if schema is None: - raise StateBackendConfigError(f"Target '{target_name}' state config must define schema") - try: - backend: StateBackendName = StateBackendName(backend_name) - except ValueError as error: - raise StateBackendConfigError(f"Unsupported state backend: {backend_name}") from error - connection: dict[str, object] = _resolve_connection_config( - raw_config=target_config.state.connection, - project_dir=project_dir, - backend=backend, - ) - return StateBackendConfig( - backend=backend, - schema=schema, - connection=connection, - allow_reset=target_config.state.allow_reset, - ) - - -def _resolve_connection_config( - *, raw_config: dict[str, object], project_dir: Path, backend: StateBackendName -) -> dict[str, object]: - config: dict[str, object] = dict(raw_config) - database: object | None = config.get("database") - if ( - backend == StateBackendName.DUCKDB - and isinstance(database, str) - and not Path(database).is_absolute() - and database != DUCKDB_MEMORY_DATABASE - ): - config["database"] = str(project_dir / database) - return config diff --git a/src/sqlbuild/virtual/state/_helpers/state_runtime/runtime.py b/src/sqlbuild/virtual/state/_helpers/state_runtime/runtime.py deleted file mode 100644 index dc88984707..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_runtime/runtime.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Runtime helpers for virtual state backend access.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.backend import build_state_backend -from sqlbuild.virtual.state._helpers.state_runtime.config import resolve_state_backend_config -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateBackendConfig - - -def build_state_runtime( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - selected_target: str | None = None, -) -> tuple[StateBackendConfig, StateBackend]: - """Resolve config and construct the configured virtual state backend.""" - - config: StateBackendConfig = resolve_state_backend_config( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - selected_target=selected_target, - ) - return config, build_state_backend(config.backend) diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/__init__.py b/src/sqlbuild/virtual/state/_helpers/state_storage/__init__.py deleted file mode 100644 index 40dedf499e..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state storage helpers.""" diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/datetime.py b/src/sqlbuild/virtual/state/_helpers/state_storage/datetime.py deleted file mode 100644 index 628e69a05c..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/datetime.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Virtual-state naive UTC wall-clock datetime boundary helpers.""" - -from __future__ import annotations - -from datetime import UTC, datetime - - -def to_naive_utc_wall_clock(value: datetime) -> datetime: - """Encode an application datetime for a naive UTC virtual-state TIMESTAMP column.""" - - if value.tzinfo is None or value.utcoffset() is None: - return value - return value.astimezone(UTC).replace(tzinfo=None) - - -def from_naive_utc_wall_clock(value: datetime) -> datetime: - """Decode a naive UTC virtual-state TIMESTAMP as an aware UTC datetime.""" - - if value.tzinfo is None or value.utcoffset() is None: - return value.replace(tzinfo=UTC) - return value.astimezone(UTC) diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/encoding.py b/src/sqlbuild/virtual/state/_helpers/state_storage/encoding.py deleted file mode 100644 index 4d74c24522..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/encoding.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Private encoding implementation for portable virtual state text payloads.""" - -from __future__ import annotations - -import base64 - - -def encode_state_text_impl(value: str) -> str: - return base64.b64encode(value.encode("utf-8")).decode("ascii") - - -def decode_state_text_impl(value: str | None) -> str | None: - if value is None: - return None - return base64.b64decode(value.encode("ascii")).decode("utf-8") diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/events.py b/src/sqlbuild/virtual/state/_helpers/state_storage/events.py deleted file mode 100644 index 6df4c20dc2..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/events.py +++ /dev/null @@ -1,18 +0,0 @@ -"""State event helper functions.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from uuid import uuid4 - - -def backup_id() -> str: - """Return a portable backup id safe for schema names.""" - - return datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ") - - -def event_id() -> str: - """Return a unique migration event id.""" - - return uuid4().hex diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py b/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py deleted file mode 100644 index 4eab0e47fd..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/locks.py +++ /dev/null @@ -1,111 +0,0 @@ -"""State lock helper services.""" - -from __future__ import annotations - -from datetime import datetime, timedelta -from typing import Any - -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateLockLease - - -def virtual_environment_lock_key(virtual_environment_name: str) -> str: - """Return the lock key for a virtual environment mutation.""" - - return f"virtual_env:{virtual_environment_name}" - - -def model_version_lock_key(*, model_name: str, version_hash: str) -> str: - """Return the lock key for a physical model version mutation.""" - - return f"model_version:{model_name}:{version_hash}" - - -def acquire_virtual_environment_lock( - *, - backend: StateBackend, - connection: Any, - schema: str, - virtual_environment_name: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire the lock for a virtual environment mutation.""" - - return acquire_state_lock( - backend=backend, - connection=connection, - schema=schema, - lock_key=virtual_environment_lock_key(virtual_environment_name), - owner_id=owner_id, - ttl=ttl, - now=now, - ) - - -def acquire_model_version_lock( - *, - backend: StateBackend, - connection: Any, - schema: str, - model_name: str, - version_hash: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire the lock for a physical model version mutation.""" - - return acquire_state_lock( - backend=backend, - connection=connection, - schema=schema, - lock_key=model_version_lock_key(model_name=model_name, version_hash=version_hash), - owner_id=owner_id, - ttl=ttl, - now=now, - ) - - -def acquire_state_lock( - *, - backend: StateBackend, - connection: Any, - schema: str, - lock_key: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire a state lock and return a lease when successful.""" - - base_time: datetime = now or datetime.now() - expires_at: datetime = base_time + ttl - acquired: bool = backend.acquire_lock( - connection=connection, - schema=schema, - lock_key=lock_key, - owner_id=owner_id, - expires_at=expires_at, - ) - if not acquired: - return None - return StateLockLease(lock_key=lock_key, owner_id=owner_id, expires_at=expires_at) - - -def release_state_lock( - *, - backend: StateBackend, - connection: Any, - schema: str, - lease: StateLockLease, -) -> bool: - """Release a previously acquired state lock lease.""" - - return backend.release_lock( - connection=connection, - schema=schema, - lock_key=lease.lock_key, - owner_id=lease.owner_id, - ) diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py b/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py deleted file mode 100644 index a9acb526ac..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/microbatch_events.py +++ /dev/null @@ -1,98 +0,0 @@ -"""DuckDB storage helpers for virtual microbatch events.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.constants import ( - MICROBATCH_COLUMNS, - MICROBATCH_GENERATION_WILDCARD, - VIRTUAL_MICROBATCH_SCOPE_KIND, -) -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult - - -def append_duckdb_microbatch_events( - *, connection: Any, qualified_table: str, events: tuple[MicrobatchEvent, ...] -) -> MicrobatchWriteResult: - """Append idempotent logical events to DuckDB state in one statement.""" - - if not events: - return MicrobatchWriteResult(total=0, inserted=0, already_existing=0) - id_placeholders: str = ", ".join("?" for _ in events) - existing_rows: list[tuple[str]] = connection.execute( - f"SELECT event_id FROM {qualified_table} WHERE event_id IN ({id_placeholders})", - [event.event_id for event in events], - ).fetchall() - existing_ids: frozenset[str] = frozenset(row[0] for row in existing_rows) - missing: tuple[MicrobatchEvent, ...] = tuple( - event for event in events if event.event_id not in existing_ids - ) - if missing: - row_placeholders: str = "(" + ", ".join("?" for _ in MICROBATCH_COLUMNS) + ")" - values_sql: str = ", ".join(row_placeholders for _ in missing) - params: list[object | None] = [] - for event in missing: - params.extend(MicrobatchEventCodec.values(event)) - columns: str = ", ".join(MICROBATCH_COLUMNS) - connection.execute( - f"INSERT INTO {qualified_table} ({columns}) SELECT {columns} FROM " - f"(VALUES {values_sql}) AS incoming ({columns}) WHERE NOT EXISTS " - f"(SELECT 1 FROM {qualified_table} existing " - "WHERE existing.event_id = incoming.event_id)", - params, - ) - return MicrobatchWriteResult( - total=len(events), inserted=len(missing), already_existing=len(events) - len(missing) - ) - - -def read_duckdb_microbatch_scope_history( - *, connection: Any, qualified_table: str, scope: MicrobatchScope -) -> tuple[MicrobatchEvent, ...]: - """Read ordered history for one DuckDB physical scope.""" - - generation_sql: str = ( - "" - if scope.physical_generation_id == MICROBATCH_GENERATION_WILDCARD - else "AND physical_generation_id = ? " - ) - params: list[object] = [scope.scope_kind, scope.scope_key] - if scope.physical_generation_id != MICROBATCH_GENERATION_WILDCARD: - params.append(scope.physical_generation_id) - rows: list[tuple[Any, ...]] = connection.execute( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM {qualified_table} " - f"WHERE scope_kind = ? AND scope_key = ? {generation_sql}" - "ORDER BY created_at, event_id", - params, - ).fetchall() - return MicrobatchEventCodec.from_rows(tuple(row) for row in rows) - - -def read_duckdb_microbatch_retention_history( - *, connection: Any, qualified_table: str -) -> tuple[MicrobatchEvent, ...]: - """Read ordered virtual-scope history from DuckDB state.""" - - rows: list[tuple[Any, ...]] = connection.execute( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM {qualified_table} " - "WHERE scope_kind = ? ORDER BY created_at, event_id", - [VIRTUAL_MICROBATCH_SCOPE_KIND], - ).fetchall() - return MicrobatchEventCodec.from_rows(tuple(row) for row in rows) - - -def read_duckdb_microbatch_model_history( - *, connection: Any, qualified_table: str, scope: MicrobatchScope -) -> tuple[MicrobatchEvent, ...]: - """Read ordered history for one model in one warehouse realm.""" - - warehouse_realm: str = scope.physical_generation_id.partition(":")[0] - rows: list[tuple[Any, ...]] = connection.execute( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM {qualified_table} " - "WHERE scope_kind = ? AND model_name = ? AND physical_generation_id LIKE ? " - "ORDER BY created_at, event_id", - [scope.scope_kind, scope.model_name, f"{warehouse_realm}:%"], - ).fetchall() - return MicrobatchEventCodec.from_rows(tuple(row) for row in rows) diff --git a/src/sqlbuild/virtual/state/_helpers/state_storage/validation.py b/src/sqlbuild/virtual/state/_helpers/state_storage/validation.py deleted file mode 100644 index 14b740d969..0000000000 --- a/src/sqlbuild/virtual/state/_helpers/state_storage/validation.py +++ /dev/null @@ -1,222 +0,0 @@ -"""State schema validation helper functions.""" - -from __future__ import annotations - -from sqlbuild.virtual.state.constants import STATE_TABLES -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import ( - StateSchemaValidationIssue, - StateSchemaValidationResult, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import ( - StateColumnType, - StateSchemaValidationIssueKind, - StateTypeMatcher, - VirtualEnvironmentStatus, -) - -_MODEL_NODE_TYPE: str = "model" -_SEED_NODE_TYPE: str = "seed" -_FUNCTION_NODE_TYPES: tuple[str, ...] = ("udf", "table_fn") - - -def build_validation_result( - *, - existing_tables: set[str], - columns_by_table: dict[str, dict[str, str]], - expected_columns: dict[str, dict[str, StateColumnType]], - type_matches: StateTypeMatcher, - expected_indexes: dict[str, dict[str, tuple[str, ...]]] | None = None, - existing_indexes_by_table: dict[str, set[str]] | None = None, -) -> StateSchemaValidationResult: - """Build validation issues for required tables and columns.""" - - issues: list[StateSchemaValidationIssue] = [] - table_name: str - for table_name in STATE_TABLES: - if table_name not in existing_tables: - issues.append( - StateSchemaValidationIssue( - kind=StateSchemaValidationIssueKind.MISSING_TABLE, - table_name=table_name, - message=f"Missing state table: {table_name}", - ) - ) - continue - column_name: str - expected_type: StateColumnType - actual_columns: dict[str, str] = columns_by_table.get(table_name, {}) - for column_name, expected_type in expected_columns[table_name].items(): - actual_type: str | None = actual_columns.get(column_name) - if actual_type is None: - issues.append( - StateSchemaValidationIssue( - kind=StateSchemaValidationIssueKind.MISSING_COLUMN, - table_name=table_name, - column_name=column_name, - message=f"Missing state column: {table_name}.{column_name}", - ) - ) - continue - if not type_matches(actual_type=actual_type, expected_type=expected_type): - issues.append( - StateSchemaValidationIssue( - kind=StateSchemaValidationIssueKind.WRONG_TYPE, - table_name=table_name, - column_name=column_name, - message=( - f"Wrong state column type: {table_name}.{column_name} " - f"expected {expected_type.value}, got {actual_type}" - ), - ) - ) - if expected_indexes is not None: - actual_indexes: set[str] = (existing_indexes_by_table or {}).get(table_name, set()) - index_name: str - for index_name in expected_indexes.get(table_name, {}): - if index_name not in actual_indexes: - issues.append( - StateSchemaValidationIssue( - kind=StateSchemaValidationIssueKind.MISSING_INDEX, - table_name=table_name, - message=f"Missing state index: {table_name}.{index_name}", - ) - ) - return StateSchemaValidationResult(issues=tuple(issues)) - - -def validate_conditional_virtual_environment_publication( - *, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - checkpoint: VirtualEnvironmentCheckpointRecord | None, - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...], - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...], -) -> None: - """Validate one conditional environment/ref/checkpoint publication before writes begin.""" - - has_checkpoint_payload: bool = checkpoint is not None or bool( - checkpoint_refs or checkpoint_function_refs or checkpoint_seed_refs - ) - if record.status == VirtualEnvironmentStatus.FINALIZED and checkpoint is None: - raise StateBackendConfigError( - "Finalized conditional virtual environment publication requires a checkpoint" - ) - if record.status != VirtualEnvironmentStatus.FINALIZED and has_checkpoint_payload: - raise StateBackendConfigError( - "Conditional checkpoint payload requires finalized virtual environment status" - ) - if checkpoint is None: - _validate_current_ref_groups(record=record, refs_by_node_type=refs_by_node_type) - return - if checkpoint.virtual_environment_name != record.virtual_environment_name: - raise StateBackendConfigError( - "Checkpoint virtual_environment_name must match the published environment" - ) - - _validate_current_ref_groups(record=record, refs_by_node_type=refs_by_node_type) - checkpoint_id: str = checkpoint.checkpoint_id - _validate_checkpoint_ids( - checkpoint_id=checkpoint_id, - refs=checkpoint_refs, - function_refs=checkpoint_function_refs, - seed_refs=checkpoint_seed_refs, - ) - _validate_exact_ref_pairs( - label="model", - current_pairs=tuple( - (ref.node_name, ref.version_hash) for ref in refs_by_node_type.get(_MODEL_NODE_TYPE, ()) - ), - checkpoint_pairs=tuple((ref.model_name, ref.version_hash) for ref in checkpoint_refs), - ) - _validate_exact_ref_pairs( - label="function", - current_pairs=_current_function_pairs(refs_by_node_type), - checkpoint_pairs=tuple( - (ref.function_name, ref.version_hash) for ref in checkpoint_function_refs - ), - ) - _validate_exact_ref_pairs( - label="seed", - current_pairs=tuple( - (ref.node_name, ref.version_hash) for ref in refs_by_node_type.get(_SEED_NODE_TYPE, ()) - ), - checkpoint_pairs=tuple((ref.seed_name, ref.version_hash) for ref in checkpoint_seed_refs), - ) - - -def _validate_current_ref_groups( - *, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], -) -> None: - node_type: str - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - for node_type, refs in refs_by_node_type.items(): - identities: list[str] = [] - ref: VirtualEnvironmentNodeRefRecord - for ref in refs: - if ref.virtual_environment_name != record.virtual_environment_name: - raise StateBackendConfigError( - "Published ref virtual_environment_name must match the environment record" - ) - if ref.node_type != node_type: - raise StateBackendConfigError("Published ref node_type must match its ref group") - identities.append(ref.node_name) - if len(identities) != len(set(identities)): - raise StateBackendConfigError( - f"Published {node_type} refs contain duplicate identities" - ) - - -def _current_function_pairs( - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], -) -> tuple[tuple[str, str], ...]: - pairs: list[tuple[str, str]] = [] - node_type: str - for node_type in _FUNCTION_NODE_TYPES: - ref: VirtualEnvironmentNodeRefRecord - for ref in refs_by_node_type.get(node_type, ()): - pairs.append((ref.node_name, ref.version_hash)) - return tuple(pairs) - - -def _validate_checkpoint_ids( - *, - checkpoint_id: str, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...], - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...], -) -> None: - actual_ids: tuple[str, ...] = tuple( - ref.checkpoint_id for ref in (*refs, *function_refs, *seed_refs) - ) - if any(actual_id != checkpoint_id for actual_id in actual_ids): - raise StateBackendConfigError( - "Every checkpoint ref checkpoint_id must match the checkpoint record" - ) - - -def _validate_exact_ref_pairs( - *, - label: str, - current_pairs: tuple[tuple[str, str], ...], - checkpoint_pairs: tuple[tuple[str, str], ...], -) -> None: - current_identities: tuple[str, ...] = tuple(identity for identity, _ in current_pairs) - checkpoint_identities: tuple[str, ...] = tuple(identity for identity, _ in checkpoint_pairs) - if len(checkpoint_identities) != len(set(checkpoint_identities)): - raise StateBackendConfigError(f"Checkpoint {label} refs contain duplicate identities") - if len(current_identities) != len(set(current_identities)): - raise StateBackendConfigError(f"Published {label} refs contain duplicate identities") - if set(current_pairs) != set(checkpoint_pairs): - raise StateBackendConfigError( - f"Checkpoint {label} refs must exactly match the published {label} refs" - ) diff --git a/src/sqlbuild/virtual/state/classes/__init__.py b/src/sqlbuild/virtual/state/classes/__init__.py deleted file mode 100644 index fb0edf1770..0000000000 --- a/src/sqlbuild/virtual/state/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state backend implementations.""" diff --git a/src/sqlbuild/virtual/state/classes/_sql_state_backend.py b/src/sqlbuild/virtual/state/classes/_sql_state_backend.py deleted file mode 100644 index c31a45cbf2..0000000000 --- a/src/sqlbuild/virtual/state/classes/_sql_state_backend.py +++ /dev/null @@ -1,1632 +0,0 @@ -"""Shared SQL for relational virtual-state backends.""" - -from __future__ import annotations - -from abc import abstractmethod -from collections.abc import Sequence -from contextlib import AbstractContextManager -from datetime import datetime -from typing import Any, ClassVar, cast - -from sqlbuild.executor.node_results.main.decode_json import decode_node_result_json -from sqlbuild.executor.node_results.main.encode_json import encode_node_result_json -from sqlbuild.executor.node_results.models import ( - NodeResultEnvelope, - NodeResultQuery, - NodeResultRecord, -) -from sqlbuild.virtual.state._helpers.state_storage.datetime import ( - from_naive_utc_wall_clock, -) -from sqlbuild.virtual.state._helpers.state_storage.events import backup_id, event_id -from sqlbuild.virtual.state._helpers.state_storage.validation import ( - validate_conditional_virtual_environment_publication, -) -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.constants import ( - CURRENT_STATE_SCHEMA_VERSION, - FUNCTION_VERSION_TABLE, - LOCK_TABLE, - MODEL_VERSION_TABLE, - NODE_RESULTS_TABLE, - NON_UNIQUE_STATE_INDEXES, - PHYSICAL_RELATION_ANCESTRY_TABLE, - PHYSICAL_RELATION_TABLE, - PYTHON_NODE_VERSION_TABLE, - RECONCILE_EVENT_TABLE, - SEED_VERSION_TABLE, - SOURCE_FRESHNESS_OBSERVATION_TABLE, - STATE_BOOLEAN_TRUE, - STATE_MIGRATION_EVENTS_TABLE, - STATE_OPERATION_EVENT_TABLE, - STATE_OPERATION_TABLE, - STATE_TABLE_COLUMNS, - STATE_TABLE_INDEXES, - STATE_TABLES, - STATE_VERSION_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE, - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE, - VIRTUAL_ENVIRONMENT_TABLE, -) -from sqlbuild.virtual.state.exceptions import ( - StateBackendConfigError, - StateBackupNotFoundError, - StateSchemaInvalidError, -) -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationAncestryRecord, - PhysicalRelationRecord, - PythonNodeVersionRecord, - ReconcileEventRecord, - SeedVersionRecord, - SourceFreshnessRecord, - StateBackupRecord, - StateLockLease, - StateLockRecord, - StateOperationEventRecord, - StateOperationRecord, - StateSchemaValidationResult, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentPythonNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentRetentionRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - StateColumnType, - StateMigrationAction, - StateMigrationStatus, - StateOperationStatus, - StateOperationType, - VirtualEnvironmentStatus, -) - - -class SqlStateBackend(StateBackend): - """Relational state backend owning SQL shared by DuckDB and Postgres.""" - - _placeholder: ClassVar[str] - - @abstractmethod - def _fetch_one( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> tuple[Any, ...] | None: - """Run one read statement and return its first row.""" - ... - - @abstractmethod - def _fetch_all( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> list[tuple[Any, ...]]: - """Run one read statement and return all rows.""" - ... - - @abstractmethod - def _write_transaction(self, *, connection: Any) -> AbstractContextManager[Any]: - """Open a write transaction and yield the executor that runs its statements.""" - ... - - @abstractmethod - def _statement_executor(self, *, connection: Any) -> AbstractContextManager[Any]: - """Open an executor for explicitly delimited statements on a connection.""" - ... - - @abstractmethod - def _execute_in( - self, *, executor: Any, sql: str, params: Sequence[object] | None = None - ) -> None: - """Run one statement on an executor.""" - ... - - @abstractmethod - def _lease_is_owned(self, *, executor: Any, schema: str, lease: StateLockLease) -> bool: - """Fence one lock lease inside an open transaction and report whether it is held.""" - ... - - @abstractmethod - def _fetch_one_in( - self, *, executor: Any, sql: str, params: Sequence[object] - ) -> tuple[Any, ...] | None: - """Run one read statement on a transaction executor and return its first row.""" - ... - - def upsert_model_version( - self, *, connection: Any, schema: str, record: ModelVersionRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=MODEL_VERSION_TABLE, - key_values={"model_name": record.model_name, "version_hash": record.version_hash}, - row_values={ - "definition_identity_hash": record.definition_identity_hash, - "identity_metadata_hash": record.identity_metadata_hash, - "definition_text_b64": record.definition_text_b64, - "identity_metadata_json_b64": record.identity_metadata_json_b64, - "compiled_sql_b64": record.compiled_sql_b64, - "status": record.status.value, - }, - ) - - def upsert_function_version( - self, *, connection: Any, schema: str, record: FunctionVersionRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=FUNCTION_VERSION_TABLE, - key_values={ - "function_name": record.function_name, - "version_hash": record.version_hash, - }, - row_values={ - "language": record.language, - "returns": record.returns, - "arguments_json_b64": record.arguments_json_b64, - "return_columns_json_b64": record.return_columns_json_b64, - "packages_json_b64": record.packages_json_b64, - "runtime_version": record.runtime_version, - "entry_point": record.entry_point, - "body_sql_b64": record.body_sql_b64, - "definition_text_b64": record.definition_text_b64, - "status": record.status.value, - }, - ) - - def upsert_seed_version( - self, *, connection: Any, schema: str, record: SeedVersionRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=SEED_VERSION_TABLE, - key_values={"seed_name": record.seed_name, "version_hash": record.version_hash}, - row_values={ - "identity_metadata_hash": record.identity_metadata_hash, - "identity_metadata_json_b64": record.identity_metadata_json_b64, - "status": record.status.value, - }, - ) - - def upsert_python_node_version( - self, *, connection: Any, schema: str, record: PythonNodeVersionRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=PYTHON_NODE_VERSION_TABLE, - key_values={ - "node_type": record.node_type, - "node_name": record.node_name, - "version_hash": record.version_hash, - }, - row_values={ - "definition_hash": record.definition_hash, - "identity_metadata_hash": record.identity_metadata_hash, - "definition_json_b64": record.definition_json_b64, - "identity_metadata_json_b64": record.identity_metadata_json_b64, - "status": record.status.value, - }, - ) - - def upsert_physical_relation( - self, *, connection: Any, schema: str, record: PhysicalRelationRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=PHYSICAL_RELATION_TABLE, - key_values={ - "artifact_type": record.artifact_type.value, - "artifact_name": record.artifact_name, - "version_hash": record.version_hash, - }, - row_values={ - "database_name": record.database_name, - "schema_name": record.schema_name, - "relation_name": record.relation_name, - "relation_type": record.relation_type, - }, - ) - - def upsert_physical_relation_ancestry( - self, *, connection: Any, schema: str, record: PhysicalRelationAncestryRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=PHYSICAL_RELATION_ANCESTRY_TABLE, - key_values={"model_name": record.model_name, "version_hash": record.version_hash}, - row_values={ - "parent_model_name": record.parent_model_name, - "parent_version_hash": record.parent_version_hash, - "seed_strategy": record.seed_strategy, - }, - ) - - def upsert_virtual_environment( - self, *, connection: Any, schema: str, record: VirtualEnvironmentRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._upsert_virtual_environment_record(executor=executor, schema=schema, record=record) - - def upsert_state_operation( - self, *, connection: Any, schema: str, record: StateOperationRecord - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=STATE_OPERATION_TABLE, - key_values={"operation_id": record.operation_id}, - row_values={ - "operation_type": record.operation_type.value, - "status": record.status.value, - "virtual_environment_name": record.virtual_environment_name, - }, - ) - - def _upsert_virtual_environment_record( - self, *, executor: Any, schema: str, record: VirtualEnvironmentRecord - ) -> None: - self._replace_row_preserving_created_at( - executor=executor, - schema=schema, - table_name=VIRTUAL_ENVIRONMENT_TABLE, - key_values={"virtual_environment_name": record.virtual_environment_name}, - row_values={ - "status": record.status.value, - "baseline_virtual_environment_name": record.baseline_virtual_environment_name, - "finalized_at": record.finalized_at, - }, - ) - - def list_virtual_environments( - self, *, connection: Any, schema: str - ) -> tuple[VirtualEnvironmentRetentionRecord, ...]: - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT virtual_environment_name, status, updated_at " - f"FROM {self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_TABLE)} " - "ORDER BY updated_at DESC, virtual_environment_name DESC", - ) - return tuple( - VirtualEnvironmentRetentionRecord( - virtual_environment_name=row[0], - status=VirtualEnvironmentStatus(row[1]), - updated_at=row[2], - ) - for row in rows - ) - - def renew_lock( - self, - *, - connection: Any, - schema: str, - lock_key: str, - owner_id: str, - expires_at: datetime, - ) -> bool: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql=f"UPDATE {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - f"SET expires_at = {p}, updated_at = CURRENT_TIMESTAMP " - f"WHERE lock_key = {p} AND owner_id = {p} AND expires_at > CURRENT_TIMESTAMP " - "RETURNING lock_key", - params=[expires_at, lock_key, owner_id], - ) - return row is not None - - def list_active_locks(self, *, connection: Any, schema: str) -> tuple[StateLockRecord, ...]: - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT lock_key, owner_id, expires_at FROM " - f"{self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE expires_at > CURRENT_TIMESTAMP ORDER BY lock_key", - ) - return tuple( - StateLockRecord(lock_key=row[0], owner_id=row[1], expires_at=row[2]) for row in rows - ) - - def list_expired_locks(self, *, connection: Any, schema: str) -> tuple[StateLockRecord, ...]: - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT lock_key, owner_id, expires_at FROM " - f"{self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE expires_at <= CURRENT_TIMESTAMP ORDER BY lock_key", - ) - return tuple( - StateLockRecord(lock_key=row[0], owner_id=row[1], expires_at=row[2]) for row in rows - ) - - @abstractmethod - def _replace_virtual_environment_node_ref_groups( - self, - *, - executor: Any, - schema: str, - virtual_environment_name: str, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - """Replace node refs per node type inside an open write transaction.""" - ... - - def replace_virtual_environment_node_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - node_type: str, - refs: tuple[VirtualEnvironmentNodeRefRecord, ...], - ) -> None: - self.replace_virtual_environment_node_ref_groups( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - refs_by_node_type={node_type: refs}, - ) - - def replace_virtual_environment_node_ref_groups( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._replace_virtual_environment_node_ref_groups( - executor=executor, - schema=schema, - virtual_environment_name=virtual_environment_name, - refs_by_node_type=refs_by_node_type, - ) - - def upsert_virtual_environment_and_replace_node_ref_groups( - self, - *, - connection: Any, - schema: str, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._upsert_virtual_environment_record(executor=executor, schema=schema, record=record) - self._replace_virtual_environment_node_ref_groups( - executor=executor, - schema=schema, - virtual_environment_name=record.virtual_environment_name, - refs_by_node_type=refs_by_node_type, - ) - - def replace_virtual_environment_model_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], - ) -> None: - self.replace_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="model", - refs=tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type="model", - node_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - ), - ) - - def upsert_virtual_environment_python_node_ref( - self, - *, - connection: Any, - schema: str, - ref: VirtualEnvironmentPythonNodeRefRecord, - ) -> None: - self.upsert_virtual_environment_node_ref( - connection=connection, - schema=schema, - ref=VirtualEnvironmentNodeRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - node_name=ref.node_name, - version_hash=ref.version_hash, - ), - ) - - def _validate_node_ref_replacement( - self, - *, - virtual_environment_name: str, - node_type: str, - refs: tuple[VirtualEnvironmentNodeRefRecord, ...], - ) -> None: - seen_node_names: set[str] = set() - ref: VirtualEnvironmentNodeRefRecord - for ref in refs: - if ref.virtual_environment_name != virtual_environment_name: - raise StateBackendConfigError( - "Node ref virtual_environment_name must match replacement " - "virtual_environment_name" - ) - if ref.node_type != node_type: - raise StateBackendConfigError("Node ref node_type must match replacement node_type") - if ref.node_name in seen_node_names: - raise StateBackendConfigError( - f"Duplicate node ref for node type '{node_type}' and name '{ref.node_name}'" - ) - seen_node_names.add(ref.node_name) - - def upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - self, - *, - connection: Any, - schema: str, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - leases: tuple[StateLockLease, ...], - checkpoint: VirtualEnvironmentCheckpointRecord | None = None, - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = (), - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = (), - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = (), - ) -> bool: - validate_conditional_virtual_environment_publication( - record=record, - refs_by_node_type=refs_by_node_type, - checkpoint=checkpoint, - checkpoint_refs=checkpoint_refs, - checkpoint_function_refs=checkpoint_function_refs, - checkpoint_seed_refs=checkpoint_seed_refs, - ) - with self._statement_executor(connection=connection) as executor: - self._execute_in(executor=executor, sql="BEGIN") - try: - lease: StateLockLease - for lease in leases: - if not self._lease_is_owned(executor=executor, schema=schema, lease=lease): - self._execute_in(executor=executor, sql="ROLLBACK") - return False - self._upsert_virtual_environment_record( - executor=executor, schema=schema, record=record - ) - self._replace_virtual_environment_node_ref_groups( - executor=executor, - schema=schema, - virtual_environment_name=record.virtual_environment_name, - refs_by_node_type=refs_by_node_type, - ) - if checkpoint is not None: - self._insert_virtual_environment_checkpoint_rows( - executor=executor, - schema=schema, - checkpoint=checkpoint, - refs=checkpoint_refs, - function_refs=checkpoint_function_refs, - seed_refs=checkpoint_seed_refs, - ) - self._execute_in(executor=executor, sql="COMMIT") - return True - except BaseException: - self._execute_in(executor=executor, sql="ROLLBACK") - raise - - def create_virtual_environment_checkpoint( - self, - *, - connection: Any, - schema: str, - checkpoint: VirtualEnvironmentCheckpointRecord, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = (), - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = (), - ) -> None: - with self._write_transaction(connection=connection) as executor: - self._insert_virtual_environment_checkpoint_rows( - executor=executor, - schema=schema, - checkpoint=checkpoint, - refs=refs, - function_refs=function_refs, - seed_refs=seed_refs, - ) - - def delete_virtual_environment_checkpoint( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> None: - p: str = self._placeholder - table_names: tuple[str, ...] = ( - VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE, - ) - with self._write_transaction(connection=connection) as executor: - table_name: str - for table_name in table_names: - self._execute_in( - executor=executor, - sql=( - f"DELETE FROM {self._qualified_name(schema=schema, table=table_name)} " - f"WHERE checkpoint_id = {p}" - ), - params=[checkpoint_id], - ) - - def _insert_virtual_environment_checkpoint_rows( - self, - *, - executor: Any, - schema: str, - checkpoint: VirtualEnvironmentCheckpointRecord, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...], - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...], - ) -> None: - p: str = self._placeholder - checkpoint_table: str = self._qualified_name( - schema=schema, table=VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE - ) - model_ref_table: str = self._qualified_name( - schema=schema, table=VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE - ) - function_ref_table: str = self._qualified_name( - schema=schema, table=VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE - ) - seed_ref_table: str = self._qualified_name( - schema=schema, table=VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE - ) - self._execute_in( - executor=executor, - sql=( - f"INSERT INTO {checkpoint_table} " - "(checkpoint_id, virtual_environment_name, created_at) " - f"VALUES ({p}, {p}, CURRENT_TIMESTAMP)" - ), - params=[checkpoint.checkpoint_id, checkpoint.virtual_environment_name], - ) - ref: VirtualEnvironmentCheckpointModelRefRecord - for ref in refs: - self._execute_in( - executor=executor, - sql=( - f"INSERT INTO {model_ref_table} " - f"(checkpoint_id, model_name, version_hash) VALUES ({p}, {p}, {p})" - ), - params=[ref.checkpoint_id, ref.model_name, ref.version_hash], - ) - function_ref: VirtualEnvironmentCheckpointFunctionRefRecord - for function_ref in function_refs: - self._execute_in( - executor=executor, - sql=( - f"INSERT INTO {function_ref_table} " - f"(checkpoint_id, function_name, version_hash) VALUES ({p}, {p}, {p})" - ), - params=[ - function_ref.checkpoint_id, - function_ref.function_name, - function_ref.version_hash, - ], - ) - seed_ref: VirtualEnvironmentCheckpointSeedRefRecord - for seed_ref in seed_refs: - self._execute_in( - executor=executor, - sql=( - f"INSERT INTO {seed_ref_table} " - f"(checkpoint_id, seed_name, version_hash) VALUES ({p}, {p}, {p})" - ), - params=[seed_ref.checkpoint_id, seed_ref.seed_name, seed_ref.version_hash], - ) - - def insert_node_result( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - record: NodeResultRecord, - ) -> None: - p: str = self._placeholder - self._execute( - connection=connection, - sql=f"INSERT INTO {self._qualified_name(schema=schema, table=NODE_RESULTS_TABLE)} " - "(virtual_environment_name, node_type, node_name, target_database, target_schema, " - "target_name, run_id, status, payload_json_b64, metadata_json_b64, error_message, " - "materialized, created_at) " - f"VALUES ({p}, {p}, {p}, {p}, {p}, {p}, {p}, {p}, {p}, {p}, {p}, {p}, {p})", - params=[ - virtual_environment_name, - record.node_type, - record.node_name, - record.target_database, - record.target_schema, - record.target_name, - record.run_id, - record.status, - encode_node_result_json( - value=record.payload, label="payload", node_name=record.node_name - ), - encode_node_result_json( - value=record.metadata, label="metadata", node_name=record.node_name - ), - record.error_message, - self._materialized_storage(record.materialized), - record.ts, - ], - ) - - def read_node_results( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - query: NodeResultQuery, - ) -> tuple[NodeResultEnvelope, ...]: - p: str = self._placeholder - if query.limit < 1: - return () - predicates: list[str] = [ - f"virtual_environment_name = {p}", - f"node_type = {p}", - f"node_name = {p}", - self._optional_equality_sql( - column="target_database", value=query.target_database, placeholder=f"{p}" - ), - self._optional_equality_sql( - column="target_schema", value=query.target_schema, placeholder=f"{p}" - ), - self._optional_equality_sql( - column="target_name", value=query.target_name, placeholder=f"{p}" - ), - ] - params: list[object] = [virtual_environment_name, query.node_type, query.node_name] - for value in (query.target_database, query.target_schema, query.target_name): - if value is not None: - params.append(value) - if query.statuses is not None: - placeholders: str = ", ".join(f"{p}" for _ in query.statuses) - predicates.append(f"status IN ({placeholders})") - params.extend(query.statuses) - if query.run_id is not None: - predicates.append(f"run_id = {p}") - params.append(query.run_id) - params.append(query.limit) - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT node_type, node_name, run_id, status, payload_json_b64, metadata_json_b64, " - "error_message, materialized, created_at " - f"FROM {self._qualified_name(schema=schema, table=NODE_RESULTS_TABLE)} " - f"WHERE {' AND '.join(predicates)} " - f"ORDER BY created_at DESC, run_id DESC LIMIT {p}", - params=params, - ) - return tuple(self._node_result_row_to_envelope(row) for row in rows) - - def create_state_operation_event( - self, *, connection: Any, schema: str, record: StateOperationEventRecord - ) -> None: - p: str = self._placeholder - self._execute( - connection=connection, - sql="INSERT INTO " - f"{self._qualified_name(schema=schema, table=STATE_OPERATION_EVENT_TABLE)} " - "(event_id, operation_id, action, status, message, created_at) " - f"VALUES ({p}, {p}, {p}, {p}, {p}, CURRENT_TIMESTAMP)", - params=[ - record.event_id, - record.operation_id, - record.action, - record.status.value, - record.message, - ], - ) - - def create_reconcile_event( - self, *, connection: Any, schema: str, record: ReconcileEventRecord - ) -> None: - p: str = self._placeholder - self._execute( - connection=connection, - sql=f"INSERT INTO {self._qualified_name(schema=schema, table=RECONCILE_EVENT_TABLE)} " - "(event_id, action, status, message, created_at) " - f"VALUES ({p}, {p}, {p}, {p}, CURRENT_TIMESTAMP)", - params=[record.event_id, record.action.value, record.status.value, record.message], - ) - - def _node_result_row_to_envelope(self, row: tuple[Any, ...]) -> NodeResultEnvelope: - node_name: str = str(row[1]) - metadata: object = decode_node_result_json( - value=str(row[5]), label="metadata", node_name=node_name - ) - normalized_metadata: dict[str, object] = ( - {str(key): value for key, value in metadata.items()} - if isinstance(metadata, dict) - else {} - ) - return NodeResultEnvelope( - node_type=str(row[0]), - node_name=node_name, - run_id=str(row[2]), - status=str(row[3]), - payload=decode_node_result_json( - value=str(row[4]), label="payload", node_name=node_name - ), - metadata=normalized_metadata, - error_message=str(row[6]) if row[6] is not None else None, - materialized=self._parse_materialized(row[7]), - ts=row[8], - ) - - def _optional_equality_sql(self, *, column: str, value: object | None, placeholder: str) -> str: - if value is None: - return f"{column} IS NULL" - return f"{column} = {placeholder}" - - def _materialized_storage(self, value: bool | None) -> str | None: - if value is None: - return None - return "true" if value else "false" - - def _parse_materialized(self, value: object) -> bool | None: - if value is None: - return None - return str(value).lower() == STATE_BOOLEAN_TRUE - - def _execute( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> None: - with self._statement_executor(connection=connection) as executor: - self._execute_in(executor=executor, sql=sql, params=params) - - def initialize(self, *, connection: Any, schema: str, sqlbuild_version: str) -> None: - p: str = self._placeholder - with self._write_transaction(connection=connection) as executor: - self._execute_in( - executor=executor, - sql=f"CREATE SCHEMA IF NOT EXISTS {self._quote_identifier(schema)}", - ) - self._execute_in( - executor=executor, - sql="CREATE TABLE IF NOT EXISTS " - f"{self._qualified_name(schema=schema, table=STATE_VERSION_TABLE)} (" - "schema_version INTEGER NOT NULL, " - "sqlbuild_version TEXT NOT NULL, " - "updated_at TIMESTAMP NOT NULL" - ")", - ) - self._execute_in( - executor=executor, - sql="CREATE TABLE IF NOT EXISTS " - f"{self._qualified_name(schema=schema, table=STATE_MIGRATION_EVENTS_TABLE)} (" - "event_id TEXT NOT NULL, " - "action TEXT NOT NULL, " - "backup_id TEXT, " - "status TEXT NOT NULL, " - "message TEXT, " - "created_at TIMESTAMP NOT NULL" - ")", - ) - self._create_additional_state_tables(executor=executor, schema=schema) - self._execute_in( - executor=executor, - sql=f"DELETE FROM {self._qualified_name(schema=schema, table=STATE_VERSION_TABLE)}", - ) - self._execute_in( - executor=executor, - sql=f"INSERT INTO {self._qualified_name(schema=schema, table=STATE_VERSION_TABLE)} " - "(schema_version, sqlbuild_version, updated_at) " - f"VALUES ({p}, {p}, CURRENT_TIMESTAMP)", - params=[CURRENT_STATE_SCHEMA_VERSION, sqlbuild_version], - ) - self._record_event( - executor=executor, - schema=schema, - action=StateMigrationAction.INIT, - backup_id_value=None, - status=StateMigrationStatus.SUCCESS, - message=None, - ) - - def create_backup(self, *, connection: Any, schema: str) -> str: - validation: StateSchemaValidationResult = self.inspect_schema( - connection=connection, schema=schema - ) - if not validation.valid: - raise StateSchemaInvalidError("Cannot backup invalid state schema") - backup_id_value: str = backup_id() - backup_schema: str = self._backup_schema_name( - schema=schema, - backup_id_value=backup_id_value, - ) - with self._write_transaction(connection=connection) as executor: - self._execute_in( - executor=executor, sql=f"CREATE SCHEMA {self._quote_identifier(backup_schema)}" - ) - table_name: str - for table_name in STATE_TABLES: - self._execute_in( - executor=executor, - sql="CREATE TABLE " - f"{self._qualified_name(schema=backup_schema, table=table_name)} " - "AS " - f"SELECT * FROM {self._qualified_name(schema=schema, table=table_name)}", - ) - self._record_event( - executor=executor, - schema=schema, - action=StateMigrationAction.BACKUP, - backup_id_value=backup_id_value, - status=StateMigrationStatus.SUCCESS, - message=None, - ) - return backup_id_value - - def rollback(self, *, connection: Any, schema: str, backup_id: str | None = None) -> str: - backup_id_value: str = backup_id or self._latest_backup_id( - connection=connection, schema=schema - ) - backup_schema: str = self._backup_schema_name( - schema=schema, - backup_id_value=backup_id_value, - ) - if not self._schema_exists(connection=connection, schema=backup_schema): - raise StateBackupNotFoundError(f"State backup schema '{backup_schema}' does not exist") - with self._write_transaction(connection=connection) as executor: - table_name: str - for table_name in STATE_TABLES: - self._execute_in( - executor=executor, - sql="DROP TABLE IF EXISTS " - f"{self._qualified_name(schema=schema, table=table_name)}", - ) - self._execute_in( - executor=executor, - sql=f"CREATE SCHEMA IF NOT EXISTS {self._quote_identifier(schema)}", - ) - for table_name in STATE_TABLES: - self._execute_in( - executor=executor, - sql=f"CREATE TABLE {self._qualified_name(schema=schema, table=table_name)} AS " - f"SELECT * FROM {self._qualified_name(schema=backup_schema, table=table_name)}", - ) - self._create_state_indexes(executor=executor, schema=schema) - self._record_event( - executor=executor, - schema=schema, - action=StateMigrationAction.ROLLBACK, - backup_id_value=backup_id_value, - status=StateMigrationStatus.SUCCESS, - message=None, - ) - return backup_id_value - - def reset(self, *, connection: Any, schema: str) -> None: - with self._write_transaction(connection=connection) as executor: - for table_name in STATE_TABLES: - self._execute_in( - executor=executor, - sql="DROP TABLE IF EXISTS " - f"{self._qualified_name(schema=schema, table=table_name)}", - ) - - def _create_additional_state_tables(self, *, executor: Any, schema: str) -> None: - table_name: str - columns: dict[str, StateColumnType] - for table_name, columns in STATE_TABLE_COLUMNS.items(): - if table_name in {STATE_VERSION_TABLE, STATE_MIGRATION_EVENTS_TABLE}: - continue - column_sql: str = ", ".join( - f"{self._quote_identifier(column_name)} {self._state_column_sql_type(column_type)}" - for column_name, column_type in columns.items() - ) - self._execute_in( - executor=executor, - sql="CREATE TABLE IF NOT EXISTS " - f"{self._qualified_name(schema=schema, table=table_name)} " - f"({column_sql})", - ) - column_name: str - column_type: StateColumnType - for column_name, column_type in columns.items(): - self._execute_in( - executor=executor, - sql=f"ALTER TABLE {self._qualified_name(schema=schema, table=table_name)} " - f"ADD COLUMN IF NOT EXISTS {self._quote_identifier(column_name)} " - f"{self._state_column_sql_type(column_type)}", - ) - self._create_state_indexes(executor=executor, schema=schema) - - def _create_state_indexes(self, *, executor: Any, schema: str) -> None: - table_name: str - indexes: dict[str, tuple[str, ...]] - for table_name, indexes in STATE_TABLE_INDEXES.items(): - index_name: str - columns: tuple[str, ...] - for index_name, columns in indexes.items(): - column_sql: str = ", ".join(self._quote_identifier(column) for column in columns) - unique_sql: str = "" if index_name in NON_UNIQUE_STATE_INDEXES else "UNIQUE " - self._execute_in( - executor=executor, - sql=f"CREATE {unique_sql}INDEX IF NOT EXISTS " - f"{self._quote_identifier(index_name)} " - f"ON {self._qualified_name(schema=schema, table=table_name)} ({column_sql})", - ) - - def _record_event( - self, - *, - executor: Any, - schema: str, - action: StateMigrationAction, - backup_id_value: str | None, - status: StateMigrationStatus, - message: str | None, - ) -> None: - p: str = self._placeholder - self._execute_in( - executor=executor, - sql="INSERT INTO " - f"{self._qualified_name(schema=schema, table=STATE_MIGRATION_EVENTS_TABLE)} " - "(event_id, action, backup_id, status, message, created_at) " - f"VALUES ({p}, {p}, {p}, {p}, {p}, CURRENT_TIMESTAMP)", - params=[event_id(), action.value, backup_id_value, status.value, message], - ) - - def list_state_backups(self, *, connection: Any, schema: str) -> tuple[StateBackupRecord, ...]: - p: str = self._placeholder - prefix: str = f"{schema}__backup_%" - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT s.schema_name, e.backup_id, MAX(e.created_at) " - "FROM information_schema.schemata s " - "LEFT JOIN " - f"{self._qualified_name(schema=schema, table=STATE_MIGRATION_EVENTS_TABLE)} e " - f"ON s.schema_name = {p} || e.backup_id " - f"WHERE s.schema_name LIKE {p} " - "GROUP BY s.schema_name, e.backup_id ORDER BY s.schema_name DESC", - params=[f"{schema}__backup_", prefix], - ) - return tuple( - StateBackupRecord( - backup_id=row[1] or str(row[0]).removeprefix(f"{schema}__backup_"), - schema_name=row[0], - created_at=row[2], - ) - for row in rows - ) - - def _latest_backup_id(self, *, connection: Any, schema: str) -> str: - p: str = self._placeholder - prefix: str = f"{schema}__backup_%" - rows: list[tuple[str]] = self._fetch_all( - connection=connection, - sql=f"SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE {p} " - "ORDER BY schema_name DESC LIMIT 1", - params=[prefix], - ) - if not rows: - raise StateBackupNotFoundError("No state backup is available for rollback") - return rows[0][0].removeprefix(f"{schema}__backup_") - - def _schema_exists(self, *, connection: Any, schema: str) -> bool: - p: str = self._placeholder - rows: list[tuple[str]] = self._fetch_all( - connection=connection, - sql=f"SELECT schema_name FROM information_schema.schemata WHERE schema_name = {p}", - params=[schema], - ) - return bool(rows) - - def _state_column_sql_type(self, column_type: StateColumnType) -> str: - match column_type: - case StateColumnType.INTEGER: - return "INTEGER" - case StateColumnType.TEXT: - return "TEXT" - case StateColumnType.TIMESTAMP: - return "TIMESTAMP" - raise StateBackendConfigError(f"Unsupported state column type: {column_type}") - - def _backup_schema_name(self, *, schema: str, backup_id_value: str) -> str: - return f"{schema}__backup_{backup_id_value}" - - def close(self, connection: Any) -> None: - connection.close() - - def delete_virtual_environment( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> None: - p: str = self._placeholder - table_names: tuple[str, ...] = ( - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE, - SOURCE_FRESHNESS_OBSERVATION_TABLE, - VIRTUAL_ENVIRONMENT_TABLE, - ) - with self._write_transaction(connection=connection) as executor: - table_name: str - for table_name in table_names: - self._execute_in( - executor=executor, - sql=( - f"DELETE FROM {self._qualified_name(schema=schema, table=table_name)} " - f"WHERE virtual_environment_name = {p}" - ), - params=[virtual_environment_name], - ) - - def count_unreferenced_python_node_versions(self, *, connection: Any, schema: str) -> int: - version_table: str = self._qualified_name(schema=schema, table=PYTHON_NODE_VERSION_TABLE) - ref_table: str = self._qualified_name( - schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE - ) - row: tuple[Any, ...] = cast( - tuple[Any, ...], - self._fetch_one( - connection=connection, - sql=( - "SELECT COUNT(*) " - f"FROM {version_table} versions " - "WHERE NOT EXISTS (" - "SELECT 1 " - f"FROM {ref_table} refs " - "WHERE refs.node_type = versions.node_type " - "AND refs.node_name = versions.node_name " - "AND refs.version_hash = versions.version_hash)" - ), - ), - ) - return int(row[0]) - - def _validate_source_freshness_records( - self, - *, - virtual_environment_name: str, - records: tuple[SourceFreshnessRecord, ...], - ) -> None: - seen_source_names: set[str] = set() - record: SourceFreshnessRecord - for record in records: - if record.virtual_environment_name != virtual_environment_name: - raise StateBackendConfigError( - "Source freshness record virtual_environment_name must match replacement " - "virtual_environment_name" - ) - if record.source_name in seen_source_names: - raise StateBackendConfigError( - f"Duplicate source freshness record for source '{record.source_name}'" - ) - seen_source_names.add(record.source_name) - - def _replace_row_preserving_created_at( - self, - *, - executor: Any, - schema: str, - table_name: str, - key_values: dict[str, object], - row_values: dict[str, object], - ) -> None: - p: str = self._placeholder - table: str = self._qualified_name(schema=schema, table=table_name) - where_sql: str = " AND ".join(f"{column} = {p}" for column in key_values) - key_params: list[object] = list(key_values.values()) - existing_created_at: datetime | None = self._created_at_for_key( - executor=executor, - schema=schema, - table_name=table_name, - where_sql=where_sql, - params=key_params, - ) - self._execute_in( - executor=executor, sql=f"DELETE FROM {table} WHERE {where_sql}", params=key_params - ) - columns: tuple[str, ...] = (*key_values, *row_values) - value_placeholders: str = ", ".join(p for _ in columns) - self._execute_in( - executor=executor, - sql=( - f"INSERT INTO {table} ({', '.join(columns)}, created_at, updated_at) " - f"VALUES ({value_placeholders}, COALESCE({p}, CURRENT_TIMESTAMP), " - "CURRENT_TIMESTAMP)" - ), - params=[*key_params, *row_values.values(), existing_created_at], - ) - - def _created_at_for_key( - self, - *, - executor: Any, - schema: str, - table_name: str, - where_sql: str, - params: list[object], - ) -> datetime | None: - row: tuple[Any, ...] | None = self._fetch_one_in( - executor=executor, - sql=( - f"SELECT created_at FROM {self._qualified_name(schema=schema, table=table_name)} " - f"WHERE {where_sql}" - ), - params=params, - ) - if row is None: - return None - return row[0] - - def _quote_identifier(self, identifier: str) -> str: - return '"' + identifier.replace('"', '""') + '"' - - def _qualified_name(self, *, schema: str, table: str) -> str: - return f"{self._quote_identifier(schema)}.{self._quote_identifier(table)}" - - def get_model_version( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> ModelVersionRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT model_name, version_hash, definition_identity_hash, " - "identity_metadata_hash, definition_text_b64, identity_metadata_json_b64, " - "compiled_sql_b64, status " - f"FROM {self._qualified_name(schema=schema, table=MODEL_VERSION_TABLE)} " - f"WHERE model_name = {p} AND version_hash = {p}", - params=[model_name, version_hash], - ) - if row is None: - return None - return ModelVersionRecord( - model_name=row[0], - version_hash=row[1], - definition_identity_hash=row[2], - identity_metadata_hash=row[3], - definition_text_b64=row[4], - identity_metadata_json_b64=row[5], - compiled_sql_b64=row[6], - status=ModelVersionStatus(row[7]), - ) - - def get_function_version( - self, *, connection: Any, schema: str, function_name: str, version_hash: str - ) -> FunctionVersionRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT function_name, version_hash, language, returns, arguments_json_b64, " - "return_columns_json_b64, packages_json_b64, runtime_version, entry_point, " - "body_sql_b64, definition_text_b64, status " - f"FROM {self._qualified_name(schema=schema, table=FUNCTION_VERSION_TABLE)} " - f"WHERE function_name = {p} AND version_hash = {p}", - params=[function_name, version_hash], - ) - if row is None: - return None - return FunctionVersionRecord( - function_name=row[0], - version_hash=row[1], - language=row[2], - returns=row[3], - arguments_json_b64=row[4], - return_columns_json_b64=row[5], - packages_json_b64=row[6], - runtime_version=row[7], - entry_point=row[8], - body_sql_b64=row[9], - definition_text_b64=row[10], - status=ModelVersionStatus(row[11]), - ) - - def get_seed_version( - self, *, connection: Any, schema: str, seed_name: str, version_hash: str - ) -> SeedVersionRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT seed_name, version_hash, identity_metadata_hash, " - "identity_metadata_json_b64, status " - f"FROM {self._qualified_name(schema=schema, table=SEED_VERSION_TABLE)} " - f"WHERE seed_name = {p} AND version_hash = {p}", - params=[seed_name, version_hash], - ) - if row is None: - return None - return SeedVersionRecord( - seed_name=row[0], - version_hash=row[1], - identity_metadata_hash=row[2], - identity_metadata_json_b64=row[3], - status=ModelVersionStatus(row[4]), - ) - - def get_python_node_version( - self, - *, - connection: Any, - schema: str, - node_type: str, - node_name: str, - version_hash: str, - ) -> PythonNodeVersionRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT node_type, node_name, version_hash, definition_hash, " - "identity_metadata_hash, definition_json_b64, identity_metadata_json_b64, status " - f"FROM {self._qualified_name(schema=schema, table=PYTHON_NODE_VERSION_TABLE)} " - f"WHERE node_type = {p} AND node_name = {p} AND version_hash = {p}", - params=[node_type, node_name, version_hash], - ) - if row is None: - return None - return PythonNodeVersionRecord( - node_type=row[0], - node_name=row[1], - version_hash=row[2], - definition_hash=row[3], - identity_metadata_hash=row[4], - definition_json_b64=row[5], - identity_metadata_json_b64=row[6], - status=ModelVersionStatus(row[7]), - ) - - def get_physical_relation_for_artifact( - self, - *, - connection: Any, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - version_hash: str, - ) -> PhysicalRelationRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT artifact_type, artifact_name, version_hash, database_name, schema_name, " - "relation_name, relation_type " - f"FROM {self._qualified_name(schema=schema, table=PHYSICAL_RELATION_TABLE)} " - f"WHERE artifact_type = {p} AND artifact_name = {p} AND version_hash = {p}", - params=[artifact_type.value, artifact_name, version_hash], - ) - if row is None: - return None - return PhysicalRelationRecord( - artifact_type=PhysicalArtifactType(row[0]), - artifact_name=row[1], - version_hash=row[2], - database_name=row[3], - schema_name=row[4], - relation_name=row[5], - relation_type=row[6], - ) - - def list_physical_relations_for_artifact( - self, - *, - connection: Any, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - ) -> tuple[PhysicalRelationRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT artifact_type, artifact_name, version_hash, database_name, schema_name, " - "relation_name, relation_type " - f"FROM {self._qualified_name(schema=schema, table=PHYSICAL_RELATION_TABLE)} " - f"WHERE artifact_type = {p} AND artifact_name = {p} " - "ORDER BY updated_at DESC, version_hash DESC", - params=[artifact_type.value, artifact_name], - ) - return tuple( - PhysicalRelationRecord( - artifact_type=PhysicalArtifactType(row[0]), - artifact_name=row[1], - version_hash=row[2], - database_name=row[3], - schema_name=row[4], - relation_name=row[5], - relation_type=row[6], - ) - for row in rows - ) - - def get_virtual_environment( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> VirtualEnvironmentRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT virtual_environment_name, status, baseline_virtual_environment_name, " - "finalized_at " - f"FROM {self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_TABLE)} " - f"WHERE virtual_environment_name = {p}", - params=[virtual_environment_name], - ) - if row is None: - return None - return VirtualEnvironmentRecord( - virtual_environment_name=row[0], - status=VirtualEnvironmentStatus(row[1]), - baseline_virtual_environment_name=row[2], - finalized_at=row[3], - ) - - def get_virtual_environment_node_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - node_type: str, - ) -> tuple[VirtualEnvironmentNodeRefRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT virtual_environment_name, node_type, node_name, version_hash " - f"FROM {self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - f"WHERE virtual_environment_name = {p} AND node_type = {p} ORDER BY node_name", - params=[virtual_environment_name, node_type], - ) - return tuple( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=row[0], - node_type=row[1], - node_name=row[2], - version_hash=row[3], - ) - for row in rows - ) - - def get_virtual_environment_model_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentModelRefRecord, ...]: - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = self.get_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="model", - ) - return tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=ref.virtual_environment_name, - model_name=ref.node_name, - version_hash=ref.version_hash, - ) - for ref in refs - ) - - def get_virtual_environment_function_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentFunctionRefRecord, ...]: - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - *self.get_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="udf", - ), - *self.get_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="table_fn", - ), - ) - return tuple( - VirtualEnvironmentFunctionRefRecord( - virtual_environment_name=ref.virtual_environment_name, - node_type=ref.node_type, - function_name=ref.node_name, - version_hash=ref.version_hash, - ) - for ref in sorted(refs, key=lambda item: (item.node_type, item.node_name)) - ) - - def get_virtual_environment_seed_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentSeedRefRecord, ...]: - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = self.get_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - node_type="seed", - ) - return tuple( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=ref.virtual_environment_name, - seed_name=ref.node_name, - version_hash=ref.version_hash, - ) - for ref in refs - ) - - def get_virtual_environment_python_node_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentPythonNodeRefRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT virtual_environment_name, node_type, node_name, version_hash " - f"FROM {self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - f"WHERE virtual_environment_name = {p} " - "AND node_type IN ('task', 'loader', 'asset', 'check', 'hook') " - "ORDER BY node_type, node_name", - params=[virtual_environment_name], - ) - return tuple( - VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=row[0], - node_type=row[1], - node_name=row[2], - version_hash=row[3], - ) - for row in rows - ) - - def get_virtual_environment_source_freshness( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[SourceFreshnessRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT virtual_environment_name, source_name, strategy, value_kind, " - "data_version, data_version_hash, observed_at " - f"FROM {self._qualified_name(schema=schema, table=SOURCE_FRESHNESS_OBSERVATION_TABLE)} " - f"WHERE virtual_environment_name = {p} ORDER BY source_name", - params=[virtual_environment_name], - ) - return tuple( - SourceFreshnessRecord( - virtual_environment_name=row[0], - source_name=row[1], - strategy=row[2], - value_kind=row[3], - data_version=row[4], - data_version_hash=row[5], - observed_at=from_naive_utc_wall_clock(row[6]), - ) - for row in rows - ) - - def list_virtual_environment_checkpoints( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentCheckpointRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT checkpoint_id, virtual_environment_name, created_at " - "FROM " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE)} " - f"WHERE virtual_environment_name = {p} ORDER BY created_at DESC, checkpoint_id DESC", - params=[virtual_environment_name], - ) - return tuple( - VirtualEnvironmentCheckpointRecord( - checkpoint_id=row[0], - virtual_environment_name=row[1], - created_at=row[2], - ) - for row in rows - ) - - def get_virtual_environment_checkpoint_model_refs( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointModelRefRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT checkpoint_id, model_name, version_hash FROM " - + self._qualified_name( - schema=schema, - table=VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE, - ) - + " " - f"WHERE checkpoint_id = {p} ORDER BY model_name", - params=[checkpoint_id], - ) - return tuple( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id=row[0], - model_name=row[1], - version_hash=row[2], - ) - for row in rows - ) - - def get_virtual_environment_checkpoint_function_refs( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...]: - p: str = self._placeholder - checkpoint_function_ref_table: str = self._qualified_name( - schema=schema, - table=VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE, - ) - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql=f"SELECT checkpoint_id, function_name, version_hash " - f"FROM {checkpoint_function_ref_table} " - f"WHERE checkpoint_id = {p} ORDER BY function_name", - params=[checkpoint_id], - ) - return tuple( - VirtualEnvironmentCheckpointFunctionRefRecord( - checkpoint_id=row[0], - function_name=row[1], - version_hash=row[2], - ) - for row in rows - ) - - def get_virtual_environment_checkpoint_seed_refs( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...]: - p: str = self._placeholder - rows: list[tuple[Any, ...]] = self._fetch_all( - connection=connection, - sql="SELECT checkpoint_id, seed_name, version_hash FROM " - + self._qualified_name( - schema=schema, - table=VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE, - ) - + " " - f"WHERE checkpoint_id = {p} ORDER BY seed_name", - params=[checkpoint_id], - ) - return tuple( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=row[0], - seed_name=row[1], - version_hash=row[2], - ) - for row in rows - ) - - def get_state_operation( - self, *, connection: Any, schema: str, operation_id: str - ) -> StateOperationRecord | None: - p: str = self._placeholder - row: tuple[Any, ...] | None = self._fetch_one( - connection=connection, - sql="SELECT operation_id, operation_type, status, virtual_environment_name " - f"FROM {self._qualified_name(schema=schema, table=STATE_OPERATION_TABLE)} " - f"WHERE operation_id = {p}", - params=[operation_id], - ) - if row is None: - return None - return StateOperationRecord( - operation_id=row[0], - operation_type=StateOperationType(row[1]), - status=StateOperationStatus(row[2]), - virtual_environment_name=row[3], - ) diff --git a/src/sqlbuild/virtual/state/classes/duckdb.py b/src/sqlbuild/virtual/state/classes/duckdb.py deleted file mode 100644 index 0f04be4598..0000000000 --- a/src/sqlbuild/virtual/state/classes/duckdb.py +++ /dev/null @@ -1,426 +0,0 @@ -"""DuckDB virtual-state backend.""" - -from __future__ import annotations - -from collections.abc import Iterator, Sequence -from contextlib import AbstractContextManager, contextmanager, nullcontext -from datetime import datetime -from typing import Any, ClassVar - -from sqlbuild.adapter.contract.classes.observed_connection import ObservedConnection -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -from sqlbuild.virtual.state._helpers.state_storage.datetime import ( - to_naive_utc_wall_clock, -) -from sqlbuild.virtual.state._helpers.state_storage.microbatch_events import ( - append_duckdb_microbatch_events, - read_duckdb_microbatch_model_history, - read_duckdb_microbatch_retention_history, - read_duckdb_microbatch_scope_history, -) -from sqlbuild.virtual.state._helpers.state_storage.validation import build_validation_result -from sqlbuild.virtual.state.classes._sql_state_backend import SqlStateBackend -from sqlbuild.virtual.state.constants import ( - DUCKDB_DATETIME_TYPE_TOKEN, - DUCKDB_INTEGER_TYPE_TOKEN, - DUCKDB_TIMESTAMP_TYPE_TOKEN, - LOCK_TABLE, - MICROBATCH_EVENT_TABLE, - PYTHON_NODE_VERSION_TABLE, - SOURCE_FRESHNESS_OBSERVATION_TABLE, - STATE_TABLE_COLUMNS, - STATE_TABLE_INDEXES, - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE, -) -from sqlbuild.virtual.state.exceptions import ( - StateBackendConfigError, -) -from sqlbuild.virtual.state.models import ( - SourceFreshnessRecord, - StateLockLease, - StateSchemaValidationResult, - VirtualEnvironmentNodeRefRecord, -) -from sqlbuild.virtual.state.types import ( - StateColumnType, -) - - -class DuckDbStateBackend(SqlStateBackend): - """DuckDB implementation for virtual state.""" - - _placeholder: ClassVar[str] = "?" - - def connect(self, config: dict[str, object]) -> Any: - import duckdb - - database: object | None = config.get("database") - if not isinstance(database, str) or not database: - raise StateBackendConfigError("DuckDB state backend requires state.connection.database") - return ObservedConnection(raw_connection=duckdb.connect(database), adapter="duckdb") - - def _fetch_one( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> tuple[Any, ...] | None: - if params is None: - return connection.execute(sql).fetchone() - return connection.execute(sql, params).fetchone() - - def _fetch_all( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> list[tuple[Any, ...]]: - if params is None: - return connection.execute(sql).fetchall() - return connection.execute(sql, params).fetchall() - - @contextmanager - def _write_transaction(self, *, connection: Any) -> Iterator[Any]: - connection.execute("BEGIN") - try: - yield connection - connection.execute("COMMIT") - except BaseException: - connection.execute("ROLLBACK") - raise - - def _statement_executor(self, *, connection: Any) -> AbstractContextManager[Any]: - return nullcontext(connection) - - def _execute_in( - self, *, executor: Any, sql: str, params: Sequence[object] | None = None - ) -> None: - if params is None: - executor.execute(sql) - return - executor.execute(sql, params) - - def _lease_is_owned(self, *, executor: Any, schema: str, lease: StateLockLease) -> bool: - owned: tuple[Any, ...] | None = executor.execute( - f"UPDATE {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "SET updated_at = CURRENT_TIMESTAMP " - "WHERE lock_key = ? AND owner_id = ? AND expires_at > CURRENT_TIMESTAMP " - "RETURNING lock_key", - [lease.lock_key, lease.owner_id], - ).fetchone() - return owned is not None - - def _fetch_one_in( - self, *, executor: Any, sql: str, params: Sequence[object] - ) -> tuple[Any, ...] | None: - return executor.execute(sql, params).fetchone() - - def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidationResult: - tables: set[str] = { - row[0] - for row in connection.execute( - "SELECT table_name FROM information_schema.tables WHERE table_schema = ?", - [schema], - ).fetchall() - } - columns_by_table: dict[str, dict[str, str]] = {} - for row in connection.execute( - "SELECT table_name, column_name, data_type FROM information_schema.columns " - "WHERE table_schema = ?", - [schema], - ).fetchall(): - columns_by_table.setdefault(row[0], {})[row[1]] = row[2] - indexes_by_table: dict[str, set[str]] = {} - for row in connection.execute( - "SELECT table_name, index_name FROM duckdb_indexes() WHERE schema_name = ?", - [schema], - ).fetchall(): - indexes_by_table.setdefault(row[0], set()).add(row[1]) - return build_validation_result( - existing_tables=tables, - columns_by_table=columns_by_table, - expected_columns=STATE_TABLE_COLUMNS, - type_matches=self._state_type_matches, - expected_indexes=STATE_TABLE_INDEXES, - existing_indexes_by_table=indexes_by_table, - ) - - def append_microbatch_events( - self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] - ) -> MicrobatchWriteResult: - return append_duckdb_microbatch_events( - connection=connection, - qualified_table=self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE), - events=events, - ) - - def read_microbatch_scope_history( - self, *, connection: Any, schema: str, scope: MicrobatchScope - ) -> tuple[MicrobatchEvent, ...]: - return read_duckdb_microbatch_scope_history( - connection=connection, - qualified_table=self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE), - scope=scope, - ) - - def read_microbatch_retention_history( - self, *, connection: Any, schema: str - ) -> tuple[MicrobatchEvent, ...]: - return read_duckdb_microbatch_retention_history( - connection=connection, - qualified_table=self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE), - ) - - def read_microbatch_model_history( - self, *, connection: Any, schema: str, scope: MicrobatchScope - ) -> tuple[MicrobatchEvent, ...]: - return read_duckdb_microbatch_model_history( - connection=connection, - qualified_table=self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE), - scope=scope, - ) - - def upsert_virtual_environment_node_ref( - self, - *, - connection: Any, - schema: str, - ref: VirtualEnvironmentNodeRefRecord, - ) -> None: - connection.execute( - "INSERT INTO " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "(virtual_environment_name, node_type, node_name, version_hash, updated_at) " - "VALUES (?, ?, ?, ?, now()) " - "ON CONFLICT (virtual_environment_name, node_type, node_name) " - "DO UPDATE SET version_hash = excluded.version_hash, updated_at = now()", - [ref.virtual_environment_name, ref.node_type, ref.node_name, ref.version_hash], - ) - - def prune_unreferenced_python_node_versions(self, *, connection: Any, schema: str) -> int: - before_count: int = self.count_unreferenced_python_node_versions( - connection=connection, schema=schema - ) - connection.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=PYTHON_NODE_VERSION_TABLE)} versions " - "WHERE NOT EXISTS (" - "SELECT 1 " - "FROM " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "refs " - "WHERE refs.node_type = versions.node_type " - "AND refs.node_name = versions.node_name " - "AND refs.version_hash = versions.version_hash)" - ) - return before_count - - def replace_virtual_environment_source_freshness( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - records: tuple[SourceFreshnessRecord, ...], - ) -> None: - self._validate_source_freshness_records( - virtual_environment_name=virtual_environment_name, - records=records, - ) - temp_table_name: str = "__sqlbuild_replace_source_freshness" - connection.execute("BEGIN") - try: - connection.execute(f"DROP TABLE IF EXISTS {temp_table_name}") - connection.execute( - f"CREATE TEMP TABLE {temp_table_name} (" - "virtual_environment_name TEXT NOT NULL, " - "source_name TEXT NOT NULL, " - "strategy TEXT NOT NULL, " - "value_kind TEXT NOT NULL, " - "data_version TEXT NOT NULL, " - "data_version_hash TEXT NOT NULL, " - "observed_at TIMESTAMP NOT NULL, " - "UNIQUE (virtual_environment_name, source_name))" - ) - record: SourceFreshnessRecord - for record in records: - connection.execute( - f"INSERT INTO {temp_table_name} " - "(virtual_environment_name, source_name, strategy, value_kind, " - "data_version, data_version_hash, observed_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?)", - [ - record.virtual_environment_name, - record.source_name, - record.strategy, - record.value_kind, - record.data_version, - record.data_version_hash, - to_naive_utc_wall_clock(record.observed_at), - ], - ) - connection.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=SOURCE_FRESHNESS_OBSERVATION_TABLE)} " - "WHERE virtual_environment_name = ? " - f"AND source_name NOT IN (SELECT source_name FROM {temp_table_name})", - [virtual_environment_name], - ) - connection.execute( - "INSERT INTO " - f"{self._qualified_name(schema=schema, table=SOURCE_FRESHNESS_OBSERVATION_TABLE)} " - "(virtual_environment_name, source_name, strategy, value_kind, data_version, " - "data_version_hash, observed_at, updated_at) " - "SELECT virtual_environment_name, source_name, strategy, value_kind, " - f"data_version, data_version_hash, observed_at, now() FROM {temp_table_name} " - "ON CONFLICT (virtual_environment_name, source_name) " - "DO UPDATE SET " - "strategy = excluded.strategy, " - "value_kind = excluded.value_kind, " - "data_version = excluded.data_version, " - "data_version_hash = excluded.data_version_hash, " - "observed_at = excluded.observed_at, " - "updated_at = now()" - ) - connection.execute(f"DROP TABLE IF EXISTS {temp_table_name}") - connection.execute("COMMIT") - except BaseException: - connection.execute("ROLLBACK") - raise - - def acquire_lock( - self, - *, - connection: Any, - schema: str, - lock_key: str, - owner_id: str, - expires_at: datetime, - ) -> bool: - connection.execute("BEGIN") - try: - connection.execute( - f"DELETE FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = ? AND expires_at <= CURRENT_TIMESTAMP", - [lock_key], - ) - connection.execute( - f"INSERT INTO {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "(lock_key, owner_id, expires_at, created_at, updated_at) " - "VALUES (?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", - [lock_key, owner_id, expires_at], - ) - connection.execute("COMMIT") - return True - except BaseException: - try: - connection.execute("ROLLBACK") - except BaseException: - pass - active_row: tuple[Any, ...] | None = connection.execute( - f"SELECT owner_id FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = ? AND expires_at > CURRENT_TIMESTAMP", - [lock_key], - ).fetchone() - if active_row is not None: - return False - raise - - def release_lock(self, *, connection: Any, schema: str, lock_key: str, owner_id: str) -> bool: - connection.execute("BEGIN") - try: - existing_row: tuple[Any, ...] | None = connection.execute( - f"SELECT owner_id FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = ? AND owner_id = ?", - [lock_key, owner_id], - ).fetchone() - if existing_row is None: - connection.execute("COMMIT") - return False - connection.execute( - f"DELETE FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = ? AND owner_id = ?", - [lock_key, owner_id], - ) - connection.execute("COMMIT") - return True - except BaseException: - connection.execute("ROLLBACK") - raise - - def delete_lock(self, *, connection: Any, schema: str, lock_key: str) -> None: - connection.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=LOCK_TABLE)} WHERE lock_key = ?", - [lock_key], - ) - - def delete_state_backup(self, *, connection: Any, schema: str, backup_id: str) -> None: - backup_schema: str = self._backup_schema_name(schema=schema, backup_id_value=backup_id) - connection.execute(f"DROP SCHEMA IF EXISTS {self._quote_identifier(backup_schema)} CASCADE") - - def _replace_virtual_environment_node_ref_groups( - self, - *, - executor: Any, - schema: str, - virtual_environment_name: str, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - node_type: str - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - for node_type, refs in refs_by_node_type.items(): - temp_table_name: str = "__sqlbuild_replace_virtual_environment_node_refs" - self._validate_node_ref_replacement( - virtual_environment_name=virtual_environment_name, - node_type=node_type, - refs=refs, - ) - executor.execute(f"DROP TABLE IF EXISTS {temp_table_name}") - executor.execute( - f"CREATE TEMP TABLE {temp_table_name} (" - "virtual_environment_name TEXT NOT NULL, " - "node_type TEXT NOT NULL, " - "node_name TEXT NOT NULL, " - "version_hash TEXT NOT NULL, " - "UNIQUE (virtual_environment_name, node_type, node_name))" - ) - ref: VirtualEnvironmentNodeRefRecord - for ref in refs: - executor.execute( - f"INSERT INTO {temp_table_name} " - "(virtual_environment_name, node_type, node_name, version_hash) " - "VALUES (?, ?, ?, ?)", - [ - ref.virtual_environment_name, - ref.node_type, - ref.node_name, - ref.version_hash, - ], - ) - executor.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "WHERE virtual_environment_name = ? AND node_type = ? " - "AND NOT EXISTS (" - f"SELECT 1 FROM {temp_table_name} incoming " - "WHERE incoming.node_name = " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)}" - ".node_name)", - [virtual_environment_name, node_type], - ) - executor.execute( - "INSERT INTO " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "(virtual_environment_name, node_type, node_name, version_hash, updated_at) " - "SELECT virtual_environment_name, node_type, node_name, version_hash, now() " - f"FROM {temp_table_name} " - "ON CONFLICT (virtual_environment_name, node_type, node_name) " - "DO UPDATE SET version_hash = excluded.version_hash, updated_at = now()" - ) - executor.execute(f"DROP TABLE IF EXISTS {temp_table_name}") - - def _state_type_matches(self, *, actual_type: str, expected_type: StateColumnType) -> bool: - actual: str = actual_type.lower() - match expected_type: - case StateColumnType.INTEGER: - return DUCKDB_INTEGER_TYPE_TOKEN in actual - case StateColumnType.TEXT: - return any(token in actual for token in ("text", "varchar", "character", "string")) - case StateColumnType.TIMESTAMP: - return DUCKDB_TIMESTAMP_TYPE_TOKEN in actual or DUCKDB_DATETIME_TYPE_TOKEN in actual - return False diff --git a/src/sqlbuild/virtual/state/classes/microbatch_store.py b/src/sqlbuild/virtual/state/classes/microbatch_store.py deleted file mode 100644 index 9380224ea0..0000000000 --- a/src/sqlbuild/virtual/state/classes/microbatch_store.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Virtual-state implementation of the shared microbatch event-store contract.""" - -from __future__ import annotations - -import threading -import time -from typing import Any - -from sqlbuild.microbatches.constants import ( - MICROBATCH_WRITE_ATTEMPTS, - MICROBATCH_WRITE_RETRY_BASE_SECONDS, -) -from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -from sqlbuild.virtual.state.classes.state_backend import StateBackend - - -class VirtualMicrobatchEventStore: - """Serialize short-lived backend connections for each event operation.""" - - def __init__( - self, - *, - backend: StateBackend, - connection_config: dict[str, object], - schema: str, - operation_lock: threading.Lock, - ) -> None: - self._backend = backend - self._connection_config = connection_config - self._schema = schema - self._operation_lock = operation_lock - - def write(self, event: MicrobatchEvent) -> None: - self.write_many((event,)) - - def write_many(self, events: tuple[MicrobatchEvent, ...]) -> MicrobatchWriteResult: - if not events: - return MicrobatchWriteResult(total=0, inserted=0, already_existing=0) - for attempt in range(MICROBATCH_WRITE_ATTEMPTS): - try: - with self._operation_lock: - connection: Any = self._backend.connect(self._connection_config) - try: - result: MicrobatchWriteResult = self._backend.append_microbatch_events( - connection=connection, schema=self._schema, events=events - ) - finally: - self._backend.close(connection) - return result - except Exception: - if attempt + 1 == MICROBATCH_WRITE_ATTEMPTS: - raise - time.sleep(MICROBATCH_WRITE_RETRY_BASE_SECONDS * (attempt + 1)) - raise MicrobatchStateError("virtual microbatch event publication exhausted retries") - - def read_scope_history(self, scope: MicrobatchScope) -> tuple[MicrobatchEvent, ...]: - with self._operation_lock: - connection: Any = self._backend.connect(self._connection_config) - try: - return self._backend.read_microbatch_scope_history( - connection=connection, schema=self._schema, scope=scope - ) - finally: - self._backend.close(connection) - - def read_model_history(self, scope: MicrobatchScope) -> tuple[MicrobatchEvent, ...]: - with self._operation_lock: - connection: Any = self._backend.connect(self._connection_config) - try: - return self._backend.read_microbatch_model_history( - connection=connection, - schema=self._schema, - scope=scope, - ) - finally: - self._backend.close(connection) diff --git a/src/sqlbuild/virtual/state/classes/postgres.py b/src/sqlbuild/virtual/state/classes/postgres.py deleted file mode 100644 index ac11fb12ec..0000000000 --- a/src/sqlbuild/virtual/state/classes/postgres.py +++ /dev/null @@ -1,489 +0,0 @@ -"""Postgres virtual-state backend.""" - -from __future__ import annotations - -from collections.abc import Iterator, Sequence -from contextlib import AbstractContextManager, contextmanager -from datetime import datetime -from typing import Any, ClassVar - -from sqlbuild.adapter.contract.classes.observed_connection import ObservedConnection -from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.constants import ( - MICROBATCH_COLUMNS, - MICROBATCH_GENERATION_WILDCARD, - VIRTUAL_MICROBATCH_SCOPE_KIND, -) -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -from sqlbuild.virtual.state._helpers.state_storage.datetime import ( - to_naive_utc_wall_clock, -) -from sqlbuild.virtual.state._helpers.state_storage.validation import ( - build_validation_result, -) -from sqlbuild.virtual.state.classes._sql_state_backend import SqlStateBackend -from sqlbuild.virtual.state.constants import ( - LOCK_TABLE, - MICROBATCH_EVENT_TABLE, - POSTGRES_INTEGER_TYPES, - POSTGRES_TEXT_TYPES, - PYTHON_NODE_VERSION_TABLE, - SOURCE_FRESHNESS_OBSERVATION_TABLE, - STATE_TABLE_COLUMNS, - STATE_TABLE_INDEXES, - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE, -) -from sqlbuild.virtual.state.exceptions import ( - StateBackendConfigError, -) -from sqlbuild.virtual.state.models import ( - SourceFreshnessRecord, - StateLockLease, - StateSchemaValidationResult, - VirtualEnvironmentNodeRefRecord, -) -from sqlbuild.virtual.state.types import ( - StateColumnType, -) - - -class PostgresStateBackend(SqlStateBackend): - """Postgres implementation for virtual state.""" - - _placeholder: ClassVar[str] = "%s" - - def connect(self, config: dict[str, object]) -> Any: - try: - import psycopg - except ImportError as error: - raise StateBackendConfigError( - "Postgres state backend requires optional dependency psycopg. " - "Install with: pip install 'psycopg[binary]' or sqlbuild[postgres]" - ) from error - - try: - return ObservedConnection( - raw_connection=psycopg.connect( - host=_optional_str(config.get("host")), - port=_optional_int(config.get("port")), - user=_optional_str(config.get("user")), - password=_optional_str(config.get("password")), - dbname=_optional_str(config.get("dbname")), - autocommit=True, - ), - adapter="postgres", - ) - except Exception as error: - raise StateBackendConfigError("Could not connect to Postgres state backend") from error - - def _fetch_one( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> tuple[Any, ...] | None: - with connection.cursor() as cursor: - if params is None: - cursor.execute(sql) - else: - cursor.execute(sql, params) - return cursor.fetchone() - - def _fetch_all( - self, *, connection: Any, sql: str, params: Sequence[object] | None = None - ) -> list[tuple[Any, ...]]: - with connection.cursor() as cursor: - if params is None: - cursor.execute(sql) - else: - cursor.execute(sql, params) - return cursor.fetchall() - - @contextmanager - def _write_transaction(self, *, connection: Any) -> Iterator[Any]: - with connection.cursor() as cursor: - cursor.execute("BEGIN") - try: - yield cursor - cursor.execute("COMMIT") - except BaseException: - cursor.execute("ROLLBACK") - raise - - def _statement_executor(self, *, connection: Any) -> AbstractContextManager[Any]: - return connection.cursor() - - def _execute_in( - self, *, executor: Any, sql: str, params: Sequence[object] | None = None - ) -> None: - if params is None: - executor.execute(sql) - return - executor.execute(sql, params) - - def _lease_is_owned(self, *, executor: Any, schema: str, lease: StateLockLease) -> bool: - lock_table: str = self._qualified_name(schema=schema, table=LOCK_TABLE) - executor.execute( - f"SELECT lock_key FROM {lock_table} " - "WHERE lock_key = %s AND owner_id = %s " - "AND expires_at > CURRENT_TIMESTAMP FOR UPDATE", - [lease.lock_key, lease.owner_id], - ) - return executor.fetchone() is not None - - def _fetch_one_in( - self, *, executor: Any, sql: str, params: Sequence[object] - ) -> tuple[Any, ...] | None: - executor.execute(sql, params) - return executor.fetchone() - - def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidationResult: - with connection.cursor() as cursor: - cursor.execute( - "SELECT table_name FROM information_schema.tables WHERE table_schema = %s", - [schema], - ) - tables: set[str] = {row[0] for row in cursor.fetchall()} - cursor.execute( - "SELECT table_name, column_name, data_type FROM information_schema.columns " - "WHERE table_schema = %s", - [schema], - ) - columns_by_table: dict[str, dict[str, str]] = {} - for row in cursor.fetchall(): - columns_by_table.setdefault(row[0], {})[row[1]] = row[2] - cursor.execute( - "SELECT tablename, indexname FROM pg_indexes WHERE schemaname = %s", - [schema], - ) - indexes_by_table: dict[str, set[str]] = {} - for row in cursor.fetchall(): - indexes_by_table.setdefault(row[0], set()).add(row[1]) - return build_validation_result( - existing_tables=tables, - columns_by_table=columns_by_table, - expected_columns=STATE_TABLE_COLUMNS, - type_matches=self._state_type_matches, - expected_indexes=STATE_TABLE_INDEXES, - existing_indexes_by_table=indexes_by_table, - ) - - def append_microbatch_events( - self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] - ) -> MicrobatchWriteResult: - if not events: - return MicrobatchWriteResult(total=0, inserted=0, already_existing=0) - table: str = self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE) - id_placeholders: str = ", ".join("%s" for _ in events) - with connection.cursor() as cursor: - cursor.execute( - f"SELECT event_id FROM {table} WHERE event_id IN ({id_placeholders})", - [event.event_id for event in events], - ) - existing_ids: frozenset[str] = frozenset(str(row[0]) for row in cursor.fetchall()) - missing: tuple[MicrobatchEvent, ...] = tuple( - event for event in events if event.event_id not in existing_ids - ) - if missing: - row_placeholders: str = ( - "(" - + ", ".join( - f"%s::{ - self._state_column_sql_type( - STATE_TABLE_COLUMNS[MICROBATCH_EVENT_TABLE][column] - ) - }" - for column in MICROBATCH_COLUMNS - ) - + ")" - ) - values_sql: str = ", ".join(row_placeholders for _ in missing) - columns: str = ", ".join(MICROBATCH_COLUMNS) - params: list[object | None] = [] - for event in missing: - params.extend(MicrobatchEventCodec.values(event)) - cursor.execute( - f"INSERT INTO {table} ({columns}) SELECT {columns} FROM " - f"(VALUES {values_sql}) AS incoming ({columns}) WHERE NOT EXISTS " - f"(SELECT 1 FROM {table} existing " - "WHERE existing.event_id = incoming.event_id)", - params, - ) - return MicrobatchWriteResult( - total=len(events), inserted=len(missing), already_existing=len(events) - len(missing) - ) - - def read_microbatch_scope_history( - self, *, connection: Any, schema: str, scope: MicrobatchScope - ) -> tuple[MicrobatchEvent, ...]: - generation_sql: str = ( - "" - if scope.physical_generation_id == MICROBATCH_GENERATION_WILDCARD - else "AND physical_generation_id = %s " - ) - params: list[object] = [scope.scope_kind, scope.scope_key] - if scope.physical_generation_id != MICROBATCH_GENERATION_WILDCARD: - params.append(scope.physical_generation_id) - with connection.cursor() as cursor: - cursor.execute( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM " - f"{self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - "WHERE scope_kind = %s AND scope_key = %s " - f"{generation_sql}ORDER BY created_at, event_id", - params, - ) - rows: list[tuple[Any, ...]] = cursor.fetchall() - return MicrobatchEventCodec.from_rows(tuple(row) for row in rows) - - def read_microbatch_retention_history( - self, *, connection: Any, schema: str - ) -> tuple[MicrobatchEvent, ...]: - with connection.cursor() as cursor: - cursor.execute( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM " - f"{self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - "WHERE scope_kind = %s ORDER BY created_at, event_id", - [VIRTUAL_MICROBATCH_SCOPE_KIND], - ) - rows: list[tuple[Any, ...]] = cursor.fetchall() - return MicrobatchEventCodec.from_rows(tuple(row) for row in rows) - - def read_microbatch_model_history( - self, *, connection: Any, schema: str, scope: MicrobatchScope - ) -> tuple[MicrobatchEvent, ...]: - warehouse_realm: str = scope.physical_generation_id.partition(":")[0] - with connection.cursor() as cursor: - cursor.execute( - f"SELECT {', '.join(MICROBATCH_COLUMNS)} FROM " - f"{self._qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - "WHERE scope_kind = %s AND model_name = %s " - "AND physical_generation_id LIKE %s ORDER BY created_at, event_id", - [scope.scope_kind, scope.model_name, f"{warehouse_realm}:%"], - ) - rows: list[tuple[Any, ...]] = cursor.fetchall() - return MicrobatchEventCodec.from_rows(tuple(row) for row in rows) - - def upsert_virtual_environment_node_ref( - self, - *, - connection: Any, - schema: str, - ref: VirtualEnvironmentNodeRefRecord, - ) -> None: - with connection.cursor() as cursor: - cursor.execute( - "INSERT INTO " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "(virtual_environment_name, node_type, node_name, version_hash, updated_at) " - "VALUES (%s, %s, %s, %s, CURRENT_TIMESTAMP) " - "ON CONFLICT (virtual_environment_name, node_type, node_name) " - "DO UPDATE SET version_hash = excluded.version_hash, " - "updated_at = CURRENT_TIMESTAMP", - [ref.virtual_environment_name, ref.node_type, ref.node_name, ref.version_hash], - ) - - def prune_unreferenced_python_node_versions(self, *, connection: Any, schema: str) -> int: - before_count: int = self.count_unreferenced_python_node_versions( - connection=connection, schema=schema - ) - with connection.cursor() as cursor: - cursor.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=PYTHON_NODE_VERSION_TABLE)} " - "AS versions " - "WHERE NOT EXISTS (" - "SELECT 1 " - "FROM " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "refs " - "WHERE refs.node_type = versions.node_type " - "AND refs.node_name = versions.node_name " - "AND refs.version_hash = versions.version_hash)" - ) - return before_count - - def replace_virtual_environment_source_freshness( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - records: tuple[SourceFreshnessRecord, ...], - ) -> None: - self._validate_source_freshness_records( - virtual_environment_name=virtual_environment_name, - records=records, - ) - with connection.cursor() as cursor: - cursor.execute("BEGIN") - try: - cursor.execute( - "DELETE FROM " - + self._qualified_name( - schema=schema, - table=SOURCE_FRESHNESS_OBSERVATION_TABLE, - ) - + " " - "WHERE virtual_environment_name = %s", - [virtual_environment_name], - ) - record: SourceFreshnessRecord - for record in records: - cursor.execute( - "INSERT INTO " - + self._qualified_name( - schema=schema, - table=SOURCE_FRESHNESS_OBSERVATION_TABLE, - ) - + " " - "(virtual_environment_name, source_name, strategy, value_kind, " - "data_version, data_version_hash, observed_at, updated_at) " - "VALUES (%s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP)", - [ - record.virtual_environment_name, - record.source_name, - record.strategy, - record.value_kind, - record.data_version, - record.data_version_hash, - to_naive_utc_wall_clock(record.observed_at), - ], - ) - cursor.execute("COMMIT") - except BaseException: - cursor.execute("ROLLBACK") - raise - - def acquire_lock( - self, - *, - connection: Any, - schema: str, - lock_key: str, - owner_id: str, - expires_at: datetime, - ) -> bool: - with connection.cursor() as cursor: - cursor.execute("BEGIN") - try: - cursor.execute( - f"DELETE FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = %s AND expires_at <= CURRENT_TIMESTAMP", - [lock_key], - ) - cursor.execute( - f"INSERT INTO {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "(lock_key, owner_id, expires_at, created_at, updated_at) " - "VALUES (%s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", - [lock_key, owner_id, expires_at], - ) - cursor.execute("COMMIT") - return True - except BaseException: - cursor.execute("ROLLBACK") - cursor.execute( - f"SELECT owner_id FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = %s AND expires_at > CURRENT_TIMESTAMP", - [lock_key], - ) - if cursor.fetchone() is not None: - return False - raise - - def release_lock(self, *, connection: Any, schema: str, lock_key: str, owner_id: str) -> bool: - with connection.cursor() as cursor: - cursor.execute("BEGIN") - try: - cursor.execute( - f"SELECT owner_id FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = %s AND owner_id = %s", - [lock_key, owner_id], - ) - if cursor.fetchone() is None: - cursor.execute("COMMIT") - return False - cursor.execute( - f"DELETE FROM {self._qualified_name(schema=schema, table=LOCK_TABLE)} " - "WHERE lock_key = %s AND owner_id = %s", - [lock_key, owner_id], - ) - cursor.execute("COMMIT") - return True - except BaseException: - cursor.execute("ROLLBACK") - raise - - def delete_lock(self, *, connection: Any, schema: str, lock_key: str) -> None: - with connection.cursor() as cursor: - cursor.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=LOCK_TABLE)} WHERE lock_key = %s", - [lock_key], - ) - connection.commit() - - def delete_state_backup(self, *, connection: Any, schema: str, backup_id: str) -> None: - backup_schema: str = self._backup_schema_name(schema=schema, backup_id_value=backup_id) - with connection.cursor() as cursor: - cursor.execute(f"DROP SCHEMA IF EXISTS {self._quote_identifier(backup_schema)} CASCADE") - connection.commit() - - def _replace_virtual_environment_node_ref_groups( - self, - *, - executor: Any, - schema: str, - virtual_environment_name: str, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - node_type: str - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - for node_type, refs in refs_by_node_type.items(): - self._validate_node_ref_replacement( - virtual_environment_name=virtual_environment_name, - node_type=node_type, - refs=refs, - ) - executor.execute( - "DELETE FROM " - f"{self._qualified_name(schema=schema, table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE)} " - "WHERE virtual_environment_name = %s AND node_type = %s", - [virtual_environment_name, node_type], - ) - ref: VirtualEnvironmentNodeRefRecord - for ref in refs: - executor.execute( - "INSERT INTO " - + self._qualified_name( - schema=schema, - table=VIRTUAL_ENVIRONMENT_NODE_REF_TABLE, - ) - + " " - "(virtual_environment_name, node_type, node_name, version_hash, " - "updated_at) VALUES (%s, %s, %s, %s, CURRENT_TIMESTAMP) " - "ON CONFLICT (virtual_environment_name, node_type, node_name) " - "DO UPDATE SET version_hash = excluded.version_hash, " - "updated_at = CURRENT_TIMESTAMP", - [ - ref.virtual_environment_name, - ref.node_type, - ref.node_name, - ref.version_hash, - ], - ) - - def _state_type_matches(self, *, actual_type: str, expected_type: StateColumnType) -> bool: - actual: str = actual_type.lower() - match expected_type: - case StateColumnType.INTEGER: - return actual in POSTGRES_INTEGER_TYPES - case StateColumnType.TEXT: - return actual in POSTGRES_TEXT_TYPES - case StateColumnType.TIMESTAMP: - return actual.startswith("timestamp") - return False - - -def _optional_str(value: object) -> str | None: - return value if isinstance(value, str) else None - - -def _optional_int(value: object) -> int | None: - return value if isinstance(value, int) else None diff --git a/src/sqlbuild/virtual/state/classes/state_backend.py b/src/sqlbuild/virtual/state/classes/state_backend.py deleted file mode 100644 index 45bcf63860..0000000000 --- a/src/sqlbuild/virtual/state/classes/state_backend.py +++ /dev/null @@ -1,572 +0,0 @@ -"""State backend contract.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from datetime import datetime -from typing import Any - -from sqlbuild.executor.node_results.models import ( - NodeResultEnvelope, - NodeResultQuery, - NodeResultRecord, -) -from sqlbuild.microbatches.exceptions import MicrobatchStateError -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationAncestryRecord, - PhysicalRelationRecord, - PythonNodeVersionRecord, - ReconcileEventRecord, - SeedVersionRecord, - SourceFreshnessRecord, - StateBackupRecord, - StateLockLease, - StateLockRecord, - StateOperationEventRecord, - StateOperationRecord, - StateSchemaValidationResult, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentPythonNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentRetentionRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import PhysicalArtifactType - - -class StateBackend(ABC): - """State backend contract for virtual mode lifecycle operations.""" - - @abstractmethod - def connect(self, config: dict[str, object]) -> Any: - """Open a state store connection.""" - ... - - @abstractmethod - def close(self, connection: Any) -> None: - """Close a state store connection.""" - ... - - @abstractmethod - def initialize(self, *, connection: Any, schema: str, sqlbuild_version: str) -> None: - """Create or migrate initial state schema objects.""" - ... - - @abstractmethod - def inspect_schema(self, *, connection: Any, schema: str) -> StateSchemaValidationResult: - """Inspect required state schema objects.""" - ... - - @abstractmethod - def create_backup(self, *, connection: Any, schema: str) -> str: - """Create a backup set and return its backup id.""" - ... - - @abstractmethod - def rollback(self, *, connection: Any, schema: str, backup_id: str | None = None) -> str: - """Restore state tables from a backup and return the used backup id.""" - ... - - @abstractmethod - def reset(self, *, connection: Any, schema: str) -> None: - """Drop state tables for the configured schema.""" - ... - - @abstractmethod - def upsert_model_version( - self, *, connection: Any, schema: str, record: ModelVersionRecord - ) -> None: - """Insert or replace a model version row.""" - ... - - @abstractmethod - def get_model_version( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> ModelVersionRecord | None: - """Return a model version row if it exists.""" - ... - - @abstractmethod - def upsert_function_version( - self, *, connection: Any, schema: str, record: FunctionVersionRecord - ) -> None: - """Insert or replace a function version row.""" - ... - - @abstractmethod - def get_function_version( - self, *, connection: Any, schema: str, function_name: str, version_hash: str - ) -> FunctionVersionRecord | None: - """Return a function version row if it exists.""" - ... - - @abstractmethod - def upsert_seed_version( - self, *, connection: Any, schema: str, record: SeedVersionRecord - ) -> None: - """Insert or replace a seed version row.""" - ... - - @abstractmethod - def get_seed_version( - self, *, connection: Any, schema: str, seed_name: str, version_hash: str - ) -> SeedVersionRecord | None: - """Return a seed version row if it exists.""" - ... - - @abstractmethod - def upsert_python_node_version( - self, *, connection: Any, schema: str, record: PythonNodeVersionRecord - ) -> None: - """Insert or replace a Python node identity version row.""" - ... - - @abstractmethod - def get_python_node_version( - self, - *, - connection: Any, - schema: str, - node_type: str, - node_name: str, - version_hash: str, - ) -> PythonNodeVersionRecord | None: - """Return a Python node identity version row if it exists.""" - ... - - @abstractmethod - def insert_node_result( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - record: NodeResultRecord, - ) -> None: - """Append one runtime node result row for a virtual environment.""" - ... - - @abstractmethod - def read_node_results( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - query: NodeResultQuery, - ) -> tuple[NodeResultEnvelope, ...]: - """Read runtime node result rows for one virtual environment identity.""" - ... - - def append_microbatch_events( - self, *, connection: Any, schema: str, events: tuple[MicrobatchEvent, ...] - ) -> MicrobatchWriteResult: - """Append logical microbatch events to virtual state in bulk.""" - raise MicrobatchStateError("virtual state backend does not support bulk microbatch events") - - def read_microbatch_scope_history( - self, *, connection: Any, schema: str, scope: MicrobatchScope - ) -> tuple[MicrobatchEvent, ...]: - """Read ordered microbatch history for one physical scope.""" - raise MicrobatchStateError("virtual state backend does not support microbatch events") - - def read_microbatch_retention_history( - self, *, connection: Any, schema: str - ) -> tuple[MicrobatchEvent, ...]: - """Read virtual microbatch history needed to derive active janitor roots.""" - raise MicrobatchStateError("virtual state backend does not support microbatch events") - - def read_microbatch_model_history( - self, *, connection: Any, schema: str, scope: MicrobatchScope - ) -> tuple[MicrobatchEvent, ...]: - """Read virtual microbatch history for one model and warehouse realm.""" - raise MicrobatchStateError("virtual state backend does not support microbatch events") - - @abstractmethod - def upsert_physical_relation( - self, *, connection: Any, schema: str, record: PhysicalRelationRecord - ) -> None: - """Insert or replace a physical relation row.""" - ... - - @abstractmethod - def get_physical_relation_for_artifact( - self, - *, - connection: Any, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - version_hash: str, - ) -> PhysicalRelationRecord | None: - """Return a physical relation row for an artifact if it exists.""" - ... - - @abstractmethod - def list_physical_relations_for_artifact( - self, - *, - connection: Any, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - ) -> tuple[PhysicalRelationRecord, ...]: - """Return tracked physical relations for one artifact, newest first when available.""" - ... - - def get_physical_relation( - self, *, connection: Any, schema: str, model_name: str, version_hash: str - ) -> PhysicalRelationRecord | None: - """Return a physical model relation row if it exists.""" - return self.get_physical_relation_for_artifact( - connection=connection, - schema=schema, - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=model_name, - version_hash=version_hash, - ) - - def list_physical_relations_for_model( - self, *, connection: Any, schema: str, model_name: str - ) -> tuple[PhysicalRelationRecord, ...]: - """Return tracked physical relations for one model, newest first when available.""" - return self.list_physical_relations_for_artifact( - connection=connection, - schema=schema, - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=model_name, - ) - - @abstractmethod - def upsert_physical_relation_ancestry( - self, *, connection: Any, schema: str, record: PhysicalRelationAncestryRecord - ) -> None: - """Insert or replace physical relation ancestry.""" - ... - - @abstractmethod - def upsert_virtual_environment( - self, *, connection: Any, schema: str, record: VirtualEnvironmentRecord - ) -> None: - """Insert or replace a virtual environment row.""" - ... - - @abstractmethod - def get_virtual_environment( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> VirtualEnvironmentRecord | None: - """Return a virtual environment row if it exists.""" - ... - - @abstractmethod - def list_virtual_environments( - self, *, connection: Any, schema: str - ) -> tuple[VirtualEnvironmentRetentionRecord, ...]: - """Return virtual environment retention metadata.""" - ... - - @abstractmethod - def delete_virtual_environment( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> None: - """Delete a virtual environment and its current refs.""" - ... - - @abstractmethod - def replace_virtual_environment_node_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - node_type: str, - refs: tuple[VirtualEnvironmentNodeRefRecord, ...], - ) -> None: - """Replace all refs for one virtual environment node type.""" - ... - - @abstractmethod - def replace_virtual_environment_node_ref_groups( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - """Replace refs for multiple virtual environment node types atomically.""" - ... - - @abstractmethod - def upsert_virtual_environment_and_replace_node_ref_groups( - self, - *, - connection: Any, - schema: str, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - """Upsert a virtual environment and replace multiple ref groups atomically.""" - ... - - @abstractmethod - def upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - self, - *, - connection: Any, - schema: str, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - leases: tuple[StateLockLease, ...], - checkpoint: VirtualEnvironmentCheckpointRecord | None = None, - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = (), - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = (), - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = (), - ) -> bool: - """Validate status/ref/checkpoint input before writes, then publish if leases are owned.""" - ... - - @abstractmethod - def get_virtual_environment_node_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - node_type: str, - ) -> tuple[VirtualEnvironmentNodeRefRecord, ...]: - """Return refs for one virtual environment node type.""" - ... - - @abstractmethod - def upsert_virtual_environment_node_ref( - self, - *, - connection: Any, - schema: str, - ref: VirtualEnvironmentNodeRefRecord, - ) -> None: - """Insert or replace one generic node ref for a virtual environment.""" - ... - - @abstractmethod - def replace_virtual_environment_model_refs( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], - ) -> None: - """Replace all refs for a virtual environment.""" - ... - - @abstractmethod - def get_virtual_environment_model_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentModelRefRecord, ...]: - """Return refs for a virtual environment.""" - ... - - @abstractmethod - def get_virtual_environment_function_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentFunctionRefRecord, ...]: - """Return function refs for a virtual environment.""" - ... - - @abstractmethod - def get_virtual_environment_seed_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentSeedRefRecord, ...]: - """Return seed refs for a virtual environment.""" - ... - - @abstractmethod - def upsert_virtual_environment_python_node_ref( - self, - *, - connection: Any, - schema: str, - ref: VirtualEnvironmentPythonNodeRefRecord, - ) -> None: - """Insert or replace one Python node ref for a virtual environment.""" - ... - - @abstractmethod - def get_virtual_environment_python_node_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentPythonNodeRefRecord, ...]: - """Return Python node refs for a virtual environment.""" - ... - - @abstractmethod - def count_unreferenced_python_node_versions(self, *, connection: Any, schema: str) -> int: - """Return count of Python node versions not referenced by any virtual environment.""" - ... - - @abstractmethod - def prune_unreferenced_python_node_versions(self, *, connection: Any, schema: str) -> int: - """Delete Python node versions not referenced by any virtual environment.""" - ... - - @abstractmethod - def replace_virtual_environment_source_freshness( - self, - *, - connection: Any, - schema: str, - virtual_environment_name: str, - records: tuple[SourceFreshnessRecord, ...], - ) -> None: - """Replace all source freshness records for a virtual environment.""" - ... - - @abstractmethod - def get_virtual_environment_source_freshness( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[SourceFreshnessRecord, ...]: - """Return source freshness records for a virtual environment.""" - ... - - @abstractmethod - def create_virtual_environment_checkpoint( - self, - *, - connection: Any, - schema: str, - checkpoint: VirtualEnvironmentCheckpointRecord, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = (), - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = (), - ) -> None: - """Create a finalized virtual environment checkpoint.""" - ... - - @abstractmethod - def list_virtual_environment_checkpoints( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentCheckpointRecord, ...]: - """Return checkpoints for a virtual environment, newest first.""" - ... - - @abstractmethod - def get_virtual_environment_checkpoint_model_refs( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointModelRefRecord, ...]: - """Return refs for a virtual environment checkpoint.""" - ... - - @abstractmethod - def get_virtual_environment_checkpoint_function_refs( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...]: - """Return function refs for a virtual environment checkpoint.""" - ... - - @abstractmethod - def get_virtual_environment_checkpoint_seed_refs( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...]: - """Return seed refs for a virtual environment checkpoint.""" - ... - - @abstractmethod - def delete_virtual_environment_checkpoint( - self, *, connection: Any, schema: str, checkpoint_id: str - ) -> None: - """Delete one virtual environment checkpoint and its refs.""" - ... - - @abstractmethod - def upsert_state_operation( - self, *, connection: Any, schema: str, record: StateOperationRecord - ) -> None: - """Insert or replace a tracked virtual operation row.""" - ... - - @abstractmethod - def get_state_operation( - self, *, connection: Any, schema: str, operation_id: str - ) -> StateOperationRecord | None: - """Return a tracked virtual operation row if it exists.""" - ... - - @abstractmethod - def create_state_operation_event( - self, *, connection: Any, schema: str, record: StateOperationEventRecord - ) -> None: - """Append one state operation event row.""" - ... - - @abstractmethod - def create_reconcile_event( - self, *, connection: Any, schema: str, record: ReconcileEventRecord - ) -> None: - """Append one reconcile event row.""" - ... - - @abstractmethod - def acquire_lock( - self, - *, - connection: Any, - schema: str, - lock_key: str, - owner_id: str, - expires_at: datetime, - ) -> bool: - """Acquire or replace an expired lock.""" - ... - - def renew_lock( - self, - *, - connection: Any, - schema: str, - lock_key: str, - owner_id: str, - expires_at: datetime, - ) -> bool: - """Renew a non-expired lock when ownership still matches.""" - - return False - - @abstractmethod - def release_lock(self, *, connection: Any, schema: str, lock_key: str, owner_id: str) -> bool: - """Release a lock owned by the given owner.""" - ... - - @abstractmethod - def list_active_locks(self, *, connection: Any, schema: str) -> tuple[StateLockRecord, ...]: - """Return non-expired locks.""" - ... - - @abstractmethod - def list_expired_locks(self, *, connection: Any, schema: str) -> tuple[StateLockRecord, ...]: - """Return expired locks.""" - ... - - @abstractmethod - def delete_lock(self, *, connection: Any, schema: str, lock_key: str) -> None: - """Delete one lock by key.""" - ... - - @abstractmethod - def list_state_backups(self, *, connection: Any, schema: str) -> tuple[StateBackupRecord, ...]: - """Return state migration backup schemas known to the state store.""" - ... - - @abstractmethod - def delete_state_backup(self, *, connection: Any, schema: str, backup_id: str) -> None: - """Delete one state migration backup schema.""" - ... diff --git a/src/sqlbuild/virtual/state/constants.py b/src/sqlbuild/virtual/state/constants.py deleted file mode 100644 index 309451c7fb..0000000000 --- a/src/sqlbuild/virtual/state/constants.py +++ /dev/null @@ -1,493 +0,0 @@ -"""Virtual state store constants.""" - -from __future__ import annotations - -from sqlbuild.virtual.state.types import StateColumnType - -DUCKDB_MEMORY_DATABASE: str = ":memory:" -STATE_BOOLEAN_TRUE: str = "true" -DUCKDB_INTEGER_TYPE_TOKEN: str = "int" -DUCKDB_TIMESTAMP_TYPE_TOKEN: str = "timestamp" -DUCKDB_DATETIME_TYPE_TOKEN: str = "datetime" -POSTGRES_INTEGER_TYPES: tuple[str, ...] = ("integer", "bigint", "smallint") -POSTGRES_TEXT_TYPES: tuple[str, ...] = ("text", "character varying", "character") - -CURRENT_STATE_SCHEMA_VERSION: int = 1 - -STATE_VERSION_TABLE: str = "state_versions" -MODEL_VERSION_TABLE: str = "model_versions" -FUNCTION_VERSION_TABLE: str = "function_versions" -SEED_VERSION_TABLE: str = "seed_versions" -PYTHON_NODE_VERSION_TABLE: str = "python_node_versions" -NODE_RESULTS_TABLE: str = "node_results" -MICROBATCH_EVENT_TABLE: str = "microbatch_events" -PHYSICAL_RELATION_TABLE: str = "physical_relations" -PHYSICAL_RELATION_ANCESTRY_TABLE: str = "physical_relation_ancestry" -VIRTUAL_ENVIRONMENT_TABLE: str = "virtual_environments" -VIRTUAL_ENVIRONMENT_NODE_REF_TABLE: str = "virtual_environment_node_refs" -SOURCE_FRESHNESS_OBSERVATION_TABLE: str = "source_freshness_observations" -VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE: str = "virtual_environment_checkpoints" -VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE: str = "virtual_environment_checkpoint_model_refs" -VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE: str = ( - "virtual_environment_checkpoint_function_refs" -) -VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE: str = "virtual_environment_checkpoint_seed_refs" -LOCK_TABLE: str = "locks" -STATE_OPERATION_TABLE: str = "state_operations" -PLAN_RUN_TABLE: str = "plan_runs" -VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_TABLE: str = "virtual_environment_model_ref_events" -RECONCILE_EVENT_TABLE: str = "reconcile_events" -STATE_MIGRATION_EVENTS_TABLE: str = "state_migration_events" -STATE_OPERATION_EVENT_TABLE: str = "state_operation_events" - -STATE_TABLES: tuple[str, ...] = ( - STATE_VERSION_TABLE, - MODEL_VERSION_TABLE, - FUNCTION_VERSION_TABLE, - SEED_VERSION_TABLE, - PYTHON_NODE_VERSION_TABLE, - NODE_RESULTS_TABLE, - MICROBATCH_EVENT_TABLE, - PHYSICAL_RELATION_TABLE, - PHYSICAL_RELATION_ANCESTRY_TABLE, - VIRTUAL_ENVIRONMENT_TABLE, - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE, - SOURCE_FRESHNESS_OBSERVATION_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE, - VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE, - LOCK_TABLE, - STATE_OPERATION_TABLE, - PLAN_RUN_TABLE, - VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_TABLE, - RECONCILE_EVENT_TABLE, - STATE_MIGRATION_EVENTS_TABLE, - STATE_OPERATION_EVENT_TABLE, -) - -STATE_VERSION_COLUMNS: dict[str, StateColumnType] = { - "schema_version": StateColumnType.INTEGER, - "sqlbuild_version": StateColumnType.TEXT, - "updated_at": StateColumnType.TIMESTAMP, -} - -STATE_MIGRATION_EVENT_COLUMNS: dict[str, StateColumnType] = { - "event_id": StateColumnType.TEXT, - "action": StateColumnType.TEXT, - "backup_id": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "message": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -MODEL_VERSION_COLUMNS: dict[str, StateColumnType] = { - "model_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "definition_identity_hash": StateColumnType.TEXT, - "identity_metadata_hash": StateColumnType.TEXT, - "definition_text_b64": StateColumnType.TEXT, - "identity_metadata_json_b64": StateColumnType.TEXT, - "compiled_sql_b64": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -FUNCTION_VERSION_COLUMNS: dict[str, StateColumnType] = { - "function_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "language": StateColumnType.TEXT, - "returns": StateColumnType.TEXT, - "arguments_json_b64": StateColumnType.TEXT, - "return_columns_json_b64": StateColumnType.TEXT, - "packages_json_b64": StateColumnType.TEXT, - "runtime_version": StateColumnType.TEXT, - "entry_point": StateColumnType.TEXT, - "body_sql_b64": StateColumnType.TEXT, - "definition_text_b64": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -SEED_VERSION_COLUMNS: dict[str, StateColumnType] = { - "seed_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "identity_metadata_hash": StateColumnType.TEXT, - "identity_metadata_json_b64": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -PYTHON_NODE_VERSION_COLUMNS: dict[str, StateColumnType] = { - "node_type": StateColumnType.TEXT, - "node_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "definition_hash": StateColumnType.TEXT, - "identity_metadata_hash": StateColumnType.TEXT, - "definition_json_b64": StateColumnType.TEXT, - "identity_metadata_json_b64": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -NODE_RESULTS_COLUMNS: dict[str, StateColumnType] = { - "virtual_environment_name": StateColumnType.TEXT, - "node_type": StateColumnType.TEXT, - "node_name": StateColumnType.TEXT, - "target_database": StateColumnType.TEXT, - "target_schema": StateColumnType.TEXT, - "target_name": StateColumnType.TEXT, - "run_id": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "payload_json_b64": StateColumnType.TEXT, - "metadata_json_b64": StateColumnType.TEXT, - "error_message": StateColumnType.TEXT, - "materialized": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -MICROBATCH_EVENT_COLUMNS: dict[str, StateColumnType] = { - "event_id": StateColumnType.TEXT, - "record_type": StateColumnType.TEXT, - "scope_kind": StateColumnType.TEXT, - "scope_key": StateColumnType.TEXT, - "model_name": StateColumnType.TEXT, - "target_database": StateColumnType.TEXT, - "target_schema": StateColumnType.TEXT, - "target_name": StateColumnType.TEXT, - "physical_generation_id": StateColumnType.TEXT, - "virtual_environment_name": StateColumnType.TEXT, - "virtual_model_version_hash": StateColumnType.TEXT, - "origin_run_id": StateColumnType.TEXT, - "origin_run_started_at": StateColumnType.TIMESTAMP, - "execution_run_id": StateColumnType.TEXT, - "execution_run_started_at": StateColumnType.TIMESTAMP, - "run_type": StateColumnType.TEXT, - "completion_type": StateColumnType.TEXT, - "run_start": StateColumnType.TEXT, - "run_end": StateColumnType.TEXT, - "partition_start": StateColumnType.TEXT, - "partition_end": StateColumnType.TEXT, - "batch_size": StateColumnType.TEXT, - "cursor_column": StateColumnType.TEXT, - "cursor_type": StateColumnType.TEXT, - "cursor_grain": StateColumnType.TEXT, - "model_version_hash": StateColumnType.TEXT, - "definition_hash": StateColumnType.TEXT, - "fingerprint_status": StateColumnType.TEXT, - "replay_requirement_id": StateColumnType.TEXT, - "required_model_version_hash": StateColumnType.TEXT, - "previous_model_version_hash": StateColumnType.TEXT, - "replay_policy": StateColumnType.TEXT, - "rows_affected": StateColumnType.INTEGER, - "completed_at": StateColumnType.TIMESTAMP, - "coverage_source": StateColumnType.TEXT, - "observed_row_count": StateColumnType.INTEGER, - "observed_at": StateColumnType.TIMESTAMP, - "synthetic_reason": StateColumnType.TEXT, - "unaccounted_policy": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -PHYSICAL_RELATION_COLUMNS: dict[str, StateColumnType] = { - "artifact_type": StateColumnType.TEXT, - "artifact_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "database_name": StateColumnType.TEXT, - "schema_name": StateColumnType.TEXT, - "relation_name": StateColumnType.TEXT, - "relation_type": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -PHYSICAL_RELATION_ANCESTRY_COLUMNS: dict[str, StateColumnType] = { - "model_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "parent_model_name": StateColumnType.TEXT, - "parent_version_hash": StateColumnType.TEXT, - "seed_strategy": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -VIRTUAL_ENVIRONMENT_COLUMNS: dict[str, StateColumnType] = { - "virtual_environment_name": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "baseline_virtual_environment_name": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, - "finalized_at": StateColumnType.TIMESTAMP, -} - -VIRTUAL_ENVIRONMENT_NODE_REF_COLUMNS: dict[str, StateColumnType] = { - "virtual_environment_name": StateColumnType.TEXT, - "node_type": StateColumnType.TEXT, - "node_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, - "updated_at": StateColumnType.TIMESTAMP, -} - -SOURCE_FRESHNESS_OBSERVATION_COLUMNS: dict[str, StateColumnType] = { - "virtual_environment_name": StateColumnType.TEXT, - "source_name": StateColumnType.TEXT, - "strategy": StateColumnType.TEXT, - "value_kind": StateColumnType.TEXT, - "data_version": StateColumnType.TEXT, - "data_version_hash": StateColumnType.TEXT, - "observed_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -VIRTUAL_ENVIRONMENT_CHECKPOINT_COLUMNS: dict[str, StateColumnType] = { - "checkpoint_id": StateColumnType.TEXT, - "virtual_environment_name": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_COLUMNS: dict[str, StateColumnType] = { - "checkpoint_id": StateColumnType.TEXT, - "model_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, -} - -VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_COLUMNS: dict[str, StateColumnType] = { - "checkpoint_id": StateColumnType.TEXT, - "function_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, -} - -VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_COLUMNS: dict[str, StateColumnType] = { - "checkpoint_id": StateColumnType.TEXT, - "seed_name": StateColumnType.TEXT, - "version_hash": StateColumnType.TEXT, -} - -LOCK_COLUMNS: dict[str, StateColumnType] = { - "lock_key": StateColumnType.TEXT, - "owner_id": StateColumnType.TEXT, - "expires_at": StateColumnType.TIMESTAMP, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -STATE_OPERATION_COLUMNS: dict[str, StateColumnType] = { - "operation_id": StateColumnType.TEXT, - "operation_type": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "virtual_environment_name": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, - "updated_at": StateColumnType.TIMESTAMP, -} - -PLAN_RUN_COLUMNS: dict[str, StateColumnType] = { - "run_id": StateColumnType.TEXT, - "command": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "virtual_environment_name": StateColumnType.TEXT, - "started_at": StateColumnType.TIMESTAMP, - "completed_at": StateColumnType.TIMESTAMP, -} - -VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_COLUMNS: dict[str, StateColumnType] = { - "event_id": StateColumnType.TEXT, - "virtual_environment_name": StateColumnType.TEXT, - "model_name": StateColumnType.TEXT, - "previous_version_hash": StateColumnType.TEXT, - "new_version_hash": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -RECONCILE_EVENT_COLUMNS: dict[str, StateColumnType] = { - "event_id": StateColumnType.TEXT, - "action": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "message": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -STATE_OPERATION_EVENT_COLUMNS: dict[str, StateColumnType] = { - "event_id": StateColumnType.TEXT, - "operation_id": StateColumnType.TEXT, - "action": StateColumnType.TEXT, - "status": StateColumnType.TEXT, - "message": StateColumnType.TEXT, - "created_at": StateColumnType.TIMESTAMP, -} - -STATE_TABLE_COLUMNS: dict[str, dict[str, StateColumnType]] = { - STATE_VERSION_TABLE: STATE_VERSION_COLUMNS, - MODEL_VERSION_TABLE: MODEL_VERSION_COLUMNS, - FUNCTION_VERSION_TABLE: FUNCTION_VERSION_COLUMNS, - SEED_VERSION_TABLE: SEED_VERSION_COLUMNS, - PYTHON_NODE_VERSION_TABLE: PYTHON_NODE_VERSION_COLUMNS, - NODE_RESULTS_TABLE: NODE_RESULTS_COLUMNS, - MICROBATCH_EVENT_TABLE: MICROBATCH_EVENT_COLUMNS, - PHYSICAL_RELATION_TABLE: PHYSICAL_RELATION_COLUMNS, - PHYSICAL_RELATION_ANCESTRY_TABLE: PHYSICAL_RELATION_ANCESTRY_COLUMNS, - VIRTUAL_ENVIRONMENT_TABLE: VIRTUAL_ENVIRONMENT_COLUMNS, - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE: VIRTUAL_ENVIRONMENT_NODE_REF_COLUMNS, - SOURCE_FRESHNESS_OBSERVATION_TABLE: SOURCE_FRESHNESS_OBSERVATION_COLUMNS, - VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE: VIRTUAL_ENVIRONMENT_CHECKPOINT_COLUMNS, - VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE: ( - VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_COLUMNS - ), - VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE: ( - VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_COLUMNS - ), - VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE: ( - VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_COLUMNS - ), - LOCK_TABLE: LOCK_COLUMNS, - STATE_OPERATION_TABLE: STATE_OPERATION_COLUMNS, - PLAN_RUN_TABLE: PLAN_RUN_COLUMNS, - VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_TABLE: VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_COLUMNS, - RECONCILE_EVENT_TABLE: RECONCILE_EVENT_COLUMNS, - STATE_MIGRATION_EVENTS_TABLE: STATE_MIGRATION_EVENT_COLUMNS, - STATE_OPERATION_EVENT_TABLE: STATE_OPERATION_EVENT_COLUMNS, -} - -STATE_TABLE_INDEXES: dict[str, dict[str, tuple[str, ...]]] = { - MODEL_VERSION_TABLE: { - "idx_sqb_model_versions_identity": ("model_name", "version_hash"), - }, - FUNCTION_VERSION_TABLE: { - "idx_sqb_function_versions_identity": ("function_name", "version_hash"), - }, - SEED_VERSION_TABLE: { - "idx_sqb_seed_versions_identity": ("seed_name", "version_hash"), - }, - PYTHON_NODE_VERSION_TABLE: { - "idx_sqb_python_node_versions_identity": ( - "node_type", - "node_name", - "version_hash", - ), - }, - NODE_RESULTS_TABLE: { - "idx_sqb_node_results_latest": ( - "virtual_environment_name", - "node_type", - "node_name", - "target_database", - "target_schema", - "target_name", - "status", - "created_at", - "run_id", - ), - "idx_sqb_node_results_run_id": ( - "virtual_environment_name", - "run_id", - "node_type", - "node_name", - "target_database", - "target_schema", - "target_name", - ), - }, - MICROBATCH_EVENT_TABLE: { - "idx_sqb_microbatch_events_scope": ( - "scope_kind", - "scope_key", - "physical_generation_id", - "created_at", - "event_id", - ), - "idx_sqb_microbatch_events_model": ("model_name", "created_at"), - "idx_sqb_microbatch_events_run": ("execution_run_id", "created_at"), - "idx_sqb_microbatch_events_requirement": ( - "replay_requirement_id", - "record_type", - "created_at", - ), - "idx_sqb_microbatch_events_partition": ( - "scope_key", - "physical_generation_id", - "partition_start", - "partition_end", - "created_at", - ), - }, - PHYSICAL_RELATION_TABLE: { - "idx_sqb_physical_relations_identity": ( - "artifact_type", - "artifact_name", - "version_hash", - ), - }, - PHYSICAL_RELATION_ANCESTRY_TABLE: { - "idx_sqb_physical_relation_ancestry_identity": ("model_name", "version_hash"), - }, - VIRTUAL_ENVIRONMENT_TABLE: { - "idx_sqb_virtual_environments_identity": ("virtual_environment_name",), - }, - VIRTUAL_ENVIRONMENT_NODE_REF_TABLE: { - "idx_sqb_virtual_environment_node_refs_identity": ( - "virtual_environment_name", - "node_type", - "node_name", - ), - }, - SOURCE_FRESHNESS_OBSERVATION_TABLE: { - "idx_sqb_source_freshness_observations_identity": ( - "virtual_environment_name", - "source_name", - ), - }, - VIRTUAL_ENVIRONMENT_CHECKPOINT_TABLE: { - "idx_sqb_virtual_environment_checkpoints_identity": ("checkpoint_id",), - }, - VIRTUAL_ENVIRONMENT_CHECKPOINT_MODEL_REF_TABLE: { - "idx_sqb_virtual_environment_checkpoint_model_refs_identity": ( - "checkpoint_id", - "model_name", - ), - }, - VIRTUAL_ENVIRONMENT_CHECKPOINT_FUNCTION_REF_TABLE: { - "idx_sqb_virtual_environment_checkpoint_function_refs_identity": ( - "checkpoint_id", - "function_name", - ), - }, - VIRTUAL_ENVIRONMENT_CHECKPOINT_SEED_REF_TABLE: { - "idx_sqb_virtual_environment_checkpoint_seed_refs_identity": ( - "checkpoint_id", - "seed_name", - ), - }, - LOCK_TABLE: { - "idx_sqb_locks_identity": ("lock_key",), - }, - STATE_OPERATION_TABLE: { - "idx_sqb_state_operations_identity": ("operation_id",), - }, - PLAN_RUN_TABLE: { - "idx_sqb_plan_runs_identity": ("run_id",), - }, - VIRTUAL_ENVIRONMENT_MODEL_REF_EVENT_TABLE: { - "idx_sqb_virtual_environment_model_ref_events_identity": ("event_id",), - }, - RECONCILE_EVENT_TABLE: { - "idx_sqb_reconcile_events_identity": ("event_id",), - }, - STATE_MIGRATION_EVENTS_TABLE: { - "idx_sqb_state_migration_events_identity": ("event_id",), - }, - STATE_OPERATION_EVENT_TABLE: { - "idx_sqb_state_operation_events_identity": ("event_id",), - }, -} - -NON_UNIQUE_STATE_INDEXES: frozenset[str] = frozenset( - { - "idx_sqb_microbatch_events_model", - "idx_sqb_microbatch_events_run", - "idx_sqb_microbatch_events_requirement", - "idx_sqb_microbatch_events_partition", - } -) diff --git a/src/sqlbuild/virtual/state/exceptions.py b/src/sqlbuild/virtual/state/exceptions.py deleted file mode 100644 index 6780294346..0000000000 --- a/src/sqlbuild/virtual/state/exceptions.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Virtual state store exceptions.""" - -from __future__ import annotations - - -class StateBackendError(RuntimeError): - """Raised when state backend operations fail.""" - - -class StateBackendConfigError(StateBackendError): - """Raised when state backend configuration is invalid.""" - - -class StateSchemaInvalidError(StateBackendError): - """Raised when an operation requires a valid state schema.""" - - -class StateBackupNotFoundError(StateBackendError): - """Raised when rollback cannot find a state backup.""" diff --git a/src/sqlbuild/virtual/state/main/__init__.py b/src/sqlbuild/virtual/state/main/__init__.py deleted file mode 100644 index 4eaf57cd05..0000000000 --- a/src/sqlbuild/virtual/state/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""State lifecycle entry package.""" diff --git a/src/sqlbuild/virtual/state/main/checkpoints/__init__.py b/src/sqlbuild/virtual/state/main/checkpoints/__init__.py deleted file mode 100644 index a82143fd01..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state checkpoint entrypoints.""" diff --git a/src/sqlbuild/virtual/state/main/checkpoints/_build_finalized_checkpoint.py b/src/sqlbuild/virtual/state/main/checkpoints/_build_finalized_checkpoint.py deleted file mode 100644 index db4e04e816..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/_build_finalized_checkpoint.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Build finalized virtual environment checkpoint rows.""" - -from __future__ import annotations - -import uuid -from datetime import UTC, datetime - -from sqlbuild.virtual.state.models import ( - FinalizedVirtualEnvironmentCheckpoint, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) - - -def build_finalized_virtual_environment_checkpoint( - *, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = (), - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = (), -) -> FinalizedVirtualEnvironmentCheckpoint: - """Build checkpoint rows for one finalized virtual environment.""" - - timestamp: str = datetime.now(UTC).strftime("%Y%m%d%H%M%S%f") - checkpoint_id: str = f"chk_{timestamp}_{uuid.uuid4().hex}" - return FinalizedVirtualEnvironmentCheckpoint( - checkpoint=VirtualEnvironmentCheckpointRecord( - checkpoint_id=checkpoint_id, - virtual_environment_name=virtual_environment_name, - ), - refs=tuple( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id=checkpoint_id, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - ), - function_refs=tuple( - VirtualEnvironmentCheckpointFunctionRefRecord( - checkpoint_id=checkpoint_id, - function_name=ref.function_name, - version_hash=ref.version_hash, - ) - for ref in function_refs - ), - seed_refs=tuple( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id=checkpoint_id, - seed_name=ref.seed_name, - version_hash=ref.version_hash, - ) - for ref in seed_refs - ), - ) diff --git a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_model_refs.py b/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_model_refs.py deleted file mode 100644 index bbd86c884e..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoint_model_refs.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Public checkpoint ref helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.backend import build_state_backend -from sqlbuild.virtual.state._helpers.state_runtime.config import resolve_state_backend_config -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import ( - StateBackendConfig, - VirtualEnvironmentCheckpointModelRefRecord, -) - - -def get_virtual_environment_checkpoint_model_refs( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - checkpoint_id: str, -) -> tuple[VirtualEnvironmentCheckpointModelRefRecord, ...]: - """Get refs for a virtual environment checkpoint.""" - - config: StateBackendConfig = resolve_state_backend_config( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - backend: StateBackend = build_state_backend(config.backend) - connection: Any = backend.connect(config.connection) - try: - return backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, - schema=config.schema, - checkpoint_id=checkpoint_id, - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoints.py b/src/sqlbuild/virtual/state/main/checkpoints/_checkpoints.py deleted file mode 100644 index eed2953521..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/_checkpoints.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Public checkpoint helpers for virtual targets.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.main.checkpoints._build_finalized_checkpoint import ( - build_finalized_virtual_environment_checkpoint, -) -from sqlbuild.virtual.state.models import ( - FinalizedVirtualEnvironmentCheckpoint, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) - - -def create_finalized_virtual_environment_checkpoint( - *, - backend: StateBackend, - connection: Any, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = (), - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = (), -) -> str: - """Persist a finalized VDE checkpoint and return its checkpoint id.""" - - payload: FinalizedVirtualEnvironmentCheckpoint = build_finalized_virtual_environment_checkpoint( - virtual_environment_name=virtual_environment_name, - refs=refs, - function_refs=function_refs, - seed_refs=seed_refs, - ) - backend.create_virtual_environment_checkpoint( - connection=connection, - schema=schema, - checkpoint=payload.checkpoint, - refs=payload.refs, - function_refs=payload.function_refs, - seed_refs=payload.seed_refs, - ) - return payload.checkpoint.checkpoint_id diff --git a/src/sqlbuild/virtual/state/main/checkpoints/checkpoint_retention.py b/src/sqlbuild/virtual/state/main/checkpoints/checkpoint_retention.py deleted file mode 100644 index 903eca664d..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/checkpoint_retention.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Public checkpoint retention inspection helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - CheckpointRetentionInspection, - PhysicalRelationRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, -) - - -def inspect_checkpoint_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, - max_checkpoints: int, -) -> CheckpointRetentionInspection: - """Inspect checkpoint retention and retained checkpoint physical relations.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - backend.list_virtual_environment_checkpoints( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ) - ) - retained_checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = checkpoints[ - :max_checkpoints - ] - prune_checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = checkpoints[ - max_checkpoints: - ] - retained_relations: dict[tuple[str | None, str | None, str], PhysicalRelationRecord] = {} - checkpoint: VirtualEnvironmentCheckpointRecord - for checkpoint in retained_checkpoints: - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, - schema=config.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - ref: VirtualEnvironmentCheckpointModelRefRecord - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is None: - continue - retained_relations[ - (relation.database_name, relation.schema_name, relation.relation_name) - ] = relation - return CheckpointRetentionInspection( - prune_checkpoints=prune_checkpoints, - retained_physical_relations=tuple( - retained_relations[key] for key in sorted(retained_relations) - ), - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/checkpoints/delete_checkpoint.py b/src/sqlbuild/virtual/state/main/checkpoints/delete_checkpoint.py deleted file mode 100644 index a91f2eff7c..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/delete_checkpoint.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Public checkpoint deletion helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime - - -def delete_virtual_environment_checkpoint( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - checkpoint_id: str, -) -> None: - """Delete one virtual environment checkpoint and its refs.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - backend.delete_virtual_environment_checkpoint( - connection=connection, - schema=config.schema, - checkpoint_id=checkpoint_id, - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/checkpoints/list_checkpoints.py b/src/sqlbuild/virtual/state/main/checkpoints/list_checkpoints.py deleted file mode 100644 index bddf54fe2d..0000000000 --- a/src/sqlbuild/virtual/state/main/checkpoints/list_checkpoints.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Public checkpoint list helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.backend import build_state_backend -from sqlbuild.virtual.state._helpers.state_runtime.config import resolve_state_backend_config -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateBackendConfig, VirtualEnvironmentCheckpointRecord - - -def list_virtual_environment_checkpoints( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, -) -> tuple[VirtualEnvironmentCheckpointRecord, ...]: - """List checkpoints for a virtual environment.""" - - config: StateBackendConfig = resolve_state_backend_config( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - backend: StateBackend = build_state_backend(config.backend) - connection: Any = backend.connect(config.connection) - try: - return backend.list_virtual_environment_checkpoints( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/encoding/__init__.py b/src/sqlbuild/virtual/state/main/encoding/__init__.py deleted file mode 100644 index 8cf7483ed6..0000000000 --- a/src/sqlbuild/virtual/state/main/encoding/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state encoding entries.""" diff --git a/src/sqlbuild/virtual/state/main/encoding/_decode_state_text.py b/src/sqlbuild/virtual/state/main/encoding/_decode_state_text.py deleted file mode 100644 index e3acfe2e7e..0000000000 --- a/src/sqlbuild/virtual/state/main/encoding/_decode_state_text.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Decode text stored in virtual state rows.""" - -from __future__ import annotations - -from sqlbuild.virtual.state._helpers.state_storage.encoding import decode_state_text_impl - - -def decode_state_text(value: str | None) -> str | None: - """Decode text stored in state rows.""" - - return decode_state_text_impl(value) diff --git a/src/sqlbuild/virtual/state/main/encoding/_encode_state_text.py b/src/sqlbuild/virtual/state/main/encoding/_encode_state_text.py deleted file mode 100644 index ad478ae4d7..0000000000 --- a/src/sqlbuild/virtual/state/main/encoding/_encode_state_text.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Encode text for storage in virtual state rows.""" - -from __future__ import annotations - -from sqlbuild.virtual.state._helpers.state_storage.encoding import encode_state_text_impl - - -def encode_state_text(value: str) -> str: - """Encode text for storage in state rows.""" - - return encode_state_text_impl(value) diff --git a/src/sqlbuild/virtual/state/main/environments/__init__.py b/src/sqlbuild/virtual/state/main/environments/__init__.py deleted file mode 100644 index 95bf50621d..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state environment entrypoints.""" diff --git a/src/sqlbuild/virtual/state/main/environments/_environment_model_refs.py b/src/sqlbuild/virtual/state/main/environments/_environment_model_refs.py deleted file mode 100644 index 02df54edbd..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/_environment_model_refs.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Public virtual environment ref helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import VirtualEnvironmentModelRefRecord - - -def get_virtual_environment_model_refs( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, -) -> tuple[VirtualEnvironmentModelRefRecord, ...]: - """Get refs for a virtual environment.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - return backend.get_virtual_environment_model_refs( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/environments/_record_operation.py b/src/sqlbuild/virtual/state/main/environments/_record_operation.py deleted file mode 100644 index 58244fe315..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/_record_operation.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Public state operation/event recording surface.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.virtual.state._helpers.state_storage.events import event_id -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateOperationEventRecord, StateOperationRecord -from sqlbuild.virtual.state.types import StateOperationStatus, StateOperationType - - -def record_state_operation( - *, - backend: StateBackend, - connection: Any, - schema: str, - operation_id: str, - operation_type: StateOperationType | None, - status: StateOperationStatus, - action: str, - virtual_environment_name: str | None, - message: str | None = None, -) -> None: - """Persist the latest state-operation status plus an append-only event.""" - - existing: StateOperationRecord | None = backend.get_state_operation( - connection=connection, - schema=schema, - operation_id=operation_id, - ) - effective_type: StateOperationType | None = ( - operation_type - if operation_type is not None - else (existing.operation_type if existing else None) - ) - effective_virtual_environment_name: str | None = ( - virtual_environment_name - if virtual_environment_name is not None - else (existing.virtual_environment_name if existing else None) - ) - if effective_type is None: - return - backend.upsert_state_operation( - connection=connection, - schema=schema, - record=StateOperationRecord( - operation_id=operation_id, - operation_type=effective_type, - status=status, - virtual_environment_name=effective_virtual_environment_name, - ), - ) - backend.create_state_operation_event( - connection=connection, - schema=schema, - record=StateOperationEventRecord( - event_id=event_id(), - operation_id=operation_id, - action=action, - status=status, - message=message, - ), - ) diff --git a/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py b/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py deleted file mode 100644 index 5dba51746c..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/_ref_model_versions.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Public helper resolving model-version records for environment refs.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.virtual.state.models import ModelVersionRecord, VirtualEnvironmentModelRefRecord - - -def read_ref_model_versions( - *, - backend: Any, - state_connection: Any, - schema: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], -) -> dict[str, ModelVersionRecord | None]: - """Return the model-version record bound by each ref, keyed by model name.""" - - return { - ref.model_name: backend.get_model_version( - connection=state_connection, - schema=schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - for ref in refs - } diff --git a/src/sqlbuild/virtual/state/main/environments/adopt.py b/src/sqlbuild/virtual/state/main/environments/adopt.py deleted file mode 100644 index dc091ccaaf..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/adopt.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public state adopt entrypoint.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_lifecycle.adopt import adopt_into_virtual_state -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateBackendConfig - - -def run_state_adopt( - *, - discovered_inputs: DiscoveredProjectInputs, - config: StateBackendConfig, - backend: StateBackend, - state_connection: Any, - adapter: BaseAdapter, - connection: Any, - allow_copy: bool = False, -) -> str: - """Adopt existing stateless targets into virtual state and logical views.""" - - return adopt_into_virtual_state( - discovered_inputs=discovered_inputs, - config=config, - backend=backend, - state_connection=state_connection, - adapter=adapter, - connection=connection, - allow_copy=allow_copy, - ) diff --git a/src/sqlbuild/virtual/state/main/environments/delete_virtual_environment.py b/src/sqlbuild/virtual/state/main/environments/delete_virtual_environment.py deleted file mode 100644 index 541055e0b5..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/delete_virtual_environment.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Public virtual environment deletion helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime - - -def delete_virtual_environment( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - virtual_environment_name: str, -) -> None: - """Delete one virtual environment and its current refs.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - backend.delete_virtual_environment( - connection=connection, - schema=config.schema, - virtual_environment_name=virtual_environment_name, - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/environments/detach.py b/src/sqlbuild/virtual/state/main/environments/detach.py deleted file mode 100644 index 4f4999c3ca..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/detach.py +++ /dev/null @@ -1,34 +0,0 @@ -"""Public state detach entrypoint.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_lifecycle.detach import detach_from_virtual_state -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateBackendConfig - - -def run_state_detach( - *, - discovered_inputs: DiscoveredProjectInputs, - config: StateBackendConfig, - backend: StateBackend, - state_connection: Any, - adapter: BaseAdapter, - connection: Any, - allow_copy: bool = False, -) -> str: - """Detach an unsuffixed virtual environment back into stateless objects.""" - - return detach_from_virtual_state( - discovered_inputs=discovered_inputs, - config=config, - backend=backend, - state_connection=state_connection, - adapter=adapter, - connection=connection, - allow_copy=allow_copy, - ) diff --git a/src/sqlbuild/virtual/state/main/environments/lifecycle.py b/src/sqlbuild/virtual/state/main/environments/lifecycle.py deleted file mode 100644 index 9fb8b1df24..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/lifecycle.py +++ /dev/null @@ -1,155 +0,0 @@ -"""State lifecycle command entrypoint.""" - -from __future__ import annotations - -import importlib.metadata -import time -from collections.abc import Callable -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.presentation.main.supports_color import supports_color -from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback -from sqlbuild.virtual.state._helpers.state_lifecycle.lifecycle_output import ( - format_state_lifecycle_summary, -) -from sqlbuild.virtual.state._helpers.state_runtime.backend import build_state_backend -from sqlbuild.virtual.state._helpers.state_runtime.config import resolve_state_backend_config -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import StateBackendConfig, StateSchemaValidationResult -from sqlbuild.virtual.state.types import StateCommand - - -def run_state_lifecycle( - *, - project_dir: Path | None, - command: StateCommand, - backup_id: str | None = None, - auto_approve: bool = False, - no_color: bool = False, - on_connection_start: Callable[[int], None] | None = None, - on_connection_complete: ConnectionElapsedCallback | None = None, - on_connection_error: ConnectionElapsedCallback | None = None, -) -> int: - """Run a virtual state lifecycle command.""" - - effective_project_dir: Path = project_dir if project_dir is not None else Path.cwd() - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_project_dir - ) - config: StateBackendConfig = resolve_state_backend_config( - discovered_inputs=discovered_inputs, - project_dir=effective_project_dir, - ) - use_color: bool = not no_color and supports_color() - backend: StateBackend = build_state_backend(config.backend) - started_at: float = time.perf_counter() - if on_connection_start is not None: - on_connection_start(1) - try: - connection: Any = backend.connect(config.connection) - except BaseException: - if on_connection_error is not None: - on_connection_error(1, elapsed_seconds=time.perf_counter() - started_at) - raise - if on_connection_complete is not None: - on_connection_complete(1, elapsed_seconds=time.perf_counter() - started_at) - try: - return _execute_state_command( - backend=backend, - connection=connection, - config=config, - command=command, - backup_id=backup_id, - auto_approve=auto_approve, - use_color=use_color, - ) - finally: - backend.close(connection) - - -def _execute_state_command( - *, - backend: StateBackend, - connection: Any, - config: StateBackendConfig, - command: StateCommand, - backup_id: str | None, - auto_approve: bool, - use_color: bool, -) -> int: - """Run one lifecycle command against the open state backend connection.""" - - if command == StateCommand.INIT: - backend.initialize( - connection=connection, - schema=config.schema, - sqlbuild_version=importlib.metadata.version("sqlbuild"), - ) - print( - format_state_lifecycle_summary( - title="Virtual State Initialized", - config=config, - use_color=use_color, - ) - ) - return 0 - if command == StateCommand.MIGRATE: - created_backup_id: str = backend.create_backup(connection=connection, schema=config.schema) - backend.initialize( - connection=connection, - schema=config.schema, - sqlbuild_version=importlib.metadata.version("sqlbuild"), - ) - print( - format_state_lifecycle_summary( - title="Virtual State Migrated", - config=config, - use_color=use_color, - backup_id=created_backup_id, - ) - ) - return 0 - if command == StateCommand.ROLLBACK: - used_backup_id: str = backend.rollback( - connection=connection, - schema=config.schema, - backup_id=backup_id, - ) - print( - format_state_lifecycle_summary( - title="Virtual State Rolled Back", - config=config, - use_color=use_color, - backup_id=used_backup_id, - ) - ) - return 0 - if command == StateCommand.RESET: - if not config.allow_reset: - raise StateBackendConfigError( - "state reset is disabled for this target. To allow it, set " - "`allow_reset = true` under `[targets..state]` in " - "sqlbuild_project.toml or sqlbuild_local.toml, then rerun with " - "--auto-approve." - ) - if not auto_approve: - raise StateBackendConfigError("state reset requires --auto-approve") - backend.reset(connection=connection, schema=config.schema) - print( - format_state_lifecycle_summary( - title="Virtual State Reset", - config=config, - use_color=use_color, - ) - ) - return 0 - validation: StateSchemaValidationResult = backend.inspect_schema( - connection=connection, - schema=config.schema, - ) - print(f"State schema valid: {validation.valid}") - return 0 diff --git a/src/sqlbuild/virtual/state/main/environments/runtime.py b/src/sqlbuild/virtual/state/main/environments/runtime.py deleted file mode 100644 index fdab95615c..0000000000 --- a/src/sqlbuild/virtual/state/main/environments/runtime.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Build virtual state runtime access.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import ( - build_state_runtime as _build_state_runtime, -) -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateBackendConfig - - -def build_state_runtime( - *, - discovered_inputs: DiscoveredProjectInputs, - project_dir: Path, - selected_target: str | None = None, -) -> tuple[StateBackendConfig, StateBackend]: - """Resolve config and construct the configured virtual state backend.""" - - return _build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - selected_target=selected_target, - ) diff --git a/src/sqlbuild/virtual/state/main/locks/__init__.py b/src/sqlbuild/virtual/state/main/locks/__init__.py deleted file mode 100644 index 0c2cc0a0d6..0000000000 --- a/src/sqlbuild/virtual/state/main/locks/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state lock entrypoints.""" diff --git a/src/sqlbuild/virtual/state/main/locks/_locks.py b/src/sqlbuild/virtual/state/main/locks/_locks.py deleted file mode 100644 index 7484c45196..0000000000 --- a/src/sqlbuild/virtual/state/main/locks/_locks.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Public state lock helpers.""" - -from __future__ import annotations - -from datetime import datetime, timedelta -from typing import Any - -from sqlbuild.virtual.state._helpers.state_storage.locks import acquire_virtual_environment_lock -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateLockLease - - -def acquire_virtual_environment_lease( - *, - backend: StateBackend, - connection: Any, - schema: str, - virtual_environment_name: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire a virtual environment mutation lease.""" - - return acquire_virtual_environment_lock( - backend=backend, - connection=connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - owner_id=owner_id, - ttl=ttl, - now=now, - ) diff --git a/src/sqlbuild/virtual/state/main/locks/_model_version_lock.py b/src/sqlbuild/virtual/state/main/locks/_model_version_lock.py deleted file mode 100644 index 90c9541eb4..0000000000 --- a/src/sqlbuild/virtual/state/main/locks/_model_version_lock.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Public model-version lock helper.""" - -from __future__ import annotations - -from datetime import datetime, timedelta -from typing import Any - -from sqlbuild.virtual.state._helpers.state_storage.locks import acquire_model_version_lock -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateLockLease - - -def acquire_model_version_lease( - *, - backend: StateBackend, - connection: Any, - schema: str, - model_name: str, - version_hash: str, - owner_id: str, - ttl: timedelta, - now: datetime | None = None, -) -> StateLockLease | None: - """Acquire a physical model version mutation lease.""" - - return acquire_model_version_lock( - backend=backend, - connection=connection, - schema=schema, - model_name=model_name, - version_hash=version_hash, - owner_id=owner_id, - ttl=ttl, - now=now, - ) diff --git a/src/sqlbuild/virtual/state/main/locks/_release_lock.py b/src/sqlbuild/virtual/state/main/locks/_release_lock.py deleted file mode 100644 index 2f2febbbfb..0000000000 --- a/src/sqlbuild/virtual/state/main/locks/_release_lock.py +++ /dev/null @@ -1,21 +0,0 @@ -"""Public state lock release entrypoint.""" - -from __future__ import annotations - -from typing import Any - -from sqlbuild.virtual.state._helpers.state_storage.locks import release_state_lock -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import StateLockLease - - -def release_state_lease( - *, - backend: StateBackend, - connection: Any, - schema: str, - lease: StateLockLease, -) -> bool: - """Release a state lock lease.""" - - return release_state_lock(backend=backend, connection=connection, schema=schema, lease=lease) diff --git a/src/sqlbuild/virtual/state/main/locks/delete_lock.py b/src/sqlbuild/virtual/state/main/locks/delete_lock.py deleted file mode 100644 index c8a501deb4..0000000000 --- a/src/sqlbuild/virtual/state/main/locks/delete_lock.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Public state lock deletion helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime - - -def delete_lock( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - lock_key: str, -) -> None: - """Delete one state lock by key.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - backend.delete_lock(connection=connection, schema=config.schema, lock_key=lock_key) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/python_identities/__init__.py b/src/sqlbuild/virtual/state/main/python_identities/__init__.py deleted file mode 100644 index 8b0f0a69ce..0000000000 --- a/src/sqlbuild/virtual/state/main/python_identities/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state Python identity entrypoints.""" diff --git a/src/sqlbuild/virtual/state/main/python_identities/_python_node_identity_read.py b/src/sqlbuild/virtual/state/main/python_identities/_python_node_identity_read.py deleted file mode 100644 index 1e2e7cca44..0000000000 --- a/src/sqlbuild/virtual/state/main/python_identities/_python_node_identity_read.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Read Python node identity observations from virtual state.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any - -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.virtual.state.main.encoding._decode_state_text import decode_state_text -from sqlbuild.virtual.state.models import ( - PythonNodeVersionRecord, - VirtualEnvironmentPythonNodeRefRecord, -) - - -def read_virtual_python_identity_fingerprints( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, -) -> dict[tuple[str, str], Fingerprint]: - """Return VDE-bound Python identities in the shared plan comparison shape.""" - - result: dict[tuple[str, str], Fingerprint] = {} - refs: tuple[VirtualEnvironmentPythonNodeRefRecord, ...] = ( - backend.get_virtual_environment_python_node_refs( - connection=state_connection, - schema=schema, - virtual_environment_name=virtual_environment_name, - ) - ) - ref: VirtualEnvironmentPythonNodeRefRecord - for ref in refs: - record: PythonNodeVersionRecord | None = backend.get_python_node_version( - connection=state_connection, - schema=schema, - node_type=ref.node_type, - node_name=ref.node_name, - version_hash=ref.version_hash, - ) - if record is None: - continue - definition_json: str | None = decode_state_text(record.definition_json_b64) - metadata_json: str | None = decode_state_text(record.identity_metadata_json_b64) - if definition_json is None or metadata_json is None: - continue - result[(record.node_type, record.node_name)] = Fingerprint( - node_type=record.node_type, - node_name=record.node_name, - target_database=None, - target_schema=None, - target_name=None, - run_id="", - definition_hash=record.definition_hash, - version_hash=record.version_hash, - schema_fingerprint="", - definition=definition_json, - metadata_json=metadata_json, - ts=datetime.fromtimestamp(0, tz=UTC), - ) - return result diff --git a/src/sqlbuild/virtual/state/main/python_identities/_python_node_identity_write.py b/src/sqlbuild/virtual/state/main/python_identities/_python_node_identity_write.py deleted file mode 100644 index 501b76f1bb..0000000000 --- a/src/sqlbuild/virtual/state/main/python_identities/_python_node_identity_write.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Persist Python node identity observations in virtual state.""" - -from __future__ import annotations - -import hashlib -import logging -from typing import Any - -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity -from sqlbuild.virtual.state.main.encoding._encode_state_text import encode_state_text -from sqlbuild.virtual.state.models import ( - PythonNodeVersionRecord, - VirtualEnvironmentPythonNodeRefRecord, -) -from sqlbuild.virtual.state.types import ModelVersionStatus - - -def try_record_virtual_python_node_identity( - *, - backend: Any, - state_connection: Any, - schema: str, - virtual_environment_name: str, - identity: PythonNodeIdentity | None, - logger: logging.Logger | None = None, -) -> None: - """Best-effort append/update of Python node identity state for one VDE.""" - - if identity is None: - return - resolved_logger: logging.Logger = logger or logging.getLogger("sqlbuild.python_identity") - try: - backend.upsert_python_node_version( - connection=state_connection, - schema=schema, - record=PythonNodeVersionRecord( - node_type=identity.node_type, - node_name=identity.node_name, - version_hash=identity.version_hash, - definition_hash=identity.definition_hash, - identity_metadata_hash=hashlib.sha256( - identity.metadata_json.encode("utf-8") - ).hexdigest(), - definition_json_b64=encode_state_text(identity.definition_json), - identity_metadata_json_b64=encode_state_text(identity.metadata_json), - status=ModelVersionStatus.READY, - ), - ) - backend.upsert_virtual_environment_python_node_ref( - connection=state_connection, - schema=schema, - ref=VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=virtual_environment_name, - node_type=identity.node_type, - node_name=identity.node_name, - version_hash=identity.version_hash, - ), - ) - except Exception as exc: - resolved_logger.warning( - "virtual python identity write failed for '%s': %s", - identity.node_name, - exc, - ) diff --git a/src/sqlbuild/virtual/state/main/python_identities/prune_python_node_identities.py b/src/sqlbuild/virtual/state/main/python_identities/prune_python_node_identities.py deleted file mode 100644 index eca69920e1..0000000000 --- a/src/sqlbuild/virtual/state/main/python_identities/prune_python_node_identities.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Prune unreferenced Python node identity versions from virtual state.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime - - -def prune_unreferenced_python_node_versions( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, -) -> int: - """Delete Python identity versions no VDE currently references.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - return backend.prune_unreferenced_python_node_versions( - connection=connection, - schema=config.schema, - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/retention/__init__.py b/src/sqlbuild/virtual/state/main/retention/__init__.py deleted file mode 100644 index dbe99cbb7d..0000000000 --- a/src/sqlbuild/virtual/state/main/retention/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Virtual state retention entrypoints.""" diff --git a/src/sqlbuild/virtual/state/main/retention/delete_state_backup.py b/src/sqlbuild/virtual/state/main/retention/delete_state_backup.py deleted file mode 100644 index c1656fdd86..0000000000 --- a/src/sqlbuild/virtual/state/main/retention/delete_state_backup.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Public state backup deletion helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime - - -def delete_state_backup( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - backup_id: str, -) -> None: - """Delete one state migration backup schema.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - backend.delete_state_backup( - connection=connection, schema=config.schema, backup_id=backup_id - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/retention/detached_environment_retention.py b/src/sqlbuild/virtual/state/main/retention/detached_environment_retention.py deleted file mode 100644 index d991975c4d..0000000000 --- a/src/sqlbuild/virtual/state/main/retention/detached_environment_retention.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Public detached VDE retention inspection helper.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_lifecycle.detached_retention import ( - build_detached_environment_inspection, -) -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - DetachedVirtualEnvironmentInspection, - PhysicalRelationRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRetentionRecord, -) - - -def inspect_detached_environment_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - retention_days: int, -) -> DetachedVirtualEnvironmentInspection: - """Inspect detached VDE cleanup and retained current-ref physical relations.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - environments: tuple[VirtualEnvironmentRetentionRecord, ...] = ( - backend.list_virtual_environments(connection=connection, schema=config.schema) - ) - refs_by_environment: dict[str, tuple[VirtualEnvironmentModelRefRecord, ...]] = {} - physical_relations_by_ref: dict[tuple[str, str], PhysicalRelationRecord] = {} - environment: VirtualEnvironmentRetentionRecord - for environment in environments: - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=config.schema, - virtual_environment_name=environment.virtual_environment_name, - ) - ) - refs_by_environment[environment.virtual_environment_name] = refs - ref: VirtualEnvironmentModelRefRecord - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is not None: - physical_relations_by_ref[(ref.model_name, ref.version_hash)] = relation - return build_detached_environment_inspection( - environments=environments, - refs_by_environment=refs_by_environment, - physical_relations_by_ref=physical_relations_by_ref, - retention_days=retention_days, - now=datetime.now(UTC), - ) - finally: - backend.close(connection) diff --git a/src/sqlbuild/virtual/state/main/retention/expired_environment_retention.py b/src/sqlbuild/virtual/state/main/retention/expired_environment_retention.py deleted file mode 100644 index 01b9ad9666..0000000000 --- a/src/sqlbuild/virtual/state/main/retention/expired_environment_retention.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Public expired VDE retention inspection helper.""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - ExpiredVirtualEnvironmentInspection, - PhysicalRelationRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRetentionRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus - - -def inspect_expired_environment_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - active_virtual_environment_name: str | None, - retention_days: int, -) -> ExpiredVirtualEnvironmentInspection: - """Inspect non-active VDE cleanup and retained current-ref physical relations.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - environments: tuple[VirtualEnvironmentRetentionRecord, ...] = ( - backend.list_virtual_environments(connection=connection, schema=config.schema) - ) - cleanup_names: set[str] = { - environment.virtual_environment_name - for environment in environments - if _eligible_for_cleanup( - environment=environment, - active_virtual_environment_name=active_virtual_environment_name, - retention_days=retention_days, - now=datetime.now(UTC), - ) - } - cleanup_relations: dict[tuple[str | None, str, str], PhysicalRelationRecord] = {} - retained_relations: dict[tuple[str | None, str, str], PhysicalRelationRecord] = {} - for environment in environments: - if environment.status == VirtualEnvironmentStatus.DETACHED: - continue - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=config.schema, - virtual_environment_name=environment.virtual_environment_name, - ) - ) - for ref in refs: - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=ref.model_name, - version_hash=ref.version_hash, - ) - if relation is None: - continue - key: tuple[str | None, str, str] = ( - relation.database_name, - relation.schema_name, - relation.relation_name, - ) - if environment.virtual_environment_name in cleanup_names: - cleanup_relations[key] = relation - else: - retained_relations[key] = relation - return ExpiredVirtualEnvironmentInspection( - cleanup_virtual_environments=tuple( - environment - for environment in environments - if environment.virtual_environment_name in cleanup_names - ), - cleanup_physical_relations=tuple( - cleanup_relations[key] - for key in sorted( - cleanup_relations, key=lambda item: (item[0] or "", item[1], item[2]) - ) - ), - retained_physical_relations=tuple( - retained_relations[key] - for key in sorted( - retained_relations, key=lambda item: (item[0] or "", item[1], item[2]) - ) - ), - ) - finally: - backend.close(connection) - - -def _eligible_for_cleanup( - *, - environment: VirtualEnvironmentRetentionRecord, - active_virtual_environment_name: str | None, - retention_days: int, - now: datetime, -) -> bool: - if environment.virtual_environment_name == active_virtual_environment_name: - return False - if environment.status == VirtualEnvironmentStatus.DETACHED: - return False - if retention_days == 0: - return True - if environment.updated_at is None: - return False - return _aware(environment.updated_at) <= _aware(now) - timedelta(days=retention_days) - - -def _aware(value: datetime) -> datetime: - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value diff --git a/src/sqlbuild/virtual/state/main/retention/microbatch_replay_retention.py b/src/sqlbuild/virtual/state/main/retention/microbatch_replay_retention.py deleted file mode 100644 index e552405aa8..0000000000 --- a/src/sqlbuild/virtual/state/main/retention/microbatch_replay_retention.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Public virtual microbatch replay retention inspection helper.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.microbatches.models import MicrobatchEvent -from sqlbuild.virtual.state._helpers.state_lifecycle.microbatch_replay_retention import ( - active_microbatch_replay_roots, -) -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - StateLockRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRetentionRecord, -) - - -def inspect_active_microbatch_replay_relations( - *, project_dir: Path, discovered_inputs: DiscoveredProjectInputs -) -> tuple[PhysicalRelationRecord, ...]: - """Resolve incomplete replay requirements to tracked physical relations.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - events: tuple[MicrobatchEvent, ...] = backend.read_microbatch_retention_history( - connection=connection, - schema=config.schema, - ) - current_versions: dict[str, set[str]] = {} - environments: tuple[VirtualEnvironmentRetentionRecord, ...] = ( - backend.list_virtual_environments(connection=connection, schema=config.schema) - ) - for environment in environments: - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=config.schema, - virtual_environment_name=environment.virtual_environment_name, - ) - ) - for ref in refs: - current_versions.setdefault(ref.model_name, set()).add(ref.version_hash) - relations: dict[tuple[str | None, str, str], PhysicalRelationRecord] = {} - for root in active_microbatch_replay_roots(events=events): - model_current_versions: set[str] = current_versions.get(root.model_name, set()) - if model_current_versions and model_current_versions.isdisjoint( - {root.version_hash, root.previous_version_hash} - ): - continue - relation: PhysicalRelationRecord | None = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=root.model_name, - version_hash=root.version_hash, - ) - if relation is not None: - relations[ - (relation.database_name, relation.schema_name, relation.relation_name) - ] = relation - locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema=config.schema, - ) - for lock in locks: - parsed: tuple[str, str] | None = _model_version_lock_identity(lock_key=lock.lock_key) - if parsed is None: - continue - model_name, version_hash = parsed - relation = backend.get_physical_relation( - connection=connection, - schema=config.schema, - model_name=model_name, - version_hash=version_hash, - ) - if relation is not None: - relations[ - (relation.database_name, relation.schema_name, relation.relation_name) - ] = relation - return tuple(relations[key] for key in sorted(relations)) - finally: - backend.close(connection) - - -def _model_version_lock_identity(*, lock_key: str) -> tuple[str, str] | None: - prefix: str = "model_version:" - if not lock_key.startswith(prefix): - return None - model_identity, separator, version_hash = lock_key.removeprefix(prefix).rpartition(":") - if not separator: - return None - _warehouse_realm, realm_separator, model_name = model_identity.partition(":") - return (model_name if realm_separator else model_identity, version_hash) diff --git a/src/sqlbuild/virtual/state/main/retention/state_janitor_retention.py b/src/sqlbuild/virtual/state/main/retention/state_janitor_retention.py deleted file mode 100644 index 70b1725a20..0000000000 --- a/src/sqlbuild/virtual/state/main/retention/state_janitor_retention.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Public state-only janitor inspection helper.""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.virtual.state._helpers.state_runtime.runtime import build_state_runtime -from sqlbuild.virtual.state.models import StateBackupRecord, StateJanitorInspection - - -def inspect_state_janitor_retention( - *, - project_dir: Path, - discovered_inputs: DiscoveredProjectInputs, - retention_days: int, -) -> StateJanitorInspection: - """Inspect state backup and expired-lock cleanup candidates.""" - - config, backend = build_state_runtime( - discovered_inputs=discovered_inputs, - project_dir=project_dir, - ) - connection: Any = backend.connect(config.connection) - try: - backups: tuple[StateBackupRecord, ...] = backend.list_state_backups( - connection=connection, - schema=config.schema, - ) - latest_backup_id: str | None = backups[0].backup_id if backups else None - return StateJanitorInspection( - schema=config.schema, - state_backups=tuple( - backup - for backup in backups - if backup.backup_id != latest_backup_id - and _backup_is_eligible( - backup=backup, - retention_days=retention_days, - now=datetime.now(UTC), - ) - ), - expired_locks=backend.list_expired_locks(connection=connection, schema=config.schema), - unreferenced_python_node_versions=backend.count_unreferenced_python_node_versions( - connection=connection, schema=config.schema - ), - ) - finally: - backend.close(connection) - - -def _backup_is_eligible(*, backup: StateBackupRecord, retention_days: int, now: datetime) -> bool: - if retention_days == 0: - return True - if backup.created_at is None: - return False - return _aware(backup.created_at) <= _aware(now) - timedelta(days=retention_days) - - -def _aware(value: datetime) -> datetime: - if value.tzinfo is None: - return value.replace(tzinfo=UTC) - return value diff --git a/src/sqlbuild/virtual/state/models.py b/src/sqlbuild/virtual/state/models.py deleted file mode 100644 index 706bf4a99a..0000000000 --- a/src/sqlbuild/virtual/state/models.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Virtual state store models.""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime - -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - ReconcileAction, - StateBackendName, - StateOperationStatus, - StateOperationType, - StateSchemaValidationIssueKind, - VirtualEnvironmentStatus, -) - - -@dataclass(frozen=True) -class StateBackendConfig: - """Resolved state backend configuration.""" - - backend: StateBackendName - schema: str - connection: dict[str, object] - allow_reset: bool = False - - -@dataclass(frozen=True) -class StateSchemaValidationIssue: - """One state schema validation issue.""" - - kind: StateSchemaValidationIssueKind - table_name: str - message: str - column_name: str | None = None - - -@dataclass(frozen=True) -class StateSchemaValidationResult: - """State schema validation result.""" - - issues: tuple[StateSchemaValidationIssue, ...] = field(default_factory=tuple) - - @property - def valid(self) -> bool: - return not self.issues - - -@dataclass(frozen=True) -class ModelVersionRecord: - """Current state row for one model version.""" - - model_name: str - version_hash: str - definition_identity_hash: str - identity_metadata_hash: str - status: ModelVersionStatus - definition_text_b64: str | None = None - identity_metadata_json_b64: str | None = None - compiled_sql_b64: str | None = None - - -@dataclass(frozen=True) -class PhysicalRelationRecord: - """Current state row for a physical relation that stores one artifact version.""" - - artifact_type: PhysicalArtifactType - artifact_name: str - version_hash: str - database_name: str | None - schema_name: str - relation_name: str - relation_type: str - - -@dataclass(frozen=True) -class PhysicalRelationAncestryRecord: - """State row linking a seeded physical relation to its source relation.""" - - model_name: str - version_hash: str - parent_model_name: str - parent_version_hash: str - seed_strategy: str - - -@dataclass(frozen=True) -class FunctionVersionRecord: - """Current state row for one function version.""" - - function_name: str - version_hash: str - language: str - returns: str - arguments_json_b64: str - return_columns_json_b64: str - packages_json_b64: str - body_sql_b64: str - definition_text_b64: str - status: ModelVersionStatus - runtime_version: str | None = None - entry_point: str | None = None - - -@dataclass(frozen=True) -class SeedVersionRecord: - """Current state row for one seed version.""" - - seed_name: str - version_hash: str - identity_metadata_hash: str - identity_metadata_json_b64: str - status: ModelVersionStatus - - -@dataclass(frozen=True) -class PythonNodeVersionRecord: - """Current state row for one Python node identity version.""" - - node_type: str - node_name: str - version_hash: str - definition_hash: str - identity_metadata_hash: str - definition_json_b64: str - identity_metadata_json_b64: str - status: ModelVersionStatus - - -@dataclass(frozen=True) -class VirtualEnvironmentRecord: - """Current state row for one virtual data environment.""" - - virtual_environment_name: str - status: VirtualEnvironmentStatus - baseline_virtual_environment_name: str | None = None - finalized_at: datetime | None = None - - -@dataclass(frozen=True) -class VirtualEnvironmentRetentionRecord: - """Virtual data environment retention metadata.""" - - virtual_environment_name: str - status: VirtualEnvironmentStatus - updated_at: datetime | None = None - - -@dataclass(frozen=True) -class VirtualEnvironmentModelRefRecord: - """Current state row mapping a VDE model ref to a model version.""" - - virtual_environment_name: str - model_name: str - version_hash: str - - -@dataclass(frozen=True) -class VirtualEnvironmentNodeRefRecord: - """Current state row mapping a VDE node ref to a version.""" - - virtual_environment_name: str - node_type: str - node_name: str - version_hash: str - - -@dataclass(frozen=True) -class VirtualEnvironmentFunctionRefRecord: - """Current state row mapping a VDE function ref to a function version.""" - - virtual_environment_name: str - node_type: str - function_name: str - version_hash: str - - -@dataclass(frozen=True) -class VirtualEnvironmentSeedRefRecord: - """Current state row mapping a VDE seed ref to a seed version.""" - - virtual_environment_name: str - seed_name: str - version_hash: str - - -@dataclass(frozen=True) -class VirtualEnvironmentPythonNodeRefRecord: - """Current state row mapping a VDE Python node ref to an identity version.""" - - virtual_environment_name: str - node_type: str - node_name: str - version_hash: str - - -@dataclass(frozen=True) -class SourceFreshnessRecord: - """Latest observed source freshness for one virtual environment.""" - - virtual_environment_name: str - source_name: str - strategy: str - value_kind: str - data_version: str - data_version_hash: str - observed_at: datetime - - -@dataclass(frozen=True) -class VirtualEnvironmentCheckpointRecord: - """Finalized checkpoint for one virtual data environment.""" - - checkpoint_id: str - virtual_environment_name: str - created_at: datetime | None = None - - -@dataclass(frozen=True) -class VirtualEnvironmentCheckpointModelRefRecord: - """Checkpoint row mapping a model ref to a model version.""" - - checkpoint_id: str - model_name: str - version_hash: str - - -@dataclass(frozen=True) -class VirtualEnvironmentCheckpointFunctionRefRecord: - """Checkpoint row mapping a function ref to a function version.""" - - checkpoint_id: str - function_name: str - version_hash: str - - -@dataclass(frozen=True) -class VirtualEnvironmentCheckpointSeedRefRecord: - """Checkpoint row mapping a seed ref to a seed version.""" - - checkpoint_id: str - seed_name: str - version_hash: str - - -@dataclass(frozen=True) -class FinalizedVirtualEnvironmentCheckpoint: - """Checkpoint rows written atomically with finalized virtual refs.""" - - checkpoint: VirtualEnvironmentCheckpointRecord - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] - function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] - - -@dataclass(frozen=True) -class CheckpointRetentionInspection: - """Checkpoint retention inspection for janitor planning.""" - - prune_checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] - retained_physical_relations: tuple[PhysicalRelationRecord, ...] - - -@dataclass(frozen=True) -class DetachedVirtualEnvironmentInspection: - """Detached VDE cleanup inspection for janitor planning.""" - - cleanup_virtual_environments: tuple[VirtualEnvironmentRetentionRecord, ...] - cleanup_physical_relations: tuple[PhysicalRelationRecord, ...] - retained_physical_relations: tuple[PhysicalRelationRecord, ...] - - -@dataclass(frozen=True) -class ExpiredVirtualEnvironmentInspection: - """Non-active VDE cleanup inspection for janitor planning.""" - - cleanup_virtual_environments: tuple[VirtualEnvironmentRetentionRecord, ...] - cleanup_physical_relations: tuple[PhysicalRelationRecord, ...] - retained_physical_relations: tuple[PhysicalRelationRecord, ...] - - -@dataclass(frozen=True) -class MicrobatchReplayRoot: - """One physical model version protected by an incomplete replay requirement.""" - - model_name: str - version_hash: str - previous_version_hash: str | None - - -@dataclass(frozen=True) -class StateJanitorInspection: - """State-only janitor cleanup candidates.""" - - schema: str - state_backups: tuple[StateBackupRecord, ...] - expired_locks: tuple[StateLockRecord, ...] - unreferenced_python_node_versions: int = 0 - - -@dataclass(frozen=True) -class StateLockRecord: - """Current state row for one active lock.""" - - lock_key: str - owner_id: str - expires_at: datetime - - -@dataclass(frozen=True) -class StateBackupRecord: - """One state migration backup schema.""" - - backup_id: str - schema_name: str - created_at: datetime | None = None - - -@dataclass(frozen=True) -class StateLockLease: - """Acquired state lock lease.""" - - lock_key: str - owner_id: str - expires_at: datetime - - -@dataclass(frozen=True) -class StateOperationRecord: - """Current state row for one tracked multi-step virtual operation.""" - - operation_id: str - operation_type: StateOperationType - status: StateOperationStatus - virtual_environment_name: str | None = None - - -@dataclass(frozen=True) -class StateOperationEventRecord: - """Append-only event row for one tracked operation.""" - - event_id: str - operation_id: str - action: str - status: StateOperationStatus - message: str | None = None - - -@dataclass(frozen=True) -class ReconcileEventRecord: - """Append-only event row for virtual reconcile actions.""" - - event_id: str - action: ReconcileAction - status: StateOperationStatus - message: str | None = None diff --git a/src/sqlbuild/virtual/state/types.py b/src/sqlbuild/virtual/state/types.py deleted file mode 100644 index e3d38de646..0000000000 --- a/src/sqlbuild/virtual/state/types.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Virtual state type declarations.""" - -from __future__ import annotations - -from enum import StrEnum -from typing import Protocol - - -class StateTypeMatcher(Protocol): - def __call__(self, *, actual_type: str, expected_type: StateColumnType) -> bool: ... - - -class StateBackendName(StrEnum): - DUCKDB = "duckdb" - POSTGRES = "postgres" - - -class StateCommand(StrEnum): - INIT = "init" - MIGRATE = "migrate" - ROLLBACK = "rollback" - RESET = "reset" - ADOPT = "adopt" - DETACH = "detach" - - -class StateMigrationAction(StrEnum): - INIT = "init" - MIGRATE = "migrate" - BACKUP = "backup" - ROLLBACK = "rollback" - RESET = "reset" - - -class StateMigrationStatus(StrEnum): - SUCCESS = "success" - FAILED = "failed" - - -class StateColumnType(StrEnum): - INTEGER = "integer" - TEXT = "text" - TIMESTAMP = "timestamp" - - -class StateSchemaValidationIssueKind(StrEnum): - MISSING_TABLE = "missing_table" - MISSING_COLUMN = "missing_column" - WRONG_TYPE = "wrong_type" - MISSING_INDEX = "missing_index" - - -class ModelVersionStatus(StrEnum): - PLANNED = "planned" - BUILDING = "building" - READY = "ready" - FAILED = "failed" - - -class PhysicalArtifactType(StrEnum): - MODEL = "model" - SEED = "seed" - - -class VirtualEnvironmentStatus(StrEnum): - ACTIVE = "active" - FINALIZING = "finalizing" - FINALIZED = "finalized" - DETACHED = "detached" - FAILED = "failed" - - -class StateOperationStatus(StrEnum): - RUNNING = "running" - SUCCEEDED = "succeeded" - FAILED = "failed" - - -class StateOperationType(StrEnum): - PROMOTE = "promote" - ADOPT = "adopt" - DETACH = "detach" - - -class ReconcileAction(StrEnum): - REPORT = "report" - REPAIR_VIEW = "repair_view" - ATTACH = "attach" diff --git a/tests/e2e/scripts/cli_preview/test_preview_cli.py b/tests/e2e/scripts/cli_preview/test_preview_cli.py index b65a802b0d..34b39d3d35 100644 --- a/tests/e2e/scripts/cli_preview/test_preview_cli.py +++ b/tests/e2e/scripts/cli_preview/test_preview_cli.py @@ -32,34 +32,6 @@ expected_fragments=("ERROR", "Failures:", "[T002]"), unexpected_fragments=("\033[", "0 mismatched", "error[T002]"), ), - PreviewCliTestCase( - description="virtual promotion scene runs a real branch lifecycle", - scene="virtual-promote", - arguments=("--no-color",), - expected_return_code=0, - expected_fragments=( - "virtual-promote: promote a branch virtual environment into dev", - "Virtual promotion complete", - "promoted models", - ), - unexpected_fragments=("\033[",), - ), - PreviewCliTestCase( - description="virtual reconcile scene reports healthy persisted state", - scene="virtual-reconcile", - arguments=("--no-color",), - expected_return_code=0, - expected_fragments=("Virtual reconcile", "Reconcile report for dev: no issues."), - unexpected_fragments=("\033[",), - ), - PreviewCliTestCase( - description="virtual janitor scene renders cleanup decisions", - scene="virtual-janitor", - arguments=("--no-color",), - expected_return_code=0, - expected_fragments=("Janitor preview", "Skipped objects", "virtual state pruned"), - unexpected_fragments=("\033[",), - ), ), ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/__init__.py b/tests/e2e/scripts/compile_performance_ratio/__init__.py similarity index 100% rename from tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/__init__.py rename to tests/e2e/scripts/compile_performance_ratio/__init__.py diff --git a/tests/e2e/scripts/compile_performance_ratio/_test_types.py b/tests/e2e/scripts/compile_performance_ratio/_test_types.py new file mode 100644 index 0000000000..a58f84cf90 --- /dev/null +++ b/tests/e2e/scripts/compile_performance_ratio/_test_types.py @@ -0,0 +1,15 @@ +"""Test types for same-runner compile performance comparisons.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CompilePerformanceRatioTestCase: + description: str + kind: str + models: int + max_ratio: str + expected_return_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py new file mode 100644 index 0000000000..e322a59273 --- /dev/null +++ b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py @@ -0,0 +1,78 @@ +"""End-to-end tests for the same-runner compile performance ratio guard.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from tests.e2e.scripts.compile_performance_ratio._test_types import ( + CompilePerformanceRatioTestCase, +) + +_REPO_ROOT: Path = Path(__file__).resolve().parents[4] + + +@pytest.mark.parametrize( + "test_case", + ( + CompilePerformanceRatioTestCase( + description="identical builds pass a generous limit", + kind="dense", + models=20, + max_ratio="3.0", + expected_return_code=0, + expected_fragments=("| Wall (s) |", "| CPU (s) |", "| analysis_native_ms |"), + ), + CompilePerformanceRatioTestCase( + description="an unattainable limit fails with the exceeded ratios", + kind="dense", + models=20, + max_ratio="0.01", + expected_return_code=1, + expected_fragments=("Compile performance regression: wall ratio",), + ), + ), + ids=lambda case: case.description, +) +def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio_limit( + test_case: CompilePerformanceRatioTestCase, +) -> None: + environment: dict[str, str] = dict(os.environ) + _ = environment.pop("GITHUB_STEP_SUMMARY", None) + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-m", + "scripts.compare_compile_performance", + "--kind", + test_case.kind, + "--models", + str(test_case.models), + "--runs", + "1", + "--base-python", + sys.executable, + "--head-python", + sys.executable, + "--max-ratio", + test_case.max_ratio, + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + env=environment, + check=False, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + for fragment in test_case.expected_fragments: + assert fragment in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py index 738fd00e29..e6a33bccfe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/_test_types.py @@ -158,35 +158,3 @@ class BigQuerySourceDeferralE2ETestCase: expected_loader_rows: tuple[tuple[object, ...], ...] command: tuple[str, ...] = ("--no-color", "build", "--select", "stg_orders") expected_return_code: int = 0 - - -@dataclass(frozen=True) -class BigQueryVirtualSeedE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_seed_strategy: str - - -@dataclass(frozen=True) -class BigQueryVirtualLifecycleE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class BigQueryReconcileE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class BigQueryJanitorDetachedVdeE2ETestCase: - description: str - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - expected_ref_count_after: int - expected_return_code: int = 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py index 9f672bf8bb..2619ba35ad 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/helpers.py @@ -53,56 +53,6 @@ def build_bigquery_project_toml(*, project_name: str, dataset_name: str) -> str: ) -def build_bigquery_virtual_seed_project_toml( - *, project_name: str, dataset_name: str, unsuffixed_virtual_env: str | None = None -) -> str: - project_id: str = str(build_bigquery_connection_config(schema=dataset_name)["project"]) - location: str = str(build_bigquery_connection_config(schema=dataset_name)["location"]) - unsuffixed_line: str = {None: ""}.get( - unsuffixed_virtual_env, - f'unsuffixed_virtual_env = "{unsuffixed_virtual_env}"\n', - ) - return ( - f'name = "{project_name}"\n' - 'adapter = "bigquery"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "\n" - "[connection]\n" - 'project = "${ENV:SQB_TEST_BIGQUERY_PROJECT}"\n' - f'location = "{location}"\n\n' - "[targets.dev]\n" - f'database = "{project_id}"\n' - f'schema = "{dataset_name}"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - f"{unsuffixed_line}\n" - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ) - - -def virtual_seed_source_yml(*, dataset_name: str) -> str: - return f"sources:\n - name: raw_orders\n schema: {dataset_name}\n table: raw_orders\n" - - -def virtual_seed_orders_model(*, amount_expression: str) -> str: - return ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - f"SELECT id, ordered_at, {amount_expression} AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ) - - def build_bigquery_source_deferral_project_toml( *, project_name: str, dev_dataset_name: str, prod_dataset_name: str ) -> str: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index 83a8448523..35065422c2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -4,7 +4,6 @@ import subprocess from collections import defaultdict from pathlib import Path -from textwrap import dedent import pytest @@ -16,10 +15,8 @@ BigQueryDiffE2ETestCase, BigQueryErrorE2ETestCase, BigQueryIntermediateDagStrategyE2ETestCase, - BigQueryJanitorDetachedVdeE2ETestCase, BigQueryModelBuildE2ETestCase, BigQueryNodeResultE2ETestCase, - BigQueryReconcileE2ETestCase, BigQueryScenarioLocalReplayE2ETestCase, BigQueryScenarioRemoteE2ETestCase, BigQuerySnapshotApplyE2ETestCase, @@ -27,8 +24,6 @@ BigQuerySourceDeferralE2ETestCase, BigQuerySourceLoaderSchemaEvolutionE2ETestCase, BigQuerySourceLoaderStrategiesE2ETestCase, - BigQueryVirtualLifecycleE2ETestCase, - BigQueryVirtualSeedE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.bigquery.helpers import ( assert_bigquery_snapshot_apply_rows, @@ -38,7 +33,6 @@ build_bigquery_local_config, build_bigquery_project_toml, build_bigquery_source_deferral_project_toml, - build_bigquery_virtual_seed_project_toml, cleanup_bigquery_dataset, ensure_bigquery_dataset_ready, execute_bigquery_sql, @@ -49,8 +43,6 @@ prepare_bigquery_source_loader_strategies, prepare_bigquery_waffle_shop, relation_name, - virtual_seed_orders_model, - virtual_seed_source_yml, write_local_environment_override, ) from tests.e2e.src.sqlbuild.cli.commands.main.load.helpers import ( @@ -73,7 +65,6 @@ build_real_warehouse_existing_snapshot_project_files, build_real_warehouse_snapshot_project_files, prepare_inline_project, - query_duckdb, run_sqb, stringify_warehouse_rows, ) @@ -292,444 +283,6 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u cleanup_bigquery_dataset(dataset_name=dataset_name) -@pytest.mark.parametrize( - "test_case", - [ - BigQueryVirtualSeedE2ETestCase( - description="virtual seeded incremental build uses clone on bigquery", - expected_rows=(("1", "10"), ("2", "21"), ("3", "31")), - expected_seed_strategy="durable_clone", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_incremental_change_when_building_on_bigquery_then_seeds_with_clone( - test_case: BigQueryVirtualSeedE2ETestCase, - tmp_path: Path, -) -> None: - dataset_name: str = build_unique_dataset_name(prefix="sqlbuild_virtual_seed") - project_id: str = str(build_bigquery_connection_config(schema=dataset_name)["project"]) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="bigquery_virtual_seed", - repo_files={ - "sqlbuild_project.toml": build_bigquery_virtual_seed_project_toml( - project_name="bigquery_virtual_seed", - dataset_name=dataset_name, - ), - "sources/raw.yml": virtual_seed_source_yml(dataset_name=dataset_name), - "models/orders.sql": virtual_seed_orders_model(amount_expression="amount_cents + 0"), - }, - ) - try: - ensure_bigquery_dataset_ready(dataset_name=dataset_name) - execute_bigquery_sql( - dataset_name=dataset_name, - sql=dedent( - f""" - CREATE OR REPLACE TABLE - {relation_name(dataset_name=dataset_name, name="raw_orders")} AS - SELECT - 1 AS id, - TIMESTAMP '2026-01-01 00:00:00 UTC' AS ordered_at, - 10 AS amount_cents - UNION ALL - SELECT 2, TIMESTAMP '2026-01-02 00:00:00 UTC', 20 - """ - ).strip(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - virtual_seed_orders_model(amount_expression="amount_cents + 1"), - encoding="utf-8", - ) - execute_bigquery_sql( - dataset_name=dataset_name, - sql=( - f"INSERT INTO {relation_name(dataset_name=dataset_name, name='raw_orders')} " - "VALUES (3, TIMESTAMP '2026-01-03 00:00:00 UTC', 30)" - ), - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - assert ( - stringify_warehouse_rows( - fetch_bigquery_rows( - dataset_name=dataset_name, - sql=( - f"SELECT id, amount_cents FROM `{project_id}.{dataset_name}__dev.orders` " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT seed_strategy FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) == [(test_case.expected_seed_strategy,)] - finally: - cleanup_bigquery_dataset(dataset_name=dataset_name) - - -@pytest.mark.parametrize( - "test_case", - [ - BigQueryReconcileE2ETestCase( - description="reconcile repair-view recreates bigquery logical view", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Repair", - "model orders", - "VDE dev", - "action recreate logical view from state", - "result repaired", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_view_when_repairing_on_bigquery_then_view_is_recreated( - test_case: BigQueryReconcileE2ETestCase, - tmp_path: Path, -) -> None: - dataset_name: str = build_unique_dataset_name(prefix="sqlbuild_virtual_reconcile") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="bigquery_virtual_reconcile", - repo_files={ - "sqlbuild_project.toml": build_bigquery_virtual_seed_project_toml( - project_name="bigquery_virtual_reconcile", - dataset_name=dataset_name, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_bigquery_dataset_ready(dataset_name=dataset_name) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_bigquery_sql( - dataset_name=dataset_name, - sql=f"DROP VIEW {relation_name(dataset_name=f'{dataset_name}__dev', name='orders')}", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - stringify_warehouse_rows( - fetch_bigquery_rows( - dataset_name=dataset_name, - sql=( - "SELECT id FROM " - f"{relation_name(dataset_name=f'{dataset_name}__dev', name='orders')} " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - finally: - cleanup_bigquery_dataset(dataset_name=dataset_name) - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__dev") - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - BigQueryReconcileE2ETestCase( - description="reconcile attach rebinds bigquery logical view", - expected_rows=(("2",),), - expected_stdout_fragments=("Attach", "model orders", "result attached"), - ) - ], - ids=lambda case: case.description, -) -def test_given_tracked_physical_relation_when_attaching_on_bigquery_then_view_is_rebound( - test_case: BigQueryReconcileE2ETestCase, - tmp_path: Path, -) -> None: - dataset_name: str = build_unique_dataset_name(prefix="sqlbuild_virtual_attach") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="bigquery_virtual_attach", - repo_files={ - "sqlbuild_project.toml": build_bigquery_virtual_seed_project_toml( - project_name="bigquery_virtual_attach", - dataset_name=dataset_name, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_bigquery_dataset_ready(dataset_name=dataset_name) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ).returncode - == 0 - ) - project_id, physical_dataset_name, physical_relation_name = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT database_name, schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - )[0] - physical_relation: str = f"`{project_id}.{physical_dataset_name}.{physical_relation_name}`" - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - physical_relation, - "--auto-approve", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - stringify_warehouse_rows( - fetch_bigquery_rows( - dataset_name=dataset_name, - sql=( - "SELECT id FROM " - f"{relation_name(dataset_name=f'{dataset_name}__dev', name='orders')} " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - finally: - cleanup_bigquery_dataset(dataset_name=dataset_name) - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__dev") - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__pr") - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - BigQueryVirtualLifecycleE2ETestCase( - description="adopt and detach preserve bigquery logical table", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Adopted 1 models into virtual environment dev.", - "Detached 1 models from virtual environment dev.", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_stateless_table_when_adopting_and_detaching_on_bigquery_then_table_is_preserved( - test_case: BigQueryVirtualLifecycleE2ETestCase, - tmp_path: Path, -) -> None: - dataset_name: str = build_unique_dataset_name(prefix="sqlbuild_virtual_lifecycle") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="bigquery_virtual_lifecycle", - repo_files={ - "sqlbuild_project.toml": build_bigquery_virtual_seed_project_toml( - project_name="bigquery_virtual_lifecycle", - dataset_name=dataset_name, - unsuffixed_virtual_env="dev", - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_bigquery_dataset_ready(dataset_name=dataset_name) - execute_bigquery_sql( - dataset_name=dataset_name, - sql=( - "CREATE OR REPLACE TABLE " - f"{relation_name(dataset_name=dataset_name, name='orders')} " - "AS SELECT 1 AS id" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert adopt_result.returncode == test_case.expected_return_code, ( - adopt_result.stdout + adopt_result.stderr - ) - assert detach_result.returncode == test_case.expected_return_code, ( - detach_result.stdout + detach_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in adopt_result.stdout + detach_result.stdout - assert ( - stringify_warehouse_rows( - fetch_bigquery_rows( - dataset_name=dataset_name, - sql=( - f"SELECT id FROM {relation_name(dataset_name=dataset_name, name='orders')} " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - finally: - cleanup_bigquery_dataset(dataset_name=dataset_name) - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - BigQueryJanitorDetachedVdeE2ETestCase( - description="janitor prunes bigquery detached VDE refs and physical versions", - expected_stdout_fragments=( - "eligible for deletion", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "state item", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_detached_vde_when_running_janitor_on_bigquery_then_refs_are_pruned( - test_case: BigQueryJanitorDetachedVdeE2ETestCase, - tmp_path: Path, -) -> None: - dataset_name: str = build_unique_dataset_name(prefix="sqlbuild_virtual_janitor") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="bigquery_virtual_janitor", - repo_files={ - "sqlbuild_project.toml": ( - build_bigquery_virtual_seed_project_toml( - project_name="bigquery_virtual_janitor", - dataset_name=dataset_name, - unsuffixed_virtual_env="dev", - ) - + "\n[janitor]\n" - + "enabled = true\n" - + "retention_days = 0\n" - + "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_bigquery_dataset_ready(dataset_name=dataset_name) - execute_bigquery_sql( - dataset_name=dataset_name, - sql=( - "CREATE OR REPLACE TABLE " - f"{relation_name(dataset_name=dataset_name, name='orders')} " - "AS SELECT 1 AS id" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ).returncode - == 0 - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_return_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environments", - ) == [(test_case.expected_virtual_environment_count_after,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model'" - ), - ) == [(test_case.expected_ref_count_after,)] - finally: - cleanup_bigquery_dataset(dataset_name=dataset_name) - cleanup_bigquery_dataset(dataset_name=f"{dataset_name}__sqb_physical") - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 0ed2a4cb58..5bc5f00a4d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -98,17 +98,6 @@ class CappedMicrobatchScenarioE2ETestCase: expected_exit_code: int -@dataclass(frozen=True) -class VirtualCappedProducerGapE2ETestCase: - """Expected physical and state results for a legal virtual capped producer graph.""" - - description: str - expected_exit_code: int - expected_ids: tuple[int, ...] - expected_gap_rows: tuple[tuple[object, ...], ...] - expected_microbatch_event_count: int - - @dataclass(frozen=True) class CappedWatermarkRejectionE2ETestCase: """Expected static rejection for a capped watermark producer.""" @@ -330,17 +319,6 @@ class PythonHooksLifecycleMatrixBuildE2ETestCase: expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] -@dataclass(frozen=True) -class VirtualPythonHooksBuildE2ETestCase: - """Test case for virtual builds with Python lifecycle hooks.""" - - description: str - expected_exit_code: int - expected_model_rows: tuple[tuple[object, ...], ...] - expected_hook_log_rows: tuple[tuple[object, ...], ...] - expected_identity_rows: tuple[tuple[object, ...], ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class DirectChangesOnlyBuildE2ETestCase: """Test case for direct build changes-only behavior.""" @@ -353,191 +331,6 @@ class DirectChangesOnlyBuildE2ETestCase: expected_failed_sql_fragment: str | None = None -@dataclass(frozen=True) -class VirtualBuildE2ETestCase: - """Test case for virtual build e2e behavior.""" - - description: str - expected_build_fragments: tuple[str, ...] - expected_plan_fragments: tuple[str, ...] - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - expected_ref_rows: tuple[tuple[object, ...], ...] - expected_physical_version_count: int | None = None - expected_default_plan_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_final_plan_fragments: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VirtualConcurrentBuildE2ETestCase: - """Test case for concurrent virtual build e2e behavior.""" - - description: str - concurrency: int - expected_model_count: int - expected_build_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class VirtualSeedBuildE2ETestCase: - """Test case for virtual seed build state behavior.""" - - description: str - expected_initial_rows: tuple[tuple[object, ...], ...] - expected_changed_rows: tuple[tuple[object, ...], ...] - expected_changed_fragments: tuple[str, ...] - expected_branch_rows: tuple[tuple[object, ...], ...] = field(default_factory=tuple) - expected_physical_seed_count: int = 0 - - -@dataclass(frozen=True) -class VirtualSeedGapE2ETestCase: - """Test case for targeted virtual seed gap coverage.""" - - description: str - expected_fragments: tuple[str, ...] = field(default_factory=tuple) - unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VirtualCustomMaterializationE2ETestCase: - """Test case for virtual custom materialization behavior.""" - - description: str - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - expected_ancestry_rows: tuple[tuple[object, ...], ...] - - -@dataclass(frozen=True) -class VirtualPythonBuildE2ETestCase: - """Test case for virtual Python-node build behavior.""" - - description: str - project_name: str - plan_command: tuple[str, ...] - build_command: tuple[str, ...] - expected_build_exit_code: int - expected_plan_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_absent_plan_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_build_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_prepared_text: str | None = None - expected_profile_text: str | None = None - expected_source_profile_text: str | None = None - expected_profile_exists: bool | None = None - - -@dataclass(frozen=True) -class VirtualPythonIdentityBuildE2ETestCase: - """Test case for virtual Python identity persistence behavior.""" - - description: str - expected_state_identity_rows: tuple[tuple[object, ...], ...] - expected_warehouse_fingerprint_table_count: int - expected_changed_plan_fragments: tuple[str, ...] - unexpected_changed_plan_fragments: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VirtualNodeResultStateE2ETestCase: - """Test case for virtual node result state persistence behavior.""" - - description: str - expected_state_rows: tuple[tuple[object, ...], ...] - expected_asset_payload: dict[str, object] - expected_loader_text: str - expected_history_text: str - expected_warehouse_result_table_count: int - expected_build_fragments: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VirtualNodeResultFailureStateE2ETestCase: - """Test case for virtual failed node result state persistence behavior.""" - - description: str - project_name: str - repo_files: dict[str, str] - command: tuple[str, ...] - expected_exit_code: int - expected_state_rows: tuple[tuple[object, ...], ...] - - -@dataclass(frozen=True) -class VirtualWaffleShopE2ETestCase: - """Test case for full waffle shop virtual build behavior.""" - - description: str - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - expected_view_names: tuple[str, ...] - expected_function_names: tuple[str, ...] - - -@dataclass(frozen=True) -class VirtualBuildSelectionGuardE2ETestCase: - """Test case for virtual build selection guard behavior.""" - - description: str - blocked_command: tuple[str, ...] - expanded_command: tuple[str, ...] - expected_blocked_fragments: tuple[str, ...] - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - - -@dataclass(frozen=True) -class VirtualSourceFreshnessBuildE2ETestCase: - """Test case for virtual source freshness build behavior.""" - - description: str - expected_initial_rows: tuple[tuple[object, ...], ...] - expected_updated_rows: tuple[tuple[object, ...], ...] - expected_error_fragment: str | None = None - - -@dataclass(frozen=True) -class VirtualPromoteE2ETestCase: - """Test case for virtual promotion behavior.""" - - description: str - promote_command: tuple[str, ...] - expected_promote_fragments: tuple[str, ...] - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - blocked_command: tuple[str, ...] = field(default_factory=tuple) - expected_blocked_fragments: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VirtualRollbackE2ETestCase: - """Test case for virtual rollback behavior.""" - - description: str - rollback_command: tuple[str, ...] - expected_rollback_fragments: tuple[str, ...] - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - expected_checkpoint_count: int - expected_exit_code: int = 0 - expected_stderr_fragments: tuple[str, ...] = field(default_factory=tuple) - - -@dataclass(frozen=True) -class VirtualExplicitCheckpointRollbackE2ETestCase: - """Test case for explicit checkpoint rollback behavior.""" - - description: str - rollback_command_prefix: tuple[str, ...] - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - - -@dataclass(frozen=True) -class VirtualPartialRollbackE2ETestCase: - """Test case for partial rollback guard behavior.""" - - description: str - blocked_command: tuple[str, ...] - allowed_command: tuple[str, ...] - expected_blocked_stderr_fragments: tuple[str, ...] - expected_allowed_stdout_fragments: tuple[str, ...] - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - - @dataclass(frozen=True) class ModelBackedCursorBuildE2ETestCase: """Test case for model-backed cursor build e2e regression coverage.""" @@ -949,15 +742,6 @@ class StatelessSerialMicrobatchE2ETestCase: expected_state_table_count: int -@dataclass(frozen=True) -class VirtualConcurrentMicrobatchE2ETestCase: - """Expected virtual concurrent storage and lock behavior.""" - - description: str - expected_exit_code: int - expected_minimum_event_count: int - - @dataclass(frozen=True) class MicrobatchReconciliationPolicyE2ETestCase: """Expected state facts after rebuilding with missing completion history.""" @@ -1022,15 +806,6 @@ class PlanCommandBuildE2ETestCase: expected_stderr_fragments: tuple[str, ...] -@dataclass(frozen=True) -class VirtualModeGuardBuildE2ETestCase: - description: str - project_toml: str - command: tuple[str, ...] - expected_exit_code: int - expected_error_fragment: str - - @dataclass(frozen=True) class RemoveColumnSemanticsBuildE2ETestCase: """Test case for remove-column mutation semantics.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index e0949c0c95..1c75ce1862 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -7,12 +7,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( DeferCloneBuildE2ETestCase, ) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, - build_virtual_plan_repo_files, -) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, prepare_inline_project, query_duckdb, run_sqb, @@ -176,73 +171,6 @@ def prepare_defer_clone_project( ) -def build_virtual_wide_dag_repo_files(*, model_count: int) -> dict[str, str]: - repo_files: dict[str, str] = { - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - } - for index in range(1, model_count + 1): - model_name: str = f"model_{index:02d}" - repo_files[f"models/{model_name}.sql"] = ( - f"MODEL (materialized table);\n\nSELECT {index} AS id, '{model_name}' AS model_name\n" - ) - return repo_files - - -def table_upstream_model_sql(*, amount: int = 100) -> str: - return f"MODEL (materialized table);\n\nSELECT id, {amount} AS amount FROM main.raw_orders\n" - - -def incremental_upstream_model_sql() -> str: - return ( - dedent( - """ - MODEL ( - materialized incremental, - incremental_strategy append, - cursor event_time, - cursor_type timestamp, - cursor_grain second, - ); - - SELECT id, amount, event_time FROM main.raw_orders - """ - ).strip() - + "\n" - ) - - -def downstream_model_sql() -> str: - return ( - "MODEL (materialized table);\n\n" - 'SELECT id, amount AS downstream_amount FROM __ref("upstream")\n' - ) - - -def raw_orders_setup_sql(*, rows_sql: str) -> str: - return ( - "CREATE TABLE main.raw_orders (id INTEGER, amount INTEGER, event_time TIMESTAMP);\n" - f"INSERT INTO main.raw_orders VALUES {rows_sql};\n" - ) - - -def replace_raw_orders_versions(*, db_path: Path, versions: tuple[int, ...]) -> None: - values_sql: str = ", ".join( - f"({index + 1}, {version})" for index, version in enumerate(versions) - ) - execute_duckdb( - db_path=db_path, - sql=( - "CREATE OR REPLACE TABLE raw_orders (id INTEGER, data_version INTEGER);\n" - f"INSERT INTO raw_orders VALUES {values_sql};\n" - ), - ) - - -def run_successful_sqb_build(*, project_dir: Path, command: tuple[str, ...]) -> None: - result: subprocess.CompletedProcess[str] = run_sqb(command=command, project_dir=project_dir) - assert result.returncode == 0, result.stdout + result.stderr - - def assert_defer_clone_build_case(*, tmp_path: Path, test_case: DeferCloneBuildE2ETestCase) -> None: """Run and assert one direct-mode defer-clone E2E case.""" @@ -305,120 +233,6 @@ def assert_defer_clone_build_case(*, tmp_path: Path, test_case: DeferCloneBuildE ) == list(test_case.expected_fingerprint_rows) -def _apply_command_target(*, project_dir: Path, command: tuple[str, ...]) -> tuple[str, ...]: - command_with_sentinel: tuple[str, ...] = (*command, "--target") - target_index: int = command_with_sentinel.index("--target") - target_names: tuple[str, ...] = command[target_index + 1 : target_index + 2] - for target_name in target_names: - (project_dir / "sqlbuild_local.toml").write_text( - f'target = "{target_name}"\n', encoding="utf-8" - ) - return (*command[:target_index], *command[target_index + 2 * len(target_names) :]) - - -def _apply_optional_dev_setup_sql(*, project_dir: Path, sql: str | None) -> None: - for setup_sql in (sql,) * int(sql is not None): - assert setup_sql is not None - execute_duckdb(db_path=project_dir / "warehouse.duckdb", sql=setup_sql) - - -def prepare_virtual_seeded_incremental_project( - *, - tmp_path: Path, - project_name: str, - incremental_strategy: str, - replay_on_change: str, -) -> Path: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - """ - ).strip() - + "\n", - "models/orders.sql": incremental_orders_model_sql( - incremental_strategy=incremental_strategy, - replay_on_change=replay_on_change, - amount_expression="amount_cents + 0", - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders ( - id INTEGER, - ordered_at TIMESTAMP, - amount_cents INTEGER - ); - INSERT INTO raw.raw_orders VALUES - (1, '2026-01-01 00:00:00', 10), - (2, '2026-01-02 00:00:00', 20); - """ - ).strip(), - ) - return project_dir - - -def build_virtual_seed_lifecycle_repo_files(*, amount_cents: int) -> dict[str, str]: - return build_virtual_plan_repo_files( - stg_orders_sql='SELECT id, amount_cents FROM __seed("order_amounts")' - ) | { - "seeds/orders.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " path: order_amounts.csv\n" - " columns:\n" - " - name: id\n" - " type: integer\n" - " - name: amount_cents\n" - " type: integer\n" - ), - "seeds/order_amounts.csv": f"id,amount_cents\n1,{amount_cents}\n", - } - - -def build_virtual_multi_seed_lifecycle_repo_files( - *, amount_cents: int, multiplier: int -) -> dict[str, str]: - return build_virtual_plan_repo_files( - stg_orders_sql=( - "SELECT a.id, a.amount_cents * m.multiplier AS amount_cents " - 'FROM __seed("order_amounts") a ' - 'JOIN __seed("amount_multipliers") m USING (id)' - ) - ) | { - "seeds/orders.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " path: order_amounts.csv\n" - " columns:\n" - " - name: id\n" - " type: integer\n" - " - name: amount_cents\n" - " type: integer\n" - " - name: amount_multipliers\n" - " path: amount_multipliers.csv\n" - " columns:\n" - " - name: id\n" - " type: integer\n" - " - name: multiplier\n" - " type: integer\n" - ), - "seeds/order_amounts.csv": f"id,amount_cents\n1,{amount_cents}\n", - "seeds/amount_multipliers.csv": f"id,multiplier\n1,{multiplier}\n", - } - - def prepare_build_test_audit_flag_project(*, tmp_path: Path, project_name: str) -> Path: return prepare_inline_project( tmp_path=tmp_path, @@ -493,144 +307,6 @@ def build_freshness_error_branch_source_yml( ) -def prepare_virtual_cursor_override_without_snapshot_project( - *, - tmp_path: Path, - project_name: str, -) -> Path: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - """ - ).strip() - + "\n", - "models/orders.sql": _cursor_override_without_snapshot_model_sql( - amount_expression="amount_cents + 0" - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders ( - id INTEGER, - ordered_at TIMESTAMP, - amount_cents INTEGER - ); - INSERT INTO raw.raw_orders VALUES - (1, '2026-01-01 00:00:00', 10), - (2, '2026-01-02 00:00:00', 20); - """ - ).strip(), - ) - return project_dir - - -def rewrite_cursor_override_without_snapshot_model( - *, project_dir: Path, amount_expression: str -) -> None: - (project_dir / "models" / "orders.sql").write_text( - _cursor_override_without_snapshot_model_sql(amount_expression=amount_expression), - encoding="utf-8", - ) - - -def initialize_virtual_seeded_project(*, project_dir: Path) -> None: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - -def rewrite_incremental_orders_model( - *, - project_dir: Path, - incremental_strategy: str, - replay_on_change: str, - amount_expression: str, -) -> None: - (project_dir / "models" / "orders.sql").write_text( - incremental_orders_model_sql( - incremental_strategy=incremental_strategy, - replay_on_change=replay_on_change, - amount_expression=amount_expression, - ), - encoding="utf-8", - ) - - -def count_virtual_physical_versions(*, project_dir: Path, schema: str = "dev__sqb_physical") -> int: - rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{schema}' AND table_name LIKE '%__v_%'" - ), - ) - return int(rows[0][0]) - - -def incremental_orders_model_sql( - *, incremental_strategy: str, replay_on_change: str, amount_expression: str -) -> str: - return ( - dedent( - f""" - MODEL ( - materialized incremental, - incremental_strategy {incremental_strategy}, - cursor ordered_at, - cursor_type timestamp, - cursor_grain day, - replay_on_change {replay_on_change} - ); - - SELECT id, ordered_at, {amount_expression} AS amount_cents - FROM __source("raw_orders") - """ - ).strip() - + "\n" - ) - - -def _cursor_override_without_snapshot_model_sql(*, amount_expression: str) -> str: - return ( - dedent( - f""" - MODEL ( - materialized incremental, - incremental_strategy delete_insert, - cursor ordered_at, - cursor_type timestamp, - cursor_grain day, - cursor_inputs (raw_orders ordered_at,), - replay_on_change bounded-7d - ); - - SELECT id, ordered_at, {amount_expression} AS amount_cents - FROM __source("raw_orders") - """ - ).strip() - + "\n" - ) - - def replay_microbatch_model_sql(*, value_expression: str, replay_policy: str = "bounded-2h") -> str: """Build a replay-on-change microbatch model used by lifecycle E2E tests.""" @@ -724,50 +400,6 @@ def timestamp_microbatch_model_sql( ) -def capped_watermark_consumer_model_sql() -> str: - """Return an illegal watermark consumer of the canonical capped orders model.""" - - return ( - dedent( - """ - MODEL ( - materialized incremental, - incremental_strategy delete_insert, - incremental_mode microbatch, - microbatch_strategy watermark, - cursor_watermark_mode all, - cursor event_time, - cursor_type timestamp, - cursor_grain hour, - cursor_start '2026-01-01T00:00:00', - cursor_inputs ( - orders (column event_time, roles [filter, watermark]), - ), - batch_size 1h, - lookback 1h, - ); - - SELECT id, event_time, value FROM __ref("orders") - """ - ).strip() - + "\n" - ) - - -def plain_capped_consumer_model_sql() -> str: - """Return a legal non-watermark consumer of the canonical capped orders model.""" - - return ( - dedent( - """ - MODEL (materialized table); - SELECT id, event_time, value FROM __ref("orders") - """ - ).strip() - + "\n" - ) - - def prepare_replay_microbatch_project( *, tmp_path: Path, project_name: str, database_name: str, replay_policy: str ) -> tuple[Path, Path]: @@ -793,77 +425,6 @@ def prepare_replay_microbatch_project( return project_dir, project_dir / database_name -def prepare_virtual_microbatch_lifecycle_project( - *, - tmp_path: Path, - project_name: str, - model_sql: str, - unaccounted_policy: str = "synthesize", - enable_janitor: bool = False, -) -> tuple[Path, Path, Path]: - """Prepare a virtual concurrent microbatch lifecycle project.""" - - project_toml: str = build_virtual_plan_project_toml().replace( - "virtual_environments = true\n", - ( - "virtual_environments = true\n" - "concurrency = 3\n" - "microbatch_concurrency = true\n" - f'microbatch_unaccounted_partition_policy = "{unaccounted_policy}"\n' - ), - ) - project_toml += { - False: "", - True: ("\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n"), - }[enable_janitor] - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": project_toml, - "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" - ), - "models/orders.sql": model_sql, - }, - ) - warehouse_path: Path = project_dir / "warehouse.duckdb" - state_path: Path = project_dir / "state.duckdb" - execute_duckdb( - db_path=warehouse_path, - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_events " - "(id INTEGER, event_time TIMESTAMP, payload VARCHAR); " - "INSERT INTO raw.raw_events VALUES " - "(1, '2026-01-01 00:30:00', '1'), " - "(2, '2026-01-01 01:30:00', '2'), " - "(3, '2026-01-01 02:30:00', '3'), " - "(4, '2026-01-01 03:30:00', '4')" - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - return project_dir, warehouse_path, state_path - - -def physical_orders_relation(*, state_path: Path) -> tuple[str, str]: - """Return the latest registered physical orders relation.""" - - schema_name, relation_name = query_duckdb( - db_path=state_path, - sql=( - "SELECT schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY created_at DESC LIMIT 1" - ), - )[0] - return str(schema_name), str(relation_name) - - def dropped_incremental_project_files(*, incremental_strategy: str) -> dict[str, str]: """Return a DuckDB project with one timestamp-cursor incremental order model.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_removed_run_command.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_removed_run_command.py index df73713b1b..f6e65a87f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_removed_run_command.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_removed_run_command.py @@ -23,19 +23,16 @@ ], ids=lambda case: case.description, ) -def test_given_virtual_mode_project_when_running_top_level_run_then_cli_rejects_command( +def test_given_project_when_running_top_level_run_then_cli_rejects_command( test_case: RunE2ETestCase, tmp_path: Path, ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, - project_name="virtual_run_project", + project_name="removed_run_project", repo_files={ "sqlbuild_project.toml": ( - 'name = "virtual_run_project"\n' - 'adapter = "duckdb"\n\n' - "[settings]\n" - "virtual_environments = true\n" + 'name = "removed_run_project"\nadapter = "duckdb"\n\n[settings]\n\n' ) }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py index 77f626f569..9ac5975d75 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_cursor_runtime_failures.py @@ -21,7 +21,7 @@ "test_case", [ CliFailureBuildE2ETestCase( - description="bad mapped cursor input column fails during planning validation", + description="bad mapped cursor input column fails during compile validation", repo_files={ "sqlbuild_project.toml": ( 'name = "cursor_runtime_project"\n' @@ -73,9 +73,8 @@ command=("--no-color", "build", "--select", "+customer_status_snapshot"), expected_exit_code=1, expected_stderr_fragments=( - "error[S302]: model 'customer_status_snapshot': cursor_inputs " - "watermark references 'fact_orders' column 'missing_column', but its enforced contract " - "does not expose the column. Declared contract columns: order_id, ordered_at", + "error[B300]: [B300] models/customer_status_snapshot.sql:9:17: cursor_inputs " + "fact_orders references unknown column 'missing_column'", ), verification_sql="SELECT COUNT(*) FROM main.customer_status_snapshot", expected_verification_rows=((0,),), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py index 3268604f80..b1a0c16b71 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py @@ -2794,8 +2794,8 @@ def test_given_timestamp_snapshot_project_when_rerunning_build_then_tracks_histo command=("--no-color", "build"), expected_exit_code=1, expected_output_fragments=( - "customer_snapshot", - "query output is missing required columns: customer_id", + "error[B300]: [B300] models/customer_snapshot.sql:3:15:", + "unique_key references unknown column 'customer_id'", ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_build_state.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_build_state.py deleted file mode 100644 index 811d9bc094..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_build_state.py +++ /dev/null @@ -1,442 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualBuildE2ETestCase, - VirtualBuildSelectionGuardE2ETestCase, - VirtualConcurrentBuildE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - build_virtual_wide_dag_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - query_duckdb, - run_sqb, - table_exists, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentBuildE2ETestCase( - description="wide virtual DAG builds with concurrent physical schema setup", - concurrency=8, - expected_model_count=8, - expected_build_fragments=( - "Execution\n command sqb build", - "target dev", - "logical schema dev__dev", - "physical schema dev__sqb_physical", - "concurrency 8 configured limit", - "selected 8 of 8 build resources", - "Scheduler ", - "Phase timings", - "connection preparation", - "schema preparation", - "execution", - "cost collection", - "total", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_wide_virtual_dag_when_building_concurrently_then_physical_schema_setup_is_safe( - test_case: VirtualConcurrentBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_concurrent_schema_setup", - repo_files=build_virtual_wide_dag_repo_files( - model_count=test_case.expected_model_count, - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - (project_dir / "sqlbuild_local.toml").write_text( - f"[settings]\nconcurrency = {test_case.concurrency}\n", - encoding="utf-8", - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--verbose"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - fragment: str - for fragment in test_case.expected_build_fragments: - assert fragment in build_result.stdout - physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'dev__sqb_physical' AND table_name LIKE '%__v_%' " - "ORDER BY table_name" - ), - ) - assert len(physical_rows) == test_case.expected_model_count - logical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id, model_name FROM dev__dev.model_08", - ) - assert logical_rows == [(8, "model_08")] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildE2ETestCase( - description="default VDE creates physical versions and queryable views", - expected_build_fragments=("Virtual environment dev (working, build required)",), - expected_plan_fragments=( - "Plan ready 0 selected", - "Virtual environment dev (finalized, up to date)", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((7,),)),), - expected_ref_rows=(("dim_customers",), ("fact_orders",), ("stg_orders",)), - expected_physical_version_count=3, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_default_vde_when_building_then_it_creates_physical_versions_and_views( - test_case: VirtualBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_build_default_vde", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 7 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stderr - fragment: str - for fragment in test_case.expected_build_fragments: - assert fragment in build_result.stdout - assert table_exists( - db_path=project_dir / "warehouse.duckdb", - schema="dev__dev", - table_name="stg_orders", - ) - assert table_exists( - db_path=project_dir / "warehouse.duckdb", - schema="dev__dev", - table_name="fact_orders", - ) - assert not table_exists( - db_path=project_dir / "warehouse.duckdb", - schema="dev__sqb_physical", - table_name="_sqlbuild_fingerprints", - ) - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) - physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'dev__sqb_physical' AND table_name LIKE '%__v_%' " - "ORDER BY table_name" - ), - ) - assert len(physical_rows) == test_case.expected_physical_version_count - assert all("__v_" in str(row[0]) for row in physical_rows) - ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - ) - assert ref_rows == list(test_case.expected_ref_rows) - ref_hash_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - ) - - default_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=project_dir, - ) - assert default_plan_result.returncode == 0, default_plan_result.stderr - assert f"Plan ready {len(test_case.expected_ref_rows)} selected" in default_plan_result.stdout - - default_repeat_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_repeat_build_result.returncode == 0, default_repeat_build_result.stderr - for (model_name,) in test_case.expected_ref_rows: - assert str(model_name) in default_repeat_build_result.stdout - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - - assert plan_result.returncode == 0, plan_result.stderr - for fragment in test_case.expected_plan_fragments: - assert fragment in plan_result.stdout - - repeat_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert repeat_build_result.returncode == 0, repeat_build_result.stderr - repeat_physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'dev__sqb_physical' AND table_name LIKE '%__v_%' " - "ORDER BY table_name" - ), - ) - assert repeat_physical_rows == physical_rows - repeat_ref_hash_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - ) - assert repeat_ref_hash_rows == ref_hash_rows - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildE2ETestCase( - description="explicit VDE graph selection diverges refs and leaves downstream working", - expected_build_fragments=("Virtual environment kevin",), - expected_plan_fragments=( - "working, build required", - "Models needing build (1)", - "downstream affected (1) orders_rollup", - ), - expected_query_results=( - ("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)), - ("SELECT id FROM dev__kevin.fact_orders ORDER BY id", ((2,),)), - ("SELECT order_count FROM dev__kevin.orders_rollup", ((1,),)), - ), - expected_ref_rows=(("fact_orders",), ("stg_orders",)), - expected_default_plan_fragments=( - "query diff:", - "-SELECT 1 AS id", - "+SELECT 2 AS id", - ), - expected_final_plan_fragments=( - "Plan ready 0 selected", - "finalized, up to date", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_explicit_virtual_env_with_graph_selection_when_building_then_refs_diverge( - test_case: VirtualBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_build_explicit_vde", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ) - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - - default_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=project_dir, - ) - assert default_plan_result.returncode == 0, default_plan_result.stderr - for fragment in test_case.expected_default_plan_fragments: - assert fragment in default_plan_result.stdout - - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--virtual-env", - "kevin", - "--select", - "+fact_orders", - "--select", - "dim_customers", - ), - project_dir=project_dir, - ) - - assert branch_build_result.returncode == 0, branch_build_result.stderr - for fragment in test_case.expected_build_fragments: - assert fragment in branch_build_result.stdout - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) - divergent_refs: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT dev.node_name " - "FROM sqlbuild_state.virtual_environment_node_refs dev " - "JOIN sqlbuild_state.virtual_environment_node_refs kevin " - "ON dev.node_name = kevin.node_name " - "WHERE dev.virtual_environment_name = 'dev' " - "AND kevin.virtual_environment_name = 'kevin' " - "AND dev.node_type = 'model' AND kevin.node_type = 'model' " - "AND dev.version_hash <> kevin.version_hash " - "ORDER BY dev.node_name" - ), - ) - assert divergent_refs == list(test_case.expected_ref_rows) - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--virtual-env", "kevin", "--changes-only"), - project_dir=project_dir, - ) - - assert plan_result.returncode == 0, plan_result.stderr - for fragment in test_case.expected_plan_fragments: - assert fragment in plan_result.stdout - - final_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "kevin", "--changes-only"), - project_dir=project_dir, - ) - assert final_build_result.returncode == 0, final_build_result.stderr - - final_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--virtual-env", "kevin", "--changes-only"), - project_dir=project_dir, - ) - assert final_plan_result.returncode == 0, final_plan_result.stderr - for fragment in test_case.expected_final_plan_fragments: - assert fragment in final_plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildSelectionGuardE2ETestCase( - description="selected downstream with stale upstream blocks then expands", - blocked_command=("--no-color", "build", "--select", "fact_orders"), - expanded_command=( - "--no-color", - "build", - "--select", - "fact_orders", - "--include-stale-upstreams", - ), - expected_blocked_fragments=("missing stale required upstream models: stg_orders",), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((2,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_build_selected_downstream_with_stale_upstream_when_running_then_it_blocks( - test_case: VirtualBuildSelectionGuardE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_build_guard", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.blocked_command, - project_dir=project_dir, - ) - - assert blocked_result.returncode != 0 - blocked_output: str = blocked_result.stdout + blocked_result.stderr - fragment: str - for fragment in test_case.expected_blocked_fragments: - assert fragment in blocked_output - - expanded_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.expanded_command, - project_dir=project_dir, - ) - - assert expanded_result.returncode == 0, expanded_result.stderr - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_incremental_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_incremental_build.py deleted file mode 100644 index 3832ef20fb..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_incremental_build.py +++ /dev/null @@ -1,751 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualBuildE2ETestCase, - VirtualConcurrentMicrobatchE2ETestCase, - VirtualCustomMaterializationE2ETestCase, - VirtualWaffleShopE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - initialize_virtual_seeded_project, - prepare_virtual_cursor_override_without_snapshot_project, - prepare_virtual_seeded_incremental_project, - rewrite_cursor_override_without_snapshot_model, - rewrite_incremental_orders_model, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - prepare_waffle_shop, - query_duckdb, - run_sqb, - table_exists, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="virtual concurrency stores state outside warehouse", - expected_exit_code=0, - expected_minimum_event_count=8, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_concurrent_microbatch_when_building_then_events_use_state_backend( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_toml: str = build_virtual_plan_project_toml().replace( - "virtual_environments = true\n", - "virtual_environments = true\nconcurrency = 3\nmicrobatch_concurrency = true\n", - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_concurrent_microbatch", - repo_files={ - "sqlbuild_project.toml": project_toml, - "sources/raw.yml": """ -sources: - - name: raw_events - schema: raw - table: raw_events -""".strip() - + "\n", - "models/orders.sql": """ -MODEL ( - materialized incremental, - incremental_strategy delete_insert, - incremental_mode microbatch, - cursor event_time, - cursor_type timestamp, - cursor_grain hour, - microbatch_strategy watermark, - cursor_watermark_mode all, - cursor_inputs ( - raw_events (column event_time, roles [filter, watermark]), - ), - batch_size 1h, - batch_concurrency 3, -); - -SELECT id, event_time, payload -FROM __source("raw_events") -WHERE event_time >= __cursor_start() - AND event_time < __cursor_end() -""".strip() - + "\n", - }, - ) - warehouse_path: Path = project_dir / "warehouse.duckdb" - execute_duckdb( - db_path=warehouse_path, - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_events " - "(id INTEGER, event_time TIMESTAMP, payload VARCHAR); " - "INSERT INTO raw.raw_events VALUES " - "(1, '2026-01-01 00:30:00', 'a'), " - "(2, '2026-01-01 01:30:00', 'b'), " - "(3, '2026-01-01 02:30:00', 'c')" - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - initial_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert initial_result.returncode == 0, initial_result.stdout + initial_result.stderr - - execute_duckdb( - db_path=warehouse_path, - sql=( - "INSERT INTO raw.raw_events VALUES " - "(4, '2026-01-01 03:30:00', 'd'), " - "(5, '2026-01-01 04:30:00', 'e'), " - "(6, '2026-01-01 05:30:00', 'f')" - ), - ) - incremental_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert incremental_result.returncode == test_case.expected_exit_code, ( - incremental_result.stdout + incremental_result.stderr - ) - - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, payload FROM dev__dev.orders ORDER BY id", - ) == [(1, "a"), (2, "b"), (3, "c"), (4, "d"), (5, "e"), (6, "f")] - virtual_history: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT scope_kind, fingerprint_status, COUNT(*) " - "FROM sqlbuild_state.microbatch_events " - "GROUP BY scope_kind, fingerprint_status" - ), - ) - assert len(virtual_history) == 1 - assert virtual_history[0][:2] == ("virtual_physical", "known") - assert int(str(virtual_history[0][2])) >= test_case.expected_minimum_event_count - assert query_duckdb( - db_path=warehouse_path, - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name = '_sqlbuild_microbatches'" - ), - ) == [(0,)] - - version_hash: str = str( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - )[0][0] - ) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=("SELECT DISTINCT virtual_model_version_hash FROM sqlbuild_state.microbatch_events"), - ) == [(version_hash,)] - scope_key: str = str( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT DISTINCT scope_key FROM sqlbuild_state.microbatch_events", - )[0][0] - ) - warehouse_realm: str = scope_key.split(":")[2] - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - f"('model_version:{warehouse_realm}:orders:{version_hash}', 'other-build', " - "CURRENT_TIMESTAMP + INTERVAL 1 HOUR, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - execute_duckdb( - db_path=warehouse_path, - sql="INSERT INTO raw.raw_events VALUES (7, '2026-01-01 06:30:00', 'g')", - ) - - conflicted_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert conflicted_result.returncode != 0 - assert "physical version is already being mutated" in ( - conflicted_result.stdout + conflicted_result.stderr - ) - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT COUNT(*) FROM dev__dev.orders", - ) == [(6,)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="failed first virtual microbatch can retry provisional physical mapping", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_failed_first_virtual_microbatch_when_retried_then_provisional_mapping_is_reused( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_toml: str = build_virtual_plan_project_toml().replace( - "virtual_environments = true\n", - "virtual_environments = true\nconcurrency = 2\nmicrobatch_concurrency = true\n", - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_failed_first_microbatch", - repo_files={ - "sqlbuild_project.toml": project_toml, - "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" - ), - "models/orders.sql": ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " incremental_mode microbatch,\n" - " microbatch_strategy watermark,\n" - " cursor_watermark_mode all,\n" - " cursor event_time,\n" - " cursor_type timestamp,\n" - " cursor_grain hour,\n" - " cursor_inputs (raw_events (column event_time, roles [filter, watermark]),),\n" - " batch_size 1h,\n" - " batch_concurrency 2,\n" - ");\n" - "SELECT id, event_time, CAST(payload AS INTEGER) AS value\n" - 'FROM __source("raw_events")\n' - "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" - ), - }, - ) - warehouse_path: Path = project_dir / "warehouse.duckdb" - execute_duckdb( - db_path=warehouse_path, - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_events " - "(id INTEGER, event_time TIMESTAMP, payload VARCHAR); " - "INSERT INTO raw.raw_events VALUES (1, '2026-01-01 00:30:00', 'bad')" - ), - ) - initialized: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert initialized.returncode == test_case.expected_exit_code, ( - initialized.stdout + initialized.stderr - ) - failed: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert failed.returncode != test_case.expected_exit_code - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders'" - ), - ) == [(1,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=("SELECT COUNT(*) FROM sqlbuild_state.locks WHERE lock_key LIKE 'model_version:%'"), - ) == [(0,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'partition_completion' AND rows_affected > 0" - ), - ) == [(0,)] - - execute_duckdb( - db_path=warehouse_path, - sql="UPDATE raw.raw_events SET payload = '1'", - ) - retried: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert retried.returncode == test_case.expected_exit_code, retried.stdout + retried.stderr - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders", - ) == [(1, 1)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildE2ETestCase( - description="delete-insert incremental seeds new physical version from prior version", - expected_build_fragments=(), - expected_plan_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.orders ORDER BY id", - ((1, 10), (2, 21), (3, 31)), - ), - ), - expected_ref_rows=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_incremental_change_when_building_then_it_seeds_new_physical_version( - test_case: VirtualBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_seeded_incremental_project( - tmp_path=tmp_path, - project_name="virtual_seeded_delete_insert", - incremental_strategy="delete_insert", - replay_on_change="bounded-7d", - ) - initialize_virtual_seeded_project(project_dir=project_dir) - - rewrite_incremental_orders_model( - project_dir=project_dir, - incremental_strategy="delete_insert", - replay_on_change="bounded-7d", - amount_expression="amount_cents + 1", - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="INSERT INTO raw.raw_orders VALUES (3, '2026-01-03 00:00:00', 30)", - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stderr - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - ancestry_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT parent_model_name, seed_strategy " - "FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) - assert ancestry_rows == [("orders", "copy")] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildE2ETestCase( - description="explicit cursor overrides work without target or upstream snapshots", - expected_build_fragments=(), - expected_plan_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.orders ORDER BY id", - ((1, 10), (2, 21), (3, 31)), - ), - ), - expected_ref_rows=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_incremental_without_cursor_snapshot_when_building_then_cli_bounds_apply( - test_case: VirtualBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_cursor_override_without_snapshot_project( - tmp_path=tmp_path, - project_name="virtual_cursor_override_without_snapshot", - ) - initialize_virtual_seeded_project(project_dir=project_dir) - - rewrite_cursor_override_without_snapshot_model( - project_dir=project_dir, - amount_expression="amount_cents + 1", - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "DELETE FROM raw.raw_orders WHERE id = 1; " - "INSERT INTO raw.raw_orders VALUES (3, '2026-01-03 00:00:00', 30)" - ), - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCustomMaterializationE2ETestCase( - description="custom materialization prepare_version seeds changed physical target", - expected_query_results=( - ( - "SELECT id, amount_cents, version_marker FROM dev__dev.orders ORDER BY id", - ( - (1, 10, "prepared"), - (2, 21, "materialized"), - (3, 30, "materialized"), - ), - ), - ), - expected_ancestry_rows=(("custom_prepare_version",),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_custom_materialization_when_model_changes_then_prepare_version_seeds_target( - test_case: VirtualCustomMaterializationE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_custom_materialization", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "materializations/merge_by_id.py": """ -from sqlbuild.executor.custom.models import ( - MaterializationContext, - MaterializationResult, - PrepareVersionContext, -) - - -def prepare_version(ctx: PrepareVersionContext) -> None: - ctx.execute_sql( - f"CREATE TABLE {ctx.destination} AS " - f"SELECT id, amount_cents, 'prepared' AS version_marker FROM {ctx.origin_relation}" - ) - - -def materialize(ctx: MaterializationContext) -> MaterializationResult: - incoming = ( - "SELECT id, amount_cents, 'materialized' AS version_marker " - f"FROM ({ctx.sql}) AS model_sql" - ) - exists = ctx.adapter.relation_exists( - connection=ctx.connection, - database=ctx.destination_database, - schema=ctx.destination_schema, - name=ctx.destination_name, - ) - if not exists: - ctx.execute_sql(f"CREATE TABLE {ctx.destination} AS {incoming}") - else: - ctx.execute_sql( - f"DELETE FROM {ctx.destination} WHERE id IN " - f"(SELECT id FROM ({ctx.sql}) AS model_sql)" - ) - ctx.execute_sql(f"INSERT INTO {ctx.destination} {incoming}") - return MaterializationResult(relation=ctx.destination) -""", - "models/orders.sql": """ -MODEL (materialized merge_by_id); - -SELECT 1 AS id, 10 AS amount_cents -UNION ALL SELECT 2 AS id, 20 AS amount_cents -""", - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert first_build_result.returncode == 0, first_build_result.stderr - - (project_dir / "models" / "orders.sql").write_text( - """ -MODEL (materialized merge_by_id); - -SELECT 2 AS id, 21 AS amount_cents -UNION ALL SELECT 3 AS id, 30 AS amount_cents -""", - encoding="utf-8", - ) - - second_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - - assert second_build_result.returncode == 0, second_build_result.stderr - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT seed_strategy FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) == list(test_case.expected_ancestry_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualWaffleShopE2ETestCase( - description="full waffle shop fixture builds in virtual mode", - expected_view_names=( - "customer_status_snapshot", - "daily_activity_rollup", - "daily_order_partitioned", - "daily_revenue", - "dim_customers", - "fact_orders", - "hourly_activity_with_daily_context", - "hourly_order_activity", - "order_status_index", - "stg_customers", - "stg_orders", - "stg_payments", - ), - expected_function_names=( - "customer_orders", - "is_completed_order", - "is_completed_order_py", - ), - expected_query_results=( - ( - "SELECT order_id, customer_id, waffle_name, order_status " - "FROM dev__dev.fact_orders ORDER BY order_id LIMIT 3", - ( - (1, 1, "Classic Belgian", "completed"), - (2, 1, "Cheddar Herb", "completed"), - (3, 2, "Chicken and Waffle", "completed"), - ), - ), - ( - "SELECT order_id, is_completed_order_py FROM dev__dev.fact_orders " - "WHERE order_id IN (1, 10) ORDER BY order_id", - ((1, True), (10, False)), - ), - ( - "SELECT order_id, waffle_name, line_total_cents, order_status, " - "is_completed_order FROM dev__dev.customer_orders(1) ORDER BY order_id", - ( - (1, "Classic Belgian", 1700, "completed", True), - (2, "Cheddar Herb", 1050, "completed", True), - (8, "Liege", 950, "completed", True), - ), - ), - ( - "SELECT CAST(order_date AS VARCHAR), order_count, waffles_ordered, " - "unique_customers FROM dev__dev.daily_order_partitioned ORDER BY order_date", - ( - ("2026-04-01", 3, 6, 2), - ("2026-04-02", 3, 3, 2), - ("2026-04-03", 2, 3, 2), - ("2026-04-04", 2, 6, 2), - ), - ), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_waffle_shop_project_when_virtual_building_then_vde_outputs_are_queryable( - test_case: VirtualWaffleShopE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_waffle_shop(tmp_path) - (project_dir / "sqlbuild_project.toml").write_text( - """ -name = "waffle_shop" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true -default_audit_severity = "warn" - -[connection] -database = "waffle_shop.duckdb" - -[defaults] -materialized = "table" - -[path_defaults.staging] -materialized = "view" - -[targets.dev] -schema = "dev" -defer_sources_to = "dev" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - encoding="utf-8", - ) - db_path: Path = project_dir / "waffle_shop.duckdb" - execution_json_path: Path = project_dir / "target" / "virtual-build.json" - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--json-output", str(execution_json_path)), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - view_name: str - for view_name in test_case.expected_view_names: - assert table_exists(db_path=db_path, schema="dev__dev", table_name=view_name) - - payload: dict[str, object] = json.loads(execution_json_path.read_text(encoding="utf-8")) - assets: list[dict[str, object]] = list(payload["assets"]) # type: ignore[arg-type] - assets_by_name: dict[str, dict[str, object]] = {str(asset["name"]): asset for asset in assets} - assert len(assets_by_name) == len(assets) - function_name: str - for function_name in test_case.expected_function_names: - assert assets_by_name[function_name]["kind"] in {"udf", "table_fn"} - assert assets_by_name[function_name]["status"] == "success" - - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=db_path, sql=query_sql) == list(expected_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildE2ETestCase( - description="append bounded incremental seeds only rows before replay window", - expected_build_fragments=(), - expected_plan_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.orders ORDER BY id", - ((1, 10), (2, 21), (3, 31)), - ), - ), - expected_ref_rows=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_append_bounded_change_when_building_then_seed_excludes_replay_window( - test_case: VirtualBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_seeded_incremental_project( - tmp_path=tmp_path, - project_name="virtual_seeded_append", - incremental_strategy="append", - replay_on_change="bounded-7d", - ) - initialize_virtual_seeded_project(project_dir=project_dir) - - rewrite_incremental_orders_model( - project_dir=project_dir, - incremental_strategy="append", - replay_on_change="bounded-7d", - amount_expression="amount_cents + 1", - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="INSERT INTO raw.raw_orders VALUES (3, '2026-01-03 00:00:00', 30)", - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stderr - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - ancestry_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT parent_model_name, seed_strategy " - "FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) - assert ancestry_rows == [("orders", "bounded_append_copy")] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_microbatch_lifecycle.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_microbatch_lifecycle.py deleted file mode 100644 index f8744c0dda..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_microbatch_lifecycle.py +++ /dev/null @@ -1,740 +0,0 @@ -"""E2E coverage for virtual microbatch replay, reconciliation, and generations.""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - CappedWatermarkRejectionE2ETestCase, - VirtualCappedProducerGapE2ETestCase, - VirtualConcurrentMicrobatchE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - capped_watermark_consumer_model_sql, - physical_orders_relation, - plain_capped_consumer_model_sql, - prepare_virtual_microbatch_lifecycle_project, - timestamp_microbatch_model_sql, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - query_duckdb, - run_sqb, - table_exists, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="virtual state loss synthesizes only virtual events", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_history_loss_when_reconciling_then_synthetic_events_stay_in_state_backend( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_microbatch_synthesis", - model_sql=timestamp_microbatch_model_sql( - value_expression="payload", - batch_concurrency=3, - replay_policy="forward_only", - ), - ) - initial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - execute_duckdb( - db_path=state_path, - sql="DELETE FROM sqlbuild_state.microbatch_events", - ) - - reconciled: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert reconciled.returncode == test_case.expected_exit_code, ( - reconciled.stdout + reconciled.stderr - ) - assert "fingerprints are unknown" in reconciled.stdout + reconciled.stderr - assert ( - query_duckdb( - db_path=state_path, - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'synthetic_completion' " - "AND scope_kind = 'virtual_physical' " - "AND fingerprint_status = 'unknown' AND model_version_hash IS NULL" - ), - )[0][0] - >= test_case.expected_minimum_event_count - ) - assert query_duckdb( - db_path=warehouse_path, - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name = '_sqlbuild_microbatches'" - ), - ) == [(0,)] - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders ORDER BY id", - ) == [(1, "1"), (2, "2"), (3, "3"), (4, "4")] - - -@pytest.mark.parametrize( - "test_case", - ( - CappedWatermarkRejectionE2ETestCase( - description="virtual capped watermark dependency is rejected", - expected_exit_code=1, - expected_error_fragment=( - "model 'downstream_orders' uses capped producer 'orders' as a watermark input; " - "capped producers cannot serve as watermark inputs" - ), - expected_absent_relations=("orders", "downstream_orders"), - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_capped_producer_as_watermark_when_building_then_plan_rejects_before_dml( - tmp_path: Path, test_case: CappedWatermarkRejectionE2ETestCase -) -> None: - project_dir, _warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_capped_watermark_rejection", - model_sql=timestamp_microbatch_model_sql( - value_expression="payload", - batch_concurrency=1, - replay_policy="forward_only", - extra_config=( - "cursor_start '2026-01-01T00:00:00',\n" - "cursor_end '2026-01-01T12:00:00',\n" - "microbatch_limit (max_batches 3, action cap_from_end)," - ), - ), - ) - (project_dir / "models" / "downstream_orders.sql").write_text( - capped_watermark_consumer_model_sql(), - encoding="utf-8", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert result.returncode == test_case.expected_exit_code - assert test_case.expected_error_fragment in result.stdout + result.stderr - registered_models: tuple[str, ...] = tuple( - str(row[0]) - for row in query_duckdb( - db_path=state_path, - sql=( - "SELECT artifact_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' ORDER BY artifact_name" - ), - ) - ) - assert set(registered_models).isdisjoint(test_case.expected_absent_relations) - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualCappedProducerGapE2ETestCase( - description="virtual capped producer preserves physical gap", - expected_exit_code=0, - expected_ids=(2, 3, 4, 8, 9, 10), - expected_gap_rows=(), - expected_microbatch_event_count=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_capped_producer_with_plain_consumer_when_watermark_jumps_then_gap_is_untouched( - tmp_path: Path, test_case: VirtualCappedProducerGapE2ETestCase -) -> None: - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_capped_producer_gap", - model_sql=timestamp_microbatch_model_sql( - value_expression="payload", - batch_concurrency=1, - replay_policy="forward_only", - extra_config=( - "cursor_start '2026-01-01T00:00:00',\n" - "cursor_end '2026-01-01T12:00:00',\n" - "microbatch_limit (max_batches 3, action cap_from_end)," - ), - ), - ) - (project_dir / "models" / "downstream_orders.sql").write_text( - plain_capped_consumer_model_sql(), - encoding="utf-8", - ) - - initial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - execute_duckdb( - db_path=warehouse_path, - sql=( - "INSERT INTO raw.raw_events VALUES " - "(8, '2026-01-01 08:30:00', '8'), " - "(9, '2026-01-01 09:30:00', '9'), " - "(10, '2026-01-01 10:30:00', '10')" - ), - ) - - jumped: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert jumped.returncode == test_case.expected_exit_code, jumped.stdout + jumped.stderr - rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_path, - sql="SELECT id FROM dev__dev.orders ORDER BY id", - ) - gap_rows: tuple[tuple[object, ...], ...] = tuple( - query_duckdb( - db_path=warehouse_path, - sql=( - "SELECT id FROM dev__dev.orders " - "WHERE event_time >= '2026-01-01T04:00:00' " - "AND event_time < '2026-01-01T08:00:00'" - ), - ) - ) - event_count: int = int( - query_duckdb( - db_path=state_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.microbatch_events", - )[0][0] - ) - - assert tuple(int(str(row[0])) for row in rows) == test_case.expected_ids - assert gap_rows == test_case.expected_gap_rows - assert event_count == test_case.expected_microbatch_event_count - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="recreated virtual physical target starts a distinct event generation", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_recreated_virtual_physical_target_when_building_then_stale_generation_is_not_inherited( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_microbatch_generation", - model_sql=timestamp_microbatch_model_sql( - value_expression="payload", - batch_concurrency=3, - replay_policy="forward_only", - ), - ) - initial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - initial_generation: object = query_duckdb( - db_path=state_path, - sql=( - "SELECT physical_generation_id FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'partition_completion' " - "ORDER BY created_at DESC LIMIT 1" - ), - )[0][0] - schema_name, relation_name = physical_orders_relation(state_path=state_path) - execute_duckdb( - db_path=warehouse_path, - sql=( - f'DROP TABLE "{schema_name}"."{relation_name}"; ' - f'CREATE TABLE "{schema_name}"."{relation_name}" ' - "(id INTEGER, event_time TIMESTAMP, value VARCHAR)" - ), - ) - - rebuilt: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - latest_generation: object = query_duckdb( - db_path=state_path, - sql=( - "SELECT physical_generation_id FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'partition_completion' " - "ORDER BY created_at DESC LIMIT 1" - ), - )[0][0] - - assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert rebuilt.returncode == test_case.expected_exit_code, rebuilt.stdout + rebuilt.stderr - assert latest_generation != initial_generation - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders ORDER BY id", - ) == [(1, "1"), (2, "2"), (3, "3"), (4, "4")] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="partial virtual bounded replay retains its requirement and resumes", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_virtual_replay_when_retried_then_same_requirement_completes_new_version( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - f1_sql: str = timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER)", - batch_concurrency=3, - replay_policy="bounded-4h", - ) - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_microbatch_partial_replay", - model_sql=f1_sql, - unaccounted_policy="recover_all", - ) - initial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - execute_duckdb( - db_path=warehouse_path, - sql="UPDATE raw.raw_events SET payload = 'bad' WHERE id = 2", - ) - (project_dir / "models" / "orders.sql").write_text( - timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER) + 10", - batch_concurrency=3, - replay_policy="bounded-4h", - ), - encoding="utf-8", - ) - partial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - requirement_before: tuple[object, ...] = query_duckdb( - db_path=state_path, - sql=( - "SELECT replay_requirement_id, run_start, run_end, " - "required_model_version_hash, physical_generation_id " - "FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'replay_requirement' " - "ORDER BY created_at DESC LIMIT 1" - ), - )[0] - execute_duckdb( - db_path=warehouse_path, - sql="UPDATE raw.raw_events SET payload = '2' WHERE id = 2", - ) - - retried: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - requirement_after: tuple[object, ...] = query_duckdb( - db_path=state_path, - sql=( - "SELECT replay_requirement_id, run_start, run_end, " - "required_model_version_hash, physical_generation_id " - "FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'replay_requirement' " - "ORDER BY created_at DESC LIMIT 1" - ), - )[0] - - assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert partial.returncode != test_case.expected_exit_code - assert retried.returncode == test_case.expected_exit_code, retried.stdout + retried.stderr - assert requirement_after == requirement_before - requirement_id: str = str(requirement_before[0]) - required_version_hash: str = str(requirement_before[3]) - assert ( - query_duckdb( - db_path=state_path, - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'partition_completion' " - f"AND replay_requirement_id = '{requirement_id}' " - f"AND model_version_hash = '{required_version_hash}'" - ), - )[0][0] - >= test_case.expected_minimum_event_count - ) - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders ORDER BY id", - ) == [(1, 11), (2, 12), (3, 13), (4, 14)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="virtual all-failed F3 supersedes F2 without resurrecting old requirement", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_all_failed_virtual_f2_and_f3_when_returning_to_f2_then_new_transition_is_created( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_microbatch_supersession", - model_sql=timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER)", - batch_concurrency=3, - replay_policy="bounded-2h", - ), - unaccounted_policy="recover_all", - ) - initial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - execute_duckdb( - db_path=warehouse_path, - sql="UPDATE raw.raw_events SET payload = 'bad'", - ) - f2_sql: str = timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER) + 10", - batch_concurrency=3, - replay_policy="bounded-2h", - ) - f3_sql: str = timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER) + 20", - batch_concurrency=3, - replay_policy="bounded-2h", - ) - model_path: Path = project_dir / "models" / "orders.sql" - model_path.write_text(f2_sql, encoding="utf-8") - failed_f2: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - model_path.write_text(f3_sql, encoding="utf-8") - failed_f3: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - model_path.write_text(f2_sql, encoding="utf-8") - returned_f2: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert failed_f2.returncode != test_case.expected_exit_code - assert failed_f3.returncode != test_case.expected_exit_code - assert returned_f2.returncode != test_case.expected_exit_code - requirement_counts: list[tuple[object, ...]] = query_duckdb( - db_path=state_path, - sql=( - "SELECT required_model_version_hash, " - "COUNT(DISTINCT replay_requirement_id) " - "FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'replay_requirement' " - "GROUP BY required_model_version_hash" - ), - ) - assert sorted(int(str(count)) for _version, count in requirement_counts) == [1, 2] - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders ORDER BY id", - ) == [(1, 1), (2, 2), (3, 3), (4, 4)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="shared history survives promotion and rollback without event copying", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_shared_virtual_microbatch_versions_when_promoting_and_rolling_back_then_events_are_unchanged( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_microbatch_promote_rollback", - model_sql=timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER)", - batch_concurrency=3, - replay_policy="bounded-2h", - ), - unaccounted_policy="recover_all", - ) - dev_build: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - shared_pr_build: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - shared_version_rows: list[tuple[object, ...]] = query_duckdb( - db_path=state_path, - sql=( - "SELECT virtual_environment_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model' AND node_name = 'orders' " - "ORDER BY virtual_environment_name" - ), - ) - assert len(shared_version_rows) == 2 - assert shared_version_rows[0][1] == shared_version_rows[1][1] - assert query_duckdb( - db_path=state_path, - sql=( - "SELECT COUNT(DISTINCT scope_key), COUNT(DISTINCT physical_generation_id) " - "FROM sqlbuild_state.microbatch_events " - f"WHERE virtual_model_version_hash = '{shared_version_rows[0][1]}'" - ), - ) == [(1, 1)] - - (project_dir / "models" / "orders.sql").write_text( - timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER) + 10", - batch_concurrency=3, - replay_policy="bounded-2h", - ), - encoding="utf-8", - ) - changed_pr_build: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - event_identities_before: list[tuple[object, ...]] = query_duckdb( - db_path=state_path, - sql=( - "SELECT event_id, scope_key, physical_generation_id, virtual_model_version_hash " - "FROM sqlbuild_state.microbatch_events ORDER BY event_id" - ), - ) - promoted: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - project_dir=project_dir, - ) - event_identities_after_promote: list[tuple[object, ...]] = query_duckdb( - db_path=state_path, - sql=( - "SELECT event_id, scope_key, physical_generation_id, virtual_model_version_hash " - "FROM sqlbuild_state.microbatch_events ORDER BY event_id" - ), - ) - promoted_rows: list[tuple[object, ...]] = query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders ORDER BY id", - ) - rolled_back: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rollback", "--virtual-env", "dev"), - project_dir=project_dir, - ) - - assert dev_build.returncode == test_case.expected_exit_code, dev_build.stdout + dev_build.stderr - assert shared_pr_build.returncode == test_case.expected_exit_code, ( - shared_pr_build.stdout + shared_pr_build.stderr - ) - assert changed_pr_build.returncode == test_case.expected_exit_code, ( - changed_pr_build.stdout + changed_pr_build.stderr - ) - assert promoted.returncode == test_case.expected_exit_code, promoted.stdout + promoted.stderr - assert rolled_back.returncode == test_case.expected_exit_code, ( - rolled_back.stdout + rolled_back.stderr - ) - assert len(event_identities_before) >= test_case.expected_minimum_event_count - assert event_identities_after_promote == event_identities_before - assert ( - query_duckdb( - db_path=state_path, - sql=( - "SELECT event_id, scope_key, physical_generation_id, virtual_model_version_hash " - "FROM sqlbuild_state.microbatch_events ORDER BY event_id" - ), - ) - == event_identities_before - ) - assert promoted_rows == [(1, 11), (2, 12), (3, 13), (4, 14)] - assert query_duckdb( - db_path=warehouse_path, - sql="SELECT id, value FROM dev__dev.orders ORDER BY id", - ) == [(1, 1), (2, 2), (3, 3), (4, 4)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentMicrobatchE2ETestCase( - description="active replay roots survive janitor until forward-only supersession", - expected_exit_code=0, - expected_minimum_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_orphan_virtual_replay_target_when_janitor_runs_then_only_active_requirement_protects_it( - test_case: VirtualConcurrentMicrobatchE2ETestCase, tmp_path: Path -) -> None: - project_dir, warehouse_path, state_path = prepare_virtual_microbatch_lifecycle_project( - tmp_path=tmp_path, - project_name="virtual_microbatch_replay_janitor_root", - model_sql=timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER)", - batch_concurrency=3, - replay_policy="bounded-2h", - ), - unaccounted_policy="recover_all", - enable_janitor=True, - ) - initial: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - execute_duckdb( - db_path=warehouse_path, - sql="UPDATE raw.raw_events SET payload = 'bad' WHERE id = 3", - ) - (project_dir / "models" / "orders.sql").write_text( - timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER) + 10", - batch_concurrency=3, - replay_policy="bounded-2h", - ), - encoding="utf-8", - ) - failed_f2: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - f2_version_hash: str = str( - query_duckdb( - db_path=state_path, - sql=( - "SELECT required_model_version_hash " - "FROM sqlbuild_state.microbatch_events " - "WHERE record_type = 'replay_requirement' " - "ORDER BY created_at DESC LIMIT 1" - ), - )[0][0] - ) - f2_schema, f2_relation = tuple( - str(value) - for value in query_duckdb( - db_path=state_path, - sql=( - "SELECT schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - f"AND version_hash = '{f2_version_hash}'" - ), - )[0] - ) - - protected_janitor: subprocess.CompletedProcess[str] = run_sqb( - command=("janitor", "--auto-approve"), project_dir=project_dir - ) - - assert initial.returncode == test_case.expected_exit_code, initial.stdout + initial.stderr - assert failed_f2.returncode != test_case.expected_exit_code - assert protected_janitor.returncode == test_case.expected_exit_code, ( - protected_janitor.stdout + protected_janitor.stderr - ) - assert table_exists( - db_path=warehouse_path, - schema=f2_schema, - table_name=f2_relation, - ) - - execute_duckdb( - db_path=warehouse_path, - sql="UPDATE raw.raw_events SET payload = CAST(id AS VARCHAR)", - ) - (project_dir / "models" / "orders.sql").write_text( - timestamp_microbatch_model_sql( - value_expression="CAST(payload AS INTEGER) + 20", - batch_concurrency=3, - replay_policy="forward_only", - ), - encoding="utf-8", - ) - successful_f3: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - warehouse_realm: str = str( - query_duckdb( - db_path=state_path, - sql=( - "SELECT scope_key FROM sqlbuild_state.microbatch_events " - f"WHERE virtual_model_version_hash = '{f2_version_hash}' LIMIT 1" - ), - )[0][0] - ).split(":")[2] - execute_duckdb( - db_path=state_path, - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - f"('model_version:{warehouse_realm}:orders:{f2_version_hash}', " - "'other-build', CURRENT_TIMESTAMP + INTERVAL 1 HOUR, " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - lease_protected_janitor: subprocess.CompletedProcess[str] = run_sqb( - command=("janitor", "--auto-approve"), project_dir=project_dir - ) - assert lease_protected_janitor.returncode == test_case.expected_exit_code, ( - lease_protected_janitor.stdout + lease_protected_janitor.stderr - ) - assert table_exists( - db_path=warehouse_path, - schema=f2_schema, - table_name=f2_relation, - ) - execute_duckdb( - db_path=state_path, - sql=( - "DELETE FROM sqlbuild_state.locks " - f"WHERE lock_key = 'model_version:{warehouse_realm}:orders:{f2_version_hash}'" - ), - ) - collecting_janitor: subprocess.CompletedProcess[str] = run_sqb( - command=("janitor", "--auto-approve"), project_dir=project_dir - ) - - assert successful_f3.returncode == test_case.expected_exit_code, ( - successful_f3.stdout + successful_f3.stderr - ) - assert collecting_janitor.returncode == test_case.expected_exit_code, ( - collecting_janitor.stdout + collecting_janitor.stderr - ) - assert not table_exists( - db_path=warehouse_path, - schema=f2_schema, - table_name=f2_relation, - ) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_mode_guard.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_mode_guard.py deleted file mode 100644 index 0c45412459..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_mode_guard.py +++ /dev/null @@ -1,87 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path -from textwrap import dedent - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualModeGuardBuildE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualModeGuardBuildE2ETestCase( - description="plan blocks defer-to in virtual mode", - project_toml=dedent( - """ - name = "virtual_plan_project" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [targets.dev] - schema = "dev" - - [targets.prod] - schema = "prod" - """ - ).strip() - + "\n", - command=("--no-color", "plan", "--defer-to", "prod"), - expected_exit_code=1, - expected_error_fragment=( - "plan does not support --defer-to when virtual_environments = true" - ), - ), - VirtualModeGuardBuildE2ETestCase( - description="build blocks defer-to in virtual mode", - project_toml=dedent( - """ - name = "virtual_build_project" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [targets.dev] - schema = "dev" - - [targets.prod] - schema = "prod" - """ - ).strip() - + "\n", - command=("--no-color", "build", "--defer-to", "prod"), - expected_exit_code=1, - expected_error_fragment=( - "build does not support --defer-to when virtual_environments = true" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_virtual_mode_defer_to_when_running_cli_then_blocks_cleanly( - test_case: VirtualModeGuardBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_mode_guard_project", - repo_files={"sqlbuild_project.toml": test_case.project_toml}, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - assert test_case.expected_error_fragment in result.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_promote.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_promote.py deleted file mode 100644 index 5abf502830..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_promote.py +++ /dev/null @@ -1,981 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualPromoteE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - build_virtual_multi_seed_lifecycle_repo_files, - build_virtual_seed_lifecycle_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, - build_virtual_plan_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="whole VDE promotion swaps target refs and views", - promote_command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=( - "\u2713 Virtual promotion complete", - "pr -> dev", - "target status finalized", - "promoted models 3", - "remaining stale models 0", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((2,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_env_when_promoting_then_it_updates_target_refs_and_views( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_whole", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert promote_result.returncode == 0, promote_result.stderr - fragment: str - for fragment in test_case.expected_promote_fragments: - assert fragment in promote_result.stdout - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints " - "WHERE virtual_environment_name = 'dev'" - ), - ) - assert len(checkpoint_rows) == 2 - operation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT operation_type, status, virtual_environment_name " - "FROM sqlbuild_state.state_operations ORDER BY created_at DESC LIMIT 1" - ), - ) - assert operation_rows == [("promote", "succeeded", "dev")] - operation_event_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT action, status FROM sqlbuild_state.state_operation_events ORDER BY created_at" - ), - ) - assert operation_event_rows[-2:] == [("start", "running"), ("finish", "succeeded")] - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="whole VDE promotion carries seed refs and views", - promote_command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.order_amounts ORDER BY id", - ((1, 200),), - ), - ( - "SELECT id, amount_cents FROM dev__dev.stg_orders ORDER BY id", - ((1, 200),), - ), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_change_when_promoting_then_it_updates_destination_seed_refs_and_views( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_seed_refs", - repo_files=build_virtual_seed_lifecycle_repo_files(amount_cents=100), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "seeds" / "order_amounts.csv").write_text( - "id,amount_cents\n1,200\n", encoding="utf-8" - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert promote_result.returncode == 0, promote_result.stderr - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT virtual_environment_name, node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name IN ('dev', 'pr') AND node_type = 'seed' " - "ORDER BY virtual_environment_name" - ), - ) - assert len(seed_ref_rows) == 2 - assert seed_ref_rows[0][2] == seed_ref_rows[1][2] - checkpoint_seed_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT seed_name FROM sqlbuild_state.virtual_environment_checkpoint_seed_refs " - "WHERE seed_name = 'order_amounts'" - ), - ) - assert checkpoint_seed_rows - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="partial promotion carries upstream seed refs and views", - promote_command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - expected_promote_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.order_amounts ORDER BY id", - ((1, 200),), - ), - ( - "SELECT id, multiplier FROM dev__dev.amount_multipliers ORDER BY id", - ((1, 2),), - ), - ("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_change_when_partially_promoting_then_it_updates_upstream_seed_refs_and_views( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_partial_seed_refs", - repo_files=build_virtual_multi_seed_lifecycle_repo_files(amount_cents=100, multiplier=1), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "seeds" / "order_amounts.csv").write_text( - "id,amount_cents\n1,200\n", encoding="utf-8" - ) - (project_dir / "seeds" / "amount_multipliers.csv").write_text( - "id,multiplier\n1,2\n", encoding="utf-8" - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert promote_result.returncode == 0, promote_result.stdout + promote_result.stderr - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT virtual_environment_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name IN ('dev', 'pr') " - "AND node_type = 'seed' " - "AND node_name IN ('order_amounts', 'amount_multipliers') " - "ORDER BY node_name, virtual_environment_name" - ), - ) - assert len(seed_ref_rows) == 4 - assert seed_ref_rows[0][1] == seed_ref_rows[1][1] - assert seed_ref_rows[2][1] == seed_ref_rows[3][1] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="whole VDE promotion carries function definitions", - promote_command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=( - "\u2713 Virtual promotion complete", - "pr -> dev", - "target status finalized", - ), - expected_query_results=( - ("SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", ((True,),)), - ("SELECT dev__dev.is_large_order(7)", ((True,),)), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_function_change_when_promoting_then_it_publishes_target_function_definition( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_function_definition", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 7 AS id") - | { - "models/fact_orders.sql": ( - "MODEL (materialized view);\n\n" - 'SELECT __udf("is_large_order")(id) AS is_large FROM __ref("stg_orders")\n' - ), - "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, " - "replay_on_change full);\n\n" - "amount > 9\n" - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT dev__dev.is_large_order(7)", - ) == [(False,)] - - (project_dir / "functions" / "sql" / "is_large_order.sql").write_text( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 5\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__pr.fact_orders ORDER BY is_large", - ) == [(True,)] - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", - ) == [(False,)] - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert promote_result.returncode == 0, promote_result.stderr - for fragment in test_case.expected_promote_fragments: - assert fragment in promote_result.stdout - function_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type IN ('udf', 'table_fn')" - ), - ) - assert len(function_ref_rows) == 1 - for sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="finalized source promotes after workspace changes again", - promote_command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=( - "\u2713 Virtual promotion complete", - "target status finalized", - ), - expected_query_results=(("SELECT id FROM dev__dev.stg_orders ORDER BY id", ((2,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_finalized_source_vde_when_workspace_changes_again_then_whole_promotion_succeeds( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_finalized_stale_workspace", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", - encoding="utf-8", - ) - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert promote_result.returncode == 0, promote_result.stdout + promote_result.stderr - for fragment in test_case.expected_promote_fragments: - assert fragment in promote_result.stdout - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="partial promotion preserves unrelated current source identity", - promote_command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "independent", - ), - expected_promote_fragments=( - "\u2713 Virtual promotion complete", - "promoted models 1", - ), - expected_query_results=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_unrelated_source_model_when_promoting_partial_then_current_identity_does_not_block( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_source_identity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " schema: raw\n" - " table: raw_orders\n" - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: integer\n" - ), - "models/source_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - "models/independent.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (1, 1)" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - dev_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert dev_build_result.returncode == 0, dev_build_result.stdout + dev_build_result.stderr - (project_dir / "models" / "independent.sql").write_text( - "MODEL (materialized table);\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - pr_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert pr_build_result.returncode == 0, pr_build_result.stdout + pr_build_result.stderr - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert promote_result.returncode == 0, promote_result.stdout + promote_result.stderr - for fragment in test_case.expected_promote_fragments: - assert fragment in promote_result.stdout, promote_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="partial promotion requires explicit working target acceptance", - blocked_command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - ), - promote_command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - expected_blocked_fragments=( - "promotion would leave target virtual environment working", - "orders_rollup", - "--allow-partial-promotion", - ), - expected_promote_fragments=( - "target status working", - "promoted models 2", - "remaining stale set: orders_rollup", - ), - expected_query_results=( - ("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((2,),)), - ("SELECT order_count FROM dev__dev.orders_rollup", ((1,),)), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_virtual_promotion_when_target_stays_working_then_it_requires_override( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_partial", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ) - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.blocked_command, - project_dir=project_dir, - ) - assert blocked_result.returncode != 0 - blocked_output: str = blocked_result.stdout + blocked_result.stderr - fragment: str - for fragment in test_case.expected_blocked_fragments: - assert fragment in blocked_output - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - assert promote_result.returncode == 0, promote_result.stderr - for fragment in test_case.expected_promote_fragments: - assert fragment in promote_result.stdout - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="direct mode promotion fails with mode error", - promote_command=("promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=("promote requires virtual_environments = true",), - expected_query_results=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_direct_mode_project_when_promoting_then_it_fails_with_mode_error( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="direct_promote_guard", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "direct_promote_guard"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n' - ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_promote_fragments: - assert fragment in result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="missing promoted physical relation fails clearly", - promote_command=( - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - expected_promote_fragments=("fact_orders",), - expected_query_results=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_promoted_physical_relation_is_missing_when_promoting_then_it_fails_clearly( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_missing_physical", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'fact_orders' " - "ORDER BY created_at DESC LIMIT 1" - ), - ) - assert physical_rows - schema_name, relation_name = physical_rows[0] - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=f'DROP TABLE "{schema_name}"."{relation_name}"', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - output: str = result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_promote_fragments: - assert fragment in output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="target VDE lock blocks promotion", - promote_command=("promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=("virtual environment 'dev' is locked",), - expected_query_results=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_target_virtual_environment_lock_when_promoting_then_it_fails_clearly( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_locked_target", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) " - "VALUES ('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - output: str = result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_promote_fragments: - assert fragment in output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description="interrupted promotion records failed operation", - promote_command=("promote", "--from", "pr", "--to", "dev"), - expected_promote_fragments=("simulated promote view refresh failure",), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders", ((1,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_view_refresh_failure_when_promoting_then_operation_is_marked_failed( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_failed_operation", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "adapters/failing_duckdb.py": ( - "from typing import Any\n" - "from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder\n" - "from sqlbuild.adapter.contract.exceptions import AdapterUserError\n" - "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n" - "class FailingDuckDbAdapter(DuckDbAdapter):\n" - " adapter_name = 'failing_duckdb'\n\n" - " def create_view_as(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " destination: str,\n" - " sql: str,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated promote view refresh failure')\n" - ) - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - (project_dir / "sqlbuild_local.toml").write_text( - 'adapter = "failing_duckdb"\n\n' - "[connection]\n" - f'database = "{project_dir / "warehouse.duckdb"}"\n' - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - assert test_case.expected_promote_fragments[0] in (result.stdout + result.stderr) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT operation_type, status, virtual_environment_name " - "FROM sqlbuild_state.state_operations WHERE operation_type = 'promote'" - ), - ) == [("promote", "failed", "dev")] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT status, message FROM sqlbuild_state.state_operation_events " - "WHERE operation_id LIKE 'promote:%' ORDER BY created_at DESC LIMIT 1" - ), - ) == [("failed", test_case.expected_promote_fragments[0])] - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPromoteE2ETestCase( - description=( - "working source blocks whole promotion but allows coherent partial promotion" - ), - blocked_command=("promote", "--from", "pr", "--to", "dev"), - promote_command=( - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - expected_blocked_fragments=( - "whole-VDE promotion requires a finalized source virtual environment", - "--select", - ), - expected_promote_fragments=("\u2713 Virtual promotion complete", "target status"), - expected_query_results=( - ("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((2,),)), - ("SELECT order_count FROM dev__dev.orders_rollup", ((1,),)), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_working_source_vde_when_promoting_then_whole_blocks_and_partial_succeeds( - test_case: VirtualPromoteE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_promote_working_source", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ) - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--virtual-env", - "pr", - "--select", - "fact_orders", - "--include-stale-upstreams", - ), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.blocked_command, - project_dir=project_dir, - ) - assert blocked_result.returncode == 1, blocked_result.stdout + blocked_result.stderr - blocked_output: str = blocked_result.stdout + blocked_result.stderr - fragment: str - for fragment in test_case.expected_blocked_fragments: - assert fragment in blocked_output - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.promote_command, - project_dir=project_dir, - ) - assert promote_result.returncode == 0, promote_result.stdout + promote_result.stderr - for fragment in test_case.expected_promote_fragments: - assert fragment in promote_result.stdout - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py deleted file mode 100644 index ebdeed1565..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py +++ /dev/null @@ -1,1110 +0,0 @@ -from __future__ import annotations - -import base64 -import json -import subprocess -from pathlib import Path -from textwrap import dedent - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualNodeResultFailureStateE2ETestCase, - VirtualNodeResultStateE2ETestCase, - VirtualPythonBuildE2ETestCase, - VirtualPythonHooksBuildE2ETestCase, - VirtualPythonIdentityBuildE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonHooksBuildE2ETestCase( - description="virtual build executes discovered Python lifecycle hook", - expected_exit_code=0, - expected_model_rows=((7,),), - expected_hook_log_rows=(("fact_orders", "post_hooks"),), - expected_identity_rows=(("hook", "log_virtual_hook"),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_build_with_python_hooks_when_building_then_hooks_execute( - test_case: VirtualPythonHooksBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_python_hooks_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "hooks/python/lifecycle.py": dedent( - """ - from sqlbuild.hooks import hook - - - @hook - def log_virtual_hook(ctx): - ctx.execute_sql( - "CREATE TABLE main.virtual_hook_log AS " - f"SELECT '{ctx.model_name}' AS model_name, '{ctx.phase}' AS phase" - ) - """ - ).strip() - + "\n", - "models/fact_orders.sql": dedent( - """ - MODEL ( - materialized table, - post_hooks [python("log_virtual_hook")] - ); - - SELECT 7 AS id - """ - ).strip() - + "\n", - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - db_path: Path = project_dir / "warehouse.duckdb" - - assert build_result.returncode == test_case.expected_exit_code, build_result.stderr - assert query_duckdb( - db_path=db_path, - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_model_rows) - assert query_duckdb( - db_path=db_path, - sql="SELECT model_name, phase FROM main.virtual_hook_log", - ) == list(test_case.expected_hook_log_rows) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_type, node_name " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type IN ('task', 'loader', 'asset', 'check', 'hook') " - "ORDER BY node_type, node_name" - ), - ) == list(test_case.expected_identity_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonBuildE2ETestCase( - description="runs loader-side and read-side Python nodes", - project_name="virtual_python_nodes_build", - plan_command=("--no-color", "plan", "--select", "+fact_orders"), - build_command=("--no-color", "build", "--select", "+fact_orders"), - expected_build_exit_code=0, - expected_plan_fragments=( - "Python ingress (1)", - "prepare_orders", - "Python read-side (2)", - "profile_fact_orders", - "profile_raw_orders", - ), - expected_prepared_text="7", - expected_profile_text="1", - expected_source_profile_text="1", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side_python( - test_case: VirtualPythonBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=test_case.project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_python_nodes_build"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n' - 'defer_sources_to = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "python/tasks/prepare.py": ( - "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" - "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('prepared.txt').write_text('7')\n" - " return ctx.result(payload={'order_id': 7})\n" - ), - "python/loaders/raw.py": ( - "from pathlib import Path\n" - "from sqlbuild.loaders import loader\n" - "from python.tasks.prepare import prepare_orders\n\n" - "@loader(depends_on=(prepare_orders,))\n" - "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" - " return [{'order_id': int(marker.read_text())}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' - ), - "python/tasks/profile.py": ( - "from pathlib import Path\n" - "from sqlbuild.refs import model, source\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" - "def profile_fact_orders(ctx):\n" - " relation = ctx.relation(model('fact_orders'))\n" - " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()[0][0]\n" - " Path(__file__).parents[2].joinpath('profile.txt').write_text(str(rows))\n" - " return ctx.result(payload={'rows': rows})\n" - "\n" - "@task(depends_on=source('raw_orders'))\n" - "def profile_raw_orders(ctx):\n" - " relation = ctx.relation(source('raw_orders'))\n" - " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()[0][0]\n" - " output = Path(__file__).parents[2].joinpath('source_profile.txt')\n" - " output.write_text(str(rows))\n" - " return ctx.result(payload={'rows': rows})\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.plan_command, - project_dir=project_dir, - ) - assert plan_result.returncode == 0, plan_result.stdout + plan_result.stderr - for fragment in test_case.expected_plan_fragments: - assert fragment in plan_result.stdout - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.build_command, - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_build_exit_code, ( - build_result.stdout + build_result.stderr - ) - assert (project_dir / "prepared.txt").read_text(encoding="utf-8") == ( - test_case.expected_prepared_text - ) - assert (project_dir / "profile.txt").read_text(encoding="utf-8") == ( - test_case.expected_profile_text - ) - assert (project_dir / "source_profile.txt").read_text(encoding="utf-8") == ( - test_case.expected_source_profile_text - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualNodeResultStateE2ETestCase( - description="persists loader task asset and check node results in virtual state", - expected_state_rows=( - ("dev", "asset", "publish_result", "success"), - ("dev", "check", "check_produce_result", "success"), - ("dev", "loader", "raw_orders", "success"), - ("dev", "task", "produce_result", "success"), - ("dev", "task", "summarize_loader", "success"), - ), - expected_asset_payload={"value": 42}, - expected_loader_text="raw_orders:raw_orders:1", - expected_history_text="42:1", - expected_warehouse_result_table_count=0, - expected_build_fragments=("check_produce_result", "PASS"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_python_result_when_building_then_persists_node_results_in_state( - test_case: VirtualNodeResultStateE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_node_results_state", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_node_results_state"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - 'defer_sources_to = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "python/loaders/orders.py": ( - "from sqlbuild.loaders import loader\n\n" - "@loader\n" - "def raw_orders(ctx):\n" - " return [{'value': 42}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: value\n" - " type: INTEGER\n" - ), - "python/tasks/results.py": ( - "from pathlib import Path\n" - "from python.loaders.orders import raw_orders\n" - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('orders'))\n" - "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 42}, metadata={'source': 'vde'})\n" - "\n" - "@task(depends_on=model('orders'))\n" - "def summarize_loader(ctx):\n" - " result = ctx.result_of(node_function=raw_orders)\n" - " history = ctx.results_of(node_function=raw_orders, limit=1)\n" - " output = Path(__file__).parents[2].joinpath('loader_result.txt')\n" - " output.write_text(\n" - " f\"{result.metadata['loader_name']}:{result.metadata['source_name']}:\"\n" - " f\"{result.metadata['rows_loaded']}\"\n" - " )\n" - " history_output = Path(__file__).parents[2].joinpath('history_result.txt')\n" - " history_output.write_text(\n" - " f\"{ctx.result_of(node_function=produce_result).payload['value']}:{len(history)}\"\n" - " )\n" - " return ctx.result(metadata={'summarized': True})\n" - ), - "python/assets/results.py": ( - "from sqlbuild.assets import asset\n" - "from python.tasks.results import produce_result\n\n" - "@asset(depends_on=produce_result)\n" - "def publish_result(ctx):\n" - " payload = ctx.result_of(node_function=produce_result).payload\n" - " return ctx.result(payload=payload, materialized=True)\n" - ), - "models/orders.sql": ( - 'MODEL (materialized table);\n\nSELECT value FROM __source("raw_orders")\n' - ), - "python/checks/results.py": ( - "from sqlbuild.checks import check\n" - "from python.assets.results import publish_result\n" - "from python.tasks.results import produce_result, summarize_loader\n\n" - "@check(depends_on=(publish_result, summarize_loader))\n" - "def check_produce_result(ctx):\n" - " return (\n" - " ctx.result_of(node_function=produce_result).payload['value'] == 42\n" - " and ctx.result_of(node_function=publish_result).payload['value'] == 42\n" - " )\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "orders"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - for fragment in test_case.expected_build_fragments: - assert fragment in build_result.stdout - state_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT virtual_environment_name, node_type, node_name, status " - "FROM sqlbuild_state.node_results " - "WHERE node_name IN (" - "'raw_orders', 'produce_result', 'summarize_loader', " - "'publish_result', 'check_produce_result') " - "ORDER BY node_type, node_name" - ), - ) - assert state_rows == list(test_case.expected_state_rows) - asset_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT payload_json_b64, materialized FROM sqlbuild_state.node_results " - "WHERE node_type = 'asset' AND node_name = 'publish_result'" - ), - ) - assert len(asset_rows) == 1 - assert json.loads(base64.b64decode(str(asset_rows[0][0])).decode("utf-8")) == ( - test_case.expected_asset_payload - ) - assert asset_rows[0][1] == "true" - assert (project_dir / "loader_result.txt").read_text(encoding="utf-8") == ( - test_case.expected_loader_text - ) - assert (project_dir / "history_result.txt").read_text(encoding="utf-8") == ( - test_case.expected_history_text - ) - warehouse_result_table_count: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name = '_sqlbuild_node_results'" - ), - ) - assert warehouse_result_table_count == [(test_case.expected_warehouse_result_table_count,)] - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualNodeResultFailureStateE2ETestCase( - description="failed virtual task persists failed state row", - project_name="virtual_failed_task_result_state", - repo_files={ - "sqlbuild_project.toml": dedent( - """ - name = "virtual_failed_task_result_state" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "warehouse.duckdb" - - [targets.dev] - schema = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ - ).strip() - + "\n", - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "python/tasks/results.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('orders'))\n" - "def produce_result(ctx):\n" - " raise RuntimeError('producer failed')\n" - ), - }, - command=("--no-color", "build", "--select", "orders"), - expected_exit_code=1, - expected_state_rows=(("task", "produce_result", "failed", "producer failed"),), - ), - VirtualNodeResultFailureStateE2ETestCase( - description="failed virtual check persists failed state row", - project_name="virtual_failed_check_result_state", - repo_files={ - "sqlbuild_project.toml": dedent( - """ - name = "virtual_failed_check_result_state" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "warehouse.duckdb" - - [targets.dev] - schema = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ - ).strip() - + "\n", - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "python/tasks/results.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('orders'))\n" - "def produce_result(ctx):\n" - " return ctx.result(payload={'value': 1})\n" - ), - "python/checks/results.py": ( - "from sqlbuild.checks import check\n" - "from python.tasks.results import produce_result\n\n" - "@check(depends_on=produce_result)\n" - "def check_produce_result(ctx):\n" - " return False\n" - ), - }, - command=("--no-color", "build", "--select", "orders"), - expected_exit_code=1, - expected_state_rows=( - ("check", "check_produce_result", "failed", ""), - ("task", "produce_result", "success", ""), - ), - ), - VirtualNodeResultFailureStateE2ETestCase( - description="skipped virtual task persists skipped state row", - project_name="virtual_skipped_task_result_state", - repo_files={ - "sqlbuild_project.toml": dedent( - """ - name = "virtual_skipped_task_result_state" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "warehouse.duckdb" - - [targets.dev] - schema = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ - ).strip() - + "\n", - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "python/tasks/results.py": ( - "from sqlbuild.compiler.python_nodes.types import SkipMode\n" - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('orders'))\n" - "def prepare_orders(ctx):\n" - " return ctx.skip(reason='not needed', mode=SkipMode.SOFT)\n" - ), - }, - command=("--no-color", "build", "--select", "orders"), - expected_exit_code=0, - expected_state_rows=(("task", "prepare_orders", "skipped", "not needed"),), - ), - VirtualNodeResultFailureStateE2ETestCase( - description="skipped virtual loader persists skipped state row", - project_name="virtual_skipped_loader_result_state", - repo_files={ - "sqlbuild_project.toml": dedent( - """ - name = "virtual_skipped_loader_result_state" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "warehouse.duckdb" - - [targets.dev] - schema = "dev" - defer_sources_to = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ - ).strip() - + "\n", - "python/tasks/prepare.py": ( - "from sqlbuild.compiler.python_nodes.types import SkipMode\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" - "def prepare_events(ctx):\n" - " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" - ), - "python/loaders/events.py": ( - "from python.tasks.prepare import prepare_events\n" - "from sqlbuild.loaders import loader\n\n" - "@loader(depends_on=(prepare_events,))\n" - "def raw_events(ctx):\n" - " return [{'event_id': 1}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_events\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: event_id\n" - " type: INTEGER\n" - ), - "models/events.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' - ), - }, - command=("--no-color", "build", "--select", "+events"), - expected_exit_code=0, - expected_state_rows=( - ("loader", "raw_events", "skipped", "Upstream node hard-skipped: prepare_events"), - ("task", "prepare_events", "skipped", "no input"), - ), - ), - VirtualNodeResultFailureStateE2ETestCase( - description="non JSON virtual payload persists failed state row", - project_name="virtual_non_json_payload_result_state", - repo_files={ - "sqlbuild_project.toml": dedent( - """ - name = "virtual_non_json_payload_result_state" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "warehouse.duckdb" - - [targets.dev] - schema = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ - ).strip() - + "\n", - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "python/tasks/results.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('orders'))\n" - "def produce_result(ctx):\n" - " return ctx.result(payload={'bad': {1, 2}})\n" - ), - }, - command=("--no-color", "build", "--select", "orders"), - expected_exit_code=1, - expected_state_rows=(("task", "produce_result", "failed", "non-JSON-serializable"),), - ), - VirtualNodeResultFailureStateE2ETestCase( - description="non JSON virtual metadata persists failed state row", - project_name="virtual_non_json_metadata_result_state", - repo_files={ - "sqlbuild_project.toml": dedent( - """ - name = "virtual_non_json_metadata_result_state" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "warehouse.duckdb" - - [targets.dev] - schema = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ - ).strip() - + "\n", - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "python/tasks/results.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('orders'))\n" - "def produce_result(ctx):\n" - " return ctx.result(payload={'ok': True}, metadata={'bad': {1, 2}})\n" - ), - }, - command=("--no-color", "build", "--select", "orders"), - expected_exit_code=1, - expected_state_rows=(("task", "produce_result", "failed", "non-JSON-serializable"),), - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_node_result_failure_when_building_then_persists_failed_state_row( - test_case: VirtualNodeResultFailureStateE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=test_case.project_name, - repo_files=test_case.repo_files, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_exit_code, ( - build_result.stdout + build_result.stderr - ) - state_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_type, node_name, status, error_message " - "FROM sqlbuild_state.node_results ORDER BY node_type, node_name" - ), - ) - assert len(state_rows) == len(test_case.expected_state_rows) - actual_row: tuple[object, ...] - expected_row: tuple[object, ...] - for actual_row, expected_row in zip(state_rows, test_case.expected_state_rows, strict=True): - assert actual_row[:3] == expected_row[:3] - assert str(expected_row[3]) in str(actual_row[3] or "") - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonIdentityBuildE2ETestCase( - description="stores Python identities in virtual state for later planning", - expected_state_identity_rows=( - ("loader", "raw_orders"), - ("task", "prepare_orders"), - ("task", "profile_fact_orders"), - ), - expected_warehouse_fingerprint_table_count=0, - expected_changed_plan_fragments=( - "Plan ready 0 selected", - "Python ingress (1)", - "prepare_orders", - "task (changed)", - "python diff:", - "source diff:", - ), - unexpected_changed_plan_fragments=( - "First run (", - "Query changed (", - "fact_orders table", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_python_identities_when_replanning_then_reads_virtual_state( - test_case: VirtualPythonIdentityBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_python_identity_state", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_python_identity_state"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n' - 'defer_sources_to = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "python/tasks/prepare.py": ( - "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" - "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('prepared.txt').write_text('7')\n" - " return ctx.result(payload={'order_id': 7})\n" - ), - "python/loaders/raw.py": ( - "from pathlib import Path\n" - "from sqlbuild.loaders import loader\n" - "from python.tasks.prepare import prepare_orders\n\n" - "@loader(depends_on=(prepare_orders,))\n" - "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" - " return [{'order_id': int(marker.read_text())}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' - ), - "python/tasks/profile.py": ( - "from pathlib import Path\n" - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" - "def profile_fact_orders(ctx):\n" - " relation = ctx.relation(model('fact_orders'))\n" - " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()[0][0]\n" - " Path(__file__).parents[2].joinpath('profile.txt').write_text(str(rows))\n" - " return ctx.result(payload={'rows': rows})\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_type, node_name " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type IN ('task', 'loader', 'asset', 'check', 'hook') " - "ORDER BY node_type, node_name" - ), - ) == list(test_case.expected_state_identity_rows) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_name = '_sqlbuild_fingerprints'" - ), - ) == [(test_case.expected_warehouse_fingerprint_table_count,)] - - (project_dir / "python" / "tasks" / "prepare.py").write_text( - "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" - "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('prepared.txt').write_text('8')\n" - " return ctx.result(payload={'order_id': 8})\n", - encoding="utf-8", - ) - changed_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--select", "+fact_orders", "--changes-only"), - project_dir=project_dir, - ) - assert changed_plan_result.returncode == 0, ( - changed_plan_result.stdout + changed_plan_result.stderr - ) - for fragment in test_case.expected_changed_plan_fragments: - assert fragment in changed_plan_result.stdout - for fragment in test_case.unexpected_changed_plan_fragments: - assert fragment not in changed_plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonBuildE2ETestCase( - description="prints read-side Python failure rows", - project_name="virtual_python_read_side_failure", - plan_command=(), - build_command=("--no-color", "build", "--select", "+fact_orders"), - expected_build_exit_code=1, - expected_build_fragments=( - "python task profile_fact_orders", - "FAIL", - "profile failed intentionally", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_read_side_python_failure_when_building_then_prints_python_failure_row( - test_case: VirtualPythonBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=test_case.project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_python_read_side_failure"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/fact_orders.sql": "MODEL (materialized table);\n\nSELECT 7 AS order_id\n", - "python/tasks/profile.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" - "def profile_fact_orders(ctx):\n" - " raise RuntimeError('profile failed intentionally')\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.build_command, - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_build_exit_code - for fragment in test_case.expected_build_fragments: - assert fragment in build_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonBuildE2ETestCase( - description="prints read-side Python skip rows", - project_name="virtual_python_read_side_skip", - plan_command=(), - build_command=("--no-color", "build", "--select", "+fact_orders"), - expected_build_exit_code=0, - expected_build_fragments=( - "python task skip_fact_orders", - "SKIP", - "profile not needed", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_read_side_python_skip_when_building_then_prints_python_skip_row( - test_case: VirtualPythonBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=test_case.project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_python_read_side_skip"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/fact_orders.sql": "MODEL (materialized table);\n\nSELECT 7 AS order_id\n", - "python/tasks/profile.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" - "def skip_fact_orders(ctx):\n" - " return ctx.skip(reason='profile not needed')\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.build_command, - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_build_exit_code - for fragment in test_case.expected_build_fragments: - assert fragment in build_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonBuildE2ETestCase( - description="no-python only runs loader-side Python nodes", - project_name="virtual_no_python_nodes_build", - plan_command=("--no-color", "plan", "--select", "+fact_orders", "--no-python"), - build_command=("--no-color", "build", "--select", "+fact_orders", "--no-python"), - expected_build_exit_code=0, - expected_plan_fragments=("Python ingress (1)", "prepare_orders"), - expected_absent_plan_fragments=("Python read-side", "profile_fact_orders"), - expected_prepared_text="7", - expected_profile_exists=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_runs( - test_case: VirtualPythonBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=test_case.project_name, - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_no_python_nodes_build"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n' - 'defer_sources_to = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "python/tasks/prepare.py": ( - "from pathlib import Path\n" - "from sqlbuild.tasks import task\n\n" - "@task\n" - "def prepare_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('prepared.txt').write_text('7')\n" - " return ctx.result(payload={'order_id': 7})\n" - ), - "python/loaders/raw.py": ( - "from pathlib import Path\n" - "from sqlbuild.loaders import loader\n" - "from python.tasks.prepare import prepare_orders\n\n" - "@loader(depends_on=(prepare_orders,))\n" - "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" - " return [{'order_id': int(marker.read_text())}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' - ), - "python/tasks/profile.py": ( - "from pathlib import Path\n" - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" - "def profile_fact_orders(ctx):\n" - " Path(__file__).parents[2].joinpath('profile.txt').write_text('ran')\n" - " return ctx.result()\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.plan_command, - project_dir=project_dir, - ) - assert plan_result.returncode == 0, plan_result.stdout + plan_result.stderr - for fragment in test_case.expected_plan_fragments: - assert fragment in plan_result.stdout - for fragment in test_case.expected_absent_plan_fragments: - assert fragment not in plan_result.stdout - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.build_command, - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_build_exit_code, ( - build_result.stdout + build_result.stderr - ) - assert (project_dir / "prepared.txt").read_text(encoding="utf-8") == ( - test_case.expected_prepared_text - ) - assert (project_dir / "profile.txt").exists() is test_case.expected_profile_exists diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_rollback.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_rollback.py deleted file mode 100644 index 3152b2e010..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_rollback.py +++ /dev/null @@ -1,899 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualExplicitCheckpointRollbackE2ETestCase, - VirtualPartialRollbackE2ETestCase, - VirtualRollbackE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - build_virtual_multi_seed_lifecycle_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="rollback blocks when checkpoint physical relation is missing", - rollback_command=("--no-color", "rollback"), - expected_exit_code=1, - expected_rollback_fragments=(), - expected_stderr_fragments=( - "error[S024]", - "checkpoint references missing warehouse relation", - ), - expected_query_results=(), - expected_checkpoint_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_checkpoint_physical_relation_missing_when_rolling_back_then_it_blocks_cleanly( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_missing_physical", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, initial_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - changed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert changed_build_result.returncode == 0, changed_build_result.stderr - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints", - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - initial_physical_relation: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT pr.schema_name, pr.relation_name " - "FROM sqlbuild_state.virtual_environment_checkpoints cp " - "JOIN sqlbuild_state.virtual_environment_checkpoint_model_refs cr " - "ON cp.checkpoint_id = cr.checkpoint_id " - "JOIN sqlbuild_state.physical_relations pr " - "ON pr.artifact_type = 'model' AND pr.artifact_name = cr.model_name " - "AND pr.version_hash = cr.version_hash " - "WHERE cp.virtual_environment_name = 'dev' AND cr.model_name = 'stg_orders' " - "ORDER BY cp.created_at ASC LIMIT 1" - ), - ) - assert initial_physical_relation - schema_name, relation_name = initial_physical_relation[0] - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=f'DROP TABLE "{schema_name}"."{relation_name}"', - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code - fragment: str - for fragment in test_case.expected_stderr_fragments: - assert fragment in rollback_result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="rollback blocks when no previous checkpoint exists", - rollback_command=("--no-color", "rollback"), - expected_exit_code=1, - expected_rollback_fragments=(), - expected_stderr_fragments=( - "error[S021]", - "no previous finalized checkpoint is available for rollback", - ), - expected_query_results=(), - expected_checkpoint_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_only_current_checkpoint_when_rolling_back_then_it_blocks_cleanly( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_no_previous", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints", - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code - fragment: str - for fragment in test_case.expected_stderr_fragments: - assert fragment in rollback_result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="rollback blocks when target VDE lock exists", - rollback_command=("rollback", "--virtual-env", "dev"), - expected_exit_code=1, - expected_rollback_fragments=(), - expected_stderr_fragments=("virtual environment 'dev' is locked",), - expected_query_results=(), - expected_checkpoint_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_target_virtual_environment_lock_when_rolling_back_then_it_fails_clearly( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_locked_target", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, initial_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - changed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert changed_build_result.returncode == 0, changed_build_result.stderr - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints", - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) " - "VALUES ('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code - fragment: str - for fragment in test_case.expected_stderr_fragments: - assert fragment in rollback_result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="checkpoint show blocks for unknown checkpoint", - rollback_command=("--no-color", "state", "checkpoints", "show", "missing"), - expected_exit_code=1, - expected_rollback_fragments=(), - expected_stderr_fragments=("error[C905]", "unknown checkpoint 'missing'"), - expected_query_results=(), - expected_checkpoint_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_unknown_checkpoint_when_showing_checkpoint_then_it_blocks_cleanly( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_checkpoint_show_unknown", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints", - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - fragment: str - for fragment in test_case.expected_stderr_fragments: - assert fragment in result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="whole VDE rollback restores previous finalized checkpoint", - rollback_command=("--no-color", "rollback"), - expected_rollback_fragments=( - "\u2713 Virtual rollback complete", - "virtual environment dev", - "status finalized", - "rolled back models 2", - "rolled back model set", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)),), - expected_checkpoint_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_finalized_checkpoints_when_rolling_back_then_it_restores_previous_refs_and_views( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_whole", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, initial_build_result.stderr - initial_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - ) - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - changed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert changed_build_result.returncode == 0, changed_build_result.stderr - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints", - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - checkpoint_id: str = str(checkpoint_rows[0][0]) - list_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "list"), - project_dir=project_dir, - ) - assert list_result.returncode == 0, list_result.stderr - assert "Virtual environment checkpoints" in list_result.stdout - assert "dev" in list_result.stdout - assert checkpoint_id in list_result.stdout - show_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "show", checkpoint_id), - project_dir=project_dir, - ) - assert show_result.returncode == 0, show_result.stderr - assert "Virtual environment checkpoint" in show_result.stdout - assert checkpoint_id in show_result.stdout - assert "stg_orders" in show_result.stdout - diff_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "diff", checkpoint_id), - project_dir=project_dir, - ) - assert diff_result.returncode == 0, diff_result.stderr - assert "Virtual environment checkpoint diff" in diff_result.stdout - assert "changed refs 2" in diff_result.stdout - assert "stg_orders" in diff_result.stdout - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stderr - fragment: str - for fragment in test_case.expected_rollback_fragments: - assert fragment in rollback_result.stdout - rolled_back_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - ) - assert rolled_back_ref_rows == initial_ref_rows - query_sql: str - expected_rows: tuple[tuple[object, ...], ...] - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=query_sql, - ) == list(expected_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="whole rollback restores seed refs and views", - rollback_command=("--no-color", "rollback"), - expected_rollback_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.order_amounts ORDER BY id", - ((1, 100),), - ), - ( - "SELECT id, multiplier FROM dev__dev.amount_multipliers ORDER BY id", - ((1, 1),), - ), - ( - "SELECT id, amount_cents FROM dev__dev.stg_orders ORDER BY id", - ((1, 100),), - ), - ), - expected_checkpoint_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_change_when_rolling_back_then_it_restores_checkpointed_seed_refs_and_views( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_seed_refs", - repo_files=build_virtual_multi_seed_lifecycle_repo_files(amount_cents=100, multiplier=1), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - initial_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - checkpoint_id: str = str( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints " - "WHERE virtual_environment_name = 'dev' ORDER BY created_at LIMIT 1" - ), - )[0][0] - ) - (project_dir / "seeds" / "order_amounts.csv").write_text( - "id,amount_cents\n1,200\n", encoding="utf-8" - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id, amount_cents FROM dev__dev.order_amounts ORDER BY id", - ) == [(1, 200)] - show_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "show", checkpoint_id), - project_dir=project_dir, - ) - assert show_result.returncode == 0, show_result.stderr - assert "Seed refs" in show_result.stdout - assert "order_amounts" in show_result.stdout - diff_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "diff", checkpoint_id), - project_dir=project_dir, - ) - assert diff_result.returncode == 0, diff_result.stderr - assert "changed seeds 1" in diff_result.stdout - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stderr - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - rolled_back_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - assert rolled_back_seed_ref_rows == initial_seed_ref_rows - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="partial rollback restores upstream seed refs and views", - rollback_command=( - "--no-color", - "rollback", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-rollback", - ), - expected_rollback_fragments=(), - expected_query_results=( - ( - "SELECT id, amount_cents FROM dev__dev.order_amounts ORDER BY id", - ((1, 100),), - ), - ), - expected_checkpoint_count=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_change_when_partial_rollback_then_restores_upstream_seed_refs_and_views( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_partial_seed_refs", - repo_files=build_virtual_multi_seed_lifecycle_repo_files(amount_cents=100, multiplier=1), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - initial_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - (project_dir / "seeds" / "order_amounts.csv").write_text( - "id,amount_cents\n1,200\n", encoding="utf-8" - ) - (project_dir / "seeds" / "amount_multipliers.csv").write_text( - "id,multiplier\n1,2\n", encoding="utf-8" - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stdout + rollback_result.stderr - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - rolled_back_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - assert rolled_back_seed_ref_rows == initial_seed_ref_rows - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRollbackE2ETestCase( - description="whole VDE rollback restores checkpointed function definitions", - rollback_command=("--no-color", "rollback"), - expected_rollback_fragments=( - "\u2713 Virtual rollback complete", - "status finalized", - ), - expected_query_results=( - ("SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", ((False,),)), - ), - expected_checkpoint_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_function_change_when_rolling_back_then_it_restores_checkpointed_definition( - test_case: VirtualRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_function_definition", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 7 AS id") - | { - "models/fact_orders.sql": ( - "MODEL (materialized view);\n\n" - 'SELECT __udf("is_large_order")(id) AS is_large FROM __ref("stg_orders")\n' - ), - "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, " - "replay_on_change full);\n\n" - "amount > 9\n" - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", - ) == [(False,)] - - (project_dir / "functions" / "sql" / "is_large_order.sql").write_text( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 5\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", - ) == [(True,)] - checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_checkpoints", - ) - checkpoint_function_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_checkpoint_function_refs", - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.rollback_command, - project_dir=project_dir, - ) - - assert int(checkpoint_rows[0][0]) == test_case.expected_checkpoint_count - assert int(checkpoint_function_ref_rows[0][0]) == 2 - assert rollback_result.returncode == 0, rollback_result.stderr - for fragment in test_case.expected_rollback_fragments: - assert fragment in rollback_result.stdout - for sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualExplicitCheckpointRollbackE2ETestCase( - description="explicit checkpoint restores selected checkpoint", - rollback_command_prefix=("--no-color", "rollback", "--checkpoint-id"), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_explicit_checkpoint_when_rolling_back_then_it_restores_that_checkpoint( - test_case: VirtualExplicitCheckpointRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_explicit_checkpoint", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - initial_checkpoint_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT checkpoint_id FROM sqlbuild_state.virtual_environment_checkpoints " - "ORDER BY created_at ASC, checkpoint_id ASC" - ), - ) - initial_checkpoint_id: str = str(initial_checkpoint_rows[0][0]) - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=(*test_case.rollback_command_prefix, initial_checkpoint_id), - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stderr - assert f"checkpoint {initial_checkpoint_id}" in rollback_result.stdout - for sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPartialRollbackE2ETestCase( - description="partial rollback requires override and marks VDE working", - blocked_command=("--no-color", "rollback", "--select", "stg_orders"), - allowed_command=( - "--no-color", - "rollback", - "--select", - "stg_orders", - "--allow-partial-rollback", - ), - expected_blocked_stderr_fragments=( - "rollback would leave target virtual environment working", - ), - expected_allowed_stdout_fragments=( - "status active", - "rolled back models 1", - ), - expected_query_results=(("SELECT id FROM dev__dev.stg_orders ORDER BY id", ((1,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_rollback_when_allowed_then_it_marks_vde_working( - test_case: VirtualPartialRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_partial", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.blocked_command, - project_dir=project_dir, - ) - assert blocked_result.returncode == 1 - for fragment in test_case.expected_blocked_stderr_fragments: - assert fragment in blocked_result.stderr - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.allowed_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stderr - for fragment in test_case.expected_allowed_stdout_fragments: - assert fragment in rollback_result.stdout - for sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPartialRollbackE2ETestCase( - description="partial rollback can include stale required upstreams", - blocked_command=( - "--no-color", - "rollback", - "--select", - "fact_orders", - "--allow-partial-rollback", - ), - allowed_command=( - "--no-color", - "rollback", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-rollback", - ), - expected_blocked_stderr_fragments=( - "selected rollback scope is missing stale required upstream models", - "stg_orders", - ), - expected_allowed_stdout_fragments=( - "status finalized", - "rolled back models 2", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_rollback_missing_stale_upstreams_when_including_them_then_it_succeeds( - test_case: VirtualPartialRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_include_stale_upstreams", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.blocked_command, - project_dir=project_dir, - ) - assert blocked_result.returncode == 1 - for fragment in test_case.expected_blocked_stderr_fragments: - assert fragment in blocked_result.stderr - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.allowed_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stderr - for fragment in test_case.expected_allowed_stdout_fragments: - assert fragment in rollback_result.stdout - for sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPartialRollbackE2ETestCase( - description="partial rollback matches checkpoint when workspace changed", - blocked_command=(), - allowed_command=( - "--no-color", - "rollback", - "--select", - "fact_orders", - "--include-stale-upstreams", - ), - expected_blocked_stderr_fragments=(), - expected_allowed_stdout_fragments=( - "status finalized", - "rolled back models 2", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_partial_rollback_matches_checkpoint_when_workspace_changed_then_no_override_needed( - test_case: VirtualPartialRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_rollback_stale_workspace", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", - encoding="utf-8", - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.allowed_command, - project_dir=project_dir, - ) - - assert rollback_result.returncode == 0, rollback_result.stdout + rollback_result.stderr - fragment: str - for fragment in test_case.expected_allowed_stdout_fragments: - assert fragment in rollback_result.stdout - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_seed_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_seed_build.py deleted file mode 100644 index 78ae10c8b9..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_seed_build.py +++ /dev/null @@ -1,689 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from collections import defaultdict -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualSeedBuildE2ETestCase, - VirtualSeedGapE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedBuildE2ETestCase( - description="virtual seed build persists seed refs and reloads changed seeds", - expected_initial_rows=((1, 100),), - expected_changed_rows=((1, 200),), - expected_changed_fragments=( - "order_amounts", - "seed_changed", - "fact_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_seed_change_when_building_changes_only_then_updates_seed_state_and_model( - test_case: VirtualSeedBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_change_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.fact_orders ORDER BY order_id", - ) == list(test_case.expected_initial_rows) - initial_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - assert len(initial_seed_ref_rows) == 1 - - default_unchanged_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_unchanged_build_result.returncode == 0, default_unchanged_build_result.stderr - assert "order_amounts" in default_unchanged_build_result.stdout - assert "fact_orders" in default_unchanged_build_result.stdout - assert "OK" in default_unchanged_build_result.stdout - assert "seed order_amounts" in default_unchanged_build_result.stdout - assert "SKIP=0" in default_unchanged_build_result.stdout - - unchanged_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert unchanged_build_result.returncode == 0, unchanged_build_result.stderr - assert "Plan ready 0 selected" in unchanged_build_result.stdout - - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", - encoding="utf-8", - ) - changed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert changed_build_result.returncode == 0, ( - changed_build_result.stdout + changed_build_result.stderr - ) - fragment: str - for fragment in test_case.expected_changed_fragments: - assert fragment in changed_build_result.stdout, changed_build_result.stdout - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.fact_orders ORDER BY order_id", - ) == list(test_case.expected_changed_rows) - changed_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash " - "FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - assert len(changed_seed_ref_rows) == 1 - assert changed_seed_ref_rows[0][0] == "order_amounts" - assert changed_seed_ref_rows[0][1] != initial_seed_ref_rows[0][1] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedBuildE2ETestCase( - description="two VDEs bind isolated seed physical versions", - expected_initial_rows=((1, 100),), - expected_changed_rows=((1, 100),), - expected_branch_rows=((1, 200),), - expected_changed_fragments=(), - expected_physical_seed_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_two_virtual_environments_when_seed_differs_then_each_reads_bound_seed_version( - test_case: VirtualSeedBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_isolation", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - dev_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert dev_build_result.returncode == 0, dev_build_result.stdout + dev_build_result.stderr - - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", - encoding="utf-8", - ) - pr_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert pr_build_result.returncode == 0, pr_build_result.stdout + pr_build_result.stderr - - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.fact_orders ORDER BY order_id", - ) == list(test_case.expected_changed_rows) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__pr.fact_orders ORDER BY order_id", - ) == list(test_case.expected_branch_rows) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.order_amounts ORDER BY order_id", - ) == list(test_case.expected_changed_rows) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__pr.order_amounts ORDER BY order_id", - ) == list(test_case.expected_branch_rows) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'seed' AND artifact_name = 'order_amounts'" - ), - ) == [(test_case.expected_physical_seed_count,)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedBuildE2ETestCase( - description="second VDE reuses existing physical seed artifact", - expected_initial_rows=((1, 100),), - expected_changed_rows=((1, 100),), - expected_changed_fragments=("order_amounts",), - expected_physical_seed_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_second_vde_when_seed_version_exists_then_uses_existing_physical_seed( - test_case: VirtualSeedBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_existing_artifact", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - pr_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - - assert pr_build_result.returncode == 0, pr_build_result.stdout + pr_build_result.stderr - for fragment in test_case.expected_changed_fragments: - assert fragment in pr_build_result.stdout, pr_build_result.stdout - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__pr.fact_orders ORDER BY order_id", - ) == list(test_case.expected_changed_rows) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__pr.order_amounts ORDER BY order_id", - ) == list(test_case.expected_initial_rows) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'seed' AND artifact_name = 'order_amounts'" - ), - ) == [(test_case.expected_physical_seed_count,)] - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedBuildE2ETestCase( - description="explicit model selection updates stale upstream seed artifact", - expected_initial_rows=((1, 100),), - expected_changed_rows=((1, 200),), - expected_changed_fragments=("order_amounts", "fact_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_explicit_model_selection_when_upstream_seed_changed_then_model_reads_new_seed( - test_case: VirtualSeedBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_explicit_model_with_changed_seed", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.fact_orders ORDER BY order_id", - ) == list(test_case.expected_initial_rows) - - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", - encoding="utf-8", - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "fact_orders"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - for fragment in test_case.expected_changed_fragments: - assert fragment in build_result.stdout, build_result.stdout - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.fact_orders ORDER BY order_id", - ) == list(test_case.expected_changed_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedGapE2ETestCase( - description="failed virtual seed reload leaves seed ref unchanged", - expected_fragments=("order_amounts", "FAIL", "\u2717 Completed with errors"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_seed_load_failure_when_building_changes_only_then_seed_state_is_unchanged( - test_case: VirtualSeedGapE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_failure_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - initial_seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' ORDER BY node_name" - ), - ) - - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,not_an_integer\n", - encoding="utf-8", - ) - failed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert failed_build_result.returncode == 1, ( - failed_build_result.stdout + failed_build_result.stderr - ) - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in failed_build_result.stdout, failed_build_result.stdout - assert ( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name, version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' " - "ORDER BY node_name" - ), - ) - == initial_seed_ref_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedGapE2ETestCase( - description="virtual seed JSON includes seed reasons", - expected_fragments=("order_amounts", "config_changed"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_seed_change_when_plan_and_build_json_then_seed_reason_is_reported( - test_case: VirtualSeedGapE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_json_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", encoding="utf-8" - ) - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("plan", "--json"), project_dir=project_dir - ) - assert plan_result.returncode == 0, plan_result.stderr - plan_payload: dict[str, object] = json.loads(plan_result.stdout) - plan_seeds: list[dict[str, object]] = list(plan_payload["seeds"]) - expected_seed_name, expected_seed_reason = test_case.expected_fragments - physical_seed_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'seed' AND artifact_name = 'order_amounts'" - ), - ) - assert len(physical_seed_rows) == 1 - physical_schema, physical_name = physical_seed_rows[0] - assert len(plan_seeds) == 1 - planned_seed: dict[str, object] = plan_seeds[0] - assert planned_seed["name"] == expected_seed_name - assert planned_seed["reason"] == expected_seed_reason - planned_qualified_name: object = planned_seed["qualified_name"] - assert isinstance(planned_qualified_name, str) - assert planned_qualified_name.startswith(f"{physical_schema}.{expected_seed_name}__v_") - assert planned_qualified_name != f"{physical_schema}.{physical_name}" - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("build", "--json"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - build_payload: dict[str, object] = json.loads(build_result.stdout) - assets_by_kind: defaultdict[object, list[dict[str, object]]] = defaultdict(list) - for asset in build_payload["assets"]: - typed_asset: dict[str, object] = dict(asset) - assets_by_kind[typed_asset.get("kind")].append(typed_asset) - seed_assets: list[dict[str, object]] = assets_by_kind["seed"] - assert len(seed_assets) == 1 - assert seed_assets[0]["name"] == expected_seed_name - assert seed_assets[0]["reason"] == expected_seed_reason - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedGapE2ETestCase( - description="virtual seed schema change reloads seed and model", - expected_fragments=( - "Plan ready 2 selected", - "order_amounts", - "seed_changed", - "fact_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_seed_schema_change_when_building_changes_only_then_reloads_seed_and_model( - test_case: VirtualSeedGapE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_schema_change_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "seeds" / "schema.yml").write_text( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: BIGINT\n", - encoding="utf-8", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedGapE2ETestCase( - description="multi seed virtual graph selects only changed seed closure", - expected_fragments=("Plan ready 2 selected", "order_amounts", "fact_orders"), - unexpected_fragments=("country_codes", "dim_countries"), - ) - ], - ids=lambda case: case.description, -) -def test_given_multi_seed_graph_when_one_seed_changes_then_only_its_closure_is_selected( - test_case: VirtualSeedGapE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_multi_seed_change_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - " - name: country_codes\n" - " columns:\n" - " - name: country_code\n" - " type: TEXT\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "seeds/country_codes.csv": "country_code\nUS\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - "models/dim_countries.sql": ( - 'MODEL (materialized table);\n\nSELECT country_code FROM __seed("country_codes")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", encoding="utf-8" - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - for fragment in test_case.unexpected_fragments: - assert fragment not in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedGapE2ETestCase( - description="virtual model change with current seed does not reload seed", - expected_fragments=("Plan ready 1 selected", "fact_orders"), - unexpected_fragments=("Seeds (", "seed order_amounts"), - ) - ], - ids=lambda case: case.description, -) -def test_given_model_change_with_current_seed_when_building_then_seed_is_not_reloaded( - test_case: VirtualSeedGapE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_model_change_current_seed_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "fact_orders.sql").write_text( - "MODEL (materialized table);\n\n" - "SELECT order_id, amount_cents, amount_cents / 100.0 AS amount_dollars " - 'FROM __seed("order_amounts")\n', - encoding="utf-8", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ) - - assert result.returncode == 0, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - for fragment in test_case.unexpected_fragments: - assert fragment not in result.stdout, result.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py deleted file mode 100644 index cb7cfb1ca4..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py +++ /dev/null @@ -1,773 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path -from textwrap import dedent - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( - VirtualSourceFreshnessBuildE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - count_virtual_physical_versions, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="unchanged source freshness skips and changed freshness reruns downstream", - expected_initial_rows=((7,),), - expected_updated_rows=((8,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_when_building_then_skips_until_data_version_changes( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: integer - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); - INSERT INTO raw.raw_orders VALUES (7, 1); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_initial_rows) - - unchanged_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert unchanged_build_result.returncode == 0, unchanged_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count - - unchanged_changes_only_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert unchanged_changes_only_build_result.returncode == 0, ( - unchanged_changes_only_build_result.stderr - ) - assert "Plan ready 0 selected" in unchanged_changes_only_build_result.stdout - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count - - explicit_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "fact_orders"), - project_dir=project_dir, - ) - assert explicit_build_result.returncode == 0, explicit_build_result.stderr - assert "fact_orders" in explicit_build_result.stdout - assert "OK" in explicit_build_result.stdout - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = 2", - ) - changed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert changed_build_result.returncode == 0, changed_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 1 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_updated_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="virtual changes-only builds runtime stale table and downstream", - expected_initial_rows=((7,),), - expected_updated_rows=((7,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_run_despite_unchanged_when_building_changes_only_then_builds_downstream( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_build_run_despite_unchanged", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: order_ts - type: timestamp - """ - ).strip() - + "\n", - "models/rolling_orders.sql": ( - "MODEL (materialized table, run_despite_unchanged 30d);\n\n" - 'SELECT id, order_ts FROM __source("raw_orders")\n' - ), - "models/orders_mart.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("rolling_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, order_ts TIMESTAMP); - INSERT INTO raw.raw_orders VALUES (7, CURRENT_TIMESTAMP); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert first_build_result.returncode == 0, first_build_result.stderr - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.orders_mart ORDER BY id", - ) == list(test_case.expected_initial_rows) - - changes_only_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert changes_only_result.returncode == 0, changes_only_result.stderr - assert "rolling_orders" in changes_only_result.stdout - assert "orders_mart" in changes_only_result.stdout - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.orders_mart ORDER BY id", - ) == list(test_case.expected_updated_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="virtual build respects timestamp source freshness lag tolerance", - expected_initial_rows=((7,),), - expected_updated_rows=((9,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_timestamp_lag_tolerance_when_building_then_skips_within_tolerance( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_build_lag_tolerance", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: timestamp - lag_tolerance: 10m - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version TIMESTAMP); - INSERT INTO raw.raw_orders VALUES (7, '2026-01-01 12:00:00'); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_initial_rows) - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = '2026-01-01 12:05:00'", - ) - within_tolerance_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert within_tolerance_build_result.returncode == 0, within_tolerance_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 9, data_version = '2026-01-01 12:11:00'", - ) - beyond_tolerance_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert beyond_tolerance_build_result.returncode == 0, beyond_tolerance_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 1 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_updated_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="unknown source freshness does not skip virtual builds", - expected_initial_rows=((7,),), - expected_updated_rows=((8,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_without_freshness_when_rebuilding_then_it_does_not_skip( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_unknown_source_freshness_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER); - INSERT INTO raw.raw_orders VALUES (7); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_initial_rows) - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8", - ) - second_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert second_build_result.returncode == 0, second_build_result.stderr - assert "fact_orders" in second_build_result.stdout - assert "OK" in second_build_result.stdout - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_updated_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="source freshness propagates through views to downstream tables", - expected_initial_rows=((7,),), - expected_updated_rows=((8,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_through_view_when_changed_then_downstream_table_reruns( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_view_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: integer - """ - ).strip() - + "\n", - "models/stg_orders.sql": ( - 'MODEL (materialized view);\n\nSELECT id FROM __source("raw_orders")\n' - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); - INSERT INTO raw.raw_orders VALUES (7, 1); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_initial_rows) - - unchanged_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ) - assert unchanged_build_result.returncode == 0, unchanged_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = 2", - ) - changed_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert changed_build_result.returncode == 0, changed_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 2 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_updated_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="managed loader freshness does not cause spurious virtual rebuilds", - expected_initial_rows=((7,),), - expected_updated_rows=((7,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_managed_source_freshness_when_unchanged_then_build_skips_downstream( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_managed_source_freshness_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml().replace( - "[targets.dev]\n", '[targets.dev]\ndefer_sources_to = "dev"\n' - ), - "python/loaders/raw.py": ( - "from pathlib import Path\n" - "from sqlbuild.loaders import loader\n\n" - "@loader\n" - "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[2].joinpath('raw_order_id.txt')\n" - " return [{'id': int(marker.read_text())}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: id\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - (project_dir / "raw_order_id.txt").write_text("7", encoding="utf-8") - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "+fact_orders"), project_dir=project_dir - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_initial_rows) - - (project_dir / "raw_order_id.txt").write_text("8", encoding="utf-8") - unchanged_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ) - assert unchanged_build_result.returncode == 0, unchanged_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_updated_rows) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="managed configured freshness conservatively rebinds after first load", - expected_initial_rows=((7,),), - expected_updated_rows=((7,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_managed_source_configured_freshness_when_building_then_rebinds_safely( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_managed_configured_source_freshness_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml().replace( - "[targets.dev]\n", '[targets.dev]\ndefer_sources_to = "dev"\n' - ), - "python/loaders/raw.py": ( - "from pathlib import Path\n" - "from sqlbuild.loaders import loader\n\n" - "@loader\n" - "def raw_orders(ctx):\n" - " root = Path(__file__).parents[2]\n" - " return [{\n" - " 'id': int(root.joinpath('raw_order_id.txt').read_text()),\n" - " 'data_version': int(root.joinpath('raw_data_version.txt').read_text()),\n" - " }]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " freshness:\n" - " strategy: sql\n" - " query: SELECT MAX(data_version) FROM dev.raw_orders\n" - " type: integer\n" - " columns:\n" - " - name: id\n" - " type: INTEGER\n" - " - name: data_version\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - (project_dir / "raw_order_id.txt").write_text("7", encoding="utf-8") - (project_dir / "raw_data_version.txt").write_text("1", encoding="utf-8") - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "+fact_orders"), project_dir=project_dir - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_initial_rows) - - rebind_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert rebind_build_result.returncode == 0, rebind_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 1 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev__dev.fact_orders ORDER BY id", - ) == list(test_case.expected_updated_rows) - - stable_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert stable_build_result.returncode == 0, stable_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 1 - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="explicit adapter freshness fails clearly on unsupported adapter", - expected_initial_rows=(), - expected_updated_rows=(), - expected_error_fragment="does not support table freshness metadata", - ) - ], - ids=lambda case: case.description, -) -def test_given_unsupported_adapter_freshness_when_building_then_it_fails_clearly( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_unsupported_adapter_source_freshness_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: adapter - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER); " - "INSERT INTO raw.raw_orders VALUES (7);" - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - - assert build_result.returncode == 1 - assert test_case.expected_error_fragment is not None - assert test_case.expected_error_fragment in (build_result.stdout + build_result.stderr) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessBuildE2ETestCase( - description="function and source freshness changes independently rerun virtual model", - expected_initial_rows=((False,),), - expected_updated_rows=((True,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_and_function_when_each_changes_then_model_reruns( - test_case: VirtualSourceFreshnessBuildE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_function_build", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: integer - """ - ).strip() - + "\n", - "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, " - "replay_on_change full);\n\n" - "amount > 9\n" - ), - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT __udf("is_large_order")(id) AS is_large ' - 'FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); - INSERT INTO raw.raw_orders VALUES (7, 1); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - first_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert first_build_result.returncode == 0, first_build_result.stderr - first_version_count: int = count_virtual_physical_versions(project_dir=project_dir) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", - ) == list(test_case.expected_initial_rows) - - (project_dir / "functions" / "sql" / "is_large_order.sql").write_text( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 5\n", - encoding="utf-8", - ) - function_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert function_build_result.returncode == 0, function_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 1 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", - ) == list(test_case.expected_updated_rows) - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 4, data_version = 2", - ) - source_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert source_build_result.returncode == 0, source_build_result.stderr - assert count_virtual_physical_versions(project_dir=project_dir) == first_version_count + 2 - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT is_large FROM dev__dev.fact_orders ORDER BY is_large", - ) == list(test_case.expected_initial_rows) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/_test_types.py index b160438540..b52db0e0e8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/_test_types.py @@ -15,7 +15,6 @@ class CheckCommandTestCase: expected_stdout_fragments: tuple[str, ...] expected_absent_fragments: tuple[str, ...] = () project_kind: str = "direct" - initialize_state: bool = False expected_file_fragments: tuple[tuple[str, tuple[str, ...]], ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py index c02364afa2..ce59cfcd15 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py @@ -2,12 +2,11 @@ from __future__ import annotations -import subprocess from collections.abc import Callable from pathlib import Path from tests.e2e.src.sqlbuild.cli.commands.main.check._test_types import CheckCommandTestCase -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project def prepare_python_check_project(*, tmp_path: Path) -> Path: @@ -136,174 +135,16 @@ def check_raw_orders_loader(ctx): ) -def prepare_read_side_python_check_project(*, tmp_path: Path) -> Path: - """Create a project whose check depends on a SQL-reading Python asset.""" - - return prepare_inline_project( - tmp_path=tmp_path, - project_name="read_side_python_check_project", - repo_files={ - "sqlbuild_project.toml": """ -name = "read_side_python_check_project" -adapter = "duckdb" - -[connection] -database = "read_side_python_check_project.duckdb" -""", - "models/fact_orders.sql": """ -MODEL (materialized table); - -SELECT 1 AS order_id -""", - "python/assets/orders.py": """ -from sqlbuild.assets import asset -from sqlbuild.refs import model - - -@asset(depends_on=model("fact_orders")) -def orders_export(ctx): - relation = ctx.relation(model("fact_orders")) - row = ctx.query(f"SELECT COUNT(*) AS order_count FROM {relation}").fetchone() - return ctx.result(payload={"order_count": row[0]}, materialized=False) -""", - "python/checks/orders.py": """ -from sqlbuild.checks import check -from python.assets.orders import orders_export - - -@check(depends_on=orders_export) -def check_orders_export(ctx): - return ctx.result_of(node_function=orders_export).payload["order_count"] == 1 -""", - }, - ) - - -def prepare_virtual_python_check_project(*, tmp_path: Path) -> Path: - """Create a virtual-mode project with a task-backed Python check.""" - - return prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_python_check_project", - repo_files={ - "sqlbuild_project.toml": """ -name = "virtual_python_check_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev] -schema = "dev" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""", - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "python/tasks/export.py": """ -from sqlbuild.refs import model -from sqlbuild.tasks import task - - -@task(depends_on=model("stg_orders")) -def export_virtual_orders(ctx): - return ctx.result(metadata={"rows": 1}) -""", - "python/checks/export.py": """ -from sqlbuild.checks import check -from python.tasks.export import export_virtual_orders - - -@check(depends_on=export_virtual_orders) -def check_virtual_orders(ctx): - return ctx.pass_(message="virtual orders exported") -""", - }, - ) - - -def prepare_virtual_failing_python_check_project(*, tmp_path: Path) -> Path: - """Create a virtual-mode project with an error-severity Python check.""" - - return prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_failing_python_check_project", - repo_files={ - "sqlbuild_project.toml": """ -name = "virtual_failing_python_check_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev] -schema = "dev" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""", - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "python/tasks/export.py": """ -from sqlbuild.refs import model -from sqlbuild.tasks import task - - -@task(depends_on=model("stg_orders")) -def export_virtual_orders(ctx): - return ctx.result(metadata={"rows": 1}) -""", - "python/checks/export.py": """ -from sqlbuild.checks import check -from python.tasks.export import export_virtual_orders - - -@check(depends_on=export_virtual_orders) -def fail_virtual_orders(ctx): - return ctx.fail(message="virtual orders failed") -""", - }, - ) - - def prepare_check_project_by_kind(*, tmp_path: Path, project_kind: str) -> Path: """Create the project fixture for a check command test case.""" project_factories: dict[str, Callable[..., Path]] = { "direct": prepare_python_check_project, "terminal_loader": prepare_terminal_loader_check_project, - "virtual": prepare_virtual_python_check_project, - "virtual_failure": prepare_virtual_failing_python_check_project, } return project_factories[project_kind](tmp_path=tmp_path) -def initialize_state_when_requested(*, project_dir: Path, test_case: CheckCommandTestCase) -> None: - """Initialize virtual state for test cases that need it.""" - - for _ in range(int(test_case.initialize_state)): - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - def resolve_check_command(*, project_dir: Path, command: tuple[str, ...]) -> tuple[str, ...]: """Resolve project-dir placeholders in command arguments.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py index 8c6207058a..fd91edd38b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py @@ -13,7 +13,6 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.check.helpers import ( assert_expected_file_fragments, - initialize_state_when_requested, prepare_check_project_by_kind, prepare_python_check_project, resolve_check_command, @@ -178,31 +177,6 @@ ), project_kind="terminal_loader", ), - CheckCommandTestCase( - description="virtual build executes relevant Python checks", - command=("--no-color", "build"), - expected_returncode=0, - expected_stdout_fragments=( - "Python checks", - "check_virtual_orders", - "PASS", - ), - project_kind="virtual", - initialize_state=True, - ), - CheckCommandTestCase( - description="virtual build fails on error Python check", - command=("--no-color", "build"), - expected_returncode=1, - expected_stdout_fragments=( - "Python checks", - "fail_virtual_orders", - "FAIL", - "virtual orders failed", - ), - project_kind="virtual_failure", - initialize_state=True, - ), CheckCommandTestCase( description="build without tests or audits still executes Python checks", command=( @@ -259,7 +233,6 @@ def test_given_python_checks_when_running_check_then_reports_expected_results( tmp_path=tmp_path, project_kind=test_case.project_kind, ) - initialize_state_when_requested(project_dir=project_dir, test_case=test_case) result: subprocess.CompletedProcess[str] = run_sqb( command=resolve_check_command(project_dir=project_dir, command=test_case.command), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py index 8f67409053..c83005ba84 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/_test_types.py @@ -22,15 +22,6 @@ class ClonePolicyErrorTestCase: expected_policy_key: str -@dataclass(frozen=True) -class VirtualCloneE2ETestCase: - description: str - command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - expected_registered_artifacts: tuple[tuple[str, str], ...] = () - - @dataclass(frozen=True) class CloneFunctionGraphE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py deleted file mode 100644 index e601d5e853..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/helpers.py +++ /dev/null @@ -1,244 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, -) - - -def prepare_virtual_clone_project(tmp_path: Path) -> Path: - return prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_clone_project", - repo_files={ - "sqlbuild_project.toml": build_virtual_clone_project_toml(), - "sqlbuild_local.toml": 'target = "dev"\n', - "seeds/orders.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " path: order_amounts.csv\n" - " columns:\n" - " - name: id\n" - " type: integer\n" - ), - "seeds/order_amounts.csv": "id\n7\n", - "models/stg_orders.sql": 'MODEL ();\n\nSELECT id FROM __seed("order_amounts")\n', - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - "models/dim_customers.sql": "MODEL ();\n\nSELECT 1 AS customer_id\n", - }, - ) - - -def prepare_virtual_source_clone_project(tmp_path: Path) -> Path: - return prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_clone_project", - repo_files={ - "sqlbuild_project.toml": build_virtual_clone_project_toml(), - "sqlbuild_local.toml": 'target = "dev"\n', - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " schema: raw\n" - " table: raw_orders\n" - " columns:\n" - " - name: id\n" - " type: integer\n" - " - name: data_version\n" - " type: integer\n" - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: integer\n" - ), - "models/source_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - - -def build_virtual_clone_project_toml() -> str: - return ( - 'name = "virtual_clone_project"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "dev.duckdb"\n\n' - "[targets.prod]\n" - 'schema = "prod"\n\n' - "[targets.prod.connection]\n" - 'database = "dev.duckdb"\n\n' - "[targets.prod.clone]\n" - "allow_as_clone_origin = true\n\n" - "[targets.prod.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.prod.state.connection]\n" - 'database = "dev_state.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.connection]\n" - 'database = "dev.duckdb"\n\n' - "[targets.dev.clone]\n" - "allow_as_clone_destination = true\n\n" - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "dev_state.duckdb"\n' - ) - - -def build_prod_source_versions(project_dir: Path) -> None: - from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb - - (project_dir / "sqlbuild_local.toml").write_text('target = "prod"\n', encoding="utf-8") - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - project_config_path: Path = project_dir / "sqlbuild_project.toml" - project_config_path.write_text( - project_config_path.read_text(encoding="utf-8") - .replace( - '[targets.prod.connection]\ndatabase = "dev.duckdb"', - "[targets.prod.connection]\n" - 'database = "${ENV:SQLBUILD_TEST_UNUSED_VIRTUAL_ORIGIN_DATABASE}"', - ) - .replace( - '[targets.prod.state.connection]\ndatabase = "dev_state.duckdb"', - "[targets.prod.state.connection]\n" - 'database = "${ENV:SQLBUILD_TEST_UNUSED_VIRTUAL_ORIGIN_STATE_DATABASE}"', - ), - encoding="utf-8", - ) - - -def build_dev_target_versions(project_dir: Path) -> None: - from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb - - (project_dir / "sqlbuild_local.toml").write_text('target = "dev"\n', encoding="utf-8") - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - - -def init_dev_state(project_dir: Path) -> None: - from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb - - (project_dir / "sqlbuild_local.toml").write_text('target = "dev"\n', encoding="utf-8") - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - - -def prod_version_hash(project_dir: Path, model_name: str) -> str: - return str( - query_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'prod' AND node_type = 'model' " - f"AND node_name = '{model_name}'" - ), - )[0][0] - ) - - -def prod_seed_version_hash(project_dir: Path, seed_name: str) -> str: - return str( - query_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'prod' AND node_type = 'seed' " - f"AND node_name = '{seed_name}'" - ), - )[0][0] - ) - - -def dev_version_hash(project_dir: Path, model_name: str) -> str: - return str( - query_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - f"AND node_name = '{model_name}'" - ), - )[0][0] - ) - - -def dev_ref_rows(project_dir: Path) -> list[tuple[object, ...]]: - return query_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "SELECT node_name, version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' ORDER BY node_name" - ), - ) - - -def dev_seed_ref_rows(project_dir: Path) -> list[tuple[object, ...]]: - return query_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "SELECT node_name, version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' ORDER BY node_name" - ), - ) - - -def target_physical_relation_count(project_dir: Path) -> int: - rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "dev.duckdb", - sql=( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'dev__sqb_physical' AND table_name LIKE '%__v_%'" - ), - ) - return len(rows) - - -def insert_dev_model_version_lock(*, project_dir: Path, model_name: str, version_hash: str) -> None: - execute_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - f"('model_version:{model_name}:{version_hash}', 'test', " - "CURRENT_TIMESTAMP + INTERVAL 1 HOUR, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - - -def registered_physical_relation_artifacts(project_dir: Path) -> tuple[tuple[str, str], ...]: - """Read (artifact_type, artifact_name) rows registered in destination state.""" - - rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "dev_state.duckdb", - sql=( - "SELECT artifact_type, artifact_name FROM sqlbuild_state.physical_relations " - "ORDER BY artifact_type, artifact_name" - ), - ) - return tuple((str(row[0]), str(row[1])) for row in rows) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_virtual_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_virtual_clone.py deleted file mode 100644 index 3fb2f6951b..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_virtual_clone.py +++ /dev/null @@ -1,359 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.clone._test_types import ( - VirtualCloneE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.clone.helpers import ( - build_dev_target_versions, - build_prod_source_versions, - dev_ref_rows, - dev_seed_ref_rows, - dev_version_hash, - init_dev_state, - insert_dev_model_version_lock, - prepare_virtual_clone_project, - prepare_virtual_source_clone_project, - prod_seed_version_hash, - prod_version_hash, - registered_physical_relation_artifacts, - target_physical_relation_count, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - query_duckdb, - run_sqb, - table_exists, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneE2ETestCase( - description="hydrates workspace hashes from origin warehouse without destination refs", - command=("--no-color", "clone", "--from", "prod", "--to", "dev"), - expected_exit_code=0, - expected_stdout_fragments=( - "Virtual clone prod -> dev", - "mode workspace fingerprints", - "origin state not used", - "destination refs unchanged", - "hydrated 4", - "missing in origin 0", - ), - expected_registered_artifacts=( - ("model", "dim_customers"), - ("model", "fact_orders"), - ("model", "stg_orders"), - ("seed", "order_amounts"), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_clone_when_source_has_workspace_versions_then_target_is_hydrated( - test_case: VirtualCloneE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_clone_project(tmp_path) - build_prod_source_versions(project_dir) - init_dev_state(project_dir) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stderr - fragment: str - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert target_physical_relation_count(project_dir) == 4 - assert query_duckdb( - db_path=project_dir / "dev.duckdb", - sql=( - "SELECT id FROM dev__sqb_physical." - f"stg_orders__v_{prod_version_hash(project_dir, 'stg_orders')[:8]}" - ), - ) == [(7,)] - assert query_duckdb( - db_path=project_dir / "dev.duckdb", - sql=( - "SELECT id FROM dev__sqb_physical." - f"order_amounts__v_{prod_seed_version_hash(project_dir, 'order_amounts')[:8]}" - ), - ) == [(7,)] - assert dev_ref_rows(project_dir) == [] - assert dev_seed_ref_rows(project_dir) == [] - assert ( - registered_physical_relation_artifacts(project_dir) - == test_case.expected_registered_artifacts - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneE2ETestCase( - description="source-dependent workspace clone uses authoritative origin identity", - command=("--no-color", "clone", "--from", "prod", "--to", "dev"), - expected_exit_code=0, - expected_stdout_fragments=( - "mode workspace fingerprints + origin VDE refs", - "origin state used", - "hydrated 1", - "missing in origin 0", - ), - expected_registered_artifacts=(("model", "source_orders"),), - ) - ], - ids=lambda case: case.description, -) -def test_given_source_dependent_origin_when_cloning_workspace_then_uses_origin_vde_identity( - test_case: VirtualCloneE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_source_clone_project(tmp_path) - execute_duckdb( - db_path=project_dir / "dev.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (7, 1)" - ), - ) - build_prod_source_versions(project_dir) - init_dev_state(project_dir) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout, result.stdout - assert registered_physical_relation_artifacts(project_dir) == ( - test_case.expected_registered_artifacts - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneE2ETestCase( - description="hydrates destination VDE refs and leaves refs/views unchanged", - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--virtual-env", - "dev", - ), - expected_exit_code=0, - expected_stdout_fragments=( - "mode destination VDE refs", - "destination VDE dev", - "destination refs unchanged", - "hydrated 1", - "already present 3", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_clone_vde_mode_when_target_artifact_missing_then_only_physical_is_restored( - test_case: VirtualCloneE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_clone_project(tmp_path) - build_prod_source_versions(project_dir) - build_dev_target_versions(project_dir) - ref_rows_before: list[tuple[object, ...]] = dev_ref_rows(project_dir) - view_rows_before: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "dev.duckdb", - sql="SELECT id FROM dev__dev.stg_orders ORDER BY id", - ) - missing_hash: str = dev_version_hash(project_dir, "stg_orders") - execute_duckdb( - db_path=project_dir / "dev.duckdb", - sql=f"DROP TABLE dev__sqb_physical.stg_orders__v_{missing_hash[:8]}", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stderr - fragment: str - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert table_exists( - db_path=project_dir / "dev.duckdb", - schema="dev__sqb_physical", - table_name=f"stg_orders__v_{missing_hash[:8]}", - ) - assert dev_ref_rows(project_dir) == ref_rows_before - assert ( - query_duckdb( - db_path=project_dir / "dev.duckdb", - sql="SELECT id FROM dev__dev.stg_orders ORDER BY id", - ) - == view_rows_before - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneE2ETestCase( - description="missing origin artifact reports missing and exits nonzero", - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--select", - "stg_orders", - ), - expected_exit_code=1, - expected_stdout_fragments=( - "missing in origin 1", - "missing: stg_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_clone_when_source_artifact_missing_then_it_reports_missing( - test_case: VirtualCloneE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_clone_project(tmp_path) - build_prod_source_versions(project_dir) - init_dev_state(project_dir) - source_hash: str = prod_version_hash(project_dir, "stg_orders") - execute_duckdb( - db_path=project_dir / "dev.duckdb", - sql=f"DROP TABLE prod__sqb_physical.stg_orders__v_{source_hash[:8]}", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - fragment: str - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert target_physical_relation_count(project_dir) == 0 - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneE2ETestCase( - description="active model-version lock blocks by default", - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--select", - "stg_orders", - ), - expected_exit_code=1, - expected_stdout_fragments=("model version 'stg_orders:", "is locked"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_clone_when_target_model_version_locked_then_it_blocks( - test_case: VirtualCloneE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_clone_project(tmp_path) - build_prod_source_versions(project_dir) - init_dev_state(project_dir) - version_hash: str = prod_version_hash(project_dir, "stg_orders") - insert_dev_model_version_lock( - project_dir=project_dir, model_name="stg_orders", version_hash=version_hash - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - fragment: str - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stderr - assert target_physical_relation_count(project_dir) == 0 - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneE2ETestCase( - description="skip locked hydrates other model versions", - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--skip-locked", - ), - expected_exit_code=0, - expected_stdout_fragments=( - "hydrated 3", - "skipped locked 1", - "skipped locked: stg_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_clone_skip_locked_when_one_version_locked_then_it_hydrates_other_models( - test_case: VirtualCloneE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_clone_project(tmp_path) - build_prod_source_versions(project_dir) - init_dev_state(project_dir) - version_hash: str = prod_version_hash(project_dir, "stg_orders") - insert_dev_model_version_lock( - project_dir=project_dir, model_name="stg_orders", version_hash=version_hash - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stderr - fragment: str - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert target_physical_relation_count(project_dir) == 3 - assert not table_exists( - db_path=project_dir / "dev.duckdb", - schema="dev__sqb_physical", - table_name=f"stg_orders__v_{version_hash[:8]}", - ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ab7d11f2b6..b5905600f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -5,6 +5,23 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class SourceSemanticBindingCase: + description: str + command: str + expected_code: str = "B002" + + +@dataclass(frozen=True) +class SemanticCorpusCase: + description: str + category: str + repo_files: dict[str, str] + expected_exit_code: int + expected_codes: tuple[str, ...] + pending_native: bool + + @dataclass(frozen=True) class VariedCompileCacheTestCase: description: str @@ -252,3 +269,13 @@ class UnrelatedPythonPackageBuildTestCase: unrelated_files: dict[str, str] repo_files: dict[str, str] expected_build_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DiagnosticPerformanceCase: + description: str + depth: int = 100 + width: int = 32 + expected_max_wall_seconds: float = 20.0 + expected_timeout_seconds: float = 65.0 + diagnostic_count: int = 100 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py new file mode 100644 index 0000000000..a15065c86c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py @@ -0,0 +1,23 @@ +"""Synthetic playground fixture for semantic compilation regression cases.""" + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.fixture(scope="session") +def semantic_playground(tmp_path_factory: pytest.TempPathFactory) -> Path: + root: Path = tmp_path_factory.mktemp("semantic_playground") + project: Path = root / "orders_project" + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=root, command=("playground", str(project)) + ) + assert result.returncode == 0, result.stdout + result.stderr + typed: Path = Path(__file__).parent / "fixtures" / "semantic" / "typed_orders.sql" + (project / "models" / "staging" / "stg_orders_typed.sql").write_text( + typed.read_text(encoding="utf-8"), encoding="utf-8" + ) + return project diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json new file mode 100644 index 0000000000..37c0499799 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/corpus.json @@ -0,0 +1,2572 @@ +{ + "valid": [ + { + "description": "c01: plain select from typed source", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "c02: lateral alias reuse", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS q, q + 1 AS q2 FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "c03: timestamp vs string literal", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > '2026-01-01'\n" + } + }, + { + "description": "c04: GROUP BY ALL", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ALL\n" + } + }, + { + "description": "c05: star exclude", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT * EXCLUDE (status) FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "c06: int * decimal", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * 2.5 AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "c07: join typed contract", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id, p.amount_cents FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + }, + { + "description": "c08: order by select alias", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS q FROM __source(\"raw__orders\") ORDER BY q\n" + } + }, + { + "description": "c09: correlated exists", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.customer_id)\n" + } + }, + { + "description": "c10: window qualify", + "category": "control", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ordered_at) = 1\n" + } + }, + { + "description": "t05: AVG(TIMESTAMP)", + "category": "type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT AVG(ordered_at) AS a FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t07: INTEGER as predicate", + "category": "type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity\n" + } + }, + { + "description": "t13: NOT INTEGER", + "category": "type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT NOT quantity AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "s02: UNION ALL TIMESTAMP vs VARCHAR", + "category": "set operation", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at AS x FROM __source(\"raw__orders\") UNION ALL SELECT email FROM __source(\"raw__customers\")\n" + } + }, + { + "description": "s04: UNION ALL INTEGER vs BOOLEAN", + "category": "set operation", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT TRUE FROM __source(\"raw__customers\")\n" + } + }, + { + "description": "x01: duplicate output column", + "category": "structure", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, id FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "x03: CTE column list count mismatch", + "category": "structure", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH c(a, b) AS (SELECT id FROM __source(\"raw__orders\")) SELECT a FROM c\n" + } + }, + { + "description": "x06: unnamed expression output", + "category": "structure", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity + 1 FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A01: SUM(BOOLEAN)", + "category": "aggregate type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status = 'x') AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O01: INTEGER AND BOOLEAN", + "category": "operator type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity AND status = 'x'\n" + } + }, + { + "description": "O12: JOIN ON INTEGER predicate", + "category": "operator type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id\n" + } + }, + { + "description": "E04: CTE column alias list too long", + "category": "cte", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH c(a, b, d) AS (SELECT id, status FROM __source(\"raw__orders\")) SELECT a FROM c\n" + } + }, + { + "description": "R05: star join duplicate names", + "category": "star", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT * FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" + } + }, + { + "description": "M05: DISTINCT with ORDER BY non-selected", + "category": "order", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT DISTINCT status FROM __source(\"raw__orders\") ORDER BY quantity\n" + } + }, + { + "description": "control: declared scalar UDF return type", + "category": "control", + "repo_files": { + "models/probe.sql": "MODEL (materialized table, contract enforced, columns (valid (type BOOLEAN)));\nSELECT __udf(\"udf__is_completed_order\")(status) AS valid FROM __source(\"raw__orders\")\n" + }, + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false + }, + { + "description": "star03: CTE star join on matching types", + "category": "type", + "expected_exit_code": 0, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.id\n" + } + } + ], + "invalid": [ + { + "description": "u01: select list, typed source, unqualified", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "u02: select list, typed source, qualified", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.qty FROM __source(\"raw__orders\") o\n" + } + }, + { + "description": "u03: model with explicit projection", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __ref(\"stg_orders\")\n" + } + }, + { + "description": "u04: enforced typed contract model", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.qty FROM __ref(\"stg_orders_typed\") o\n" + } + }, + { + "description": "u05: source defined by expression", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT amount_usd FROM __source(\"raw__payments\")\n" + } + }, + { + "description": "u06: seed", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT price_usd FROM __seed(\"waffle_types\")\n" + } + }, + { + "description": "u07: WHERE", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE qty > 1\n" + } + }, + { + "description": "u08: GROUP BY", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY stat\n" + } + }, + { + "description": "u09: ORDER BY", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY qty\n" + } + }, + { + "description": "u10: JOIN ON", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer = c.id\n" + } + }, + { + "description": "u11: window PARTITION BY", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) AS rn FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "u12: QUALIFY", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY ROW_NUMBER() OVER (PARTITION BY cust ORDER BY ordered_at) = 1\n" + } + }, + { + "description": "u13: HAVING", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status HAVING SUM(qty) > 1\n" + } + }, + { + "description": "u14: CASE", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN qty > 1 THEN 'a' END AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "u15: function argument", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(stat) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "u16: column not exposed by CTE", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT id FROM __source(\"raw__orders\")) SELECT quantity FROM x\n" + } + }, + { + "description": "u17: column not exposed by subquery", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT s.quantity FROM (SELECT id FROM __source(\"raw__orders\")) s\n" + } + }, + { + "description": "u18: unknown table alias", + "category": "unknown alias", + "expected_exit_code": 1, + "expected_codes": [ + "B004" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT z.id FROM __source(\"raw__orders\") o\n" + } + }, + { + "description": "u21: correlated EXISTS", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = o.cust)\n" + } + }, + { + "description": "u22: IN subquery", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT cust_id FROM __source(\"raw__customers\"))\n" + } + }, + { + "description": "u23: second join input", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE p.amount > 0\n" + } + }, + { + "description": "u24: JOIN USING", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c USING (cust_id)\n" + } + }, + { + "description": "u25: inside macro argument", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/probe.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" + } + }, + { + "description": "u26: inside UDF argument", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(stat) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "a01: unqualified in select list", + "category": "ambiguous column", + "expected_exit_code": 1, + "expected_codes": [ + "B003" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + }, + { + "description": "a02: source join, id on both sides", + "category": "ambiguous column", + "expected_exit_code": 1, + "expected_codes": [ + "B003" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.customer_id = c.id\n" + } + }, + { + "description": "a03: in JOIN condition", + "category": "ambiguous column", + "expected_exit_code": 1, + "expected_codes": [ + "B003" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON id = c.id\n" + } + }, + { + "description": "a04: in WHERE", + "category": "ambiguous column", + "expected_exit_code": 1, + "expected_codes": [ + "B003" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id WHERE order_id > 1\n" + } + }, + { + "description": "t28: UDF argument type", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [ + "B301" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "g01: column not in GROUP BY", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B230" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, quantity, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status\n" + } + }, + { + "description": "g02: aggregate in WHERE", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B231" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE SUM(quantity) > 1\n" + } + }, + { + "description": "g03: nested aggregate", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B231" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(COUNT(*)) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "g04: window in WHERE", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B232" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ROW_NUMBER() OVER () = 1\n" + } + }, + { + "description": "g05: mixed aggregate without GROUP BY", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B230" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, COUNT(*) AS n FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "g06: HAVING non-grouped column", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B230" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING quantity > 1\n" + } + }, + { + "description": "g07: GROUP BY position out of range", + "category": "aggregation", + "expected_exit_code": 1, + "expected_codes": [ + "B230" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY 3\n" + } + }, + { + "description": "f03: UDF wrong arity", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [ + "B102" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status, id) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "y01: self-contained CTE unknown column", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT 1 AS id) SELECT quantity FROM x\n" + } + }, + { + "description": "y03: typed contract, WHERE unknown", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE qty > 1\n" + } + }, + { + "description": "y04: typed contract, JOIN ON unknown", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle = w.waffle_type_id\n" + } + }, + { + "description": "y05: typed contract + seed ambiguous", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B003" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT waffle_type_id FROM __ref(\"stg_orders_typed\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" + } + }, + { + "description": "y06: CTE over typed contract", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT order_id FROM __ref(\"stg_orders_typed\")) SELECT quantity FROM x\n" + } + }, + { + "description": "y07: typed contract, unqualified select", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT qty FROM __ref(\"stg_orders_typed\")\n" + } + }, + { + "description": "y09: seed joined with source, unknown on seed", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT w.price_usd FROM __source(\"raw__orders\") o JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\n" + } + }, + { + "description": "y10: macro arg unknown column (marts dir)", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(amount_usd)') AS d FROM __source(\"raw__payments\")\n" + } + }, + { + "description": "k01: enforced VARCHAR for SUM(INTEGER)", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K002" + ], + "pending_native": false, + "repo_files": { + "models/marts/daily_revenue.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Daily revenue aggregation.\",\n audits [\n expression_is_true (\n name \"revenue_is_non_negative\",\n expression \"total_revenue_cents >= 0\",\n ),\n ],\n contract enforced,\n columns (\n revenue_date (nullable false),\n order_count (type BIGINT),\n waffles_sold (type HUGEINT),\n total_revenue_cents (type VARCHAR),\n total_revenue_dollars (type DOUBLE),\n avg_order_value_cents (type DOUBLE),\n ),\n);\n\nSELECT\n CAST(o.ordered_at AS DATE) AS revenue_date,\n COUNT(DISTINCT o.order_id) AS order_count,\n SUM(o.quantity) AS waffles_sold,\n SUM(p.amount_cents) AS total_revenue_cents,\n @cents_to_dollars('SUM(p.amount_cents)') AS total_revenue_dollars,\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\nFROM __ref(\"stg_orders\") o\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY CAST(o.ordered_at AS DATE)\n" + } + }, + { + "description": "k02: enforced typed passthrough type mismatch", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K002" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type VARCHAR),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "k03: upstream projection drops used column", + "category": "propagation", + "expected_exit_code": 1, + "expected_codes": [ + "B002", + "B302" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_payments.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned payment records.\",\n columns (\n payment_id (audits [not_null, unique]),\n order_id (\n audits [not_null, relationships (to __source(\"raw__orders\"), field id)],\n ),\n ),\n);\n\nSELECT\n id AS payment_id,\n order_id,\n payment_method,\n paid_at,\n status AS payment_status\nFROM __source(\"raw__payments\")\n" + } + }, + { + "description": "k04: upstream enforced model renames column", + "category": "propagation", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n qty (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type VARCHAR),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity AS qty,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n", + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity FROM __ref(\"stg_orders_typed\")\n" + } + }, + { + "description": "k06: fact_orders unknown column (ticket case 1)", + "category": "propagation", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/fact_orders.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Order fact table with waffle and payment details.\",\n columns (\n order_id (nullable false, audits [not_null]),\n ),\n);\n\nSELECT\n o.order_id,\n o.customer_id,\n o.waffle_type_id,\n w.waffle_name,\n w.category AS waffle_category,\n o.quantity,\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\n o.ordered_at,\n o.status AS order_status,\n __udf(\"udf__is_completed_order\")(o.status) AS is_completed_order,\n p.payment_method,\n p.payment_status,\n p.amount_usd AS payment_amount_cents\nFROM __ref(\"stg_orders\") o\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + }, + { + "description": "k07: nullable false but NULL literal", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K004" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n CAST(NULL AS INTEGER) AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A09: aggregate in GROUP BY", + "category": "aggregate", + "expected_exit_code": 1, + "expected_codes": [ + "B231" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY SUM(quantity)\n" + } + }, + { + "description": "A11: FILTER unknown column", + "category": "aggregate", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) FILTER (WHERE qty > 1) AS n FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A12: aggregate in JOIN ON", + "category": "aggregate", + "expected_exit_code": 1, + "expected_codes": [ + "B231" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON SUM(o.customer_id) = c.id\n" + } + }, + { + "description": "A13: window inside aggregate", + "category": "aggregate", + "expected_exit_code": 1, + "expected_codes": [ + "B232" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(ROW_NUMBER() OVER ()) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N05: window in GROUP BY", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [ + "B232" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY ROW_NUMBER() OVER ()\n" + } + }, + { + "description": "N07: ORDER BY unknown in window", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT LAG(quantity) OVER (ORDER BY ordered) AS l FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F16: UPPER two args", + "category": "function arity", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(status, status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "Q04: subquery without alias column use", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [ + "B004" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT x.id FROM (SELECT id FROM __source(\"raw__orders\"))\n" + } + }, + { + "description": "Q06: correlated unknown outer alias", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [ + "B004" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") o WHERE EXISTS (SELECT 1 FROM __source(\"raw__customers\") c WHERE c.id = z.customer_id)\n" + } + }, + { + "description": "Q07: LATERAL unknown column", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id, l.x FROM __source(\"raw__orders\") o, LATERAL (SELECT o.qty AS x) l\n" + } + }, + { + "description": "J03: USING column missing on right", + "category": "join", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT a.id FROM (SELECT id, status FROM __source(\"raw__orders\")) a JOIN (SELECT id FROM __source(\"raw__customers\")) b USING (status)\n" + } + }, + { + "description": "S04: UNION BY NAME unknown column in outer", + "category": "set operation", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT z FROM (SELECT id FROM __source(\"raw__orders\") UNION ALL BY NAME SELECT id FROM __source(\"raw__customers\")) u\n" + } + }, + { + "description": "M04: ORDER BY unknown in aggregate query", + "category": "order", + "expected_exit_code": 1, + "expected_codes": [ + "B230" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status, COUNT(*) AS n FROM __source(\"raw__orders\") GROUP BY status ORDER BY quantity\n" + } + }, + { + "description": "U02: UDF with no args", + "category": "udf", + "expected_exit_code": 1, + "expected_codes": [ + "B102" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")() AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "U03: table function unknown output column", + "category": "table function", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.amount FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" + } + }, + { + "description": "U05: table function wrong arity", + "category": "table function", + "expected_exit_code": 1, + "expected_codes": [ + "P001" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")(1, 2) t\n" + } + }, + { + "description": "U07: macro expands unknown column (marts)", + "category": "macro", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT @cents_to_dollars('SUM(p.amount_usd)') AS d FROM __ref(\"stg_payments\") p\n" + } + }, + { + "description": "K01: unique_key references missing column", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/marts/dim_customers.sql": "MODEL (\n materialized table,\n unique_key [customer_key],\n row_diff_exclude_columns [email],\n tags [marts],\n description \"Customer dimension with lifetime metrics.\",\n columns (\n customer_id (nullable false, audits [not_null, unique]),\n ),\n);\n\nSELECT\n c.customer_id,\n c.first_name,\n c.last_name,\n c.email,\n c.created_at AS customer_since,\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\nFROM __ref(\"stg_customers\") c\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at\n" + } + }, + { + "description": "K02: cursor column missing from output", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/intermediate/order_status_index.sql": "MODEL (\n materialized incremental,\n incremental_strategy delete_insert,\n unique_key [order_id],\n cursor order_key,\n cursor_type integer,\n on_schema_change append_new_columns,\n replay_on_change bounded-7d,\n tags [intermediate, acceptance],\n description \"Integer-cursor incremental projection over fact orders.\",\n columns (\n order_id (audits [not_null, unique]),\n ),\n);\n\nSELECT\n order_id,\n customer_id,\n order_status,\n ordered_at,\n line_total_cents\nFROM __ref(\"fact_orders\")\n" + } + }, + { + "description": "K03: cursor_type integer on TIMESTAMP column", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B301" + ], + "pending_native": false, + "repo_files": { + "models/intermediate/order_status_index.sql": "MODEL (\n materialized incremental,\n incremental_strategy delete_insert,\n unique_key [order_id],\n cursor ordered_at,\n cursor_type integer,\n on_schema_change append_new_columns,\n replay_on_change bounded-7d,\n tags [intermediate, acceptance],\n description \"Integer-cursor incremental projection over fact orders.\",\n columns (\n order_id (audits [not_null, unique]),\n ),\n);\n\nSELECT\n order_id,\n customer_id,\n order_status,\n ordered_at,\n line_total_cents\nFROM __ref(\"fact_orders\")\n" + } + }, + { + "description": "K04: cursor_inputs column missing upstream", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/intermediate/customer_status_snapshot.sql": "MODEL (\n materialized incremental,\n incremental_strategy merge,\n unique_key [customer_id],\n cursor last_ordered_at,\n cursor_type timestamp,\n cursor_grain second,\n row_diff_exclude_columns [latest_order_status],\n row_diff_tolerances (\n by_column (\n total_revenue_cents (\n absolute 1,\n ),\n ),\n ),\n cursor_inputs (\n fact_orders ordered_ts,\n ),\n on_schema_change sync_all_columns,\n replay_on_change full,\n tags [intermediate, acceptance],\n description \"Merge-based customer snapshot with timestamp cursor and explicit backfill policies.\",\n columns (\n customer_id (audits [not_null, unique]),\n ),\n);\n\nSELECT\n customer_id,\n MAX(ordered_at) AS last_ordered_at,\n MAX(order_status) AS latest_order_status,\n COUNT(*) AS total_orders,\n SUM(line_total_cents) AS total_revenue_cents\nFROM __ref(\"fact_orders\")\nGROUP BY customer_id\n" + } + }, + { + "description": "K05: row_diff_exclude_columns unknown", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/intermediate/customer_status_snapshot.sql": "MODEL (\n materialized incremental,\n incremental_strategy merge,\n unique_key [customer_id],\n cursor last_ordered_at,\n cursor_type timestamp,\n cursor_grain second,\n row_diff_exclude_columns [latest_status],\n row_diff_tolerances (\n by_column (\n total_revenue_cents (\n absolute 1,\n ),\n ),\n ),\n cursor_inputs (\n fact_orders ordered_at,\n ),\n on_schema_change sync_all_columns,\n replay_on_change full,\n tags [intermediate, acceptance],\n description \"Merge-based customer snapshot with timestamp cursor and explicit backfill policies.\",\n columns (\n customer_id (audits [not_null, unique]),\n ),\n);\n\nSELECT\n customer_id,\n MAX(ordered_at) AS last_ordered_at,\n MAX(order_status) AS latest_order_status,\n COUNT(*) AS total_orders,\n SUM(line_total_cents) AS total_revenue_cents\nFROM __ref(\"fact_orders\")\nGROUP BY customer_id\n" + } + }, + { + "description": "K06: row_diff_tolerances unknown column", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/intermediate/customer_status_snapshot.sql": "MODEL (\n materialized incremental,\n incremental_strategy merge,\n unique_key [customer_id],\n cursor last_ordered_at,\n cursor_type timestamp,\n cursor_grain second,\n row_diff_exclude_columns [latest_order_status],\n row_diff_tolerances (\n by_column (\n total_revenue (\n absolute 1,\n ),\n ),\n ),\n cursor_inputs (\n fact_orders ordered_at,\n ),\n on_schema_change sync_all_columns,\n replay_on_change full,\n tags [intermediate, acceptance],\n description \"Merge-based customer snapshot with timestamp cursor and explicit backfill policies.\",\n columns (\n customer_id (audits [not_null, unique]),\n ),\n);\n\nSELECT\n customer_id,\n MAX(ordered_at) AS last_ordered_at,\n MAX(order_status) AS latest_order_status,\n COUNT(*) AS total_orders,\n SUM(line_total_cents) AS total_revenue_cents\nFROM __ref(\"fact_orders\")\nGROUP BY customer_id\n" + } + }, + { + "description": "K07: partition_column missing from output", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/marts/daily_order_partitioned.sql": "MODEL (\n materialized partition_tracked,\n tags [marts],\n placeholders (\n partition_start \"'2026-04-01'\",\n partition_end \"'2026-04-05'\",\n ),\n config (\n tracking_table partition_state,\n partition_column order_day,\n date_range_start 2026-04-01,\n date_range_end 2026-04-05,\n ),\n description \"Partition-tracked daily order summary using custom materialization.\",\n columns (\n order_date (audits [not_null]),\n ),\n audits [\n expression_is_true (\n name \"waffles_ordered_is_positive\",\n expression \"waffles_ordered > 0\",\n ),\n ],\n);\n\nSELECT\n CAST(o.ordered_at AS DATE) AS order_date,\n COUNT(DISTINCT o.order_id) AS order_count,\n SUM(o.quantity) AS waffles_ordered,\n COUNT(DISTINCT o.customer_id) AS unique_customers\nFROM __ref(\"stg_orders\") o\nWHERE CAST(o.ordered_at AS DATE) >= CAST(@@@partition_start AS DATE)\n AND CAST(o.ordered_at AS DATE) < CAST(@@@partition_end AS DATE)\nGROUP BY CAST(o.ordered_at AS DATE)\n" + } + }, + { + "description": "K08: source cursor_column missing", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "sources/raw.yml": "sources:\n - name: raw__customers\n contract: enforced\n managed: true\n write_strategy: table\n columns:\n - name: id\n type: INTEGER\n - name: first_name\n type: VARCHAR\n - name: last_name\n type: VARCHAR\n - name: email\n type: VARCHAR\n - name: created_at\n type: TIMESTAMP\n\n - name: raw__orders\n contract: enforced\n managed: true\n write_strategy: delete_insert\n cursor_column: ordered_ts\n columns:\n - name: id\n type: INTEGER\n - name: customer_id\n type: INTEGER\n - name: waffle_type_id\n type: INTEGER\n - name: quantity\n type: INTEGER\n - name: ordered_at\n type: TIMESTAMP\n - name: status\n type: VARCHAR\n\n - name: raw__payments\n expression: |\n SELECT 1 AS id, 1 AS order_id, 1700 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n UNION ALL SELECT 2 AS id, 2 AS order_id, 1050 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-01 09:16:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n UNION ALL SELECT 3 AS id, 3 AS order_id, 4350 AS amount_cents, 'cash' AS payment_method, CAST('2026-04-01 10:31:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n UNION ALL SELECT 4 AS id, 4 AS order_id, 950 AS amount_cents, 'credit_card' AS payment_method, CAST('2026-04-02 08:46:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n UNION ALL SELECT 5 AS id, 5 AS order_id, 850 AS amount_cents, 'debit_card' AS payment_method, CAST('2026-04-02 12:01:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n UNION ALL SELECT 6 AS id, 6 AS order_id, 750 AS amount_cents, 'debit_card' AS payment_method, CAST('2026-04-02 12:01:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n UNION ALL SELECT 7 AS id, 7 AS order_id, 2200 AS amount_cents, 'credit_card' AS payment_method, CAST(NULL AS TIMESTAMP) AS paid_at, 'failed' AS status\n UNION ALL SELECT 8 AS id, 8 AS order_id, 950 AS amount_cents, 'cash' AS payment_method, CAST('2026-04-03 11:01:00' AS TIMESTAMP) AS paid_at, 'success' AS status\n columns:\n - name: amount_cents\n type: INTEGER\n" + } + }, + { + "description": "K09: dim_customers row_diff_exclude unknown", + "category": "config", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/marts/dim_customers.sql": "MODEL (\n materialized table,\n unique_key [customer_id],\n row_diff_exclude_columns [e_mail],\n tags [marts],\n description \"Customer dimension with lifetime metrics.\",\n columns (\n customer_id (nullable false, audits [not_null, unique]),\n ),\n);\n\nSELECT\n c.customer_id,\n c.first_name,\n c.last_name,\n c.email,\n c.created_at AS customer_since,\n COUNT(DISTINCT o.order_id) AS lifetime_orders,\n COALESCE(SUM(p.amount_cents), 0) AS lifetime_spend_cents\nFROM __ref(\"stg_customers\") c\nLEFT JOIN __ref(\"stg_orders\") o ON c.customer_id = o.customer_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY c.customer_id, c.first_name, c.last_name, c.email, c.created_at\n" + } + }, + { + "description": "D01: expression_is_true unknown column", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/daily_revenue.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Daily revenue aggregation.\",\n audits [\n expression_is_true (\n name \"revenue_is_non_negative\",\n expression \"total_revenue >= 0\",\n ),\n ],\n columns (\n revenue_date (nullable false),\n ),\n);\n\nSELECT\n CAST(o.ordered_at AS DATE) AS revenue_date,\n COUNT(DISTINCT o.order_id) AS order_count,\n SUM(o.quantity) AS waffles_sold,\n SUM(p.amount_cents) AS total_revenue_cents,\n @cents_to_dollars('SUM(p.amount_cents)') AS total_revenue_dollars,\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\nFROM __ref(\"stg_orders\") o\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY CAST(o.ordered_at AS DATE)\n" + } + }, + { + "description": "D03: relationships to unknown field", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [ + "B300" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "D04: relationships to unknown target", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [ + "P001" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_kinds\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "D05: accepted_values string list on INTEGER", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [ + "B301" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null, accepted_values (values [\"a\", \"b\"])]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "D06: column audit on column not in output", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [ + "K001" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n order_ref (audits [not_null]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "D07: relationships target type mismatch", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [ + "B301" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_name)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "K10: enforced contract omits produced column", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K005" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "K11: enforced contract declares unknown type", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K002" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders_typed.sql": "MODEL (\n materialized view,\n contract enforced,\n columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n waffle_type_id (type INTEGER),\n quantity (type INTEGER),\n ordered_at (type TIMESTAMP),\n status (type STRINGY),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "K12: enforced: SUM(INTEGER) declared TIMESTAMP", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K002" + ], + "pending_native": false, + "repo_files": { + "models/marts/daily_revenue.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Daily revenue aggregation.\",\n audits [\n expression_is_true (\n name \"revenue_is_non_negative\",\n expression \"total_revenue_cents >= 0\",\n ),\n ],\n contract enforced,\n columns (\n revenue_date (type DATE),\n order_count (type BIGINT),\n waffles_sold (type HUGEINT),\n total_revenue_cents (type TIMESTAMP),\n total_revenue_dollars (type DOUBLE),\n avg_order_value_cents (type DOUBLE),\n ),\n);\n\nSELECT\n CAST(o.ordered_at AS DATE) AS revenue_date,\n COUNT(DISTINCT o.order_id) AS order_count,\n SUM(o.quantity) AS waffles_sold,\n SUM(p.amount_cents) AS total_revenue_cents,\n @cents_to_dollars('SUM(p.amount_cents)') AS total_revenue_dollars,\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\nFROM __ref(\"stg_orders\") o\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY CAST(o.ordered_at AS DATE)\n" + } + }, + { + "description": "K13: enforced: CAST(... AS DATE) declared INTEGER", + "category": "contract", + "expected_exit_code": 1, + "expected_codes": [ + "K002" + ], + "pending_native": false, + "repo_files": { + "models/marts/daily_revenue.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Daily revenue aggregation.\",\n audits [\n expression_is_true (\n name \"revenue_is_non_negative\",\n expression \"total_revenue_cents >= 0\",\n ),\n ],\n contract enforced,\n columns (\n revenue_date (type INTEGER),\n order_count (type BIGINT),\n waffles_sold (type HUGEINT),\n total_revenue_cents (type HUGEINT),\n total_revenue_dollars (type DOUBLE),\n avg_order_value_cents (type DOUBLE),\n ),\n);\n\nSELECT\n CAST(o.ordered_at AS DATE) AS revenue_date,\n COUNT(DISTINCT o.order_id) AS order_count,\n SUM(o.quantity) AS waffles_sold,\n SUM(p.amount_cents) AS total_revenue_cents,\n @cents_to_dollars('SUM(p.amount_cents)') AS total_revenue_dollars,\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\nFROM __ref(\"stg_orders\") o\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY CAST(o.ordered_at AS DATE)\n" + } + }, + { + "description": "T01: unit test expects unknown output column", + "category": "sql test", + "expected_exit_code": 1, + "expected_codes": [ + "B302" + ], + "pending_native": false, + "repo_files": { + "tests/unit/test_stg_orders.sql": "TEST();\n\nWITH\n__source__raw__orders AS (\n SELECT\n 1 AS id,\n 100 AS customer_id,\n 2 AS waffle_type_id,\n 3 AS quantity,\n CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at,\n 'completed' AS status\n),\n__expected__stg_orders AS (\n SELECT\n 1 AS order_key,\n 100 AS customer_id,\n 2 AS waffle_type_id,\n 3 AS quantity,\n CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at,\n 'completed' AS status\n),\n__assert__order_ids_are_not_null AS (\n SELECT *\n FROM __ref(\"stg_orders\")\n WHERE order_id IS NULL\n)\nSELECT 1\n" + } + }, + { + "description": "ticket case 1: unknown qualified payment amount", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/fact_orders.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Order fact table with waffle and payment details.\",\n columns (\n order_id (nullable false, audits [not_null]),\n ),\n);\n\nSELECT\n o.order_id,\n o.customer_id,\n o.waffle_type_id,\n w.waffle_name,\n w.category AS waffle_category,\n o.quantity,\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\n o.ordered_at,\n o.status AS order_status,\n __udf(\"udf__is_completed_order\")(o.status) AS is_completed_order,\n p.payment_method,\n p.payment_status,\n p.amount_usd AS payment_amount_cents\nFROM __ref(\"stg_orders\") o\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + }, + { + "description": "ticket case 2: unknown unqualified payment amount", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/fact_orders.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Order fact table with waffle and payment details.\",\n columns (\n order_id (nullable false, audits [not_null]),\n ),\n);\n\nSELECT\n o.order_id,\n o.customer_id,\n o.waffle_type_id,\n w.waffle_name,\n w.category AS waffle_category,\n o.quantity,\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\n o.ordered_at,\n o.status AS order_status,\n __udf(\"udf__is_completed_order\")(o.status) AS is_completed_order,\n p.payment_method,\n p.payment_status,\n amount_usd AS payment_amount_cents\nFROM __ref(\"stg_orders\") o\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + }, + { + "description": "ticket case 3: unknown expression source column", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_payments.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned payment records.\",\n columns (\n payment_id (audits [not_null, unique]),\n order_id (\n audits [not_null, relationships (to __source(\"raw__orders\"), field id)],\n ),\n ),\n);\n\nSELECT\n id AS payment_id,\n order_id,\n amount_usd,\n payment_method,\n paid_at,\n status AS payment_status\nFROM __source(\"raw__payments\")\n" + } + }, + { + "description": "ticket case 4: unknown contracted source column", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n qty,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "ticket case 5: website missing-column placeholder", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [ + "B002" + ], + "pending_native": false, + "repo_files": { + "models/marts/fact_orders.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Order fact table with waffle and payment details.\",\n columns (\n order_id (nullable false, audits [not_null]),\n ),\n);\n\nSELECT\n o.order_id,\n o.customer_id,\n o.waffle_type_id,\n w.waffle_name,\n w.category AS waffle_category,\n o.qty,\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\n o.ordered_at,\n o.status AS order_status,\n __udf(\"udf__is_completed_order\")(o.status) AS is_completed_order,\n p.payment_method,\n p.payment_status,\n p.amount_cents AS payment_amount_cents\nFROM __ref(\"stg_orders_typed\") o\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + }, + { + "description": "metadata: SQL mock unknown column", + "category": "sql test", + "repo_files": { + "tests/unit/test_stg_orders.sql": "TEST();\n\nWITH\n__source__raw__orders AS (\n SELECT\n 1 AS order_key,\n 100 AS customer_id,\n 2 AS waffle_type_id,\n 3 AS quantity,\n CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at,\n 'completed' AS status\n),\n__expected__stg_orders AS (\n SELECT\n 1 AS order_id,\n 100 AS customer_id,\n 2 AS waffle_type_id,\n 3 AS quantity,\n CAST('2026-04-01 10:00:00' AS TIMESTAMP) AS ordered_at,\n 'completed' AS status\n),\n__assert__order_ids_are_not_null AS (\n SELECT *\n FROM __ref(\"stg_orders\")\n WHERE order_id IS NULL\n)\nSELECT 1\n" + }, + "expected_exit_code": 1, + "expected_codes": [ + "B302" + ], + "pending_native": false + }, + { + "description": "metadata-udf: scalar UDF declared argument type", + "category": "udf", + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\nSELECT __udf(\"udf__is_completed_order\")(quantity) AS valid FROM __source(\"raw__orders\")\n" + }, + "expected_exit_code": 1, + "expected_codes": [ + "B301" + ], + "pending_native": false + }, + { + "description": "star01: CTE star join INTEGER = VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [ + "B217" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH o AS (SELECT * FROM __source(\"raw__orders\")), c AS (SELECT * FROM __source(\"raw__customers\")) SELECT o.id FROM o JOIN c ON o.id = c.email\n" + } + }, + { + "description": "star02: derived-table star join INTEGER = VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [ + "B217" + ], + "pending_native": false, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM (SELECT * FROM __source(\"raw__orders\")) AS o JOIN __source(\"raw__customers\") AS c ON o.id = c.email\n" + } + } + ], + "pending": [ + { + "description": "u19: through SELECT * CTE", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH x AS (SELECT * FROM __source(\"raw__orders\")) SELECT qty FROM x\n" + } + }, + { + "description": "u20: EXCLUDE of missing column", + "category": "unknown column", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT * EXCLUDE (qty) FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t01: TIMESTAMP > INTEGER", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > 5\n" + } + }, + { + "description": "t02: INTEGER * VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t03: join INTEGER = VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON o.id = c.email\n" + } + }, + { + "description": "t04: SUM(VARCHAR)", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t06: VARCHAR > INTEGER literal", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status > 5\n" + } + }, + { + "description": "t08: VARCHAR as predicate", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status\n" + } + }, + { + "description": "t09: CASE branches INTEGER vs VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN quantity > 1 THEN quantity ELSE status END AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t10: COALESCE(INTEGER, VARCHAR)", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COALESCE(quantity, status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t11: INTEGER IN (strings)", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity IN ('a', 'b')\n" + } + }, + { + "description": "t12: INTEGER LIKE", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity LIKE '1%'\n" + } + }, + { + "description": "t14: DATE_TRUNC on VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_TRUNC('day', status) AS d FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t15: TIMESTAMP + VARCHAR", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at + status AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t16: LENGTH(INTEGER)", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT LENGTH(quantity) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t17: TIMESTAMP = invalid literal", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at = 'not a date'\n" + } + }, + { + "description": "t18: INTEGER - TIMESTAMP", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity - ordered_at AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t19: TIMESTAMP BETWEEN integers", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at BETWEEN 1 AND 5\n" + } + }, + { + "description": "t20: TIMESTAMP vs VARCHAR scalar subquery", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o WHERE o.ordered_at > (SELECT MAX(email) FROM __source(\"raw__customers\"))\n" + } + }, + { + "description": "t21: join key vs enforced typed contract", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.order_id FROM __ref(\"stg_orders_typed\") o JOIN __ref(\"stg_payments\") p ON o.order_id = p.payment_method\n" + } + }, + { + "description": "t22: SUM(VARCHAR) window", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(p.payment_method) OVER () AS s FROM __ref(\"stg_payments\") p\n" + } + }, + { + "description": "t23: UPPER(INTEGER)", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT UPPER(quantity) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t24: VARCHAR = TRUE", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status = TRUE\n" + } + }, + { + "description": "t25: CAST TIMESTAMP AS INTEGER", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(ordered_at AS INTEGER) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "t26: TIMESTAMP > INTEGER, typed contract", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT order_id FROM __ref(\"stg_orders_typed\") WHERE ordered_at > 5\n" + } + }, + { + "description": "t27: INTEGER * VARCHAR via model", + "category": "type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders\")\n" + } + }, + { + "description": "s01: UNION ALL column count mismatch", + "category": "set operation", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\")\n" + } + }, + { + "description": "s03: INTERSECT column count mismatch", + "category": "set operation", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") INTERSECT SELECT id, email FROM __source(\"raw__customers\")\n" + } + }, + { + "description": "f01: unknown function", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT not_a_function(id) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "f02: DATE_TRUNC wrong arity", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_TRUNC('day') AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "f04: ROUND(VARCHAR, 2)", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROUND(status, 2) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "f05: SUBSTRING wrong arity", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUBSTRING(status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "x02: ORDER BY position out of range", + "category": "structure", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") ORDER BY 5\n" + } + }, + { + "description": "x04: undefined CTE", + "category": "structure", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")) SELECT * FROM b\n" + } + }, + { + "description": "x05: raw table reference", + "category": "structure", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM raw_orders\n" + } + }, + { + "description": "y02: literal TIMESTAMP > INTEGER", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST('2020-01-01' AS TIMESTAMP) > 5 AS x\n" + } + }, + { + "description": "y08: typed contract, type error in select", + "category": "boundary", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity * status AS q FROM __ref(\"stg_orders_typed\")\n" + } + }, + { + "description": "k05: upstream changes column type used in arithmetic", + "category": "propagation", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/staging/stg_payments.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned payment records.\",\n columns (\n payment_id (audits [not_null, unique]),\n order_id (\n audits [not_null, relationships (to __source(\"raw__orders\"), field id)],\n ),\n ),\n);\n\nSELECT\n id AS payment_id,\n order_id,\n CAST(amount_cents AS VARCHAR) AS amount_cents,\n payment_method,\n paid_at,\n status AS payment_status\nFROM __source(\"raw__payments\")\n" + } + }, + { + "description": "A02: SUM(TIMESTAMP)", + "category": "aggregate type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(ordered_at) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A03: AVG(VARCHAR)", + "category": "aggregate type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT AVG(status) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A04: SUM(DATE)", + "category": "aggregate type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(CAST(ordered_at AS DATE)) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A05: SUM with two arguments", + "category": "aggregate arity", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity, id) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A06: STDDEV(VARCHAR)", + "category": "aggregate type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT STDDEV(status) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "A07: SUM(VARCHAR) from typed contract", + "category": "aggregate type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(status) AS s FROM __ref(\"stg_orders_typed\")\n" + } + }, + { + "description": "A08: SUM(VARCHAR) from seed", + "category": "aggregate type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(waffle_name) AS s FROM __seed(\"waffle_types\")\n" + } + }, + { + "description": "A10: FILTER with VARCHAR condition", + "category": "aggregate", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COUNT(*) FILTER (WHERE status) AS n FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N01: ranking function without OVER", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROW_NUMBER() AS rn FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N02: LAG offset VARCHAR", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT LAG(quantity, status) OVER (ORDER BY id) AS l FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N03: NTILE VARCHAR", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT NTILE('a') OVER (ORDER BY id) AS t FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N04: RANGE frame over VARCHAR order", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity) OVER (ORDER BY status RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N06: unknown named window", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(quantity) OVER w AS s FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "N08: QUALIFY without window", + "category": "window", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") QUALIFY quantity > 1\n" + } + }, + { + "description": "F01: ABS(VARCHAR)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ABS(status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F02: ROUND(TIMESTAMP)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ROUND(ordered_at) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F03: EXTRACT from VARCHAR", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT EXTRACT(year FROM status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F04: DATE_DIFF with VARCHAR", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT DATE_DIFF('day', status, ordered_at) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F05: SPLIT_PART(INTEGER)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SPLIT_PART(quantity, ',', 1) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F06: REGEXP_MATCHES(INTEGER)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT REGEXP_MATCHES(quantity, 'a') AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F07: SQRT(VARCHAR)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SQRT(status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F08: GREATEST(INTEGER, VARCHAR)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT GREATEST(quantity, status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F09: LEAST(TIMESTAMP, INTEGER)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT LEAST(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F10: NULLIF(INTEGER, VARCHAR)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT NULLIF(quantity, status) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F11: IFNULL(TIMESTAMP, INTEGER)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT IFNULL(ordered_at, quantity) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F12: STRFTIME on INTEGER", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT STRFTIME(quantity, '%Y') AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F13: LOWER(TIMESTAMP)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT LOWER(ordered_at) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F14: LEFT(INTEGER, 2)", + "category": "function type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT LEFT(quantity, 2) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F15: COALESCE with no args", + "category": "function arity", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COALESCE() AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "F17: unknown function in WHERE", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE is_valid(status)\n" + } + }, + { + "description": "F18: unknown aggregate", + "category": "function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT not_an_agg(quantity) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O02: VARCHAR OR BOOLEAN", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status OR quantity > 1\n" + } + }, + { + "description": "O03: TIMESTAMP * INTEGER", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at * 2 AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O04: VARCHAR % INTEGER", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status % 2 AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O05: IS TRUE on VARCHAR", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE status IS TRUE\n" + } + }, + { + "description": "O06: TIMESTAMP LIKE", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at LIKE '2026%'\n" + } + }, + { + "description": "O07: INTEGER BETWEEN strings", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE quantity BETWEEN 'a' AND 'b'\n" + } + }, + { + "description": "O08: CASE WHEN VARCHAR condition", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CASE WHEN status THEN 1 END AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O09: unary minus VARCHAR", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT -status AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O10: TIMESTAMP - VARCHAR", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at - status AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "O11: HAVING VARCHAR predicate", + "category": "operator type", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT status FROM __source(\"raw__orders\") GROUP BY status HAVING status\n" + } + }, + { + "description": "L01: CAST to unknown type", + "category": "cast", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(quantity AS NOT_A_TYPE) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "L02: CAST BOOLEAN to TIMESTAMP", + "category": "cast", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(status = 'x' AS TIMESTAMP) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "L03: CAST TIMESTAMP to BOOLEAN", + "category": "cast", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT CAST(ordered_at AS BOOLEAN) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "L04: invalid DATE literal", + "category": "literal", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at > DATE '2026-13-45'\n" + } + }, + { + "description": "L05: invalid INTERVAL literal", + "category": "literal", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT ordered_at + INTERVAL 'banana' AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "Q01: IN subquery with two columns", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE customer_id IN (SELECT id, email FROM __source(\"raw__customers\"))\n" + } + }, + { + "description": "Q02: scalar subquery two columns", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT (SELECT id, email FROM __source(\"raw__customers\") LIMIT 1) AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "Q03: IN subquery type mismatch", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE ordered_at IN (SELECT email FROM __source(\"raw__customers\"))\n" + } + }, + { + "description": "Q05: row value arity mismatch", + "category": "subquery", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") WHERE (id, customer_id) = (1, 2, 3)\n" + } + }, + { + "description": "J01: duplicate table alias", + "category": "join", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") o ON o.id = o.id\n" + } + }, + { + "description": "J02: USING column type mismatch", + "category": "join", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT * FROM (SELECT id FROM __source(\"raw__orders\")) a JOIN (SELECT email AS id FROM __source(\"raw__customers\")) b USING (id)\n" + } + }, + { + "description": "J04: self join without aliases", + "category": "join", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") JOIN __source(\"raw__orders\") ON id = id\n" + } + }, + { + "description": "J05: ON references later table", + "category": "join", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT o.id FROM __source(\"raw__orders\") o JOIN __source(\"raw__customers\") c ON c.id = p.order_id JOIN __source(\"raw__payments\") p ON p.order_id = o.id\n" + } + }, + { + "description": "E01: duplicate CTE names", + "category": "cte", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM __source(\"raw__orders\")), a AS (SELECT id FROM __source(\"raw__customers\")) SELECT id FROM a\n" + } + }, + { + "description": "E02: recursive CTE type mismatch", + "category": "cte", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT 'x' FROM r WHERE n < 3) SELECT n FROM r\n" + } + }, + { + "description": "E03: CTE forward reference", + "category": "cte", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nWITH a AS (SELECT id FROM b), b AS (SELECT id FROM __source(\"raw__orders\")) SELECT id FROM a\n" + } + }, + { + "description": "S01: UNION ORDER BY non-output column", + "category": "set operation", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") UNION ALL SELECT id FROM __source(\"raw__customers\") ORDER BY status\n" + } + }, + { + "description": "S02: EXCEPT column count mismatch", + "category": "set operation", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id, status FROM __source(\"raw__orders\") EXCEPT SELECT id FROM __source(\"raw__customers\")\n" + } + }, + { + "description": "S03: UNION INTEGER vs TIMESTAMP", + "category": "set operation", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT quantity AS x FROM __source(\"raw__orders\") UNION ALL SELECT created_at FROM __source(\"raw__customers\")\n" + } + }, + { + "description": "R01: qualified star of unknown alias", + "category": "star", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT z.* FROM __source(\"raw__orders\") o\n" + } + }, + { + "description": "R02: REPLACE of missing column", + "category": "star", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT * REPLACE (1 AS qty) FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "R03: COLUMNS regex matching nothing", + "category": "star", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT COLUMNS('^nothing_') FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "R04: star from model then unknown", + "category": "star", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT x.qty FROM (SELECT * FROM __ref(\"stg_orders\")) x\n" + } + }, + { + "description": "M01: LIMIT VARCHAR", + "category": "limit", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT 'x'\n" + } + }, + { + "description": "M02: LIMIT negative", + "category": "limit", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT -1\n" + } + }, + { + "description": "M03: LIMIT column reference", + "category": "limit", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT id FROM __source(\"raw__orders\") LIMIT quantity\n" + } + }, + { + "description": "U01: UDF result used as integer", + "category": "udf", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT __udf(\"udf__is_completed_order\")(status) + 1 AS x FROM __source(\"raw__orders\")\n" + } + }, + { + "description": "U04: table function argument VARCHAR", + "category": "table function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT t.order_id FROM __table_fn(\"table_fn__customer_orders\")('abc') t\n" + } + }, + { + "description": "U06: table function output type misuse", + "category": "table function", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/probe.sql": "MODEL (materialized table);\n\nSELECT SUM(t.waffle_name) AS s FROM __table_fn(\"table_fn__customer_orders\")(1) t\n" + } + }, + { + "description": "U08: macro result type misuse (marts)", + "category": "macro", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/marts/probe_m.sql": "MODEL (materialized table);\n\nSELECT UPPER(@cents_to_dollars('p.amount_cents')) AS d FROM __ref(\"stg_payments\") p\n" + } + }, + { + "description": "D02: expression_is_true type error", + "category": "audit", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/marts/daily_revenue.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Daily revenue aggregation.\",\n audits [\n expression_is_true (\n name \"revenue_is_non_negative\",\n expression \"revenue_date > 5\",\n ),\n ],\n columns (\n revenue_date (nullable false),\n ),\n);\n\nSELECT\n CAST(o.ordered_at AS DATE) AS revenue_date,\n COUNT(DISTINCT o.order_id) AS order_count,\n SUM(o.quantity) AS waffles_sold,\n SUM(p.amount_cents) AS total_revenue_cents,\n @cents_to_dollars('SUM(p.amount_cents)') AS total_revenue_dollars,\n SUM(p.amount_cents) / COUNT(DISTINCT o.order_id) AS avg_order_value_cents\nFROM __ref(\"stg_orders\") o\nINNER JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id AND p.payment_status = 'success'\nGROUP BY CAST(o.ordered_at AS DATE)\n" + } + }, + { + "description": "ticket case 6: website timestamp comparison placeholder", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\nWHERE ordered_at > 5\n" + } + }, + { + "description": "ticket case 7: integer multiplied by string", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/staging/stg_orders.sql": "MODEL (\n materialized view,\n tags [staging],\n description \"Cleaned order records.\",\n columns (\n order_id (nullable false, audits [not_null, unique]),\n customer_id (nullable false, audits [not_null]),\n waffle_type_id (\n audits [relationships (to __seed(\"waffle_types\"), field waffle_type_id)],\n ),\n status (\n audits [\n accepted_values (values [\"placed\", \"preparing\", \"ready\", \"completed\", \"cancelled\"]),\n ],\n ),\n ),\n);\n\nSELECT\n id AS order_id,\n customer_id,\n waffle_type_id,\n quantity * status AS quantity,\n ordered_at,\n status\nFROM __source(\"raw__orders\")\n" + } + }, + { + "description": "ticket case 8: incompatible join keys", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/marts/fact_orders.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Order fact table with waffle and payment details.\",\n columns (\n order_id (nullable false, audits [not_null]),\n ),\n);\n\nSELECT\n o.order_id,\n o.customer_id,\n o.waffle_type_id,\n w.waffle_name,\n w.category AS waffle_category,\n o.quantity,\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\n o.ordered_at,\n o.status AS order_status,\n __udf(\"udf__is_completed_order\")(o.status) AS is_completed_order,\n p.payment_method,\n p.payment_status,\n p.amount_cents AS payment_amount_cents\nFROM __ref(\"stg_orders\") o\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.payment_method\n" + } + }, + { + "description": "ticket case 9: string window aggregate", + "category": "ticket regression", + "expected_exit_code": 1, + "expected_codes": [], + "pending_native": true, + "repo_files": { + "models/marts/fact_orders.sql": "MODEL (\n materialized table,\n tags [marts],\n description \"Order fact table with waffle and payment details.\",\n columns (\n order_id (nullable false, audits [not_null]),\n ),\n);\n\nSELECT\n o.order_id,\n o.customer_id,\n o.waffle_type_id,\n w.waffle_name,\n w.category AS waffle_category,\n o.quantity,\n @line_total_cents(\"w.price_cents\", \"o.quantity\") AS line_total_cents,\n o.ordered_at,\n o.status AS order_status,\n __udf(\"udf__is_completed_order\")(o.status) AS is_completed_order,\n p.payment_method,\n p.payment_status,\n SUM(p.payment_method) OVER () AS payment_amount_cents\nFROM __ref(\"stg_orders\") o\nLEFT JOIN __seed(\"waffle_types\") w ON o.waffle_type_id = w.waffle_type_id\nLEFT JOIN __ref(\"stg_payments\") p ON o.order_id = p.order_id\n" + } + } + ] +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/native_outcomes.json b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/native_outcomes.json new file mode 100644 index 0000000000..e5b41ee5f6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/native_outcomes.json @@ -0,0 +1,16 @@ +{ + "invalid:B002": ["u19", "u20", "x02", "S01", "R02", "R03", "R04"], + "invalid:B217": ["t01", "t06", "t12", "t19", "t20", "t26", "y02", "O06", "Q03", "D02", "ticket case 6", "t03", "t21", "t24", "J02", "ticket case 8"], + "invalid:B212": ["t02", "t15", "t18", "t27", "y08", "O03", "O04", "O09", "O10", "ticket case 7"], + "invalid:B213": ["t04", "t09", "t10", "t14", "t16", "t22", "t23", "f04", "k05", "A02", "A03", "A04", "A06", "A07", "A08", "F01", "F02", "F03", "F04", "F05", "F06", "F07", "F08", "F09", "F11", "F12", "F13", "F14", "U06", "U08", "ticket case 9", "N02", "N03", "F10"], + "invalid:B216": ["s01", "s03", "Q01", "Q02", "Q05", "S02"], + "invalid:B101": ["f01", "F17", "F18"], + "invalid:B102": ["f02", "f05", "A05", "F15"], + "invalid:B232": ["N01", "N04", "N06", "N08"], + "invalid:B233": ["J01", "J04", "E01"], + "invalid:B234": ["M01", "M02", "M03"], + "invalid:B301": ["t28", "metadata-udf", "D05", "D07"], + "invalid:B211": ["t08", "A10", "O02", "O05", "O08", "O11"], + "invalid:B218": ["t11", "t17", "t25", "O07", "L01", "L02", "L03", "L04", "L05"], + "invalid:B215": ["E02", "S03"] +} diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql new file mode 100644 index 0000000000..3b2df3811b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/fixtures/semantic/typed_orders.sql @@ -0,0 +1,21 @@ +MODEL ( + materialized view, + contract enforced, + columns ( + order_id (type INTEGER), + customer_id (type INTEGER), + waffle_type_id (type INTEGER), + quantity (type INTEGER), + ordered_at (type TIMESTAMP), + status (type VARCHAR), + ), +); + +SELECT + id AS order_id, + customer_id, + waffle_type_id, + quantity, + ordered_at, + status +FROM __source("raw__orders") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 2f79a13c91..9c7cb59ba8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -5,6 +5,7 @@ import json import os import re +import shutil import signal import statistics import subprocess @@ -16,16 +17,18 @@ from io import StringIO from pathlib import Path from types import FrameType -from typing import Any, NamedTuple +from typing import Any, NamedTuple, cast import pytest +from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, ) from sqlbuild.cli.commands.main.entrypoint.entry import main from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, + SemanticCorpusCase, ) _DBT_SHAPED_SQL_SIZE_PROFILE: tuple[tuple[float, int], ...] = ( @@ -43,6 +46,43 @@ _PERFORMANCE_SAFETY_TIMEOUT_MULTIPLIER: float = 2.0 +def semantic_corpus_cases(*, group: str) -> list[dict[str, Any]]: + path: Path = Path(__file__).parent / "fixtures" / "semantic" / "corpus.json" + raw: dict[str, list[dict[str, Any]]] = json.loads(path.read_text(encoding="utf-8")) + outcomes: dict[str, list[str]] = json.loads( + path.with_name("native_outcomes.json").read_text(encoding="utf-8") + ) + resolved: dict[str, list[dict[str, Any]]] = {name: [] for name in raw} + overrides: dict[str, tuple[str, str]] = {} + for outcome, identifiers in outcomes.items(): + destination, code = outcome.split(":") + overrides.update(dict.fromkeys(identifiers, (destination, code))) + for original_group, cases in raw.items(): + for case in cases: + identifier: str = case["description"].split(":", maxsplit=1)[0] + destination, code = overrides.get(identifier, (original_group, "")) + resolved[destination].append( + { + **case, + "expected_exit_code": int(destination in {"invalid", "pending"}), + "expected_codes": [code] * bool(code) or case["expected_codes"], + "pending_native": destination == "pending", + } + ) + return resolved[group] + + +def prepare_semantic_corpus_project( + *, base: Path, tmp_path: Path, test_case: SemanticCorpusCase +) -> Path: + project: Path = tmp_path / "orders_project" + shutil.copytree(base, project) + for relative_path, contents in test_case.repo_files.items(): + path: Path = project / relative_path + path.write_text(contents, encoding="utf-8") + return project + + class CompileBenchmarkMeasurement(NamedTuple): elapsed_seconds: float timings_ms: dict[str, int] @@ -76,6 +116,19 @@ class FreshProcessCompileBenchmarkResult(NamedTuple): peak_rss_bytes: int semantic_fingerprint: str payload: dict[str, object] + cpu_seconds: float + major_page_faults: int + minor_page_faults: int + + def __repr__(self) -> str: + return ( + f"FreshProcessCompileBenchmarkResult(elapsed_seconds={self.elapsed_seconds}, " + f"cpu_seconds={self.cpu_seconds}, peak_rss_bytes={self.peak_rss_bytes}, " + f"major_page_faults={self.major_page_faults}, " + f"minor_page_faults={self.minor_page_faults}, " + f"semantic_fingerprint={self.semantic_fingerprint!r}, " + f"compile_timings={self.payload.get('compile_timings')!r})" + ) class FreshProcessCompileCacheBenchmarkResult(NamedTuple): @@ -558,7 +611,7 @@ def _run_fresh_process_compile_benchmark( "--output", str(measurement_path), "--format", - "%e %M", + "%e %M %U %S %F %R", str(Path(sys.executable).with_name("sqb")), "--project-dir", str(project_dir), @@ -567,28 +620,36 @@ def _run_fresh_process_compile_benchmark( "--json", *compile_args, ] - with ( - output_path.open("wb") as output_file, - stderr_path.open("wb") as stderr_file, - ): - with subprocess.Popen( - command, - stdout=output_file, - stderr=stderr_file, - start_new_session=True, - ) as process: - try: - returncode: int = process.wait(timeout=expected_max_wall_seconds + 10.0) - except subprocess.TimeoutExpired: - os.killpg(process.pid, signal.SIGKILL) - process.wait() - raise + started: float = time.monotonic() + try: + with ( + output_path.open("wb") as output_file, + stderr_path.open("wb") as stderr_file, + ): + with subprocess.Popen( + command, + stdout=output_file, + stderr=stderr_file, + start_new_session=True, + ) as process: + try: + returncode: int = process.wait(timeout=expected_max_wall_seconds + 10.0) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise + finally: + payload_object, measurement = read_compile_measurement( + label=label, + measurement_path=measurement_path, + output_path=output_path, + elapsed_seconds=time.monotonic() - started, + ) assert returncode == 0, stderr_path.read_text(encoding="utf-8") - elapsed_text, peak_rss_kib_text = measurement_path.read_text(encoding="utf-8").split() + elapsed_text, peak_rss_kib_text, user_text, system_text, major_text, minor_text = measurement elapsed_seconds: float = float(elapsed_text) - payload_object: object = json.loads(output_path.read_bytes()) assert isinstance(payload_object, dict) - payload: dict[str, object] = payload_object + payload: dict[str, object] = cast(dict[str, object], payload_object) peak_rss_bytes: int = int(peak_rss_kib_text) * 1024 compiled_dir: Path = project_dir / "target" / "compiled" semantic_fingerprint: str = semantic_compile_fingerprint( @@ -599,6 +660,9 @@ def _run_fresh_process_compile_benchmark( peak_rss_bytes=peak_rss_bytes, semantic_fingerprint=semantic_fingerprint, payload=payload, + cpu_seconds=float(user_text) + float(system_text), + major_page_faults=int(major_text), + minor_page_faults=int(minor_text), ) @@ -1575,7 +1639,7 @@ def _semantic_regular_model_sql( False: f'__ref("model_{index - 1:05d}")', }[_layered_is_base_model(index=index)] metric_expressions: str = "".join( - f",\n CAST(COALESCE(CASE WHEN id % {column_index % 11 + 2} = 0 " + f",\n CAST(COALESCE(CASE WHEN input.id % {column_index % 11 + 2} = 0 " f"THEN amount + {column_index} WHEN status = 'priority' " f"THEN amount * {column_index % 7 + 1} ELSE amount - {column_index} END, 0) AS DOUBLE) " f"AS metric_{column_index:04d}" @@ -1593,13 +1657,16 @@ def _semantic_regular_model_sql( }[index % _FUNCTION_INTERVAL == 0] macro_index: int = (index // macro_call_interval) % macro_count id_expression: str = { - True: f'@macro_{macro_index:05d}("id")', - False: "id", + True: f'@macro_{macro_index:05d}("input.id")', + False: "input.id", }[index % macro_call_interval == 0] + status_expression: str = ( + "CAST(CASE WHEN input.id % 2 = 0 THEN 'even' ELSE 'odd' END AS VARCHAR)" + ) direct_sql: str = f"""SELECT CAST({id_expression} AS INTEGER) AS id, CAST({amount_expression} AS DOUBLE) AS amount, - CAST(CASE WHEN id % 2 = 0 THEN 'even' ELSE 'odd' END AS VARCHAR) AS status{metric_expressions} + {status_expression} AS status{metric_expressions} FROM {relation_sql} AS input{seed_join_sql} """ with_sql: str = f"WITH transformed AS (\n{direct_sql.rstrip()}\n)\nSELECT * FROM transformed\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index d2f7b47db8..8afbbb6b96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -1,6 +1,5 @@ """Fresh-process guards for dense query graphs and the complete built-in ruleset.""" -import logging from itertools import filterfalse from operator import attrgetter from pathlib import Path @@ -26,7 +25,6 @@ measure_model_sql_bytes, ) -_LOGGER: logging.Logger = logging.getLogger(__name__) _GIB: int = 1024 * 1024 * 1024 @@ -40,28 +38,28 @@ 1000, 14.0, 3 * _GIB // 2, - "99f2f5c698cca3c7f0d5a2defbd31935d98813843c899286a23a8e7d4441a736", + "093fb28b99d2da6aa98158628b758338cbfd8742fdbf43cbe1beb4a161681370", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, 34.0, 11 * _GIB // 4, - "1243f3de95857de4e7d1d8563ae6766aa9d2a192cf7525cec8ef5562b12704dc", + "3d387c37e25d7241c4c79b495df6f2737d57bf839fc9bf97cebc23c1c0ed1e70", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, 55.0, 13 * _GIB // 4, - "956135597584ec369d37c23ab91b01bc6bf2df9ba78dce602311e152d451fa90", + "dfe3610ddca5cfd995b860ac28995d75958a70626304c9879de50a380e9abe65", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, 145.0, 4 * _GIB, - "92bf43fc45084c67de0eab3d4f32160e8303f99e1b951f6a7a1da5660f0eb66b", + "dbf8679c49a22a2869511bb08123f13ab0a079bb1722cc471ec52c115f4d4794", ), ), ids=lambda case: case.description, @@ -93,14 +91,10 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ expected_max_wall_seconds=test_case.expected_max_wall_seconds, compile_args=("--no-cache",), ) - _LOGGER.info( - "dense compile models=%d builtin_rules=%d wall=%.3fs peak_rss_bytes=%d fingerprint=%s timings=%s", - test_case.model_count, - len(builtin_codes), - result.elapsed_seconds, - result.peak_rss_bytes, - result.semantic_fingerprint, - result.payload["compile_timings"], + print( + f"dense compile models={test_case.model_count} builtin_rules={len(builtin_codes)} " + f"fingerprint={result.semantic_fingerprint}", + flush=True, ) assert result.payload["diagnostics"] == [] assert result.payload["has_errors"] is False diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py new file mode 100644 index 0000000000..afed55eecd --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py @@ -0,0 +1,108 @@ +"""Bound the diagnostic-only paths missed by clean-project performance guards.""" + +import logging +from collections import Counter +from pathlib import Path +from statistics import median +from typing import Any + +import pytest + +from scripts.cold_compile_performance._helpers.diagnostic_measurement import ( + measure_diagnostic_compile, + measure_position_mapping, +) +from scripts.cold_compile_performance._helpers.diagnostic_project import ( + write_cascade_project, + write_diagnostic_project, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import DiagnosticPerformanceCase + +_LOGGER: logging.Logger = logging.getLogger(__name__) +pytestmark: list[pytest.MarkDecorator] = [ + pytest.mark.performance, + pytest.mark.cold_compile_performance, +] + + +@pytest.mark.parametrize( + "test_case", + [DiagnosticPerformanceCase("large_models_many_diagnostics")], + ids=lambda case: case.description, +) +def test_given_large_invalid_models_when_compiling_then_bounds_diagnostic_work( + test_case: DiagnosticPerformanceCase, tmp_path: Path +) -> None: + write_diagnostic_project(project_dir=tmp_path, depth=test_case.depth, width=test_case.width) + for path in (tmp_path / "models").glob("large_orders_*.sql"): + assert len(path.read_text().splitlines()) >= 3000 + elapsed, payload = measure_diagnostic_compile( + project_dir=tmp_path, timeout=test_case.expected_timeout_seconds + ) + counts: Counter[tuple[str, str]] = Counter( + (item.get("resource_name", ""), item["severity"]) for item in payload["diagnostics"] + ) + for index in range(3): + assert counts[(f"large_orders_{index}", "error")] >= 100 + assert f"downstream_orders_{index}" in payload["semantic_checks_partial"] + _LOGGER.info("large diagnostic project wall=%.3fs counts=%s", elapsed, counts) + assert elapsed < test_case.expected_max_wall_seconds + + +@pytest.mark.parametrize( + "test_case", + [DiagnosticPerformanceCase("mapping_scales_with_diagnostics", expected_max_wall_seconds=8.0)], + ids=lambda case: case.description, +) +def test_given_many_spans_on_large_sql_when_mapping_then_scales_linearly( + test_case: DiagnosticPerformanceCase, +) -> None: + cold: float = measure_position_mapping(count=1) + single: float = median( + measure_position_mapping(count=test_case.diagnostic_count) for _ in range(3) + ) + doubled: float = median( + measure_position_mapping(count=test_case.diagnostic_count * 2) for _ in range(3) + ) + _LOGGER.info("mapping cold=%.3fs N=%.3fs 2N=%.3fs", cold, single, doubled) + assert cold + doubled < test_case.expected_max_wall_seconds + assert doubled <= single * 2.8 + 0.1 + + +@pytest.mark.parametrize( + "test_case", + [ + DiagnosticPerformanceCase( + "deep_poisoned_dag", depth=40, width=50, expected_max_wall_seconds=35.0 + ) + ], + ids=lambda case: case.description, +) +def test_given_deep_poisoned_dag_when_recovering_then_bounds_scaling( + test_case: DiagnosticPerformanceCase, tmp_path: Path +) -> None: + timings: list[float] = [] + for multiplier in (1, 2): + project: Path = tmp_path / f"depth_{multiplier}" + write_cascade_project( + project_dir=project, depth=test_case.depth * multiplier, width=test_case.width + ) + elapsed, payload = measure_diagnostic_compile( + project_dir=project, timeout=test_case.expected_max_wall_seconds + 5 + ) + diagnostics: list[dict[str, Any]] = payload["diagnostics"] + assert len(diagnostics) == test_case.width + assert all(item["code"] == "B212" for item in diagnostics) + assert len(payload["semantic_checks_partial"]) == test_case.depth * multiplier + 1 + assert all(item.get("notes") for item in diagnostics) + expected_note: str = f"{test_case.depth * multiplier} downstream output uses were not type-checked because of this error" + for diagnostic in diagnostics: + assert expected_note in diagnostic["notes"] + timings.append(elapsed) + _LOGGER.info("cascade depth=%d wall=%s", test_case.depth, timings) + assert max(timings) < test_case.expected_max_wall_seconds + assert timings[1] <= timings[0] * 3.0 + 1.0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index f586c285ae..2946b2e19a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -43,24 +43,24 @@ macro_count=500, test_count=2_945, audit_count=5_056, - expected_cold_max_wall_seconds=14.5, - expected_warm_max_wall_seconds=8.25, - expected_edit_max_wall_seconds=7.25, + expected_cold_max_wall_seconds=17.0, + expected_warm_max_wall_seconds=9.5, + expected_edit_max_wall_seconds=8.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( - "3a955baeac3aaef85d1bcfe25eddce899abf8e542ad168510faa8c6cb53b9a30" + "e40512a79290e70e9170599ea526950e7b8c8745f534ee5e385ed2195f69fded" ), expected_leaf_edit_fingerprint=( - "37171d29e89f3ed9db9b29b21d9af810a7c9bf3555cc497159c2d4ef87ae8f9d" + "610664f9cb4d04db8c947f32cd57dad57e1b04064880dfd3700767a25981f849" ), expected_macro_edit_fingerprint=( - "e37d388dc19e21da66fcb2f6fad2c3ef8db288724e7ab65031a75180e1535949" + "f31227fbcf12586be22198e3dc499a9f572ef185267706c64df134b12e93403a" ), expected_project_config_fingerprint=( - "cbbe850cd2288979fd7682180176dfa9cda00bb48968ddb3b6c16ecd5728a389" + "3a7a8cab7a4e42cfffb9952130c1fdd6b4c573fc0b3a5ae99116eda1f12dabf1" ), macro_call_interval=6, scoped_macros=True, @@ -75,24 +75,24 @@ macro_count=834, test_count=4_908, audit_count=8_427, - expected_cold_max_wall_seconds=23.5, + expected_cold_max_wall_seconds=24.0, expected_warm_max_wall_seconds=13.5, - expected_edit_max_wall_seconds=12.0, + expected_edit_max_wall_seconds=12.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( - "b39d33782ca1bc09235a6b6190dbc6a35cec43a161ef29d459aafd3f6be90166" + "71d3f0d6d40cec6484a6b307ae589377767945c5ec04e0e6bcc7bd18d5c68089" ), expected_leaf_edit_fingerprint=( - "1bbf19080469c9716003c718fad5f8c196619778d20a44a4464a0af1c9f291d6" + "677f073370570f0035a41a1ec1d47d7459444da6eed11ccbafba4850708e05ce" ), expected_macro_edit_fingerprint=( - "394e5ee7d30c4cccbf608ea9d0b0e3a37d81b3948804d83a867244eccd9f3135" + "7cb89115a3fbd771eae4b9a5406f4b6f125b51f3cd3270ce0495f5db35fb28c2" ), expected_project_config_fingerprint=( - "4e2e86138b95fab8d4e45f97a990984f5057ad4686c4e05f0ba1de2de2ebe280" + "9abf2059b8a71892559bbd2fe80b5d164ae9b6ddaf375706802979543e8fed22" ), macro_call_interval=6, scoped_macros=True, @@ -107,24 +107,24 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, - expected_cold_max_wall_seconds=42.5, - expected_warm_max_wall_seconds=24.5, - expected_edit_max_wall_seconds=22.0, + expected_cold_max_wall_seconds=47.5, + expected_warm_max_wall_seconds=25.0, + expected_edit_max_wall_seconds=25.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( - "1e3a63aab6d47bd1425c23d6f03c084d5207086caad6ce0da7564be0ce048158" + "f29cb502ded5df2cd32157c5d83363d7218b7335f61944f3c034230c94b6a46e" ), expected_leaf_edit_fingerprint=( - "a853c671586612047923ffd1b8f0a5ef9d9a7c2d6edb634ad731432e5b641b16" + "291a9cd967733f6446755c5f332aeffab64881c82abb01ec8decafe249658b92" ), expected_macro_edit_fingerprint=( - "25793047e0ae59c5c9d3a67ace435990de502e0d2c2be9d25ba2b5589ecf778d" + "ed4ad652e2f02b2ed609fb07ed94b616c434f4e8f68843ed12bfd23b5cbfaaa9" ), expected_project_config_fingerprint=( - "1cd1b9d836d9f5dd27c632cc8d9ab68464855dc22457c68ed44ad7efa7eb44af" + "9e5a05f1ae2d6d4a2daaa79e43cdd51f76b3d8908b2d9146a46a36d362c13aeb" ), macro_call_interval=6, scoped_macros=True, @@ -222,18 +222,15 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in 1, 0, ) - assert fresh_process_compile_cache_metrics(result.after_leaf_edit) == ( - test_case.model_count - 1, - 1, - 0, - 0, + assert_complete_compile_cache_hit( + measurement=result.after_leaf_edit, + model_count=test_case.model_count, ) macro_batch_hits, macro_entry_hits, macro_misses, macro_bypasses = ( fresh_process_compile_cache_metrics(result.macro_edit) ) assert macro_batch_hits + macro_entry_hits + macro_misses == test_case.model_count assert macro_batch_hits > 0 - assert macro_entry_hits > 0 assert macro_misses == test_case.expected_macro_edit_misses assert macro_bypasses == 0 assert_complete_compile_cache_hit( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 057d8a1af4..8970bce3fe 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -36,10 +36,10 @@ macro_count=37, test_count=2_945, audit_count=5_056, - expected_max_wall_seconds=11.0, + expected_max_wall_seconds=12.5, expected_max_rss_bytes=int(1.5 * _GIB), expected_semantic_fingerprint=( - "e31803bd257d02a76188a9e6d3bf3b08feb50395c48f01f5535120041c58c875" + "8a6f8015c53c12d152ff137b3ef6ff89bc98e2b9471ecb3898e69e35cd895025" ), ), FreshProcessCompilePerformanceGuardTestCase( @@ -51,10 +51,10 @@ macro_count=61, test_count=4_908, audit_count=8_427, - expected_max_wall_seconds=15.5, + expected_max_wall_seconds=18.0, expected_max_rss_bytes=int(1.75 * _GIB), expected_semantic_fingerprint=( - "144f480668acba715d02ecca6712397150b5f35280a963a099b7f0cc814dbe0f" + "8c91b4a742697d8bced8ae7874a8566c3467b24f35d1fc33128cae35fcd6f97c" ), ), FreshProcessCompilePerformanceGuardTestCase( @@ -66,10 +66,10 @@ macro_count=123, test_count=9_816, audit_count=16_855, - expected_max_wall_seconds=29.5, + expected_max_wall_seconds=35.0, expected_max_rss_bytes=2 * _GIB, expected_semantic_fingerprint=( - "89fea6392fbd130552781d84f8b982037a56edef6b98e4345c9769e57c7e56a7" + "aa2dc552eae6a8f26546fc9e1d40b194da813a8cd3dec60c6f85bb696c1692c6" ), ), ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_semantic_corpus.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_semantic_corpus.py new file mode 100644 index 0000000000..c5a7e26eb4 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_semantic_corpus.py @@ -0,0 +1,84 @@ +"""Single-change semantic probes exercised through the real CLI and DuckDB.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import SemanticCorpusCase +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + prepare_semantic_corpus_project, + semantic_corpus_cases, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [SemanticCorpusCase(**case) for case in semantic_corpus_cases(group="invalid")], + ids=lambda case: case.description, +) +def test_given_invalid_semantics_when_compiling_then_errors_precede_execution( + test_case: SemanticCorpusCase, + semantic_playground: Path, + tmp_path: Path, +) -> None: + project: Path = prepare_semantic_corpus_project( + base=semantic_playground, tmp_path=tmp_path, test_case=test_case + ) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=("compile", "--no-cache", "--json") + ) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + for code in test_case.expected_codes: + assert code in result.stdout + result.stderr + + +@pytest.mark.xfail( + strict=True, + reason="External relation, scope ordering, or project-function type coverage pending", +) +@pytest.mark.parametrize( + "test_case", + [SemanticCorpusCase(**case) for case in semantic_corpus_cases(group="pending")], + ids=lambda case: case.description, +) +def test_given_pending_native_semantics_when_compiling_then_invalid_sql_should_error( + test_case: SemanticCorpusCase, + semantic_playground: Path, + tmp_path: Path, +) -> None: + project: Path = prepare_semantic_corpus_project( + base=semantic_playground, tmp_path=tmp_path, test_case=test_case + ) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=("compile", "--no-cache", "--json") + ) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + [SemanticCorpusCase(**case) for case in semantic_corpus_cases(group="valid")], + ids=lambda case: case.description, +) +def test_given_dialect_valid_control_when_compiling_and_building_then_both_succeed( + test_case: SemanticCorpusCase, + semantic_playground: Path, + tmp_path: Path, +) -> None: + project: Path = prepare_semantic_corpus_project( + base=semantic_playground, tmp_path=tmp_path, test_case=test_case + ) + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=("compile", "--no-cache", "--json") + ) + assert compiled.returncode == test_case.expected_exit_code, compiled.stdout + compiled.stderr + built: subprocess.CompletedProcess[str] = run_sqb(project_dir=project, command=("build",)) + assert built.returncode == test_case.expected_exit_code, built.stdout + built.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py new file mode 100644 index 0000000000..a81c41c446 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_source_semantic_binding.py @@ -0,0 +1,92 @@ +"""Warehouse schemas close uncontracted sources before model execution.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import SourceSemanticBindingCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, + table_exists, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SourceSemanticBindingCase( + description="plan checks inspected source columns", command="plan" + ), + SourceSemanticBindingCase( + description="build checks before executing models", command="build" + ), + ], + ids=lambda case: case.description, +) +def test_given_open_source_when_planning_then_live_columns_reject_missing_reference( + test_case: SourceSemanticBindingCase, + tmp_path: Path, +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="source_binding", + repo_files={ + "sqlbuild_project.toml": 'name = "source_binding"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', + "sources/orders.yml": "sources:\n - name: raw_orders\n table: raw_orders\n", + "models/orders.sql": 'MODEL (materialized table);\nSELECT missing FROM __source("raw_orders")\n', + }, + ) + database: Path = project / "orders.duckdb" + execute_duckdb(db_path=database, sql="CREATE TABLE raw_orders AS SELECT 1 AS id") + compiled: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=("compile", "--no-cache") + ) + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + planned: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=(test_case.command,) + ) + assert planned.returncode != 0 + assert f"error[{test_case.expected_code}]" in planned.stdout + planned.stderr + assert "models/orders.sql:2:" in planned.stdout + planned.stderr + assert not table_exists(db_path=database, table_name="orders") + + +@pytest.mark.parametrize( + "test_case", + [ + SourceSemanticBindingCase( + description="plan stops before connection", command="plan", expected_code="B300" + ), + SourceSemanticBindingCase( + description="build stops before connection", command="build", expected_code="B300" + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_metadata_when_planning_then_no_warehouse_connection_is_opened( + test_case: SourceSemanticBindingCase, + tmp_path: Path, +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="metadata_binding", + repo_files={ + "sqlbuild_project.toml": 'name = "metadata_binding"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', + "models/orders.sql": "MODEL (materialized table, unique_key [missing]);\nSELECT 1 AS id\n", + }, + ) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=(test_case.command,) + ) + assert result.returncode != 0 + assert f"error[{test_case.expected_code}]" in result.stdout + result.stderr + assert not (project / "orders.duckdb").exists() + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py index 572b6a61d2..581d1a3f4c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py @@ -44,10 +44,10 @@ 150, 250, ( - "de60ac45bddfc86fc9726e49514315d3a2945df949b2be9e8fc4cd835bc34200", - "c8cda76a18f96c345fbcbe22f05982c61f5837d4577fd7f2b45559b115756494", - "a107585c3754afc00d7119196a1421168d39d0336df64a41c89f400453591c3a", - "80d33bb0d3c0710e761658e344bfa895a5c351991eb4c8e94f1f7df7e85b4650", + "ba45ce336d0d636f7676204165e8e6b1b5622a140511c79c94234a149f9cf6ee", + "3994a72953957311ac82d5743a7bcaa93a9b39f8fdcf7efdc85ed1fc28db7a10", + "6002880e310c08aad7a28da3f397fdc88bbd477599b04a6755344fd1af5411bd", + "1f08064c1c53f511f2acfd32f06fbfc468c156540c8a4ea3649cb826f2b3a561", ), ), ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py index a3409f036c..32f2480162 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/_test_types.py @@ -132,36 +132,3 @@ class DatabricksSnapshotApplyE2ETestCase: expected_current_delete_rows: tuple[tuple[object, ...], ...] expected_historical_timestamp_rows: tuple[tuple[object, ...], ...] expected_historical_check_rows: tuple[tuple[object, ...], ...] - - -@dataclass(frozen=True) -class DatabricksVirtualLifecycleE2ETestCase: - description: str - expected_rows: tuple[tuple[str, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class DatabricksVirtualSeedE2ETestCase: - description: str - expected_rows: tuple[tuple[str, ...], ...] - expected_seed_strategy: str - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class DatabricksReconcileE2ETestCase: - description: str - expected_rows: tuple[tuple[str, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class DatabricksJanitorDetachedVdeE2ETestCase: - description: str - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - expected_ref_count_after: int - expected_return_code: int = 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py index 078412de19..62291c6de3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/helpers.py @@ -60,38 +60,6 @@ def build_databricks_project_toml(*, project_name: str, schema_name: str) -> str ) -def build_databricks_virtual_project_toml( - *, project_name: str, schema_name: str, unsuffixed_virtual_env: str | None = None -) -> str: - catalog_name: str = str(build_databricks_connection_config(schema=schema_name)["catalog"]) - unsuffixed_line: str = {None: ""}.get( - unsuffixed_virtual_env, - f'unsuffixed_virtual_env = "{unsuffixed_virtual_env}"\n', - ) - return ( - f'name = "{project_name}"\n' - 'adapter = "databricks"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "\n" - "[connection]\n" - 'server_hostname = "${ENV:SQB_TEST_DATABRICKS_SERVER_HOSTNAME}"\n' - 'http_path = "${ENV:SQB_TEST_DATABRICKS_HTTP_PATH}"\n' - 'token = "${ENV:SQB_TEST_DATABRICKS_TOKEN}"\n' - 'catalog = "${ENV:SQB_TEST_DATABRICKS_CATALOG}"\n\n' - "[targets.dev]\n" - f'database = "{catalog_name}"\n' - f'schema = "{schema_name}"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - f"{unsuffixed_line}\n" - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ) - - def prepare_databricks_waffle_shop(*, tmp_path: Path) -> tuple[Path, str]: """Prepare a Waffle Shop project wired to a unique Databricks schema.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 36438d4293..9f3b1c19f7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -14,23 +14,18 @@ DatabricksDiffE2ETestCase, DatabricksErrorE2ETestCase, DatabricksIntermediateDagStrategyE2ETestCase, - DatabricksJanitorDetachedVdeE2ETestCase, DatabricksNodeResultE2ETestCase, - DatabricksReconcileE2ETestCase, DatabricksScenarioLocalReplayE2ETestCase, DatabricksScenarioRemoteE2ETestCase, DatabricksSnapshotApplyE2ETestCase, DatabricksSnapshotE2ETestCase, DatabricksSourceLoaderStrategiesE2ETestCase, - DatabricksVirtualLifecycleE2ETestCase, - DatabricksVirtualSeedE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.databricks.helpers import ( assert_current_databricks_snapshot_rows, assert_databricks_snapshot_apply_rows, assert_databricks_snapshot_matrix_rows, build_databricks_project_toml, - build_databricks_virtual_project_toml, cleanup_databricks_schema, databricks_e2e_timing, databricks_relation_row_count, @@ -64,7 +59,6 @@ build_real_warehouse_existing_snapshot_project_files, build_real_warehouse_snapshot_project_files, prepare_inline_project, - query_duckdb, run_sqb, stringify_warehouse_rows, ) @@ -285,363 +279,6 @@ def test_given_physical_source_without_freshness_when_running_on_databricks_then cleanup_databricks_schema(schema_name=schema_name) -@pytest.mark.parametrize( - "test_case", - [ - DatabricksVirtualSeedE2ETestCase( - description="virtual seeded incremental build uses deep clone on databricks", - expected_rows=(("1", "10"), ("2", "21"), ("3", "31")), - expected_seed_strategy="durable_clone", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_incremental_change_when_building_on_databricks_then_seeds_with_deep_clone( - tmp_path: Path, - test_case: DatabricksVirtualSeedE2ETestCase, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_seed") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="databricks_virtual_seed", - repo_files={ - "sqlbuild_project.toml": build_databricks_virtual_project_toml( - project_name="databricks_virtual_seed", - schema_name=schema_name, - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - f" schema: {schema_name}\n" - " table: raw_orders\n" - ), - "models/orders.sql": ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - "SELECT id, ordered_at, amount_cents + 0 AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ), - }, - ) - try: - ensure_databricks_schema_ready(schema_name=schema_name) - execute_databricks_sql( - schema_name=schema_name, - sql=( - "CREATE OR REPLACE TABLE " - f"{relation_name(schema_name=schema_name, name='raw_orders')} AS " - "SELECT 1 AS id, TIMESTAMP '2026-01-01 00:00:00' AS ordered_at, " - "10 AS amount_cents UNION ALL " - "SELECT 2, TIMESTAMP '2026-01-02 00:00:00', 20" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - "SELECT id, ordered_at, amount_cents + 1 AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ), - encoding="utf-8", - ) - execute_databricks_sql( - schema_name=schema_name, - sql=( - f"INSERT INTO {relation_name(schema_name=schema_name, name='raw_orders')} " - "VALUES (3, TIMESTAMP '2026-01-03 00:00:00', 30)" - ), - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - rows: tuple[tuple[object, ...], ...] = fetch_databricks_rows( - schema_name=schema_name, - sql=( - f"SELECT id, amount_cents FROM " - f"{relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT seed_strategy FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) == [(test_case.expected_seed_strategy,)] - finally: - cleanup_databricks_schema(schema_name=schema_name) - cleanup_databricks_schema(schema_name=f"{schema_name}__dev") - cleanup_databricks_schema(schema_name=f"{schema_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - DatabricksReconcileE2ETestCase( - description="reconcile repair-view recreates databricks logical view", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Repair", - "model orders", - "VDE dev", - "action recreate logical view from state", - "result repaired", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_view_when_repairing_on_databricks_then_view_is_recreated( - tmp_path: Path, - test_case: DatabricksReconcileE2ETestCase, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_reconcile") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="databricks_virtual_reconcile", - repo_files={ - "sqlbuild_project.toml": build_databricks_virtual_project_toml( - project_name="databricks_virtual_reconcile", - schema_name=schema_name, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_databricks_schema_ready(schema_name=schema_name) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_databricks_sql( - schema_name=schema_name, - sql=f"DROP VIEW {relation_name(schema_name=f'{schema_name}__dev', name='orders')}", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - rows: tuple[tuple[object, ...], ...] = fetch_databricks_rows( - schema_name=schema_name, - sql=( - f"SELECT id FROM {relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - finally: - cleanup_databricks_schema(schema_name=schema_name) - cleanup_databricks_schema(schema_name=f"{schema_name}__dev") - cleanup_databricks_schema(schema_name=f"{schema_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - DatabricksVirtualLifecycleE2ETestCase( - description="adopt and detach preserve databricks logical table", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Adopted 1 models into virtual environment dev.", - "Detached 1 models from virtual environment dev.", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_stateless_table_when_adopting_and_detaching_on_databricks_then_table_is_preserved( - tmp_path: Path, - test_case: DatabricksVirtualLifecycleE2ETestCase, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_lifecycle") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="databricks_virtual_lifecycle", - repo_files={ - "sqlbuild_project.toml": build_databricks_virtual_project_toml( - project_name="databricks_virtual_lifecycle", - schema_name=schema_name, - unsuffixed_virtual_env="dev", - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_databricks_schema_ready(schema_name=schema_name) - execute_databricks_sql( - schema_name=schema_name, - sql=( - f"CREATE OR REPLACE TABLE {relation_name(schema_name=schema_name, name='orders')} " - "AS SELECT 1 AS id" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert adopt_result.returncode == test_case.expected_return_code, ( - adopt_result.stdout + adopt_result.stderr - ) - assert detach_result.returncode == test_case.expected_return_code, ( - detach_result.stdout + detach_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in adopt_result.stdout + detach_result.stdout - rows: tuple[tuple[object, ...], ...] = fetch_databricks_rows( - schema_name=schema_name, - sql=( - f"SELECT id FROM {relation_name(schema_name=schema_name, name='orders')} " - "ORDER BY id" - ), - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - finally: - cleanup_databricks_schema(schema_name=schema_name) - cleanup_databricks_schema(schema_name=f"{schema_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - DatabricksJanitorDetachedVdeE2ETestCase( - description="janitor prunes databricks detached VDE refs and physical versions", - expected_stdout_fragments=( - "eligible for deletion", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "state item", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_detached_vde_when_running_janitor_on_databricks_then_refs_are_pruned( - tmp_path: Path, - test_case: DatabricksJanitorDetachedVdeE2ETestCase, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_janitor") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="databricks_virtual_janitor", - repo_files={ - "sqlbuild_project.toml": ( - build_databricks_virtual_project_toml( - project_name="databricks_virtual_janitor", - schema_name=schema_name, - unsuffixed_virtual_env="dev", - ) - + "\n[janitor]\n" - + "enabled = true\n" - + "retention_days = 0\n" - + "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - ensure_databricks_schema_ready(schema_name=schema_name) - execute_databricks_sql( - schema_name=schema_name, - sql=( - f"CREATE OR REPLACE TABLE {relation_name(schema_name=schema_name, name='orders')} " - "AS SELECT 1 AS id" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ).returncode - == 0 - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_return_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environments", - ) == [(test_case.expected_virtual_environment_count_after,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model'" - ), - ) == [(test_case.expected_ref_count_after,)] - finally: - cleanup_databricks_schema(schema_name=schema_name) - cleanup_databricks_schema(schema_name=f"{schema_name}__sqb_physical") - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/diff/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/diff/_test_types.py index 20ef106b9a..5bc7644286 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/diff/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/diff/_test_types.py @@ -44,16 +44,6 @@ class DiffKeyFailureE2ETestCase: expected_stderr_fragment: str -@dataclass(frozen=True) -class VirtualDiffE2ETestCase: - description: str - command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_stderr_fragments: tuple[str, ...] = field(default_factory=tuple) - unexpected_stdout_fragments: tuple[str, ...] = field(default_factory=tuple) - - @dataclass(frozen=True) class DiffSamplingPrecedenceE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py b/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py index 89f56ffe4b..18bbf2c089 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/diff/test_diff.py @@ -18,7 +18,6 @@ DiffSamplingSeedE2ETestCase, QueryDiffOutcomeE2ETestCase, SingleDiffE2ETestCase, - VirtualDiffE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.diff.helpers import ( build_both_environments, @@ -28,11 +27,7 @@ reject_nonfinite_json_constant, run_sampled_diff, ) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, - build_virtual_plan_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb @pytest.mark.parametrize( @@ -1602,238 +1597,3 @@ def test_given_invalid_dev_keys_when_running_diff_then_it_fails_clearly( assert result.returncode == 1, result.stdout + result.stderr assert test_case.expected_stderr_fragment in result.stderr, result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualDiffE2ETestCase( - description="whole VDE diff allows finalized VDE stale against workspace", - command=("--no-color", "diff", "dev:pr", "--schema-only"), - expected_exit_code=0, - expected_stdout_fragments=( - "Virtual diff", - "working VDEs no", - "selected models 3", - "compared models 2", - "unchanged refs skipped 1", - "SQLBuild Diff Summary", - ), - ), - VirtualDiffE2ETestCase( - description="allow partial diff compares working VDEs", - command=( - "--no-color", - "diff", - "dev:pr", - "--schema-only", - "--allow-partial-diff", - ), - expected_exit_code=0, - expected_stdout_fragments=( - "Virtual diff", - "working VDEs no", - "selected models 3", - "compared models 2", - "unchanged refs skipped 1", - "SQLBuild Diff Summary", - ), - ), - VirtualDiffE2ETestCase( - description="unchanged virtual refs are skipped", - command=( - "--no-color", - "diff", - "dev:pr", - "--schema-only", - "--allow-partial-diff", - "--select", - "dim_customers", - ), - expected_exit_code=0, - expected_stdout_fragments=( - "selected models 1", - "compared models 0", - "unchanged refs skipped 1", - "No VDE ref differences in selected scope.", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_virtual_diff_with_working_vde_when_running_then_it_respects_partial_guard( - test_case: VirtualDiffE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_diff_guard", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout, result.stdout + result.stderr - for fragment in test_case.expected_stderr_fragments: - assert fragment in result.stderr, result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualDiffE2ETestCase( - description="verbose virtual diff keeps persisted source identity current", - command=( - "--no-color", - "diff", - "dev:pr", - "--schema-only", - "--verbose", - "--select", - "source_orders", - ), - expected_exit_code=0, - expected_stdout_fragments=( - "selected models 1", - "unchanged refs skipped 1", - ), - unexpected_stdout_fragments=("not current with workspace",), - ) - ], - ids=lambda case: case.description, -) -def test_given_source_dependent_vdes_when_diffing_verbose_then_refs_are_workspace_current( - test_case: VirtualDiffE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_diff_source_identity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " schema: raw\n" - " table: raw_orders\n" - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: integer\n" - ), - "models/source_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (1, 1)" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - dev_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert dev_build_result.returncode == 0, dev_build_result.stdout + dev_build_result.stderr - pr_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr", "--changes-only"), - project_dir=project_dir, - ) - assert pr_build_result.returncode == 0, pr_build_result.stdout + pr_build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout, result.stdout - for fragment in test_case.unexpected_stdout_fragments: - assert fragment not in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualDiffE2ETestCase( - description="active VDE requires partial diff flag", - command=("--no-color", "diff", "dev:pr", "--schema-only"), - expected_exit_code=1, - expected_stderr_fragments=( - "whole-VDE virtual diff requires finalized VDEs", - "non-finalized VDEs: pr", - "--allow-partial-diff", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_diff_with_active_vde_when_running_whole_diff_then_requires_partial_flag( - test_case: VirtualDiffE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_diff_active_guard", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - default_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert default_build_result.returncode == 0, default_build_result.stderr - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - branch_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr", "--select", "stg_orders"), - project_dir=project_dir, - ) - assert branch_build_result.returncode == 0, branch_build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_stderr_fragments: - assert fragment in result.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py index b1f49e70ac..3283901d5c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py @@ -2,11 +2,7 @@ import subprocess from pathlib import Path -from textwrap import dedent -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -75,50 +71,6 @@ def prepare_multi_schema_freshness_project(*, tmp_path: Path) -> Path: ) -def prepare_virtual_freshness_project( - *, tmp_path: Path, raw_orders_freshness: str | None = None -) -> Path: - freshness: str = raw_orders_freshness or ( - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: integer\n" - ) - return prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_freshness_command", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - f""" - sources: - - name: raw_orders - schema: raw - table: raw_orders -{freshness.rstrip()} - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - - -def persist_virtual_source_freshness(*, project_dir: Path) -> None: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - - def freshness_sources_yml( *, raw_orders_query: str = "SELECT 1 AS data_version", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/test_freshness.py b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/test_freshness.py index ebef18b93c..476a79a4f9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/test_freshness.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/test_freshness.py @@ -13,10 +13,8 @@ from tests.e2e.src.sqlbuild.cli.commands.main.freshness.helpers import ( freshness_sources_yml, persist_direct_source_freshness, - persist_virtual_source_freshness, prepare_freshness_project, prepare_multi_schema_freshness_project, - prepare_virtual_freshness_project, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, @@ -719,429 +717,3 @@ def test_given_missing_previous_freshness_when_running_state_fail_on_stale_then_ fragment: str for fragment in test_case.expected_fragments: assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison reports unchanged source freshness", - expected_fragments=( - "Unchanged (1)", - "raw_orders previous 1 current 1", - "OBSERVED=0 CHANGED=0 UNCHANGED=1 TOLERATED=0 UNKNOWN=0 ERROR=0", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_state_when_running_state_then_reports_unchanged( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project(tmp_path=tmp_path) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (7, 1)" - ), - ) - persist_virtual_source_freshness(project_dir=project_dir) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - ), - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison fails when previous state is missing", - expected_fragments=( - "Unknown (1)", - "raw_orders previous source freshness state missing", - "OBSERVED=0 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=1 ERROR=0", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_virtual_freshness_state_when_state_then_returns_nonzero( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project(tmp_path=tmp_path) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (7, 1)" - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - ), - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison fails on changed source freshness", - expected_fragments=( - "Changed (1)", - "raw_orders previous 1 current 2", - "OBSERVED=0 CHANGED=1 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=0", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_changes_when_state_then_returns_nonzero( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project(tmp_path=tmp_path) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (7, 1)" - ), - ) - persist_virtual_source_freshness(project_dir=project_dir) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = 2", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - ), - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - json_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - "--json", - ), - project_dir=project_dir, - ) - payload: dict[str, Any] = json.loads(json_result.stdout) - assert json_result.returncode == 1, json_result.stdout + json_result.stderr - assert payload["summary"]["changed"] == 1 - assert payload["sources"][0]["status"] == "changed" - assert payload["sources"][0]["previous_data_version"] == "1" - assert payload["sources"][0]["current_data_version"] == "2" - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison tolerates timestamp movement within tolerance", - expected_fragments=( - "Tolerated (1)", - "raw_orders previous 2026-01-01T00:00:00+00:00 current 2026-01-01T00:05:00+00:00", - "tolerance 10m", - "OBSERVED=0 CHANGED=0 UNCHANGED=0 TOLERATED=1 UNKNOWN=0 ERROR=0", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_timestamp_freshness_within_tolerance_when_state_then_reports_tolerated( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project( - tmp_path=tmp_path, - raw_orders_freshness=( - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: timestamp\n" - " lag_tolerance: 10m\n" - ), - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version TIMESTAMP); " - "INSERT INTO raw.raw_orders VALUES (7, TIMESTAMP '2026-01-01 00:00:00')" - ), - ) - persist_virtual_source_freshness(project_dir=project_dir) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET data_version = TIMESTAMP '2026-01-01 00:05:00'", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - ), - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison fails beyond timestamp tolerance", - expected_fragments=( - "Changed (1)", - "raw_orders previous 2026-01-01T00:00:00+00:00 current 2026-01-01T00:11:00+00:00", - "tolerance 10m", - "OBSERVED=0 CHANGED=1 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=0", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_timestamp_freshness_beyond_tolerance_when_state_then_returns_nonzero( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project( - tmp_path=tmp_path, - raw_orders_freshness=( - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: timestamp\n" - " lag_tolerance: 10m\n" - ), - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version TIMESTAMP); " - "INSERT INTO raw.raw_orders VALUES (7, TIMESTAMP '2026-01-01 00:00:00')" - ), - ) - persist_virtual_source_freshness(project_dir=project_dir) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET data_version = TIMESTAMP '2026-01-01 00:11:00'", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - ), - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison fails on backwards timestamp movement", - expected_fragments=( - "Changed (1)", - "raw_orders previous 2026-01-01T00:10:00+00:00 current 2026-01-01T00:05:00+00:00", - "tolerance 10m", - "OBSERVED=0 CHANGED=1 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=0", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_timestamp_freshness_moves_backwards_when_state_then_returns_nonzero( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project( - tmp_path=tmp_path, - raw_orders_freshness=( - " freshness:\n" - " strategy: column\n" - " column: data_version\n" - " type: timestamp\n" - " lag_tolerance: 10m\n" - ), - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version TIMESTAMP); " - "INSERT INTO raw.raw_orders VALUES (7, TIMESTAMP '2026-01-01 00:10:00')" - ), - ) - persist_virtual_source_freshness(project_dir=project_dir) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET data_version = TIMESTAMP '2026-01-01 00:05:00'", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - ), - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - FreshnessE2ETestCase( - description="virtual state comparison fails on explicit observation error", - expected_fragments=( - "Errors (1)", - "raw_orders source 'raw_orders' freshness query failed: " - 'Binder Error: Referenced column "missing_column"', - "OBSERVED=0 CHANGED=0 UNCHANGED=0 TOLERATED=0 UNKNOWN=0 ERROR=1", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_error_when_state_then_returns_nonzero( - test_case: FreshnessE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_freshness_project(tmp_path=tmp_path) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); " - "INSERT INTO raw.raw_orders VALUES (7, 1)" - ), - ) - persist_virtual_source_freshness(project_dir=project_dir) - (project_dir / "sources" / "raw.yml").write_text( - ( - "sources:\n" - " - name: raw_orders\n" - " schema: raw\n" - " table: raw_orders\n" - " freshness:\n" - " strategy: column\n" - " column: missing_column\n" - " type: integer\n" - ), - encoding="utf-8", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "freshness", - "--select", - "raw_orders", - "--state", - "--virtual-env", - "dev", - "--fail-on-stale", - ), - project_dir=project_dir, - ) - - assert result.returncode == 1, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py index d12e339fac..ff616913c9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/_test_types.py @@ -40,94 +40,6 @@ class JanitorSourceOverlapE2ETestCase: expected_existing_relations: tuple[tuple[str, str], ...] -@dataclass(frozen=True) -class JanitorMicrobatchHistoryProtectionE2ETestCase: - """Test case for immutable virtual microbatch history during janitor cleanup.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - - -@dataclass(frozen=True) -class JanitorCheckpointProtectionE2ETestCase: - """Test case for virtual checkpoint-protected janitor cleanup behavior.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorCheckpointRetentionE2ETestCase: - """Test case for virtual checkpoint pruning behavior.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_checkpoint_count_before: int - expected_checkpoint_count_after: int - expected_stdout_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorDetachedVirtualEnvironmentE2ETestCase: - """Test case for detached VDE cleanup behavior.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - expected_ref_count_after: int - - -@dataclass(frozen=True) -class JanitorDetachedVirtualEnvironmentRetentionE2ETestCase: - """Test case for detached VDE retention age behavior.""" - - description: str - retention_days: int - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - - -@dataclass(frozen=True) -class JanitorActiveVirtualEnvironmentProtectionE2ETestCase: - """Test case for active VDE ref protection behavior.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorExpiredVirtualEnvironmentE2ETestCase: - """Test case for expired non-active VDE cleanup behavior.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_names_after: tuple[str, ...] - - -@dataclass(frozen=True) -class JanitorStateCleanupE2ETestCase: - """Test case for state-only janitor cleanup behavior.""" - - description: str - janitor_command: tuple[str, ...] - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - expected_backup_schema_count_after: int - expected_lock_count_after: int - - @dataclass(frozen=True) class JanitorInvalidConfigE2ETestCase: """Test case for invalid janitor config behavior.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 21eef5918e..201fc414a1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -9,19 +9,11 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.janitor._test_types import ( - JanitorActiveVirtualEnvironmentProtectionE2ETestCase, - JanitorCheckpointProtectionE2ETestCase, - JanitorCheckpointRetentionE2ETestCase, JanitorCleanupE2ETestCase, - JanitorDetachedVirtualEnvironmentE2ETestCase, - JanitorDetachedVirtualEnvironmentRetentionE2ETestCase, JanitorDirectStatePruningE2ETestCase, JanitorDisabledE2ETestCase, - JanitorExpiredVirtualEnvironmentE2ETestCase, JanitorInvalidConfigE2ETestCase, - JanitorMicrobatchHistoryProtectionE2ETestCase, JanitorSourceOverlapE2ETestCase, - JanitorStateCleanupE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.janitor.helpers import ( create_direct_state_history, @@ -29,10 +21,6 @@ create_janitor_scenario_relations, prepare_janitor_project, ) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, - build_virtual_plan_repo_files, -) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( execute_duckdb, prepare_inline_project, @@ -259,108 +247,6 @@ def test_given_source_in_managed_schema_when_running_direct_janitor_then_blocks_ ) -@pytest.mark.parametrize( - "test_case", - [ - JanitorMicrobatchHistoryProtectionE2ETestCase( - description="virtual janitor never prunes microbatch event history", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_microbatch_history_when_running_janitor_then_events_remain_unchanged( - test_case: JanitorMicrobatchHistoryProtectionE2ETestCase, - tmp_path: Path, -) -> None: - project_toml: str = build_virtual_plan_project_toml().replace( - "virtual_environments = true\n", - "virtual_environments = true\nconcurrency = 2\nmicrobatch_concurrency = true\n", - ) - project_toml += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - delete_tracked_only = false - """ - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_microbatch_history", - repo_files={ - "sqlbuild_project.toml": project_toml, - "sources/raw.yml": ( - "sources:\n - name: raw_events\n schema: raw\n table: raw_events\n" - ), - "models/orders.sql": dedent( - """ - MODEL ( - materialized incremental, - incremental_strategy delete_insert, - incremental_mode microbatch, - microbatch_strategy watermark, - cursor_watermark_mode all, - cursor event_time, - cursor_type timestamp, - cursor_grain hour, - cursor_inputs ( - raw_events (column event_time, roles [filter, watermark]), - ), - batch_size 1h, - batch_concurrency 2, - ); - - SELECT id, event_time - FROM __source("raw_events") - WHERE event_time >= __cursor_start() - AND event_time < __cursor_end() - """ - ).strip() - + "\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw; " - "CREATE TABLE raw.raw_events (id INTEGER, event_time TIMESTAMP); " - "INSERT INTO raw.raw_events VALUES " - "(1, '2026-01-01 00:30:00'), (2, '2026-01-01 01:30:00')" - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - state_db_path: Path = project_dir / "state.duckdb" - event_count_before: object = query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.microbatch_events", - )[0][0] - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - assert int(str(event_count_before)) > 0 - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - assert "microbatch_events" not in janitor_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.microbatch_events", - ) == [(event_count_before,)] - - @pytest.mark.parametrize( "test_case", [ @@ -540,730 +426,6 @@ def test_given_direct_state_history_when_running_janitor_then_it_prunes_history( ) -@pytest.mark.parametrize( - "test_case", - [ - JanitorCheckpointProtectionE2ETestCase( - description="virtual janitor preserves checkpoint referenced physical versions", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_stdout_fragments=( - "eligible for deletion 0", - "relation is referenced by a retained virtual checkpoint", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_checkpoint_refs_when_janitor_then_preserves_physical_versions( - test_case: JanitorCheckpointProtectionE2ETestCase, - tmp_path: Path, -) -> None: - repo_files: dict[str, str] = build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - repo_files["sqlbuild_project.toml"] += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - delete_tracked_only = false - """ - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_checkpoint_protection", - repo_files=repo_files, - ) - state_db_path: Path = project_dir / "state.duckdb" - warehouse_db_path: Path = project_dir / "warehouse.duckdb" - - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - protected_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=state_db_path, - sql=( - "SELECT relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'stg_orders' " - "ORDER BY created_at ASC LIMIT 1" - ), - ) - protected_relation_name: str = str(protected_relation_rows[0][0]) - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert table_exists( - db_path=warehouse_db_path, - schema="dev__sqb_physical", - table_name=protected_relation_name, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorExpiredVirtualEnvironmentE2ETestCase( - description="virtual janitor prunes expired non-active VDEs", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_stdout_fragments=( - "expired VDEs pruned", - "Eligible expired VDEs", - "pr expired virtual environment", - "Deleted 0 objects, deleted 1 state item, and pruned 0 direct state tables.", - ), - expected_virtual_environment_names_after=("dev",), - ) - ], - ids=lambda case: case.description, -) -def test_given_non_active_vde_when_running_janitor_then_it_prunes_expired_environment( - test_case: JanitorExpiredVirtualEnvironmentE2ETestCase, - tmp_path: Path, -) -> None: - repo_files: dict[str, str] = build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - repo_files["sqlbuild_project.toml"] += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - delete_tracked_only = false - """ - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_expired_vde", - repo_files=repo_files, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - pr_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - assert pr_build_result.returncode == 0, pr_build_result.stdout + pr_build_result.stderr - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT virtual_environment_name FROM sqlbuild_state.virtual_environments " - "ORDER BY virtual_environment_name" - ), - ) == [(name,) for name in test_case.expected_virtual_environment_names_after] - assert not table_exists( - db_path=project_dir / "warehouse.duckdb", - schema="dev__pr", - table_name="fact_orders", - ) - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorStateCleanupE2ETestCase( - description="virtual janitor prunes old backups and expired locks", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_stdout_fragments=( - "state backups pruned", - "expired locks pruned", - "Eligible state backups", - "Eligible expired locks", - "Deleted 0 objects, deleted 2 state items, and pruned 0 direct state tables.", - ), - expected_backup_schema_count_after=1, - expected_lock_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_state_backups_and_expired_locks_when_running_janitor_then_state_is_pruned( - test_case: JanitorStateCleanupE2ETestCase, - tmp_path: Path, -) -> None: - repo_files: dict[str, str] = build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - repo_files["sqlbuild_project.toml"] += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - delete_tracked_only = false - """ - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_state_cleanup", - repo_files=repo_files, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("state", "migrate"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql="UPDATE sqlbuild_state.state_versions SET schema_version = 2", - ) - assert run_sqb(command=("state", "migrate"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:stale', 'owner', TIMESTAMP '2000-01-01 00:00:00', " - "TIMESTAMP '2000-01-01 00:00:00', TIMESTAMP '2000-01-01 00:00:00')" - ), - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM information_schema.schemata " - "WHERE schema_name LIKE 'sqlbuild_state__backup_%'" - ), - ) == [(test_case.expected_backup_schema_count_after,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.locks", - ) == [(test_case.expected_lock_count_after,)] - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorStateCleanupE2ETestCase( - description="virtual janitor keeps state cleanup refs when warehouse cleanup fails", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=1, - expected_stdout_fragments=("simulated janitor drop failure",), - expected_backup_schema_count_after=0, - expected_lock_count_after=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_warehouse_cleanup_failure_when_running_janitor_then_state_cleanup_is_skipped( - test_case: JanitorStateCleanupE2ETestCase, - tmp_path: Path, -) -> None: - repo_files: dict[str, str] = build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - repo_files["sqlbuild_project.toml"] += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - delete_tracked_only = false - """ - ) - repo_files["adapters/failing_janitor_duckdb.py"] = ( - "from typing import Any\n" - "from sqlbuild.adapter.contract.exceptions import AdapterUserError\n" - "from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder\n" - "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n" - "class FailingJanitorDuckDbAdapter(DuckDbAdapter):\n" - " adapter_name = 'failing_janitor_duckdb'\n\n" - " def drop(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " destination: str,\n" - " if_exists: bool = True,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated janitor drop failure')\n" - "\n" - " def drop_view(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " destination: str,\n" - " if_exists: bool = True,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated janitor drop failure')\n" - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_warehouse_cleanup_failure", - repo_files=repo_files, - ) - state_db_path: Path = project_dir / "state.duckdb" - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=state_db_path, - sql=( - "INSERT INTO sqlbuild_state.virtual_environments " - "(virtual_environment_name, status, baseline_virtual_environment_name, " - "created_at, updated_at, finalized_at) VALUES " - "('stale', 'finalized', NULL, TIMESTAMP '2000-01-01 00:00:00', " - "TIMESTAMP '2000-01-01 00:00:00', TIMESTAMP '2000-01-01 00:00:00')" - ), - ) - execute_duckdb( - db_path=state_db_path, - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:stale', 'owner', TIMESTAMP '2000-01-01 00:00:00', " - "TIMESTAMP '2000-01-01 00:00:00', TIMESTAMP '2000-01-01 00:00:00')" - ), - ) - (project_dir / "sqlbuild_local.toml").write_text( - 'adapter = "failing_janitor_duckdb"\n', encoding="utf-8" - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code - combined_output: str = janitor_result.stdout + janitor_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - assert query_duckdb( - db_path=state_db_path, - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environments " - "WHERE virtual_environment_name = 'stale'" - ), - ) == [(1,)] - assert query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.locks WHERE lock_key = 'virtual_env:stale'", - ) == [(test_case.expected_lock_count_after,)] - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorCheckpointRetentionE2ETestCase( - description="virtual janitor prunes old checkpoints and newly unprotected physicals", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_checkpoint_count_before=3, - expected_checkpoint_count_after=1, - expected_stdout_fragments=( - "checkpoints pruned", - "Eligible checkpoints", - "Deleted ", - "2 checkpoints.", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_checkpoints_over_limit_when_running_janitor_then_it_prunes_old_history( - test_case: JanitorCheckpointRetentionE2ETestCase, - tmp_path: Path, -) -> None: - repo_files: dict[str, str] = build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - repo_files["sqlbuild_project.toml"] += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - max_checkpoints = 1 - delete_tracked_only = false - """ - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_checkpoint_retention", - repo_files=repo_files, - ) - state_db_path: Path = project_dir / "state.duckdb" - warehouse_db_path: Path = project_dir / "warehouse.duckdb" - - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - first_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=state_db_path, - sql=( - "SELECT physical_relations.relation_name " - "FROM sqlbuild_state.virtual_environment_node_refs AS refs " - "JOIN sqlbuild_state.physical_relations AS physical_relations " - "ON physical_relations.artifact_type = 'model' " - "AND physical_relations.artifact_name = refs.node_name " - "AND physical_relations.version_hash = refs.version_hash " - "WHERE refs.virtual_environment_name = 'dev' " - "AND refs.node_type = 'model' AND refs.node_name = 'stg_orders'" - ), - ) - first_relation_name: str = str(first_relation_rows[0][0]) - - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 3 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - latest_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=state_db_path, - sql=( - "SELECT physical_relations.relation_name " - "FROM sqlbuild_state.virtual_environment_node_refs AS refs " - "JOIN sqlbuild_state.physical_relations AS physical_relations " - "ON physical_relations.artifact_type = 'model' " - "AND physical_relations.artifact_name = refs.node_name " - "AND physical_relations.version_hash = refs.version_hash " - "WHERE refs.virtual_environment_name = 'dev' " - "AND refs.node_type = 'model' AND refs.node_name = 'stg_orders'" - ), - ) - latest_relation_name: str = str(latest_relation_rows[0][0]) - checkpoint_rows_before: list[tuple[object, ...]] = query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_checkpoints", - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert int(checkpoint_rows_before[0][0]) == test_case.expected_checkpoint_count_before - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - checkpoint_rows_after: list[tuple[object, ...]] = query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_checkpoints", - ) - assert int(checkpoint_rows_after[0][0]) == test_case.expected_checkpoint_count_after - assert not table_exists( - db_path=warehouse_db_path, - schema="dev__sqb_physical", - table_name=first_relation_name, - ) - assert table_exists( - db_path=warehouse_db_path, - schema="dev__sqb_physical", - table_name=latest_relation_name, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorDetachedVirtualEnvironmentE2ETestCase( - description="virtual janitor prunes detached VDE refs and physicals", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_stdout_fragments=( - "eligible for deletion 1", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "Deleted 1 object, deleted 1 state item, and pruned 0 direct state tables.", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_detached_vde_when_running_janitor_then_it_cleans_refs_and_physical_versions( - test_case: JanitorDetachedVirtualEnvironmentE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_detached_vde", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_janitor_detached_vde"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n\n' - "[janitor]\n" - "enabled = true\n" - "retention_days = 0\n" - "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - warehouse_db_path: Path = project_dir / "warehouse.duckdb" - execute_duckdb( - db_path=warehouse_db_path, - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - assert adopt_result.returncode == 0, adopt_result.stdout + adopt_result.stderr - physical_relation_name: str = str( - query_duckdb( - db_path=state_db_path, - sql=( - "SELECT relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders'" - ), - )[0][0] - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - assert detach_result.returncode == 0, detach_result.stdout + detach_result.stderr - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environments", - ) == [(test_case.expected_virtual_environment_count_after,)] - assert query_duckdb( - db_path=state_db_path, - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model'" - ), - ) == [(test_case.expected_ref_count_after,)] - assert not table_exists( - db_path=warehouse_db_path, - schema="dev__sqb_physical", - table_name=physical_relation_name, - ) - assert table_exists(db_path=warehouse_db_path, schema="dev", table_name="orders") - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorDetachedVirtualEnvironmentRetentionE2ETestCase( - description="virtual janitor prunes only detached VDEs older than retention", - retention_days=7, - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_stdout_fragments=( - "7 days", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "Deleted 0 objects, deleted 1 state item, and pruned 0 direct state tables.", - ), - expected_virtual_environment_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_old_detached_vde_when_running_janitor_then_retention_allows_cleanup( - test_case: JanitorDetachedVirtualEnvironmentRetentionE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_detached_vde_retention", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_janitor_detached_vde_retention"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n\n' - "[janitor]\n" - "enabled = true\n" - f"retention_days = {test_case.retention_days}\n" - "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - warehouse_db_path: Path = project_dir / "warehouse.duckdb" - execute_duckdb( - db_path=warehouse_db_path, - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - assert adopt_result.returncode == 0, adopt_result.stdout + adopt_result.stderr - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - assert detach_result.returncode == 0, detach_result.stdout + detach_result.stderr - execute_duckdb( - db_path=state_db_path, - sql=( - "UPDATE sqlbuild_state.virtual_environments " - "SET updated_at = TIMESTAMP '2026-01-01 00:00:00' " - "WHERE virtual_environment_name = 'dev'" - ), - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environments", - ) == [(test_case.expected_virtual_environment_count_after,)] - - -@pytest.mark.parametrize( - "test_case", - [ - JanitorActiveVirtualEnvironmentProtectionE2ETestCase( - description="virtual janitor preserves active working VDE refs", - janitor_command=("janitor", "--auto-approve"), - expected_exit_code=0, - expected_stdout_fragments=( - "eligible for deletion 0", - "relation is referenced by an active or retained virtual environment", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_active_vde_ref_when_running_janitor_then_it_preserves_physical_version( - test_case: JanitorActiveVirtualEnvironmentProtectionE2ETestCase, - tmp_path: Path, -) -> None: - repo_files: dict[str, str] = build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - repo_files["sqlbuild_project.toml"] += dedent( - """ - - [janitor] - enabled = true - retention_days = 0 - delete_tracked_only = false - """ - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_janitor_active_vde_protection", - repo_files=repo_files, - ) - state_db_path: Path = project_dir / "state.duckdb" - warehouse_db_path: Path = project_dir / "warehouse.duckdb" - - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "stg_orders"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - protected_relation_name: str = str( - query_duckdb( - db_path=state_db_path, - sql=( - "SELECT relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'stg_orders'" - ), - )[0][0] - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.janitor_command, - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert table_exists( - db_path=warehouse_db_path, - schema="dev__sqb_physical", - table_name=protected_relation_name, - ) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py index 26a69b025e..7ea97ccdd3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/_test_types.py @@ -2,65 +2,28 @@ @dataclass(frozen=True) -class DirectModeVirtualFlagGuardE2ETestCase: +class RemovedInterfaceTestCase: description: str command: tuple[str, ...] - expected_error_fragment: str - expected_exit_code: int = 1 - - -@dataclass(frozen=True) -class VirtualPlanE2ETestCase: - description: str - seed_matching_refs: bool - command: tuple[str, ...] - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () + expected_error: str + expected_exit_code: int = 2 @dataclass(frozen=True) -class VirtualPlanJsonE2ETestCase: +class RemovedConfigTestCase: description: str - expected_json_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class VirtualPlanSelectionGuardE2ETestCase: - description: str - command: tuple[str, ...] - expected_exit_code: int - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () - - -@dataclass(frozen=True) -class VirtualChangesOnlyCurrentSeedParityE2ETestCase: - description: str - expected_plan_selected_fragment: str - expected_kept_model: str - expected_pruned_seed: str - - -@dataclass(frozen=True) -class VirtualPlanBuildParityE2ETestCase: - description: str - expected_plan_exit_code: int - expected_build_exit_code: int - expected_fragments: tuple[str, ...] - unexpected_plan_fragments: tuple[str, ...] = () + content: str + expected_key: str + filename: str = "sqlbuild_project.toml" + prefix: str = 'name = "orders"\nadapter = "duckdb"\n' + expected_exit_code: int = 1 @dataclass(frozen=True) -class VirtualSourceFreshnessPlanE2ETestCase: +class RemovedHookTestCase: description: str - expected_unchanged_fragments: tuple[str, ...] - expected_fragments: tuple[str, ...] - unexpected_fragments: tuple[str, ...] = () - project_suffix: str | None = None - data_version_sql: str | None = None - include_freshness: bool = True - source_freshness_type: str = "timestamp" - warehouse_column_type: str = "TIMESTAMP" + expected_error: str + expected_exit_code: int = 1 @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py deleted file mode 100644 index 4df324596c..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ /dev/null @@ -1,279 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path -from typing import Any, cast - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.pipeline.main.graph import build_project_graph -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.virtual.planner._helpers.planning import ( - build_expected_local_hashes, - build_expected_version_hashes, - build_model_fingerprint_metadata_jsons, -) -from sqlbuild.virtual.state._helpers.state_runtime.backend import build_state_backend -from sqlbuild.virtual.state._helpers.state_runtime.config import resolve_state_backend_config -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.main.encoding._encode_state_text import encode_state_text -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - StateBackendConfig, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import ModelVersionStatus, VirtualEnvironmentStatus -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -def build_virtual_plan_project_toml() -> str: - return ( - 'name = "virtual_plan_project"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ) - - -def direct_model_version_hashes(*, db_path: Path, model_name: str) -> list[tuple[object, ...]]: - return query_duckdb( - db_path=db_path, - sql=( - "SELECT version_hash FROM main._sqlbuild_fingerprints " - f"WHERE node_name = '{model_name}' ORDER BY ts" - ), - ) - - -def only_json_model(payload: dict[str, object]) -> dict[str, object]: - models: list[dict[str, object]] = cast(list[dict[str, object]], payload["models"]) - assert len(models) == 1, payload - return models[0] - - -def build_virtual_plan_repo_files( - *, stg_orders_sql: str, dim_customers_sql: str = "SELECT 1 AS customer_id" -) -> dict[str, str]: - return { - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "models/stg_orders.sql": f"MODEL ();\n\n{stg_orders_sql}\n", - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - "models/dim_customers.sql": f"MODEL ();\n\n{dim_customers_sql}\n", - } - - -def prepare_virtual_run_despite_unchanged_project( - *, - tmp_path: Path, - project_name: str, - run_despite_unchanged: str, - data_version_sql: str, - include_freshness: bool = True, - source_freshness_type: str = "timestamp", - warehouse_column_type: str = "TIMESTAMP", -) -> Path: - freshness_fragment: str = { - False: "", - True: ( - "\n" - " freshness:\n" - " strategy: column\n" - " column: order_ts\n" - f" type: {source_freshness_type}" - ), - }[include_freshness] - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name=project_name, - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " schema: raw\n" - " table: raw_orders" - f"{freshness_fragment}\n" - ), - "models/rolling_orders.sql": ( - f"MODEL (materialized table, run_despite_unchanged {run_despite_unchanged});\n\n" - 'SELECT id, order_ts FROM __source("raw_orders")\n' - ), - "models/orders_mart.sql": ( - 'MODEL (materialized table);\n\nSELECT id, order_ts FROM __ref("rolling_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA raw;\n" - f"CREATE TABLE raw.raw_orders (id INTEGER, order_ts {warehouse_column_type});\n" - f"INSERT INTO raw.raw_orders VALUES (7, {data_version_sql});" - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - return project_dir - - -def prepare_python_lifecycle_plan_project(*, tmp_path: Path) -> Path: - return prepare_inline_project( - tmp_path=tmp_path, - project_name="python_lifecycle_plan_project", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "python_lifecycle_plan_project"\n' - 'adapter = "duckdb"\n\n' - "[connection]\n" - 'database = "warehouse.duckdb"\n' - ), - "python/loaders/window.py": ( - "from sqlbuild.loaders import loader\n\n" - "@loader(\n" - " destination='window_orders',\n" - " write_strategy='table',\n" - " columns=[{'name': 'order_id', 'type': 'INTEGER'}],\n" - ")\n" - "def load_window_orders(ctx):\n" - " return [{'order_id': 7}]\n" - ), - "python/tasks/prepare.py": ( - "from python.loaders.window import load_window_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=load_window_orders)\n" - "def prepare_orders(ctx):\n" - " return ctx.result(payload={'order_id': 7})\n" - ), - "python/assets/prepare.py": ( - "from python.tasks.prepare import prepare_orders\n" - "from sqlbuild.assets import asset\n\n" - "@asset(depends_on=prepare_orders)\n" - "def publish_prepared_orders(ctx):\n" - " return ctx.result(\n" - " payload=ctx.result_of(node_function=prepare_orders).payload," - " materialized=True\n" - " )\n" - ), - "python/loaders/raw.py": ( - "from python.assets.prepare import publish_prepared_orders\n" - "from sqlbuild.loaders import loader\n\n" - "@loader(depends_on=(publish_prepared_orders,))\n" - "def raw_orders(ctx):\n" - " return [{'order_id': 7}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' - ), - "python/tasks/profile.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=model('fact_orders'))\n" - "def profile_fact_orders(ctx):\n" - " return ctx.result(payload={'rows': 1})\n" - ), - "python/tasks/notify.py": ( - "from python.tasks.profile import profile_fact_orders\n" - "from sqlbuild.tasks import task\n\n" - "@task(depends_on=profile_fact_orders)\n" - "def notify_fact_orders(ctx):\n" - " return ctx.result(" - "metadata=ctx.result_of(node_function=profile_fact_orders).payload)\n" - ), - }, - ) - - -def seed_matching_virtual_refs( - *, project_dir: Path, source_project_dir: Path | None = None -) -> None: - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - adapter: DuckDbAdapter = DuckDbAdapter() - effective_source_project_dir: Path = source_project_dir or project_dir - source_discovered_inputs: DiscoveredProjectInputs = discover_project_inputs( - project_dir=effective_source_project_dir - ) - graph: ProjectGraph = build_project_graph( - discovered_inputs=source_discovered_inputs, - adapter=adapter, - ) - expected_local_hashes: dict[str, str] = build_expected_local_hashes(graph=graph) - expected_hashes: dict[str, str] = build_expected_version_hashes( - graph=graph, - expected_local_hashes=expected_local_hashes, - ) - metadata_jsons: dict[str, str] = build_model_fingerprint_metadata_jsons(graph=graph) - config: StateBackendConfig = resolve_state_backend_config( - discovered_inputs=discovered_inputs, project_dir=project_dir - ) - backend: StateBackend = build_state_backend(config.backend) - connection: Any = backend.connect(config.connection) - try: - query_sqls: dict[str, str] = {model.name: model.query_sql for model in graph.project.models} - model_names: tuple[str, ...] = tuple(model.name for model in graph.project.models) - model_name: str - for model_name in model_names: - backend.upsert_model_version( - connection=connection, - schema=config.schema, - record=ModelVersionRecord( - model_name=model_name, - version_hash=expected_hashes[model_name], - definition_identity_hash=expected_local_hashes[model_name], - identity_metadata_hash=expected_hashes[model_name], - status=ModelVersionStatus.READY, - definition_text_b64=encode_state_text(query_sqls[model_name]), - identity_metadata_json_b64=encode_state_text(metadata_jsons[model_name]), - ), - ) - backend.upsert_virtual_environment( - connection=connection, - schema=config.schema, - record=VirtualEnvironmentRecord( - virtual_environment_name="dev", - status=VirtualEnvironmentStatus.FINALIZED, - ), - ) - backend.replace_virtual_environment_model_refs( - connection=connection, - schema=config.schema, - virtual_environment_name="dev", - refs=tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name="dev", - model_name=model_name, - version_hash=expected_hashes[model_name], - ) - for model_name in model_names - ), - ) - finally: - backend.close(connection) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_direct_mode_virtual_flag_guards.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_direct_mode_virtual_flag_guards.py deleted file mode 100644 index ce5fda6c8d..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_direct_mode_virtual_flag_guards.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - DirectModeVirtualFlagGuardE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - DirectModeVirtualFlagGuardE2ETestCase( - description="plan rejects --virtual-env on a direct-mode project", - command=("--no-color", "plan", "--virtual-env", "pr_123"), - expected_error_fragment=( - "plan does not support --virtual-env unless virtual_environments = true" - ), - ), - DirectModeVirtualFlagGuardE2ETestCase( - description="plan rejects --include-stale-upstreams on a direct-mode project", - command=("--no-color", "plan", "--include-stale-upstreams"), - expected_error_fragment=( - "plan does not support --include-stale-upstreams unless virtual_environments = true" - ), - ), - DirectModeVirtualFlagGuardE2ETestCase( - description="build rejects --virtual-env on a direct-mode project", - command=("--no-color", "build", "--virtual-env", "pr_123"), - expected_error_fragment=( - "build does not support --virtual-env unless virtual_environments = true" - ), - ), - DirectModeVirtualFlagGuardE2ETestCase( - description="build rejects --include-stale-upstreams on a direct-mode project", - command=("--no-color", "build", "--include-stale-upstreams"), - expected_error_fragment=( - "build does not support --include-stale-upstreams unless " - "virtual_environments = true" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_direct_mode_project_when_passing_virtual_only_flags_then_command_fails( - test_case: DirectModeVirtualFlagGuardE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="direct_mode_flag_guard", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "direct_mode_flag_guard"\n' - 'adapter = "duckdb"\n\n' - "[connection]\n" - 'database = "warehouse.duckdb"\n' - ), - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS order_id\n", - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - combined_output: str = result.stdout + result.stderr - assert test_case.expected_error_fragment in combined_output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py new file mode 100644 index 0000000000..5a90a99e36 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_physical_binding.py @@ -0,0 +1,57 @@ +"""Physical source closure must reach downstream models before execution.""" + +import subprocess +from pathlib import Path + +import duckdb +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import RemovedInterfaceTestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + RemovedInterfaceTestCase("plan rejects downstream missing field", ("plan",), "B002", 1), + RemovedInterfaceTestCase("build rejects before creating models", ("build",), "B002", 1), + ], + ids=lambda case: case.description, +) +def test_given_physical_source_shape_when_rebinding_then_rejects_downstream_before_mutation( + test_case: RemovedInterfaceTestCase, + tmp_path: Path, +) -> None: + database: Path = tmp_path / "orders.duckdb" + with duckdb.connect(str(database)) as connection: + connection.execute("CREATE TABLE raw_orders (id INTEGER)") + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "{database}"\n[rules]\nselect = []\n' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: raw_orders\n table: raw_orders\n schema: main\n" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + 'MODEL (materialized table, schema analytics); SELECT * FROM __source("raw_orders")' + ) + (tmp_path / "models/report.sql").write_text( + 'MODEL (materialized table, schema analytics); SELECT missing FROM __ref("orders")' + ) + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=tmp_path + ) + assert result.returncode == test_case.expected_exit_code + assert test_case.expected_error in result.stdout + result.stderr + with duckdb.connect(str(database)) as connection: + assert ( + connection.execute( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'analytics'" + ).fetchall() + == [] + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py new file mode 100644 index 0000000000..783b12cb8b --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_removed_interfaces.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( + RemovedConfigTestCase, + RemovedHookTestCase, + RemovedInterfaceTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [ + RemovedInterfaceTestCase( + "clone lock flag removed", + ("clone", "--from", "local", "--skip-locked"), + "unrecognized arguments: --skip-locked", + ), + RemovedInterfaceTestCase("promote removed", ("promote",), "invalid choice: 'promote'"), + RemovedInterfaceTestCase("rollback removed", ("rollback",), "invalid choice: 'rollback'"), + RemovedInterfaceTestCase( + "reconcile removed", ("reconcile",), "invalid choice: 'reconcile'" + ), + RemovedInterfaceTestCase("state removed", ("state",), "invalid choice: 'state'"), + RemovedInterfaceTestCase( + "plan environment removed", + ("plan", "--virtual-env"), + "unrecognized arguments: --virtual-env", + ), + RemovedInterfaceTestCase( + "plan stale upstreams removed", + ("plan", "--include-stale-upstreams"), + "unrecognized arguments: --include-stale-upstreams", + ), + RemovedInterfaceTestCase( + "plan changes only removed", + ("plan", "--changes-only"), + "unrecognized arguments: --changes-only", + ), + RemovedInterfaceTestCase( + "build environment removed", + ("build", "--virtual-env"), + "unrecognized arguments: --virtual-env", + ), + RemovedInterfaceTestCase( + "build stale upstreams removed", + ("build", "--include-stale-upstreams"), + "unrecognized arguments: --include-stale-upstreams", + ), + RemovedInterfaceTestCase( + "build changes only removed", + ("build", "--changes-only"), + "unrecognized arguments: --changes-only", + ), + RemovedInterfaceTestCase( + "partial diff removed", + ("diff", "--allow-partial-diff"), + "unrecognized arguments: --allow-partial-diff", + ), + ], + ids=lambda case: case.description, +) +def test_given_removed_interface_when_running_cli_then_reports_unknown_argument( + test_case: RemovedInterfaceTestCase, tmp_path: Path +) -> None: + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), project_dir=tmp_path + ) + assert result.returncode == test_case.expected_exit_code + assert test_case.expected_error in result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + RemovedConfigTestCase( + "enabled virtual setting", + "[settings]\nvirtual_environments = true\n", + "virtual_environments", + ), + RemovedConfigTestCase( + "removed unchanged default", + '[defaults]\nrun_despite_unchanged = "always"\n', + "run_despite_unchanged", + ), + RemovedConfigTestCase( + "removed unchanged path default", + '[path_defaults.staging]\nrun_despite_unchanged = "30d"\n', + "run_despite_unchanged", + ), + RemovedConfigTestCase( + "removed unchanged MODEL field", + "MODEL (materialized table, run_despite_unchanged always); SELECT 1 AS order_id", + "run_despite_unchanged", + "models/orders.sql", + "", + ), + RemovedConfigTestCase( + "disabled virtual setting", + "[settings]\nvirtual_environments = false\n", + "virtual_environments", + ), + RemovedConfigTestCase( + "changes only setting", "[settings]\nchanges_only = false\n", "changes_only" + ), + RemovedConfigTestCase( + "target changes only", "[targets.dev]\nchanges_only = false\n", "changes_only" + ), + RemovedConfigTestCase("target state", "[targets.dev.state]\nbackend = 'duckdb'\n", "state"), + RemovedConfigTestCase( + "checkpoint retention", "[janitor]\nmax_checkpoints = 20\n", "max_checkpoints" + ), + RemovedConfigTestCase( + "local enabled virtual setting", + "[settings]\nvirtual_environments = true\n", + "virtual_environments", + "sqlbuild_local.toml", + "", + ), + RemovedConfigTestCase( + "local disabled virtual setting", + "[settings]\nvirtual_environments = false\n", + "virtual_environments", + "sqlbuild_local.toml", + "", + ), + RemovedConfigTestCase( + "local changes only setting", + "[settings]\nchanges_only = false\n", + "changes_only", + "sqlbuild_local.toml", + "", + ), + RemovedConfigTestCase( + "local target changes only", + "[targets.dev]\nchanges_only = false\n", + "changes_only", + "sqlbuild_local.toml", + "", + ), + RemovedConfigTestCase( + "local target state", + "[targets.dev.state]\nbackend = 'duckdb'\n", + "state", + "sqlbuild_local.toml", + "", + ), + ], + ids=lambda case: case.description, +) +def test_given_removed_config_when_running_cli_then_explains_direct_mode( + test_case: RemovedConfigTestCase, tmp_path: Path +) -> None: + files: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/orders.sql": "MODEL (materialized view);\nSELECT 1 AS order_id", + } + files[test_case.filename] = test_case.prefix + test_case.content + project: Path = prepare_inline_project( + tmp_path=tmp_path, project_name="removed_config", repo_files=files + ) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "plan"), project_dir=project + ) + assert result.returncode == test_case.expected_exit_code + assert test_case.expected_key in result.stdout + result.stderr + assert ( + "were removed with virtual environments; projects run in direct mode" + in result.stdout + result.stderr + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RemovedHookTestCase( + "removed materialization hook", + "'prepare_version', which was removed with virtual environments", + ) + ], + ids=lambda case: case.description, +) +def test_given_prepare_version_hook_when_running_cli_then_rejects_removed_hook( + test_case: RemovedHookTestCase, tmp_path: Path +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="removed_hook", + repo_files={ + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n', + "models/orders.sql": "MODEL (materialized custom_order);\nSELECT 1 AS order_id", + "materializations/custom_order.py": ( + "from sqlbuild.executor.custom.models import PrepareVersionContext\n\n" + "def prepare_version(ctx: PrepareVersionContext):\n pass\n\n" + "def materialize(ctx):\n pass\n" + ), + }, + ) + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project + ) + assert result.returncode == test_case.expected_exit_code + assert test_case.expected_error in result.stdout + result.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_freshness.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_freshness.py deleted file mode 100644 index 96f66ce588..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_freshness.py +++ /dev/null @@ -1,476 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from pathlib import Path -from textwrap import dedent - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - VirtualPlanJsonE2ETestCase, - VirtualSourceFreshnessPlanE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual plan observes current source freshness before build persistence", - expected_unchanged_fragments=( - "Plan ready 0 selected", - "Source freshness (1 of 1 checked)", - "unchanged (1) raw_orders", - ), - expected_fragments=( - "Plan ready 1 selected", - "Source freshness (1 of 1 checked)", - "new or changed (1) raw_orders", - "Models needing build (1)", - "downstream affected (1) fact_orders", - "fact_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_change_when_planning_then_selects_downstream_model( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_plan", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: integer - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); - INSERT INTO raw.raw_orders VALUES (7, 1); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - unchanged_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=project_dir, - ) - assert unchanged_plan_result.returncode == 0, unchanged_plan_result.stderr - assert "Plan ready 1 selected" in unchanged_plan_result.stdout - assert "fact_orders" in unchanged_plan_result.stdout - for fragment in test_case.expected_unchanged_fragments[1:]: - assert fragment in unchanged_plan_result.stdout, unchanged_plan_result.stdout - - unchanged_changes_only_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert unchanged_changes_only_plan_result.returncode == 0, ( - unchanged_changes_only_plan_result.stderr - ) - fragment: str - for fragment in test_case.expected_unchanged_fragments: - assert fragment in unchanged_changes_only_plan_result.stdout, ( - unchanged_changes_only_plan_result.stdout - ) - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = 2", - ) - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - - assert plan_result.returncode == 0, plan_result.stderr - output: str = plan_result.stdout - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual plan respects timestamp source freshness lag tolerance", - expected_unchanged_fragments=( - "Plan ready 0 selected", - "Source freshness (1 of 1 checked)", - "unchanged (1) raw_orders", - ), - expected_fragments=("Plan ready 1 selected", "fact_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_timestamp_lag_tolerance_when_planning_then_skips_within_tolerance( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_plan_lag_tolerance", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: timestamp - lag_tolerance: 10m - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version TIMESTAMP); - INSERT INTO raw.raw_orders VALUES (7, '2026-01-01 12:00:00'); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = '2026-01-01 12:05:00'", - ) - within_tolerance_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), project_dir=project_dir - ) - - assert within_tolerance_plan_result.returncode == 0, within_tolerance_plan_result.stderr - fragment: str - for fragment in test_case.expected_unchanged_fragments: - assert fragment in within_tolerance_plan_result.stdout, within_tolerance_plan_result.stdout - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 9, data_version = '2026-01-01 12:11:00'", - ) - beyond_tolerance_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), project_dir=project_dir - ) - - assert beyond_tolerance_plan_result.returncode == 0, beyond_tolerance_plan_result.stderr - for fragment in test_case.expected_fragments: - assert fragment in beyond_tolerance_plan_result.stdout, beyond_tolerance_plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual plan keeps unknown source freshness stale", - expected_unchanged_fragments=( - "Plan ready 1 selected", - "Source freshness (0 of 1 checked)", - "not verifiable (1) raw_orders", - "affected models (1) fact_orders", - "Models needing build (1)", - ), - expected_fragments=("Plan ready 1 selected", "fact_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_without_freshness_when_planning_then_downstream_stays_stale( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_unknown_source_freshness_plan", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER); - INSERT INTO raw.raw_orders VALUES (7); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), project_dir=project_dir - ) - - assert plan_result.returncode == 0, plan_result.stderr - fragment: str - for fragment in test_case.expected_unchanged_fragments: - assert fragment in plan_result.stdout, plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual plan propagates source freshness through views", - expected_unchanged_fragments=("Plan ready 0 selected",), - expected_fragments=( - "Plan ready 2 selected", - "Source freshness (1 of 1 checked)", - "new or changed (1) raw_orders", - "Models needing build (2)", - "downstream affected (2) fact_orders, stg_orders", - "stg_orders", - "fact_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_through_view_when_planning_then_selects_downstream_path( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_view_plan", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: integer - """ - ).strip() - + "\n", - "models/stg_orders.sql": ( - 'MODEL (materialized view);\n\nSELECT id FROM __source("raw_orders")\n' - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __ref("stg_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); - INSERT INTO raw.raw_orders VALUES (7, 1); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - unchanged_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), project_dir=project_dir - ) - assert unchanged_plan_result.returncode == 0, unchanged_plan_result.stderr - fragment: str - for fragment in test_case.expected_unchanged_fragments: - assert fragment in unchanged_plan_result.stdout, unchanged_plan_result.stdout - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = 2", - ) - changed_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), project_dir=project_dir - ) - - assert changed_plan_result.returncode == 0, changed_plan_result.stderr - for fragment in test_case.expected_fragments: - assert fragment in changed_plan_result.stdout, changed_plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanJsonE2ETestCase( - description="virtual plan json explains source freshness currentness", - expected_json_fragments=( - '"virtual_environment_name": "dev"', - '"virtual_environment_status": "working"', - '"virtual_source_freshness_observed_source_names": [', - '"raw_orders"', - '"virtual_source_freshness_incomplete_source_names": []', - '"virtual_source_freshness_incomplete_model_names": []', - '"virtual_stale_model_names": [', - '"virtual_stale_root_names": []', - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_source_freshness_when_planning_json_then_metadata_reports_currentness( - test_case: VirtualPlanJsonE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_source_freshness_plan_json", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_orders - schema: raw - table: raw_orders - freshness: - strategy: column - column: data_version - type: integer - """ - ).strip() - + "\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT id FROM __source("raw_orders")\n' - ), - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_orders (id INTEGER, data_version INTEGER); - INSERT INTO raw.raw_orders VALUES (7, 1); - """ - ).strip(), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - assert build_result.returncode == 0, build_result.stderr - unchanged_result: subprocess.CompletedProcess[str] = run_sqb( - command=("plan", "--json", "--changes-only"), project_dir=project_dir - ) - assert unchanged_result.returncode == 0, unchanged_result.stderr - unchanged_payload: dict[str, object] = json.loads(unchanged_result.stdout) - assert unchanged_payload["metadata"] == { - "virtual_environment_name": "dev", - "virtual_environment_status": "finalized", - "virtual_mode": True, - "virtual_source_freshness_observed_source_names": ["raw_orders"], - "virtual_source_freshness_unchanged_source_names": ["raw_orders"], - "virtual_source_freshness_incomplete_source_names": [], - "virtual_source_freshness_incomplete_model_names": [], - "virtual_stale_model_names": [], - "virtual_stale_root_names": [], - "virtual_remaining_stale_model_names": [], - } - - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="UPDATE raw.raw_orders SET id = 8, data_version = 2", - ) - changed_result: subprocess.CompletedProcess[str] = run_sqb( - command=("plan", "--json"), project_dir=project_dir - ) - - assert changed_result.returncode == 0, changed_result.stderr - output: str = changed_result.stdout - parsed: dict[str, object] = json.loads(output) - assert "metadata" in parsed - fragment: str - for fragment in test_case.expected_json_fragments: - assert fragment in output, output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_functions.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_functions.py deleted file mode 100644 index 0f378475e1..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_functions.py +++ /dev/null @@ -1,195 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - VirtualPlanE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, - seed_matching_virtual_refs, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan shows function-driven query changed root", - seed_matching_refs=True, - command=("--no-color", "plan", "--changes-only"), - expected_fragments=( - "Plan ready 3 selected", - "Models needing build (2)", - "directly affected (1) fact_orders", - "downstream affected (1) orders_rollup", - "Changed functions (1)", - "is_large_order", - "Query changed (1)", - "fact_orders", - "Upstream changed (1)", - "orders_rollup", - "cause is_large_order (function changed)", - ), - unexpected_fragments=("cause fact_orders", "stg_orders", "First run"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_with_function_change_when_running_cli_then_it_marks_dependents_stale( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - fact_orders_sql: str = ( - 'MODEL ();\n\nSELECT __udf("is_large_order")(id) AS id FROM __ref("stg_orders")\n' - ) - orders_rollup_sql: str = ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ) - baseline_function_sql: str = ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 9\n" - ) - current_function_sql: str = ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 5\n" - ) - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_function_baseline", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "models/fact_orders.sql": fact_orders_sql, - "models/orders_rollup.sql": orders_rollup_sql, - "functions/sql/is_large_order.sql": baseline_function_sql, - }, - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_function_current", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "models/fact_orders.sql": fact_orders_sql, - "models/orders_rollup.sql": orders_rollup_sql, - "functions/sql/is_large_order.sql": current_function_sql, - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "CREATE SCHEMA IF NOT EXISTS dev; " - "CREATE TABLE IF NOT EXISTS dev.stg_orders AS SELECT 1 AS id" - ), - ) - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan after build shows changed function diff", - seed_matching_refs=False, - command=("--no-color", "plan", "--changes-only"), - expected_fragments=( - "Changed functions (1)", - "is_large_order", - "policy replay_on_change=full", - "query diff:", - "--- previous", - "+++ current", - "-amount > 9", - "+amount > 5", - "Upstream changed (1)", - "orders_rollup", - "cause is_large_order (function changed)", - ), - unexpected_fragments=("reason: first run", "cause fact_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_build_then_function_change_when_running_plan_then_it_shows_function_diff( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - initial_function_sql: str = ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 9\n" - ) - changed_function_sql: str = ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, replay_on_change full);\n\n" - "amount > 5\n" - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_function_diff_project", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id") - | { - "models/fact_orders.sql": ( - 'MODEL ();\n\nSELECT __udf("is_large_order")(id) AS id FROM __ref("stg_orders")\n' - ), - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ), - "functions/sql/is_large_order.sql": initial_function_sql, - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - (project_dir / "functions" / "sql" / "is_large_order.sql").write_text( - changed_function_sql, - encoding="utf-8", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py deleted file mode 100644 index e11a8ee0fa..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py +++ /dev/null @@ -1,270 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path -from textwrap import dedent - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - initialize_virtual_seeded_project, - prepare_virtual_seeded_incremental_project, - rewrite_incremental_orders_model, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - VirtualPlanBuildParityE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanBuildParityE2ETestCase( - description="bounded incremental replay and cursor range match build", - expected_plan_exit_code=0, - expected_build_exit_code=0, - expected_fragments=( - "rebuild last 7d", - "policy replay_on_change=bounded-7d", - "range 2026-01-02 \u2192 2026-01-04", - ), - unexpected_plan_fragments=("policy replay_on_change=full",), - ) - ], - ids=lambda case: case.description, -) -def test_given_changed_virtual_incremental_when_planning_bounds_then_matches_build_action( - test_case: VirtualPlanBuildParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_seeded_incremental_project( - tmp_path=tmp_path, - project_name="virtual_bounded_replay_plan_build_parity", - incremental_strategy="delete_insert", - replay_on_change="bounded-7d", - ) - initialize_virtual_seeded_project(project_dir=project_dir) - rewrite_incremental_orders_model( - project_dir=project_dir, - incremental_strategy="delete_insert", - replay_on_change="bounded-7d", - amount_expression="amount_cents + 1", - ) - cursor_arguments: tuple[str, ...] = ( - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ) - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", *cursor_arguments), - project_dir=project_dir, - ) - - assert plan_result.returncode == test_case.expected_plan_exit_code, ( - plan_result.stdout + plan_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in plan_result.stdout, plan_result.stdout - for fragment in test_case.unexpected_plan_fragments: - assert fragment not in plan_result.stdout, plan_result.stdout - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", *cursor_arguments), - project_dir=project_dir, - ) - assert build_result.returncode == test_case.expected_build_exit_code, ( - build_result.stdout + build_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in build_result.stdout, build_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanBuildParityE2ETestCase( - description="denied snapshot full refresh is rejected by plan and build", - expected_plan_exit_code=1, - expected_build_exit_code=1, - expected_fragments=( - "error[C238]", - "full refresh is denied for snapshot model 'customer_snapshot'", - "snapshot_full_refresh policy", - ), - unexpected_plan_fragments=("Plan ready",), - ) - ], - ids=lambda case: case.description, -) -def test_given_denied_virtual_snapshot_full_refresh_when_planning_then_matches_build_policy( - test_case: VirtualPlanBuildParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_snapshot_full_refresh_plan_build_parity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "sources/raw.yml": dedent( - """ - sources: - - name: raw_customers - schema: raw - table: raw_customers - """ - ).strip() - + "\n", - "models/customer_snapshot.sql": dedent( - """ - MODEL ( - materialized snapshot, - unique_key [customer_id], - snapshot_strategy timestamp, - updated_at updated_at - ); - - SELECT customer_id, plan, updated_at - FROM __source("raw_customers") - """ - ).strip() - + "\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=dedent( - """ - CREATE SCHEMA raw; - CREATE TABLE raw.raw_customers AS - SELECT 1 AS customer_id, 'basic' AS plan, - TIMESTAMP '2024-01-01 00:00:00' AS updated_at; - """ - ).strip(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--full-refresh"), - project_dir=project_dir, - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--full-refresh"), - project_dir=project_dir, - ) - - assert plan_result.returncode == test_case.expected_plan_exit_code - assert build_result.returncode == test_case.expected_build_exit_code - for result in (plan_result, build_result): - output: str = result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_plan_fragments: - assert fragment not in plan_result.stdout, plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanBuildParityE2ETestCase( - description="managed source without prior observation uses run identity in plan and build", - expected_plan_exit_code=0, - expected_build_exit_code=0, - expected_fragments=( - "Plan ready 1 selected, 1 source to load", - "new or changed (1) raw_orders", - "raw_orders", - "fact_orders", - ), - unexpected_plan_fragments=("not verifiable", "First run"), - ) - ], - ids=lambda case: case.description, -) -def test_given_managed_source_without_observation_when_planning_then_matches_build_identity( - test_case: VirtualPlanBuildParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_managed_source_plan_build_parity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml().replace( - '[targets.dev]\nschema = "dev"\n\n', - '[targets.dev]\nschema = "dev"\ndefer_sources_to = "dev"\n\n', - ), - "python/loaders/raw.py": ( - "from sqlbuild.loaders import loader\n\n" - "@loader\n" - "def raw_orders(ctx):\n" - " return [{'order_id': 7}]\n" - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - ), - "models/fact_orders.sql": ( - 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "DELETE FROM sqlbuild_state.source_freshness_observations " - "WHERE virtual_environment_name = 'dev'" - ), - ) - - command: tuple[str, ...] = ("--no-color", "plan", "--changes-only", "--load") - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=command, - project_dir=project_dir, - ) - - assert plan_result.returncode == test_case.expected_plan_exit_code, ( - plan_result.stdout + plan_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in plan_result.stdout, plan_result.stdout - for fragment in test_case.unexpected_plan_fragments: - assert fragment not in plan_result.stdout, plan_result.stdout - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only", "--load"), - project_dir=project_dir, - ) - assert build_result.returncode == test_case.expected_build_exit_code, ( - build_result.stdout + build_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in build_result.stdout, build_result.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_parity.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_parity.py deleted file mode 100644 index da6f68121a..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_parity.py +++ /dev/null @@ -1,440 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - VirtualChangesOnlyCurrentSeedParityE2ETestCase, - VirtualPlanBuildParityE2ETestCase, - VirtualPlanE2ETestCase, - VirtualSourceFreshnessPlanE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, - build_virtual_plan_repo_files, - seed_matching_virtual_refs, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan shows query changed root and upstream changed descendants", - seed_matching_refs=True, - command=("--no-color", "plan", "--changes-only"), - expected_fragments=( - "Plan ready 2 selected", - "Virtual environment dev (working, build required)", - "Models needing build (2)", - "directly affected (1) stg_orders", - "downstream affected (1) fact_orders", - "Query changed (1)", - "stg_orders", - "Upstream changed (1)", - "fact_orders", - "cause stg_orders (query changed)", - ), - unexpected_fragments=("dim_customers", "First run"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_with_seeded_baseline_when_running_cli_then_it_uses_virtual_reasons( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_project_baseline", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_project", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 2 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual seed change selects seed and downstream model", - expected_unchanged_fragments=("Plan ready 0 selected",), - expected_fragments=( - "Plan ready 2 selected", - "fact_orders", - "order_amounts (seed_changed)", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_seed_change_when_planning_changes_only_then_selects_dependent_model( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_change_plan", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - "SELECT order_id, amount_cents / 100.0 AS amount_dollars " - 'FROM __seed("order_amounts")\n' - ), - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - - default_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=project_dir, - ) - assert default_plan_result.returncode == 0, default_plan_result.stderr - assert "Plan ready 2 selected" in default_plan_result.stdout - assert "order_amounts" in default_plan_result.stdout - assert "fact_orders" in default_plan_result.stdout - - unchanged_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert unchanged_plan_result.returncode == 0, unchanged_plan_result.stderr - for fragment in test_case.expected_unchanged_fragments: - assert fragment in unchanged_plan_result.stdout, unchanged_plan_result.stdout - assert "order_amounts" not in unchanged_plan_result.stdout - assert "fact_orders" not in unchanged_plan_result.stdout - - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", - encoding="utf-8", - ) - changed_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - - assert changed_plan_result.returncode == 0, changed_plan_result.stderr - for fragment in test_case.expected_fragments: - assert fragment in changed_plan_result.stdout, changed_plan_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualChangesOnlyCurrentSeedParityE2ETestCase( - description="virtual changes-only plan matches build when a current seed is pruned", - expected_plan_selected_fragment="Plan ready 1 selected", - expected_kept_model="fact_orders", - expected_pruned_seed="order_amounts", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_current_seed_when_planning_changes_only_then_matches_build( - test_case: VirtualChangesOnlyCurrentSeedParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_changes_only_current_seed_parity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - 'MODEL (materialized table, run_despite_unchanged "always");\n\n' - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert plan_result.returncode == 0, plan_result.stdout + plan_result.stderr - assert test_case.expected_plan_selected_fragment in plan_result.stdout - assert test_case.expected_kept_model in plan_result.stdout - assert "Plan ready 0 selected" not in plan_result.stdout - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - assert test_case.expected_kept_model in build_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanBuildParityE2ETestCase( - description="detached virtual environment is rejected by plan and build", - expected_plan_exit_code=1, - expected_build_exit_code=1, - expected_fragments=("error[S028]", "detached"), - ) - ], - ids=lambda case: case.description, -) -def test_given_detached_virtual_environment_when_planning_and_building_then_both_reject( - test_case: VirtualPlanBuildParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_detached_plan_build_parity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml().replace( - 'schema = "sqlbuild_state"\n\n', - 'schema = "sqlbuild_state"\nunsuffixed_virtual_env = "dev"\n\n', - ), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - assert detach_result.returncode == 0, detach_result.stdout + detach_result.stderr - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - - assert plan_result.returncode == test_case.expected_plan_exit_code - assert build_result.returncode == test_case.expected_build_exit_code - for result in (plan_result, build_result): - output: str = result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanBuildParityE2ETestCase( - description="new virtual environment inherits current baseline read only", - expected_plan_exit_code=0, - expected_build_exit_code=0, - expected_fragments=("Plan ready 0 selected",), - unexpected_plan_fragments=("First run",), - ) - ], - ids=lambda case: case.description, -) -def test_given_new_virtual_environment_when_planning_then_inherits_baseline_without_writing( - test_case: VirtualPlanBuildParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_new_environment_plan_build_parity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - state_db_path: Path = project_dir / "state.duckdb" - pr_ref_count_sql: str = ( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'pr'" - ) - assert query_duckdb(db_path=state_db_path, sql=pr_ref_count_sql) == [(0,)] - - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--virtual-env", "pr", "--changes-only"), - project_dir=project_dir, - ) - - assert plan_result.returncode == test_case.expected_plan_exit_code, ( - plan_result.stdout + plan_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in plan_result.stdout, plan_result.stdout - for fragment in test_case.unexpected_plan_fragments: - assert fragment not in plan_result.stdout, plan_result.stdout - assert query_duckdb(db_path=state_db_path, sql=pr_ref_count_sql) == [(0,)] - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr", "--changes-only"), - project_dir=project_dir, - ) - assert build_result.returncode == test_case.expected_build_exit_code, ( - build_result.stdout + build_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in build_result.stdout, build_result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanBuildParityE2ETestCase( - description="explicit model selection includes stale upstream seed", - expected_plan_exit_code=0, - expected_build_exit_code=0, - expected_fragments=("Plan ready 2 selected", "order_amounts", "fact_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_stale_seed_upstream_when_planning_selected_model_then_matches_build_closure( - test_case: VirtualPlanBuildParityE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_selected_model_stale_seed_plan_build_parity", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - initial_build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert initial_build_result.returncode == 0, ( - initial_build_result.stdout + initial_build_result.stderr - ) - (project_dir / "seeds" / "order_amounts.csv").write_text( - "order_id,amount_cents\n1,200\n", - encoding="utf-8", - ) - - command: tuple[str, ...] = ("--no-color", "plan", "--select", "fact_orders") - plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=command, - project_dir=project_dir, - ) - - assert plan_result.returncode == test_case.expected_plan_exit_code, ( - plan_result.stdout + plan_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in plan_result.stdout, plan_result.stdout - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--select", "fact_orders"), - project_dir=project_dir, - ) - assert build_result.returncode == test_case.expected_build_exit_code, ( - build_result.stdout + build_result.stderr - ) - for fragment in test_case.expected_fragments: - assert fragment in build_result.stdout, build_result.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_runtime_policy.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_runtime_policy.py deleted file mode 100644 index 1e7231f223..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_runtime_policy.py +++ /dev/null @@ -1,384 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from pathlib import Path -from typing import cast - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - VirtualPlanE2ETestCase, - VirtualSourceFreshnessPlanE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, - prepare_virtual_run_despite_unchanged_project, - seed_matching_virtual_refs, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan shows config changed root without query diff", - seed_matching_refs=True, - command=("--no-color", "plan", "--changes-only"), - expected_fragments=( - "Plan ready 2 selected", - "directly affected (1) stg_orders", - "Config changed (1)", - "stg_orders", - "config diff:", - '"materialized": "view"', - '"materialized": "table"', - "Upstream changed (1)", - "fact_orders", - "cause stg_orders (config changed)", - ), - unexpected_fragments=("Query changed", "query diff:"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_with_config_change_when_running_cli_then_it_uses_config_reason( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_config_baseline", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id")[ - "sqlbuild_project.toml" - ], - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - }, - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_config_current", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id")[ - "sqlbuild_project.toml" - ], - "models/stg_orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual changes-only runs configured table despite unchanged freshness", - expected_unchanged_fragments=( - "Plan ready 2 selected", - "Runs despite unchanged (1)", - ), - expected_fragments=( - "Plan ready 2 selected", - "Runs despite unchanged (1)", - "rolling_orders", - "run_despite_unchanged 30d", - "Upstream changed (1)", - "orders_mart", - "cause rolling_orders ran despite unchanged inputs", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_run_despite_unchanged_when_planning_changes_only_then_selects_downstream( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_run_despite_unchanged_project( - tmp_path=tmp_path, - project_name="virtual_run_despite_unchanged_plan", - run_despite_unchanged="30d", - data_version_sql="CURRENT_TIMESTAMP", - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - unchanged_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=project_dir, - ) - assert unchanged_plan_result.returncode == 0, unchanged_plan_result.stderr - fragment: str - for fragment in test_case.expected_unchanged_fragments: - assert fragment in unchanged_plan_result.stdout, unchanged_plan_result.stdout - - changes_only_plan_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert changes_only_plan_result.returncode == 0, changes_only_plan_result.stderr - output: str = changes_only_plan_result.stdout - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual changes-only skips expired run_despite_unchanged duration", - expected_unchanged_fragments=("Plan ready 0 selected",), - expected_fragments=("Plan ready 0 selected",), - unexpected_fragments=("Runs despite unchanged", "rolling_orders", "orders_mart"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_expired_run_despite_unchanged_when_planning_then_skips_table( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_run_despite_unchanged_project( - tmp_path=tmp_path, - project_name="virtual_run_despite_unchanged_expired_plan", - run_despite_unchanged="1d", - data_version_sql="TIMESTAMP '2026-01-01 00:00:00'", - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert result.returncode == 0, result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - for fragment in test_case.unexpected_fragments: - assert fragment not in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual changes-only always runs configured unchanged table", - expected_unchanged_fragments=(), - expected_fragments=( - "Plan ready 2 selected", - "Runs despite unchanged (1)", - "rolling_orders", - "run_despite_unchanged always", - "orders_mart", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_run_despite_unchanged_always_when_planning_then_selects_downstream( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_run_despite_unchanged_project( - tmp_path=tmp_path, - project_name="virtual_run_despite_unchanged_always_plan", - run_despite_unchanged="always", - data_version_sql="TIMESTAMP '2026-01-01 00:00:00'", - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert result.returncode == 0, result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual JSON exposes run_despite_unchanged metadata", - expected_unchanged_fragments=(), - expected_fragments=("rolling_orders", "30d", "raw_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_run_despite_unchanged_when_planning_json_then_outputs_metadata( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_run_despite_unchanged_project( - tmp_path=tmp_path, - project_name="virtual_run_despite_unchanged_json_plan", - run_despite_unchanged="30d", - data_version_sql="CURRENT_TIMESTAMP", - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--json", "--changes-only"), - project_dir=project_dir, - ) - assert result.returncode == 0, result.stdout + result.stderr - payload: dict[str, object] = cast(dict[str, object], json.loads(result.stdout)) - models: list[dict[str, object]] = cast(list[dict[str, object]], payload["models"]) - models_by_name: dict[str, tuple[dict[str, object], ...]] = { - str(model["name"]): (model,) for model in models - } - assert len(models_by_name) == len(models) - root_model: dict[str, object] = next(iter(models_by_name.get("rolling_orders", ()))) - run_metadata: dict[str, object] = cast(dict[str, object], root_model["run_despite_unchanged"]) - - assert root_model["name"] == test_case.expected_fragments[0] - assert payload["selected_count"] == 2 - assert root_model["reason"] == "run_despite_unchanged" - assert run_metadata["mode"] == "duration" - assert run_metadata["duration"] == test_case.expected_fragments[1] - assert run_metadata["newest_source_name"] == test_case.expected_fragments[2] - assert isinstance(run_metadata["newest_source_data_age_seconds"], int) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual scoped runtime stale root leaves downstream remaining stale", - expected_unchanged_fragments=(), - expected_fragments=( - "Plan ready 1 selected", - "Runs despite unchanged (1)", - "rolling_orders", - "outside this plan (1) orders_mart", - ), - unexpected_fragments=("Upstream changed (1)",), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_scoped_run_despite_unchanged_when_planning_then_downstream_remains_stale( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_virtual_run_despite_unchanged_project( - tmp_path=tmp_path, - project_name="virtual_run_despite_unchanged_scoped_plan", - run_despite_unchanged="30d", - data_version_sql="CURRENT_TIMESTAMP", - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == 0, build_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--select", "rolling_orders", "--changes-only"), - project_dir=project_dir, - ) - assert result.returncode == 0, result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout, result.stdout - for fragment in test_case.unexpected_fragments: - assert fragment not in result.stdout, result.stdout - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual duration mode fails without source freshness", - expected_unchanged_fragments=(), - expected_fragments=("cannot determine upstream source freshness age",), - project_suffix="missing", - data_version_sql="TIMESTAMP '2026-06-01 00:00:00'", - include_freshness=False, - source_freshness_type="timestamp", - warehouse_column_type="TIMESTAMP", - ), - VirtualSourceFreshnessPlanE2ETestCase( - description="virtual duration mode fails with integer source freshness", - expected_unchanged_fragments=(), - expected_fragments=("requires timestamp source freshness",), - project_suffix="integer", - data_version_sql="1", - include_freshness=True, - source_freshness_type="integer", - warehouse_column_type="INTEGER", - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_duration_without_timestamp_freshness_when_planning_then_fails( - test_case: VirtualSourceFreshnessPlanE2ETestCase, - tmp_path: Path, -) -> None: - assert test_case.project_suffix is not None - assert test_case.data_version_sql is not None - project_dir: Path = prepare_virtual_run_despite_unchanged_project( - tmp_path=tmp_path, - project_name=f"virtual_run_despite_unchanged_error_{test_case.project_suffix}", - run_despite_unchanged="30d", - data_version_sql=test_case.data_version_sql, - include_freshness=test_case.include_freshness, - source_freshness_type=test_case.source_freshness_type, - warehouse_column_type=test_case.warehouse_column_type, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert result.returncode != 0, result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr, result.stdout + result.stderr diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_selection.py deleted file mode 100644 index 00a22192be..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_selection.py +++ /dev/null @@ -1,364 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan._test_types import ( - VirtualPlanE2ETestCase, - VirtualPlanJsonE2ETestCase, - VirtualPlanSelectionGuardE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, - seed_matching_virtual_refs, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan selects nothing when all bound refs match expected hashes", - seed_matching_refs=True, - command=("--no-color", "plan", "--changes-only"), - expected_fragments=( - "Plan ready 0 selected", - "Virtual environment dev (finalized, up to date)", - "Models needing build (0)", - ), - unexpected_fragments=("First run", "fact_orders", "dim_customers", "stg_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_with_matching_refs_when_running_cli_then_it_selects_nothing( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_project", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs(project_dir=project_dir) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan with multiple stale roots excludes unaffected branches", - seed_matching_refs=True, - command=("--no-color", "plan", "--changes-only"), - expected_fragments=( - "Plan ready 3 selected", - "Models needing build (3)", - "directly affected (2) dim_customers, stg_orders", - "downstream affected (1) fact_orders", - "Query changed (2)", - "stg_orders", - "dim_customers", - "Upstream changed (1)", - "fact_orders", - ), - unexpected_fragments=("static_model",), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_with_multiple_stale_roots_when_running_cli_then_it_excludes_unaffected( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_multi_root_baseline", - repo_files=build_virtual_plan_repo_files( - stg_orders_sql="SELECT 1 AS id", - dim_customers_sql="SELECT 1 AS customer_id", - ) - | {"models/static_model.sql": "MODEL ();\n\nSELECT 99 AS static_value\n"}, - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_multi_root", - repo_files=build_virtual_plan_repo_files( - stg_orders_sql="SELECT 2 AS id", - dim_customers_sql="SELECT 2 AS customer_id", - ) - | {"models/static_model.sql": "MODEL ();\n\nSELECT 99 AS static_value\n"}, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanE2ETestCase( - description="virtual plan explicit select bypasses stale-driven selection", - seed_matching_refs=True, - command=("--no-color", "plan", "--select", "dim_customers"), - expected_fragments=( - "Plan ready 1 selected", - "Virtual environment", - "directly affected (1) stg_orders", - "dim_customers", - ), - unexpected_fragments=("Upstream changed (1)",), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_with_explicit_select_when_running_cli_then_it_bypasses_default_scope( - test_case: VirtualPlanE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_select_baseline", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_select_current", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 2 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanSelectionGuardE2ETestCase( - description="selected downstream with stale upstream blocks", - command=("--no-color", "plan", "--select", "fact_orders", "--changes-only"), - expected_exit_code=1, - expected_fragments=( - "missing stale required upstream models: stg_orders", - "--include-stale-upstreams", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_selected_downstream_with_stale_upstream_when_running_then_it_blocks( - test_case: VirtualPlanSelectionGuardE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_guard_baseline", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_guard_current", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 2 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - output: str = result.stdout + result.stderr - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanSelectionGuardE2ETestCase( - description="include stale upstreams with default pruning narrows scope", - command=( - "--no-color", - "plan", - "--select", - "fact_orders", - "--select", - "dim_customers", - "--include-stale-upstreams", - "--changes-only", - ), - expected_exit_code=0, - expected_fragments=("Plan ready 2 selected", "stg_orders", "fact_orders"), - unexpected_fragments=("dim_customers",), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_include_stale_upstreams_when_running_then_it_expands_minimally( - test_case: VirtualPlanSelectionGuardE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_include_baseline", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_include_current", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 2 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stderr - output: str = result.stdout - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output - for fragment in test_case.unexpected_fragments: - assert fragment not in output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlanJsonE2ETestCase( - description="virtual plan json includes metadata fields", - expected_json_fragments=( - '"metadata"', - '"virtual_environment_name": "dev"', - '"virtual_environment_status": "working"', - '"virtual_stale_model_names"', - '"virtual_stale_root_names"', - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_plan_json_when_running_cli_then_it_includes_virtual_metadata( - test_case: VirtualPlanJsonE2ETestCase, - tmp_path: Path, -) -> None: - baseline_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_json_baseline", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_plan_json_current", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 2 AS id"), - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - seed_matching_virtual_refs( - project_dir=project_dir, - source_project_dir=baseline_project_dir, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("plan", "--json"), - project_dir=project_dir, - ) - - assert result.returncode == 0, result.stderr - output: str = result.stdout - parsed: dict[str, object] = json.loads(output) - assert "metadata" in parsed - fragment: str - for fragment in test_case.expected_json_fragments: - assert fragment in output, output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py index 3925f0d5e7..ea616428b6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -1,19 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class VirtualPlaygroundLifecycleTestCase: - description: str - project_name: str - expected_build_fragments: tuple[str, ...] - expected_test_fragments: tuple[str, ...] - expected_audit_fragments: tuple[str, ...] - expected_scenario_fragments: tuple[str, ...] - expected_branch_fragments: tuple[str, ...] - expected_diff_fragments: tuple[str, ...] - expected_promote_fragments: tuple[str, ...] - - @dataclass(frozen=True) class PythonNodesPlaygroundLifecycleTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py index 13fc14a02a..a0376e58ec 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -10,138 +10,10 @@ from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( PlaygroundCompileBuildTestCase, PythonNodesPlaygroundLifecycleTestCase, - VirtualPlaygroundLifecycleTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb -@pytest.mark.parametrize( - "test_case", - [ - VirtualPlaygroundLifecycleTestCase( - description="virtual playground runs state build branch and promote lifecycle", - project_name="virtual_waffle_shop", - expected_build_fragments=( - "Virtual State Initialized", - "Seeds (1)", - "waffle_price_tiers", - "\u2713 Completed successfully", - ), - expected_test_fragments=( - "Test ready 1 selected", - "test_fact_waffle_orders", - "PASS=1", - ), - expected_audit_fragments=( - "Audit (3 selected", - "not_null (order_id)", - "unique (order_id)", - "PASS=3", - ), - expected_scenario_fragments=( - "Scenario", - "customer_revenue_minimal", - "PASS=1", - ), - expected_branch_fragments=( - "Virtual environment", - "pr (working, build required)", - "\u2713 Completed successfully", - ), - expected_diff_fragments=( - "Virtual diff", - "No schema differences", - ), - expected_promote_fragments=( - "\u2713 Virtual promotion complete", - "pr -> dev", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_playground_when_running_lifecycle_then_it_succeeds( - test_case: VirtualPlaygroundLifecycleTestCase, - tmp_path: Path, -) -> None: - assert ( - run_playground( - PlaygroundCommandRequest( - project_dir=tmp_path, target_path=test_case.project_name, template="virtual" - ) - ) - == 0 - ) - project_dir: Path = tmp_path / test_case.project_name - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), project_dir=project_dir - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), project_dir=project_dir - ) - test_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "test"), project_dir=project_dir - ) - audit_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "audit"), project_dir=project_dir - ) - scenario_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "scenario", "test"), project_dir=project_dir - ) - branch_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ) - diff_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "diff", "dev:pr", "--schema-only"), - project_dir=project_dir, - ) - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - project_dir=project_dir, - ) - - build_output: str = ( - init_result.stdout + init_result.stderr + build_result.stdout + build_result.stderr - ) - assert init_result.returncode == 0 - assert build_result.returncode == 0 - expected_fragment: str - for expected_fragment in test_case.expected_build_fragments: - assert expected_fragment in build_output - - test_output: str = test_result.stdout + test_result.stderr - assert test_result.returncode == 0 - for expected_fragment in test_case.expected_test_fragments: - assert expected_fragment in test_output - - audit_output: str = audit_result.stdout + audit_result.stderr - assert audit_result.returncode == 0 - for expected_fragment in test_case.expected_audit_fragments: - assert expected_fragment in audit_output - - scenario_output: str = scenario_result.stdout + scenario_result.stderr - assert scenario_result.returncode == 0 - for expected_fragment in test_case.expected_scenario_fragments: - assert expected_fragment in scenario_output - - branch_output: str = branch_result.stdout + branch_result.stderr - assert branch_result.returncode == 0 - for expected_fragment in test_case.expected_branch_fragments: - assert expected_fragment in branch_output - - diff_output: str = diff_result.stdout + diff_result.stderr - assert diff_result.returncode == 0 - for expected_fragment in test_case.expected_diff_fragments: - assert expected_fragment in diff_output - - promote_output: str = promote_result.stdout + promote_result.stderr - assert promote_result.returncode == 0 - for expected_fragment in test_case.expected_promote_fragments: - assert expected_fragment in promote_output - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py index 10f65d581e..f76a553705 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/_test_types.py @@ -127,123 +127,6 @@ class PostgresDbtProfileE2ETestCase: expected_return_code: int = 0 -@dataclass(frozen=True) -class PostgresStateLifecycleE2ETestCase: - description: str - expected_exit_code: int - expected_schema_version: int - expected_init_fragment: str = "Virtual State Initialized" - expected_migrate_fragment: str = "Virtual State Migrated" - expected_rollback_fragment: str = "Virtual State Rolled Back" - expected_reset_fragment: str = "Virtual State Reset" - - -@dataclass(frozen=True) -class PostgresStateAdoptDetachE2ETestCase: - description: str - expected_exit_code: int - expected_rows_after_adopt: tuple[tuple[object, ...], ...] - expected_rows_after_detach: tuple[tuple[object, ...], ...] - expected_detached_status: str - - -@dataclass(frozen=True) -class PostgresStateAdoptDetachErrorE2ETestCase: - description: str - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class PostgresReconcileE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_exit_code: int = 0 - input_text: str = "" - - -@dataclass(frozen=True) -class PostgresVirtualSeedE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_seed_strategy: str - incremental_strategy: str = "delete_insert" - expected_exit_code: int = 0 - - -@dataclass(frozen=True) -class PostgresVirtualRollbackE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_checkpoint_count: int - expected_exit_code: int = 0 - - -@dataclass(frozen=True) -class PostgresVirtualParityE2ETestCase: - description: str - expected_stdout_fragments: tuple[str, ...] - expected_rows: tuple[tuple[object, ...], ...] = field(default_factory=tuple) - expected_exit_code: int = 0 - - -@dataclass(frozen=True) -class PostgresJanitorDetachedVdeE2ETestCase: - description: str - expected_exit_code: int - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - expected_ref_count_after: int - - -@dataclass(frozen=True) -class PostgresStateLifecycleErrorE2ETestCase: - description: str - allow_reset: bool - command: tuple[str, ...] - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class PostgresStateExplicitRollbackE2ETestCase: - description: str - expected_exit_code: int - expected_schema_version: int - expected_rollback_fragment: str = "Virtual State Rolled Back" - - -@dataclass(frozen=True) -class PostgresStateLocalOverrideE2ETestCase: - description: str - expected_exit_code: int - expected_schema_version: int - - -@dataclass(frozen=True) -class PostgresStateConnectionErrorE2ETestCase: - description: str - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class PostgresStateResetInvalidE2ETestCase: - description: str - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class PostgresStateSchemaCorruptionE2ETestCase: - description: str - mutation_sql_template: str - expected_exit_code: int - expected_error_fragment: str - - @dataclass(frozen=True) class PostgresDbtSeedChangeE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py index dae8b3c63a..a30f2848ac 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/helpers.py @@ -113,114 +113,6 @@ def build_postgres_project_toml( ) -def build_postgres_virtual_project_toml( - *, - project_name: str, - config: dict[str, object], - state_schema: str, - warehouse_schema: str, - unsuffixed_virtual_env: str | None = None, -) -> str: - unsuffixed_line: str = {None: ""}.get( - unsuffixed_virtual_env, - f'unsuffixed_virtual_env = "{unsuffixed_virtual_env}"\n', - ) - return ( - f'name = "{project_name}"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'dbname = "{config["dbname"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n' - f"{unsuffixed_line}\n" - "[targets.dev.state.connection]\n" - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'dbname = "{config["dbname"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n' - ) - - -def build_postgres_virtual_plan_repo_files( - *, - project_name: str, - config: dict[str, object], - state_schema: str, - warehouse_schema: str, - stg_orders_sql: str, - dim_customers_sql: str = "SELECT 1 AS customer_id", - extra_project_toml: str = "", -) -> dict[str, str]: - return { - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name=project_name, - config=config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ) - + extra_project_toml, - "models/stg_orders.sql": f"MODEL ();\n\n{stg_orders_sql}\n", - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - "models/dim_customers.sql": f"MODEL ();\n\n{dim_customers_sql}\n", - } - - -def build_postgres_virtual_clone_project_toml( - *, - project_name: str, - config: dict[str, object], - prod_state_schema: str, - dev_state_schema: str, - prod_warehouse_schema: str, - dev_warehouse_schema: str, -) -> str: - connection_toml: str = ( - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'dbname = "{config["dbname"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n' - ) - return ( - f'name = "{project_name}"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f"{connection_toml}\n" - "[targets.prod]\n" - f'schema = "{prod_warehouse_schema}"\n\n' - "[targets.prod.clone]\n" - "allow_as_clone_origin = true\n\n" - "[targets.prod.state]\n" - 'backend = "postgres"\n' - f'schema = "{prod_state_schema}"\n\n' - "[targets.prod.state.connection]\n" - f"{connection_toml}\n" - "[targets.dev]\n" - f'schema = "{dev_warehouse_schema}"\n\n' - "[targets.dev.clone]\n" - "allow_as_clone_destination = true\n\n" - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{dev_state_schema}"\n\n' - "[targets.dev.state.connection]\n" - f"{connection_toml}" - ) - - def build_postgres_source_deferral_project_toml( *, project_name: str, dev_schema_name: str, prod_schema_name: str, config: dict[str, object] ) -> str: @@ -262,30 +154,6 @@ def cleanup_postgres_schema(*, schema_name: str, config: dict[str, object]) -> N adapter.close(connection) -def cleanup_postgres_state_schemas(*, schema_name: str, config: dict[str, object]) -> None: - adapter: PostgresAdapter = PostgresAdapter() - connection: Any = adapter.connect(config) - try: - adapter.execute( - connection=connection, - sql=f"DROP SCHEMA IF EXISTS {quote_identifier(schema_name)} CASCADE", - ) - cursor: Any = adapter.execute( - connection=connection, - sql="SELECT schema_name FROM information_schema.schemata " - f"WHERE schema_name LIKE '{schema_name}__backup_%'", - ) - backup_schemas: tuple[str, ...] = tuple(str(row[0]) for row in cursor.fetchall()) - backup_schema: str - for backup_schema in backup_schemas: - adapter.execute( - connection=connection, - sql=f"DROP SCHEMA IF EXISTS {quote_identifier(backup_schema)} CASCADE", - ) - finally: - adapter.close(connection) - - def fetch_postgres_rows(*, sql: str, config: dict[str, object]) -> tuple[tuple[object, ...], ...]: adapter: PostgresAdapter = PostgresAdapter() connection: Any = adapter.connect(config) @@ -309,66 +177,6 @@ def relation_name(*, schema_name: str, name: str) -> str: return f"{schema_name}.{name}" -def quote_identifier(identifier: str) -> str: - return '"' + identifier.replace('"', '""') + '"' - - -def quoted_relation_name(*, schema_name: str, name: str) -> str: - return f"{quote_identifier(schema_name)}.{quote_identifier(name)}" - - -def postgres_relation_row_count(*, schema_name: str, name: str, config: dict[str, object]) -> int: - rows: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=f"SELECT COUNT(*) FROM {relation_name(schema_name=schema_name, name=name)}", - config=config, - ) - return int(str(rows[0][0])) - - -def prepare_postgres_diff_project( - *, tmp_path: Path, config: dict[str, object] -) -> tuple[Path, str, str]: - prod_schema: str = build_unique_schema_name(prefix="sqb_diff_prod") - dev_schema: str = build_unique_schema_name(prefix="sqb_diff_dev") - project_dir: Path = tmp_path / "postgres_diff_project" - project_dir.mkdir(parents=True, exist_ok=True) - project_contents: str = ( - 'name = "postgres_diff_project"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[connection]\n" - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'dbname = "{config["dbname"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{dev_schema}"\n\n' - "[targets.dev.clone]\n" - "allow_as_clone_origin = true\n" - "allow_as_clone_destination = true\n\n" - "[targets.prod]\n" - f'schema = "{prod_schema}"\n\n' - "[targets.prod.clone]\n" - "allow_as_clone_origin = true\n" - "allow_as_clone_destination = false\n\n" - "[defaults]\n" - 'materialized = "table"\n' - ) - (project_dir / "sqlbuild_project.toml").write_text(project_contents, encoding="utf-8") - models_dir: Path = project_dir / "models" / "staging" - models_dir.mkdir(parents=True, exist_ok=True) - (models_dir / "stg_orders.sql").write_text( - "MODEL (materialized table, unique_key [order_id]);\n\n" - "SELECT * FROM (" - "SELECT 1 AS order_id, 1 AS customer_id, 100 AS amount_cents UNION ALL " - "SELECT 2 AS order_id, 2 AS customer_id, 200 AS amount_cents" - ")", - encoding="utf-8", - ) - return project_dir, prod_schema, dev_schema - - def assert_current_postgres_snapshot_rows_from_case( *, schema_name: str, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_state.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_state.py deleted file mode 100644 index 45986322d4..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_state.py +++ /dev/null @@ -1,5208 +0,0 @@ -from __future__ import annotations - -import json -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.postgres._test_types import ( - PostgresJanitorDetachedVdeE2ETestCase, - PostgresReconcileE2ETestCase, - PostgresStateAdoptDetachE2ETestCase, - PostgresStateAdoptDetachErrorE2ETestCase, - PostgresStateConnectionErrorE2ETestCase, - PostgresStateExplicitRollbackE2ETestCase, - PostgresStateLifecycleE2ETestCase, - PostgresStateLifecycleErrorE2ETestCase, - PostgresStateLocalOverrideE2ETestCase, - PostgresStateResetInvalidE2ETestCase, - PostgresStateSchemaCorruptionE2ETestCase, - PostgresVirtualParityE2ETestCase, - PostgresVirtualRollbackE2ETestCase, - PostgresVirtualSeedE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.postgres.helpers import ( - build_postgres_virtual_clone_project_toml, - build_postgres_virtual_plan_repo_files, - build_postgres_virtual_project_toml, - build_unique_schema_name, - cleanup_postgres_state_schemas, - execute_postgres_sql, - fetch_postgres_rows, - quote_identifier, - quoted_relation_name, -) -from tests.e2e.src.sqlbuild.cli.commands.main.state.helpers import ( - assert_state_cli_error, - build_postgres_local_state_connection_toml, - build_postgres_state_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateLifecycleE2ETestCase( - description="postgres state lifecycle commands manage state tables", - expected_exit_code=0, - expected_schema_version=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_config_when_running_state_lifecycle_then_state_store_is_updated( - test_case: PostgresStateLifecycleE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_project", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_project", - config=postgres_e2e_config, - state_schema=state_schema, - allow_reset=True, - ) - }, - ) - - try: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == test_case.expected_exit_code, ( - init_result.stdout + init_result.stderr - ) - assert test_case.expected_init_fragment in init_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT schema_version FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_schema_version,),) - - migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - assert migrate_result.returncode == test_case.expected_exit_code, ( - migrate_result.stdout + migrate_result.stderr - ) - assert test_case.expected_migrate_fragment in migrate_result.stdout - - execute_postgres_sql( - sql=( - "DELETE FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')}" - ), - config=postgres_e2e_config, - ) - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback"), - project_dir=project_dir, - ) - assert rollback_result.returncode == test_case.expected_exit_code, ( - rollback_result.stdout + rollback_result.stderr - ) - assert test_case.expected_rollback_fragment in rollback_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT schema_version FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_schema_version,),) - - reset_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "reset", "--auto-approve"), - project_dir=project_dir, - ) - assert reset_result.returncode == test_case.expected_exit_code, ( - reset_result.stdout + reset_result.stderr - ) - assert test_case.expected_reset_fragment in reset_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{state_schema}' AND table_name = 'state_versions'" - ), - config=postgres_e2e_config, - ) == ((0,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachE2ETestCase( - description="postgres adopt and detach preserve logical table name", - expected_exit_code=0, - expected_rows_after_adopt=((1,),), - expected_rows_after_detach=((1,),), - expected_detached_status="detached", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_virtual_state_when_adopting_and_detaching_then_state_is_detached( - test_case: PostgresStateAdoptDetachE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_adopt_detach", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_state_adopt_detach"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - assert adopt_result.returncode == test_case.expected_exit_code, ( - adopt_result.stdout + adopt_result.stderr - ) - assert ( - fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{quoted_relation_name(schema_name=warehouse_schema, name='orders')}" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows_after_adopt - ) - - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - assert detach_result.returncode == test_case.expected_exit_code, ( - detach_result.stdout + detach_result.stderr - ) - assert ( - fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{quoted_relation_name(schema_name=warehouse_schema, name='orders')}" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows_after_detach - ) - assert fetch_postgres_rows( - sql=( - "SELECT status FROM " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')} " - "WHERE virtual_environment_name = 'dev'" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_detached_status,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachErrorE2ETestCase( - description="postgres adopt blocks without unsuffixed virtual env", - expected_exit_code=1, - expected_error_fragment="unsuffixed_virtual_env", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_missing_unsuffixed_virtual_env_when_adopting_then_it_blocks( - test_case: PostgresStateAdoptDetachErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_adopt_missing_unsuffixed", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_adopt_missing_unsuffixed", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachErrorE2ETestCase( - description="postgres adopt blocks view copy fallback without explicit permission", - expected_exit_code=1, - expected_error_fragment="requires --allow-copy", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_copy_fallback_required_when_adopting_without_allow_copy_then_it_blocks( - test_case: PostgresStateAdoptDetachErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_adopt_allow_copy_required", - repo_files={ - "sqlbuild_project.toml": ( - build_postgres_virtual_project_toml( - project_name="postgres_adopt_allow_copy_required", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - unsuffixed_virtual_env="dev", - ) - ), - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE VIEW {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachErrorE2ETestCase( - description="postgres interrupted detach records failed operation", - expected_exit_code=1, - expected_error_fragment="simulated detach copy failure", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_detach_copy_failure_when_detaching_then_operation_is_marked_failed( - test_case: PostgresStateAdoptDetachErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_detach_failed_operation", - repo_files={ - "sqlbuild_project.toml": ( - build_postgres_virtual_project_toml( - project_name="postgres_detach_failed_operation", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - unsuffixed_virtual_env="dev", - ) - ), - "sqlbuild_local.toml": "", - "adapters/failing_postgres.py": ( - "from typing import Any\n" - "from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder\n" - "from sqlbuild.adapter.contract.exceptions import AdapterUserError\n" - "from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter\n\n" - "class FailingPostgresAdapter(PostgresAdapter):\n" - " adapter_name = 'failing_postgres'\n\n" - " def move_or_copy_relation(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " origin: str,\n" - " destination: str,\n" - " remove_origin: bool,\n" - " allow_copy_fallback: bool,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated detach copy failure')\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - (project_dir / "sqlbuild_local.toml").write_text('adapter = "failing_postgres"\n') - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - assert fetch_postgres_rows( - sql=( - "SELECT status FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_operations')} " - "WHERE operation_id = 'detach:dev'" - ), - config=postgres_e2e_config, - ) == (("failed",),) - assert fetch_postgres_rows( - sql=( - "SELECT action, status, message FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_operation_events')} " - "WHERE operation_id = 'detach:dev' ORDER BY created_at" - ), - config=postgres_e2e_config, - ) == ( - ("start", "running", "starting detach"), - ("fail", "failed", test_case.expected_error_fragment), - ) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}' AND table_name = 'orders'" - ), - config=postgres_e2e_config, - ) == ((0,),) - physical_relation_name: str = str( - fetch_postgres_rows( - sql=( - "SELECT relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders'" - ), - config=postgres_e2e_config, - )[0][0] - ) - assert fetch_postgres_rows( - sql=( - "SELECT table_type FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - f"AND table_name = '{physical_relation_name}'" - ), - config=postgres_e2e_config, - ) == (("BASE TABLE",),) - assert fetch_postgres_rows( - sql=( - "SELECT status FROM " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')} " - "WHERE virtual_environment_name = 'dev'" - ), - config=postgres_e2e_config, - ) == (("finalized",),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachErrorE2ETestCase( - description="postgres detach recreates view models as views", - expected_exit_code=0, - expected_error_fragment="VIEW", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_view_model_when_detaching_then_stateless_target_remains_a_view( - test_case: PostgresStateAdoptDetachErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_detach_view_model", - repo_files={ - "sqlbuild_project.toml": ( - build_postgres_virtual_project_toml( - project_name="postgres_detach_view_model", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - unsuffixed_virtual_env="dev", - ) - ), - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE VIEW {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == test_case.expected_exit_code - ) - - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert detach_result.returncode == test_case.expected_exit_code, ( - detach_result.stdout + detach_result.stderr - ) - assert fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{quoted_relation_name(schema_name=warehouse_schema, name='orders')}" - ), - config=postgres_e2e_config, - ) == ((1,),) - assert fetch_postgres_rows( - sql=( - "SELECT table_type FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}' AND table_name = 'orders'" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_error_fragment,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachErrorE2ETestCase( - description="postgres adopt cancels on wrong confirmation", - expected_exit_code=1, - expected_error_fragment="cancelled", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_wrong_confirmation_when_adopting_then_it_cancels( - test_case: PostgresStateAdoptDetachErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_adopt_cancelled", - repo_files={ - "sqlbuild_project.toml": ( - build_postgres_virtual_project_toml( - project_name="postgres_adopt_cancelled", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - unsuffixed_virtual_env="dev", - ) - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="nope\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateAdoptDetachErrorE2ETestCase( - description="postgres detach blocks non-finalized virtual environment", - expected_exit_code=1, - expected_error_fragment="requires a finalized virtual environment", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_non_finalized_virtual_environment_when_detaching_then_it_blocks( - test_case: PostgresStateAdoptDetachErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_detach_not_finalized", - repo_files={ - "sqlbuild_project.toml": ( - build_postgres_virtual_project_toml( - project_name="postgres_detach_not_finalized", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - unsuffixed_virtual_env="dev", - ) - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')} " - "(virtual_environment_name, status, created_at, updated_at) " - "VALUES ('dev', 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor prunes detached VDE refs and physical versions", - expected_exit_code=0, - expected_stdout_fragments=( - "eligible for deletion", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "state item", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_detached_vde_when_running_janitor_then_refs_and_physical_are_pruned( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_detached_vde", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_janitor_detached_vde"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[janitor]\n" - "enabled = true\n" - "retention_days = 0\n" - "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - assert adopt_result.returncode == test_case.expected_exit_code, ( - adopt_result.stdout + adopt_result.stderr - ) - physical_relation_name: str = str( - fetch_postgres_rows( - sql=( - "SELECT relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders'" - ), - config=postgres_e2e_config, - )[0][0] - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - assert detach_result.returncode == test_case.expected_exit_code, ( - detach_result.stdout + detach_result.stderr - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_virtual_environment_count_after,),) - model_ref_table_name: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - assert fetch_postgres_rows( - sql=(f"SELECT COUNT(*) FROM {model_ref_table_name}"), - config=postgres_e2e_config, - ) == ((test_case.expected_ref_count_after,),) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - f"AND table_name = '{physical_relation_name}'" - ), - config=postgres_e2e_config, - ) == ((0,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor prunes detached VDE after positive retention", - expected_exit_code=0, - expected_stdout_fragments=( - "7 days", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "state item", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_old_detached_vde_when_running_janitor_then_retention_allows_cleanup( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_detached_retention", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_janitor_detached_retention", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - unsuffixed_virtual_env="dev", - ) - + "\n[janitor]\nenabled = true\nretention_days = 7\ndelete_tracked_only = false\n", - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {quoted_relation_name(schema_name=warehouse_schema, name='orders')} " - "AS SELECT 1 AS id" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ).returncode - == 0 - ) - execute_postgres_sql( - sql=( - "UPDATE " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')} " - "SET updated_at = TIMESTAMP '2026-01-01' " - "WHERE virtual_environment_name = 'dev'" - ), - config=postgres_e2e_config, - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, janitor_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_virtual_environment_count_after,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor prunes expired non-active VDE", - expected_exit_code=0, - expected_stdout_fragments=( - "expired VDEs pruned", - "Eligible expired VDEs", - "pr expired virtual environment", - "state item", - ), - expected_virtual_environment_count_after=1, - expected_ref_count_after=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_non_active_vde_when_running_janitor_then_it_prunes_expired_environment( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_expired_vde", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_janitor_expired_vde", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ) - + "\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n", - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_virtual_environment_count_after,),) - model_ref_table_name: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - assert fetch_postgres_rows( - sql=(f"SELECT COUNT(*) FROM {model_ref_table_name}"), - config=postgres_e2e_config, - ) == ((test_case.expected_ref_count_after,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor prunes old state backups and expired locks", - expected_exit_code=0, - expected_stdout_fragments=( - "state backups pruned", - "expired locks pruned", - "Eligible state backups", - "Eligible expired locks", - "state item", - ), - expected_virtual_environment_count_after=1, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backups_and_expired_locks_when_running_janitor_then_state_is_pruned( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_state_cleanup", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_janitor_state_cleanup", - config=postgres_e2e_config, - state_schema=state_schema, - ) - + "\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n" - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb(command=("--no-color", "state", "migrate"), project_dir=project_dir).returncode - == 0 - ) - execute_postgres_sql( - sql=( - "UPDATE " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')} " - "SET schema_version = 2" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "migrate"), project_dir=project_dir).returncode - == 0 - ) - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:stale', 'owner', TIMESTAMP '2000-01-01', " - "TIMESTAMP '2000-01-01', TIMESTAMP '2000-01-01')" - ), - config=postgres_e2e_config, - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.schemata " - f"WHERE schema_name LIKE '{state_schema}__backup_%'" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_virtual_environment_count_after,),) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{quoted_relation_name(schema_name=state_schema, name='locks')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_ref_count_after,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor skips state cleanup when warehouse cleanup fails", - expected_exit_code=1, - expected_stdout_fragments=("simulated janitor drop failure",), - expected_virtual_environment_count_after=1, - expected_ref_count_after=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_warehouse_cleanup_failure_when_janitor_then_state_cleanup_is_skipped( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_warehouse_failure", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_janitor_warehouse_failure", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ) - + "\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n", - "sqlbuild_local.toml": "", - "adapters/failing_janitor_postgres.py": ( - "from typing import Any\n" - "from sqlbuild.adapter.contract.exceptions import AdapterUserError\n" - "from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder\n" - "from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter\n\n" - "class FailingJanitorPostgresAdapter(PostgresAdapter):\n" - " adapter_name = 'failing_janitor_postgres'\n\n" - " def drop(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " destination: str,\n" - " if_exists: bool = True,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated janitor drop failure')\n\n" - " def drop_view(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " destination: str,\n" - " if_exists: bool = True,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated janitor drop failure')\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text("MODEL ();\n\nSELECT 2 AS id\n") - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ).returncode - == 0 - ) - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:stale', 'owner', TIMESTAMP '2000-01-01', " - "TIMESTAMP '2000-01-01', TIMESTAMP '2000-01-01')" - ), - config=postgres_e2e_config, - ) - (project_dir / "sqlbuild_local.toml").write_text('adapter = "failing_janitor_postgres"\n') - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code - combined_output: str = janitor_result.stdout + janitor_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{quoted_relation_name(schema_name=state_schema, name='virtual_environments')} " - "WHERE virtual_environment_name = 'pr'" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_virtual_environment_count_after,),) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "WHERE lock_key = 'virtual_env:stale'" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_ref_count_after,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor prunes old checkpoints and newly unprotected physicals", - expected_exit_code=0, - expected_stdout_fragments=( - "checkpoints pruned", - "Eligible checkpoints", - "Deleted ", - "2 checkpoints.", - ), - expected_virtual_environment_count_after=1, - expected_ref_count_after=3, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_checkpoints_over_limit_when_running_janitor_then_it_prunes_old_history( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_janitor") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_checkpoint_retention", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_janitor_checkpoint_retention", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 1 AS id", - extra_project_toml=( - "\n[janitor]\n" - "enabled = true\n" - "retention_days = 0\n" - "max_checkpoints = 1\n" - "delete_tracked_only = false\n" - ), - ), - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - physical_relations_relation: str = quoted_relation_name( - schema_name=state_schema, - name="physical_relations", - ) - checkpoints_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_checkpoints", - ) - first_relation_name: str = str( - fetch_postgres_rows( - sql=( - "SELECT physical_relations.relation_name FROM " - f"{refs_relation} AS refs JOIN " - f"{physical_relations_relation} AS physical_relations " - "ON physical_relations.artifact_type = 'model' " - "AND physical_relations.artifact_name = refs.node_name " - "AND physical_relations.version_hash = refs.version_hash " - "WHERE refs.virtual_environment_name = 'dev' " - "AND refs.node_type = 'model' AND refs.node_name = 'stg_orders'" - ), - config=postgres_e2e_config, - )[0][0] - ) - (project_dir / "models" / "stg_orders.sql").write_text("MODEL ();\n\nSELECT 2 AS id\n") - assert ( - run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ).returncode - == 0 - ) - (project_dir / "models" / "stg_orders.sql").write_text("MODEL ();\n\nSELECT 3 AS id\n") - assert ( - run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ).returncode - == 0 - ) - latest_relation_name: str = str( - fetch_postgres_rows( - sql=( - "SELECT physical_relations.relation_name FROM " - f"{refs_relation} AS refs JOIN " - f"{physical_relations_relation} AS physical_relations " - "ON physical_relations.artifact_type = 'model' " - "AND physical_relations.artifact_name = refs.node_name " - "AND physical_relations.version_hash = refs.version_hash " - "WHERE refs.virtual_environment_name = 'dev' " - "AND refs.node_type = 'model' AND refs.node_name = 'stg_orders'" - ), - config=postgres_e2e_config, - )[0][0] - ) - assert fetch_postgres_rows( - sql=f"SELECT COUNT(*) FROM {checkpoints_relation}", - config=postgres_e2e_config, - ) == ((test_case.expected_ref_count_after,),) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, janitor_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert fetch_postgres_rows( - sql=f"SELECT COUNT(*) FROM {checkpoints_relation}", - config=postgres_e2e_config, - ) == ((test_case.expected_virtual_environment_count_after,),) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - f"AND table_name = '{first_relation_name}'" - ), - config=postgres_e2e_config, - ) == ((0,),) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - f"AND table_name = '{latest_relation_name}'" - ), - config=postgres_e2e_config, - ) == ((1,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", config=postgres_e2e_config - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresJanitorDetachedVdeE2ETestCase( - description="postgres janitor preserves active VDE refs", - expected_exit_code=0, - expected_stdout_fragments=( - "eligible for deletion 0", - "relation is referenced by an active or retained virtual environment", - ), - expected_virtual_environment_count_after=1, - expected_ref_count_after=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_active_vde_ref_when_running_janitor_then_it_preserves_physical_version( - test_case: PostgresJanitorDetachedVdeE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_janitor") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_active_ref", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_janitor_active_ref", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 1 AS id", - extra_project_toml=( - "\n[janitor]\nenabled = true\nretention_days = 0\ndelete_tracked_only = false\n" - ), - ), - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "build", "--select", "stg_orders"), - project_dir=project_dir, - ).returncode - == 0 - ) - protected_relation_name: str = str( - fetch_postgres_rows( - sql=( - "SELECT relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'stg_orders'" - ), - config=postgres_e2e_config, - )[0][0] - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_exit_code, janitor_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - f"AND table_name = '{protected_relation_name}'" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_ref_count_after,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", config=postgres_e2e_config - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateLifecycleErrorE2ETestCase( - description="postgres tracked-only janitor requires query tracking", - allow_reset=False, - command=("--no-color", "janitor", "--auto-approve"), - expected_exit_code=1, - expected_error_fragment="janitor.delete_tracked_only requires", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_query_tracking_disabled_when_running_tracked_only_janitor_then_it_errors( - test_case: PostgresStateLifecycleErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_janitor") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_janitor_invalid_config", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_janitor_invalid_config", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ).replace( - "virtual_environments = true\n", - "virtual_environments = true\nquery_change_tracking = false\n", - ) - + "\n[janitor]\nenabled = true\n" - }, - ) - - try: - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in result.stderr - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres rollback restores previous finalized checkpoint", - expected_rows=((1,),), - expected_stdout_fragments=( - "\u2713 Virtual rollback complete", - "virtual environment dev", - "status finalized", - "rolled back models 1", - ), - expected_checkpoint_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_finalized_checkpoints_when_rolling_back_then_refs_and_view_restore( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_rollback", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_virtual_rollback"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n\n' - "[targets.dev.state.connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - refs_table: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - initial_ref_rows: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - "SELECT node_name, version_hash FROM " - f"{refs_table} " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - config=postgres_e2e_config, - ) - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - checkpoints_table: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_checkpoints", - ) - checkpoint_rows: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=(f"SELECT checkpoint_id FROM {checkpoints_table} ORDER BY created_at"), - config=postgres_e2e_config, - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - checkpoint_id: str = str(checkpoint_rows[0][0]) - list_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "list"), - project_dir=project_dir, - ) - assert list_result.returncode == test_case.expected_exit_code, list_result.stderr - assert checkpoint_id in list_result.stdout - show_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "show", checkpoint_id), - project_dir=project_dir, - ) - assert show_result.returncode == test_case.expected_exit_code, show_result.stderr - assert "orders" in show_result.stdout - diff_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "diff", checkpoint_id), - project_dir=project_dir, - ) - assert diff_result.returncode == test_case.expected_exit_code, diff_result.stderr - assert "changed refs 1" in diff_result.stdout - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rollback"), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code, ( - rollback_result.stdout + rollback_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in rollback_result.stdout - assert ( - fetch_postgres_rows( - sql=( - "SELECT node_name, version_hash FROM " - f"{refs_table} " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - config=postgres_e2e_config, - ) - == initial_ref_rows - ) - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - assert ( - fetch_postgres_rows( - sql=f"SELECT id FROM {logical_relation} ORDER BY id", - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres rollback blocks when checkpoint physical relation is missing", - expected_rows=(), - expected_stdout_fragments=( - "error[S024]", - "checkpoint references missing warehouse relation", - ), - expected_checkpoint_count=2, - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_checkpoint_physical_relation_missing_when_rolling_back_then_it_blocks( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_rollback_missing_physical", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_virtual_rollback_missing_physical", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - checkpoints_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_checkpoints", - ) - checkpoint_model_refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_checkpoint_model_refs", - ) - physical_relations_relation: str = quoted_relation_name( - schema_name=state_schema, - name="physical_relations", - ) - checkpoint_rows: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=f"SELECT checkpoint_id FROM {checkpoints_relation}", - config=postgres_e2e_config, - ) - assert len(checkpoint_rows) == test_case.expected_checkpoint_count - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - f"SELECT pr.schema_name, pr.relation_name FROM {checkpoints_relation} cp " - f"JOIN {checkpoint_model_refs_relation} cr " - "ON cp.checkpoint_id = cr.checkpoint_id JOIN " - f"{physical_relations_relation} pr " - "ON pr.artifact_type = 'model' AND pr.artifact_name = cr.model_name " - "AND pr.version_hash = cr.version_hash " - "WHERE cp.virtual_environment_name = 'dev' AND cr.model_name = 'orders' " - "ORDER BY cp.created_at ASC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - missing_relation: str = quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ) - execute_postgres_sql( - sql=f"DROP TABLE {missing_relation} CASCADE", - config=postgres_e2e_config, - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rollback"), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code - combined_output: str = rollback_result.stdout + rollback_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres rollback blocks when no previous checkpoint exists", - expected_rows=(), - expected_stdout_fragments=( - "error[S021]", - "no previous finalized checkpoint is available for rollback", - ), - expected_checkpoint_count=1, - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_only_current_checkpoint_when_rolling_back_then_it_blocks( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_rollback_no_previous", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_virtual_rollback_no_previous", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - checkpoints_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_checkpoints", - ) - assert ( - len( - fetch_postgres_rows( - sql=f"SELECT checkpoint_id FROM {checkpoints_relation}", - config=postgres_e2e_config, - ) - ) - == test_case.expected_checkpoint_count - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rollback"), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code - combined_output: str = rollback_result.stdout + rollback_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres rollback blocks when target VDE lock exists", - expected_rows=(), - expected_stdout_fragments=("virtual environment 'dev' is locked",), - expected_checkpoint_count=2, - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_target_virtual_environment_lock_when_rolling_back_then_it_blocks( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_rollback_locked_target", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_virtual_rollback_locked_target", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - config=postgres_e2e_config, - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rollback", "--virtual-env", "dev"), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code - combined_output: str = rollback_result.stdout + rollback_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres checkpoint show blocks for unknown checkpoint", - expected_rows=(), - expected_stdout_fragments=("error[C905]", "unknown checkpoint 'missing'"), - expected_checkpoint_count=1, - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_unknown_checkpoint_when_showing_checkpoint_then_it_blocks( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_checkpoint_unknown", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_virtual_checkpoint_unknown", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "checkpoints", "show", "missing"), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code - combined_output: str = result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres partial rollback requires override and marks VDE working", - expected_rows=((1,),), - expected_stdout_fragments=( - "rollback would leave target virtual environment working", - "status active", - "rolled back models 1", - ), - expected_checkpoint_count=2, - expected_exit_code=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_partial_rollback_when_allowed_then_it_marks_vde_working( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_rollback_partial", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_virtual_rollback_partial", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 1 AS id", - ), - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ).returncode - == 0 - ) - checkpoints_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_checkpoints", - ) - assert ( - len( - fetch_postgres_rows( - sql=f"SELECT checkpoint_id FROM {checkpoints_relation}", - config=postgres_e2e_config, - ) - ) - == test_case.expected_checkpoint_count - ) - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "rollback", "--select", "stg_orders"), - project_dir=project_dir, - ) - assert blocked_result.returncode == 1 - assert test_case.expected_stdout_fragments[0] in blocked_result.stderr - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "rollback", - "--select", - "stg_orders", - "--allow-partial-rollback", - ), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code, rollback_result.stderr - for fragment in test_case.expected_stdout_fragments[1:]: - assert fragment in rollback_result.stdout - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="stg_orders", - ) - assert ( - fetch_postgres_rows( - sql=f"SELECT id FROM {logical_relation} ORDER BY id", - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", config=postgres_e2e_config - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualRollbackE2ETestCase( - description="postgres partial rollback can include stale required upstreams", - expected_rows=((1,),), - expected_stdout_fragments=( - "selected rollback scope is missing stale required upstream models", - "stg_orders", - "status finalized", - "rolled back models 2", - ), - expected_checkpoint_count=2, - expected_exit_code=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_partial_rollback_missing_stale_upstreams_when_including_then_succeeds( - test_case: PostgresVirtualRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_rollback") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_rollback_include_stale", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_virtual_rollback_include_stale", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 1 AS id", - ), - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--changes-only"), project_dir=project_dir - ).returncode - == 0 - ) - - blocked_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "rollback", - "--select", - "fact_orders", - "--allow-partial-rollback", - ), - project_dir=project_dir, - ) - assert blocked_result.returncode == 1 - for fragment in test_case.expected_stdout_fragments[:2]: - assert fragment in blocked_result.stderr - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "rollback", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-rollback", - ), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code, rollback_result.stderr - for fragment in test_case.expected_stdout_fragments[2:]: - assert fragment in rollback_result.stdout - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="fact_orders", - ) - assert ( - fetch_postgres_rows( - sql=f"SELECT id FROM {logical_relation} ORDER BY id", - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", config=postgres_e2e_config - ) - - -@pytest.mark.parametrize( - "test_case", - ( - PostgresVirtualSeedE2ETestCase( - description="postgres virtual seeded incremental build uses copy", - expected_rows=((1, 10), (2, 21), (3, 31)), - expected_seed_strategy="copy", - ), - PostgresVirtualSeedE2ETestCase( - description="postgres virtual append bounded seeded build uses bounded copy", - expected_rows=((1, 10), (2, 21), (3, 31)), - expected_seed_strategy="bounded_append_copy", - incremental_strategy="append", - ), - ), - ids=lambda case: case.description, -) -def test_given_postgres_virtual_incremental_change_when_building_then_seeds_with_copy( - test_case: PostgresVirtualSeedE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_seed") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_seed", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_virtual_seed"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n\n' - "[targets.dev.state.connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n' - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - f" schema: {warehouse_schema}\n" - " table: raw_orders\n" - ), - "models/orders.sql": ( - "MODEL (\n" - " materialized incremental,\n" - f" incremental_strategy {test_case.incremental_strategy},\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - "SELECT id, ordered_at, amount_cents + 0 AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ), - }, - ) - - try: - raw_orders_relation: str = quoted_relation_name( - schema_name=warehouse_schema, - name="raw_orders", - ) - execute_postgres_sql( - sql=( - f"CREATE SCHEMA {quote_identifier(warehouse_schema)}; " - f"CREATE TABLE {raw_orders_relation} " - "(id INTEGER, ordered_at TIMESTAMP, amount_cents INTEGER); " - f"INSERT INTO {raw_orders_relation} " - "VALUES (1, TIMESTAMP '2026-01-01 00:00:00', 10), " - "(2, TIMESTAMP '2026-01-02 00:00:00', 20)" - ), - config=postgres_e2e_config, - ) - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - (project_dir / "models" / "orders.sql").write_text( - ( - "MODEL (\n" - " materialized incremental,\n" - f" incremental_strategy {test_case.incremental_strategy},\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - "SELECT id, ordered_at, amount_cents + 1 AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ), - encoding="utf-8", - ) - execute_postgres_sql( - sql=( - f"INSERT INTO {raw_orders_relation} VALUES (3, TIMESTAMP '2026-01-03 00:00:00', 30)" - ), - config=postgres_e2e_config, - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_exit_code, ( - build_result.stdout + build_result.stderr - ) - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - assert ( - fetch_postgres_rows( - sql=f"SELECT id, amount_cents FROM {logical_relation} ORDER BY id", - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - ancestry_table: str = quoted_relation_name( - schema_name=state_schema, - name="physical_relation_ancestry", - ) - assert fetch_postgres_rows( - sql=(f"SELECT seed_strategy FROM {ancestry_table} WHERE model_name = 'orders'"), - config=postgres_e2e_config, - ) == ((test_case.expected_seed_strategy,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualParityE2ETestCase( - description="postgres virtual plan and partial build parity", - expected_stdout_fragments=( - "Plan ready 0 selected", - "finalized, up to date", - "Query changed (1)", - "directly affected (1) stg_orders", - "cause stg_orders (query changed)", - "missing stale required upstream models: stg_orders", - "Plan ready 2 selected", - '"virtual_environment_name": "dev"', - "working, build required", - "outside this plan", - "Plan ready 0 selected", - ), - expected_rows=((2,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_virtual_plan_and_partial_build_when_running_then_matches_duckdb_parity( - test_case: PostgresVirtualParityE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_parity") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_plan_build_parity", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_virtual_plan_build_parity", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 1 AS id", - ) - | { - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ) - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build"), - project_dir=project_dir, - ) - assert build_result.returncode == test_case.expected_exit_code, build_result.stderr - assert "Virtual environment dev" in build_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__dev', - name='fact_orders', - ) - }" - ), - config=postgres_e2e_config, - ) == ((1,),) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - "AND table_name = '_sqlbuild_fingerprints'" - ), - config=postgres_e2e_config, - ) == ((0,),) - physical_rows_before: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - "SELECT table_name FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - "AND table_name LIKE '%__v_%' ORDER BY table_name" - ), - config=postgres_e2e_config, - ) - repeat_build: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - assert repeat_build.returncode == test_case.expected_exit_code, repeat_build.stderr - assert ( - fetch_postgres_rows( - sql=( - "SELECT table_name FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__sqb_physical' " - "AND table_name LIKE '%__v_%' ORDER BY table_name" - ), - config=postgres_e2e_config, - ) - == physical_rows_before - ) - matching_plan: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert matching_plan.returncode == test_case.expected_exit_code, matching_plan.stderr - for fragment in test_case.expected_stdout_fragments[:2]: - assert fragment in matching_plan.stdout - - (project_dir / "models" / "stg_orders.sql").write_text("MODEL ();\n\nSELECT 2 AS id\n") - stale_plan: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--changes-only"), - project_dir=project_dir, - ) - assert stale_plan.returncode == test_case.expected_exit_code, stale_plan.stderr - for fragment in test_case.expected_stdout_fragments[2:5]: - assert fragment in stale_plan.stdout - assert "dim_customers" not in stale_plan.stdout - selected_block: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--select", "fact_orders", "--changes-only"), - project_dir=project_dir, - ) - assert selected_block.returncode == 1 - assert test_case.expected_stdout_fragments[5] in selected_block.stderr - include_plan: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "plan", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--changes-only", - ), - project_dir=project_dir, - ) - assert include_plan.returncode == test_case.expected_exit_code, include_plan.stderr - assert test_case.expected_stdout_fragments[6] in include_plan.stdout - json_plan: subprocess.CompletedProcess[str] = run_sqb( - command=("plan", "--json", "--changes-only"), - project_dir=project_dir, - ) - assert json_plan.returncode == test_case.expected_exit_code, json_plan.stderr - parsed: dict[str, object] = json.loads(json_plan.stdout) - assert "metadata" in parsed - assert test_case.expected_stdout_fragments[7] in json_plan.stdout - - partial_build: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--virtual-env", - "pr", - "--select", - "+fact_orders", - ), - project_dir=project_dir, - ) - assert partial_build.returncode == test_case.expected_exit_code, partial_build.stderr - for fragment in test_case.expected_stdout_fragments[8:10]: - assert fragment in partial_build.stdout - assert ( - fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__pr', - name='fact_orders', - ) - }" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - final_build: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--virtual-env", "pr", "--changes-only"), - project_dir=project_dir, - ) - assert final_build.returncode == test_case.expected_exit_code, final_build.stderr - final_plan: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan", "--virtual-env", "pr", "--changes-only"), - project_dir=project_dir, - ) - assert final_plan.returncode == test_case.expected_exit_code, final_plan.stderr - assert test_case.expected_stdout_fragments[10] in final_plan.stdout - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualParityE2ETestCase( - description="postgres virtual config and function change plan parity", - expected_stdout_fragments=( - "Config changed (1)", - "config diff:", - '"materialized": "view"', - '"materialized": "table"', - "Changed functions (1)", - "is_large_order", - "query diff:", - "cause is_large_order (function changed)", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_config_and_function_changes_when_planning_then_reasons_match_parity( - test_case: PostgresVirtualParityE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - config_state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - config_warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_config") - function_state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - function_warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_function") - config_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_config_plan", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_virtual_config_plan", - config=postgres_e2e_config, - state_schema=config_state_schema, - warehouse_schema=config_warehouse_schema, - ), - "models/stg_orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - }, - ) - function_project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_function_plan", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_virtual_function_plan", - config=postgres_e2e_config, - state_schema=function_state_schema, - warehouse_schema=function_warehouse_schema, - stg_orders_sql="SELECT 7 AS id", - ) - | { - "models/fact_orders.sql": ( - "MODEL ();\n\n" - 'SELECT __udf("is_large_order")(id) AS is_large FROM __ref("stg_orders")\n' - ), - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ), - "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, " - "replay_on_change full);\n\n" - "SELECT amount > 9\n" - ), - }, - ) - - try: - assert ( - run_sqb( - command=("--no-color", "state", "init"), project_dir=config_project_dir - ).returncode - == 0 - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=config_project_dir).returncode == 0 - ) - (config_project_dir / "models" / "stg_orders.sql").write_text( - "MODEL (materialized table);\n\nSELECT 1 AS id\n" - ) - config_plan: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=config_project_dir, - ) - assert config_plan.returncode == test_case.expected_exit_code, config_plan.stderr - for fragment in test_case.expected_stdout_fragments[:3]: - assert fragment in config_plan.stdout - assert "Query changed" not in config_plan.stdout - - assert ( - run_sqb( - command=("--no-color", "state", "init"), project_dir=function_project_dir - ).returncode - == 0 - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=function_project_dir).returncode - == 0 - ) - (function_project_dir / "functions" / "sql" / "is_large_order.sql").write_text( - "FUNCTION (" - "arguments (amount INTEGER), returns BOOLEAN, replay_on_change full" - ");\n\n" - "SELECT amount > 5\n" - ) - function_plan: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "plan"), - project_dir=function_project_dir, - ) - assert function_plan.returncode == test_case.expected_exit_code, function_plan.stderr - for fragment in test_case.expected_stdout_fragments[4:]: - assert fragment in function_plan.stdout - finally: - cleanup_postgres_state_schemas(schema_name=config_state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=config_warehouse_schema, config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{config_warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{config_warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=function_state_schema, config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=function_warehouse_schema, config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{function_warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{function_warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualParityE2ETestCase( - description="postgres virtual diff and promotion parity", - expected_stdout_fragments=( - "whole-VDE virtual diff requires finalized VDEs", - "Virtual diff", - "\u2713 Virtual promotion complete", - "pr_full -> dev", - "target status finalized", - "promoted models 4", - "promotion would leave target virtual environment working", - "whole-VDE promotion requires a finalized source virtual environment", - "virtual environment 'dev' is locked", - "target status working", - "promoted models 2", - "remaining stale set: orders_rollup", - "fact_orders", - ), - expected_rows=((3,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_virtual_diff_and_promotion_when_running_then_matches_duckdb_parity( - test_case: PostgresVirtualParityE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_promote") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_promote_parity", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_virtual_promote_parity", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 1 AS id", - ) - | { - "models/orders_rollup.sql": ( - 'MODEL ();\n\nSELECT COUNT(*) AS order_count FROM __ref("fact_orders")\n' - ) - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text("MODEL ();\n\nSELECT 2 AS id\n") - assert ( - run_sqb( - command=( - "--no-color", - "build", - "--virtual-env", - "pr_full", - "--changes-only", - ), - project_dir=project_dir, - ).returncode - == 0 - ) - whole_promote: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "promote", "--from", "pr_full", "--to", "dev"), - project_dir=project_dir, - ) - assert whole_promote.returncode == test_case.expected_exit_code, whole_promote.stderr - for fragment in test_case.expected_stdout_fragments[2:6]: - assert fragment in whole_promote.stdout - - (project_dir / "models" / "stg_orders.sql").write_text("MODEL ();\n\nSELECT 3 AS id\n") - assert ( - run_sqb( - command=( - "--no-color", - "build", - "--virtual-env", - "pr", - "--select", - "+fact_orders", - "--changes-only", - ), - project_dir=project_dir, - ).returncode - == 0 - ) - blocked_diff: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "diff", "dev:pr", "--schema-only"), - project_dir=project_dir, - ) - assert blocked_diff.returncode == 1 - assert test_case.expected_stdout_fragments[0] in blocked_diff.stderr - allowed_diff: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "diff", "dev:pr", "--schema-only", "--allow-partial-diff"), - project_dir=project_dir, - ) - assert allowed_diff.returncode == test_case.expected_exit_code, allowed_diff.stderr - assert test_case.expected_stdout_fragments[1] in allowed_diff.stdout - whole_working_source: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - project_dir=project_dir, - ) - assert whole_working_source.returncode == 1 - assert test_case.expected_stdout_fragments[7] in ( - whole_working_source.stdout + whole_working_source.stderr - ) - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - config=postgres_e2e_config, - ) - locked_promote: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - project_dir=project_dir, - ) - assert locked_promote.returncode == 1 - assert test_case.expected_stdout_fragments[8] in ( - locked_promote.stdout + locked_promote.stderr - ) - execute_postgres_sql( - sql=( - "DELETE FROM " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "WHERE lock_key = 'virtual_env:dev'" - ), - config=postgres_e2e_config, - ) - blocked_promote: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - ), - project_dir=project_dir, - ) - assert blocked_promote.returncode == 1 - assert test_case.expected_stdout_fragments[6] in ( - blocked_promote.stdout + blocked_promote.stderr - ) - partial_promote: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - project_dir=project_dir, - ) - assert partial_promote.returncode == test_case.expected_exit_code, partial_promote.stderr - for fragment in test_case.expected_stdout_fragments[9:12]: - assert fragment in partial_promote.stdout - assert ( - fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__dev', - name='fact_orders', - ) - }" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT physical_relations.schema_name, physical_relations.relation_name FROM " - f"{ - quoted_relation_name( - schema_name=state_schema, - name='virtual_environment_node_refs', - ) - } " - "AS refs JOIN " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "AS physical_relations ON physical_relations.artifact_type = 'model' " - "AND physical_relations.artifact_name = refs.node_name " - "AND physical_relations.version_hash = refs.version_hash " - "WHERE refs.virtual_environment_name = 'pr' " - "AND refs.node_type = 'model' AND refs.node_name = 'fact_orders'" - ), - config=postgres_e2e_config, - )[0] - execute_postgres_sql( - sql=( - "DROP TABLE " - f"{ - quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ) - } " - "CASCADE" - ), - config=postgres_e2e_config, - ) - missing_physical_promote: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - ), - project_dir=project_dir, - ) - assert missing_physical_promote.returncode == 1 - assert test_case.expected_stdout_fragments[12] in ( - missing_physical_promote.stdout + missing_physical_promote.stderr - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr_full", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualParityE2ETestCase( - description="postgres virtual promotion publishes function definitions", - expected_stdout_fragments=( - "\u2713 Virtual promotion complete", - "pr -> dev", - "target status finalized", - ), - expected_rows=((True,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_function_change_when_promoting_then_publishes_function_definition( - test_case: PostgresVirtualParityE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_func_promote") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_function_promote", - repo_files=build_postgres_virtual_plan_repo_files( - project_name="postgres_virtual_function_promote", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - stg_orders_sql="SELECT 7 AS id", - ) - | { - "models/fact_orders.sql": ( - "MODEL (materialized view);\n\n" - 'SELECT __udf("is_large_order")(id) AS is_large FROM __ref("stg_orders")\n' - ), - "functions/sql/is_large_order.sql": ( - "FUNCTION (arguments (amount INTEGER), returns BOOLEAN, " - "replay_on_change full);\n\n" - "SELECT amount > 9\n" - ), - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - assert fetch_postgres_rows( - sql=( - "SELECT " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__dev', - name='is_large_order', - ) - }" - "(7)" - ), - config=postgres_e2e_config, - ) == ((False,),) - function_refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - initial_dev_function_hash: str = str( - fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {function_refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'udf' AND node_name = 'is_large_order'" - ), - config=postgres_e2e_config, - )[0][0] - ) - - (project_dir / "functions" / "sql" / "is_large_order.sql").write_text( - "FUNCTION (" - "arguments (amount INTEGER), returns BOOLEAN, replay_on_change full" - ");\n\n" - "SELECT amount > 5\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr", "--changes-only"), - project_dir=project_dir, - ).returncode - == 0 - ) - assert ( - fetch_postgres_rows( - sql=( - "SELECT is_large FROM " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__pr', - name='fact_orders', - ) - }" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - assert fetch_postgres_rows( - sql=( - "SELECT is_large FROM " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__dev', - name='fact_orders', - ) - }" - ), - config=postgres_e2e_config, - ) == ((False,),) - pr_function_hash: str = str( - fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {function_refs_relation} " - "WHERE virtual_environment_name = 'pr' " - "AND node_type = 'udf' AND node_name = 'is_large_order'" - ), - config=postgres_e2e_config, - )[0][0] - ) - assert pr_function_hash != initial_dev_function_hash - - promote_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - project_dir=project_dir, - ) - assert promote_result.returncode == test_case.expected_exit_code, promote_result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in promote_result.stdout - assert fetch_postgres_rows( - sql=( - f"SELECT node_name, version_hash FROM {function_refs_relation} " - "WHERE virtual_environment_name = 'dev' AND node_type = 'udf'" - ), - config=postgres_e2e_config, - ) == (("is_large_order", pr_function_hash),) - assert ( - fetch_postgres_rows( - sql=( - "SELECT is_large FROM " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__dev', - name='fact_orders', - ) - }" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - assert ( - fetch_postgres_rows( - sql=( - "SELECT " - f"{ - quoted_relation_name( - schema_name=f'{warehouse_schema}__dev', - name='is_large_order', - ) - }" - "(7)" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr", config=postgres_e2e_config - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualParityE2ETestCase( - description="postgres direct mode promotion guard", - expected_stdout_fragments=("promote requires virtual_environments = true",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_direct_mode_project_when_promoting_then_fails_with_mode_error( - test_case: PostgresVirtualParityE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - warehouse_schema: str = build_unique_schema_name(prefix="sqb_direct_promote") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_direct_promote_guard", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_direct_promote_guard"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n' - ), - "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "promote", "--from", "pr", "--to", "dev"), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_stdout_fragments[0] in result.stderr - finally: - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresVirtualParityE2ETestCase( - description="postgres virtual clone parity", - expected_stdout_fragments=( - "mode workspace fingerprints", - "origin state not used", - "hydrated 3", - "mode destination VDE refs", - "destination VDE dev", - "hydrated 1", - "already present 2", - "missing in origin 0", - "missing: stg_orders", - "is locked", - "skipped locked 1", - "skipped locked: stg_orders", - ), - expected_rows=((7,),), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_virtual_clone_when_running_then_matches_duckdb_parity( - test_case: PostgresVirtualParityE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - prod_state_schema: str = build_unique_schema_name(prefix="sqb_prod_state_e2e") - dev_state_schema: str = build_unique_schema_name(prefix="sqb_dev_state_e2e") - prod_warehouse_schema: str = build_unique_schema_name(prefix="sqb_clone_prod") - dev_warehouse_schema: str = build_unique_schema_name(prefix="sqb_clone_dev") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_virtual_clone_parity", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_clone_project_toml( - project_name="postgres_virtual_clone_parity", - config=postgres_e2e_config, - prod_state_schema=prod_state_schema, - dev_state_schema=dev_state_schema, - prod_warehouse_schema=prod_warehouse_schema, - dev_warehouse_schema=dev_warehouse_schema, - ), - "sqlbuild_local.toml": 'target = "prod"\n', - "models/stg_orders.sql": "MODEL ();\n\nSELECT 7 AS id\n", - "models/fact_orders.sql": 'MODEL ();\n\nSELECT id FROM __ref("stg_orders")\n', - "models/dim_customers.sql": "MODEL ();\n\nSELECT 1 AS customer_id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "sqlbuild_local.toml").write_text('target = "dev"\n') - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - clone_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "clone", "--from", "prod", "--to", "dev"), - project_dir=project_dir, - ) - assert clone_result.returncode == test_case.expected_exit_code, clone_result.stderr - for fragment in test_case.expected_stdout_fragments[:3]: - assert fragment in clone_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{dev_warehouse_schema}__sqb_physical' " - "AND table_name LIKE '%__v_%'" - ), - config=postgres_e2e_config, - ) == ((3,),) - stg_hash: str = str( - fetch_postgres_rows( - sql=( - "SELECT version_hash FROM " - f"{ - quoted_relation_name( - schema_name=prod_state_schema, - name='virtual_environment_node_refs', - ) - } " - "WHERE virtual_environment_name = 'prod' " - "AND node_type = 'model' AND node_name = 'stg_orders'" - ), - config=postgres_e2e_config, - )[0][0] - ) - assert ( - fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{ - quoted_relation_name( - schema_name=f'{dev_warehouse_schema}__sqb_physical', - name=f'stg_orders__v_{stg_hash[:8]}', - ) - }" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM " - f"{ - quoted_relation_name( - schema_name=dev_state_schema, - name='virtual_environment_node_refs', - ) - }" - ), - config=postgres_e2e_config, - ) == ((0,),) - - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - dev_ref_rows_before: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - "SELECT node_name, version_hash FROM " - f"{ - quoted_relation_name( - schema_name=dev_state_schema, - name='virtual_environment_node_refs', - ) - } " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - config=postgres_e2e_config, - ) - execute_postgres_sql( - sql=( - "DROP TABLE " - f"{ - quoted_relation_name( - schema_name=f'{dev_warehouse_schema}__sqb_physical', - name=f'stg_orders__v_{stg_hash[:8]}', - ) - } " - "CASCADE" - ), - config=postgres_e2e_config, - ) - target_ref_clone: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--virtual-env", - "dev", - ), - project_dir=project_dir, - ) - assert target_ref_clone.returncode == test_case.expected_exit_code, target_ref_clone.stderr - for fragment in test_case.expected_stdout_fragments[3:8]: - assert fragment in target_ref_clone.stdout - assert ( - fetch_postgres_rows( - sql=( - "SELECT node_name, version_hash FROM " - f"{ - quoted_relation_name( - schema_name=dev_state_schema, - name='virtual_environment_node_refs', - ) - } " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "ORDER BY node_name" - ), - config=postgres_e2e_config, - ) - == dev_ref_rows_before - ) - assert ( - fetch_postgres_rows( - sql=( - "SELECT id FROM " - f"{ - quoted_relation_name( - schema_name=f'{dev_warehouse_schema}__sqb_physical', - name=f'stg_orders__v_{stg_hash[:8]}', - ) - }" - ), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - - execute_postgres_sql( - sql=( - "DROP TABLE " - f"{ - quoted_relation_name( - schema_name=f'{prod_warehouse_schema}__sqb_physical', - name=f'stg_orders__v_{stg_hash[:8]}', - ) - }" - " CASCADE" - ), - config=postgres_e2e_config, - ) - missing_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--select", - "stg_orders", - ), - project_dir=project_dir, - ) - assert missing_result.returncode == 1 - assert test_case.expected_stdout_fragments[8] in missing_result.stdout - execute_postgres_sql( - sql=( - "CREATE TABLE " - f"{ - quoted_relation_name( - schema_name=f'{prod_warehouse_schema}__sqb_physical', - name=f'stg_orders__v_{stg_hash[:8]}', - ) - } " - "AS SELECT 7 AS id" - ), - config=postgres_e2e_config, - ) - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=dev_state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - f"('model_version:stg_orders:{stg_hash}', 'test', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - config=postgres_e2e_config, - ) - locked_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "clone", - "--from", - "prod", - "--to", - "dev", - "--select", - "stg_orders", - ), - project_dir=project_dir, - ) - assert locked_result.returncode == 1 - for fragment in test_case.expected_stdout_fragments[9:10]: - assert fragment in locked_result.stderr - skip_locked_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "clone", "--from", "prod", "--to", "dev", "--skip-locked"), - project_dir=project_dir, - ) - assert skip_locked_result.returncode == test_case.expected_exit_code, ( - skip_locked_result.stderr - ) - for fragment in test_case.expected_stdout_fragments[10:]: - assert fragment in skip_locked_result.stdout - finally: - cleanup_postgres_state_schemas(schema_name=prod_state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=dev_state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=prod_warehouse_schema, config=postgres_e2e_config - ) - cleanup_postgres_state_schemas(schema_name=dev_warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{prod_warehouse_schema}__prod", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{dev_warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{prod_warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{dev_warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres reconcile repair-view recreates logical view", - expected_rows=((1,),), - expected_stdout_fragments=( - "Repair", - "model orders", - "VDE dev", - "action recreate logical view from state", - "result repaired", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_missing_logical_view_when_repairing_then_view_is_recreated( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_repair", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_reconcile_repair"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n\n' - "[targets.dev.state.connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - execute_postgres_sql( - sql=f"DROP VIEW {logical_relation}", - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - fetch_postgres_rows( - sql=f"SELECT id FROM {logical_relation} ORDER BY id", - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres reconcile attach rebinds logical view", - expected_rows=((2,),), - expected_stdout_fragments=("Attach", "model orders", "result attached"), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_tracked_physical_relation_when_attaching_then_view_is_rebound( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_attach", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "postgres_reconcile_attach"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{warehouse_schema}"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n\n' - "[targets.dev.state.connection]\n" - f'host = "{postgres_e2e_config["host"]}"\n' - f"port = {postgres_e2e_config['port']}\n" - f'dbname = "{postgres_e2e_config["dbname"]}"\n' - f'user = "{postgres_e2e_config["user"]}"\n' - f'password = "{postgres_e2e_config["password"]}"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - assert ( - run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode - == test_case.expected_exit_code - ) - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ).returncode - == test_case.expected_exit_code - ) - _database_name, physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT database_name, schema_name, relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - physical_relation: str = quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - physical_relation, - "--auto-approve", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - assert ( - fetch_postgres_rows( - sql=(f"SELECT id FROM {logical_relation} ORDER BY id"), - config=postgres_e2e_config, - ) - == test_case.expected_rows - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__pr", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres reconcile repair-view blocks missing physical relation", - expected_rows=(), - expected_stdout_fragments=( - "missing physical relation: orders", - "error[C252]: missing physical relation for 'orders'", - ), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_missing_physical_relation_when_repairing_then_it_blocks( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_missing_physical", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_missing_physical", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT schema_name, relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - missing_relation: str = quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ) - execute_postgres_sql( - sql=f"DROP TABLE {missing_relation} CASCADE", - config=postgres_e2e_config, - ) - - report_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "reconcile", "--virtual-env", "dev"), - project_dir=project_dir, - ) - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - combined_output: str = ( - report_result.stdout + report_result.stderr + result.stdout + result.stderr - ) - assert result.returncode == test_case.expected_exit_code, combined_output - for fragment in test_case.expected_stdout_fragments: - assert fragment in combined_output - assert "Traceback" not in combined_output - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres reconcile attach blocks untracked physical relation", - expected_rows=(), - expected_stdout_fragments=("is not a tracked relation",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_untracked_physical_relation_when_attaching_then_it_blocks( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_attach_untracked", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_attach_untracked", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - physical_schema: str = f"{warehouse_schema}__sqb_physical" - untracked_relation: str = quoted_relation_name( - schema_name=physical_schema, - name="untracked_orders", - ) - execute_postgres_sql( - sql=f"CREATE TABLE {untracked_relation} AS SELECT 9 AS id", - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - untracked_relation, - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres reconcile attach blocks wrong-model tracked physical relation", - expected_rows=(), - expected_stdout_fragments=("is not a tracked relation for 'orders'",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_wrong_model_physical_relation_when_attaching_then_refs_are_unchanged( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_attach_wrong_model", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_attach_wrong_model", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "models/customers.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - original_refs: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT schema_name, relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'customers' " - "ORDER BY updated_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ), - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - assert ( - fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - == original_refs - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres reconcile attach cancellation leaves refs unchanged", - expected_rows=(), - expected_stdout_fragments=("reconcile attach cancelled",), - expected_exit_code=1, - input_text="nope\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_wrong_confirmation_when_attaching_then_refs_are_unchanged( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_attach_cancelled", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_attach_cancelled", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - original_refs: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT schema_name, relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ), - ), - project_dir=project_dir, - input_text=test_case.input_text, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - assert ( - fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - == original_refs - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres repair-view blocks logical target table", - expected_rows=(), - expected_stdout_fragments=("logical target for 'orders' is a table",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_logical_target_table_when_repairing_then_it_blocks( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_repair_table_block", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_repair_table_block", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - execute_postgres_sql( - sql=f"DROP VIEW {logical_relation}; CREATE TABLE {logical_relation} AS SELECT 1 AS id", - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres attach blocks logical target table before ref update", - expected_rows=(), - expected_stdout_fragments=("logical target for 'orders' is a table",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_logical_target_table_when_attaching_then_refs_are_unchanged( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_attach_table_block", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_attach_table_block", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - original_refs: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT schema_name, relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - execute_postgres_sql( - sql=f"DROP VIEW {logical_relation}; CREATE TABLE {logical_relation} AS SELECT 1 AS id", - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ), - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - assert ( - fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - == original_refs - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres repair-view blocks when target vde is locked", - expected_rows=(), - expected_stdout_fragments=("virtual environment 'dev' is locked",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_target_virtual_environment_lock_when_repairing_then_it_blocks( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_repair_locked", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_repair_locked", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - logical_relation: str = quoted_relation_name( - schema_name=f"{warehouse_schema}__dev", - name="orders", - ) - execute_postgres_sql( - sql=f"DROP VIEW {logical_relation}", - config=postgres_e2e_config, - ) - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - assert fetch_postgres_rows( - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - f"WHERE table_schema = '{warehouse_schema}__dev' AND table_name = 'orders'" - ), - config=postgres_e2e_config, - ) == ((0,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresReconcileE2ETestCase( - description="postgres attach blocks when target vde is locked", - expected_rows=(), - expected_stdout_fragments=("virtual environment 'dev' is locked",), - expected_exit_code=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_target_virtual_environment_lock_when_attaching_then_refs_are_unchanged( - test_case: PostgresReconcileE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - warehouse_schema: str = build_unique_schema_name(prefix="sqb_virtual_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_reconcile_attach_locked", - repo_files={ - "sqlbuild_project.toml": build_postgres_virtual_project_toml( - project_name="postgres_reconcile_attach_locked", - config=postgres_e2e_config, - state_schema=state_schema, - warehouse_schema=warehouse_schema, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - refs_relation: str = quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ) - original_refs: tuple[tuple[object, ...], ...] = fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - physical_schema_name, physical_relation_name = fetch_postgres_rows( - sql=( - "SELECT schema_name, relation_name FROM " - f"{quoted_relation_name(schema_name=state_schema, name='physical_relations')} " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0] - execute_postgres_sql( - sql=( - "INSERT INTO " - f"{quoted_relation_name(schema_name=state_schema, name='locks')} " - "(lock_key, owner_id, expires_at, created_at, updated_at) VALUES " - "('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - quoted_relation_name( - schema_name=str(physical_schema_name), - name=str(physical_relation_name), - ), - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout + result.stderr - assert ( - fetch_postgres_rows( - sql=( - f"SELECT version_hash FROM {refs_relation} " - "WHERE virtual_environment_name = 'dev' " - "AND node_type = 'model' AND node_name = 'orders'" - ), - config=postgres_e2e_config, - ) - == original_refs - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas(schema_name=warehouse_schema, config=postgres_e2e_config) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__dev", - config=postgres_e2e_config, - ) - cleanup_postgres_state_schemas( - schema_name=f"{warehouse_schema}__sqb_physical", - config=postgres_e2e_config, - ) - - -@pytest.mark.parametrize( - "test_case", - ( - PostgresStateLifecycleErrorE2ETestCase( - description="reset blocks when allow reset is false", - allow_reset=False, - command=("--no-color", "state", "reset", "--auto-approve"), - expected_exit_code=1, - expected_error_fragment=("set `allow_reset = true` under `[targets..state]`"), - ), - PostgresStateLifecycleErrorE2ETestCase( - description="reset blocks without auto approve", - allow_reset=True, - command=("--no-color", "state", "reset"), - expected_exit_code=1, - expected_error_fragment="state reset requires --auto-approve", - ), - PostgresStateLifecycleErrorE2ETestCase( - description="rollback blocks before backup exists", - allow_reset=False, - command=("--no-color", "state", "rollback"), - expected_exit_code=1, - expected_error_fragment="No state backup is available for rollback", - ), - ), - ids=lambda case: case.description, -) -def test_given_postgres_state_config_when_running_blocked_state_command_then_cli_reports_error( - test_case: PostgresStateLifecycleErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_project", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_project", - config=postgres_e2e_config, - state_schema=state_schema, - allow_reset=test_case.allow_reset, - ) - }, - ) - - try: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - ( - PostgresStateSchemaCorruptionE2ETestCase( - description="postgres migrate blocks cleanly when required state table is missing", - mutation_sql_template="DROP TABLE {virtual_environment_node_refs}", - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ), - PostgresStateSchemaCorruptionE2ETestCase( - description="postgres migrate blocks cleanly when required state column is missing", - mutation_sql_template='ALTER TABLE {state_versions} DROP COLUMN "updated_at"', - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ), - PostgresStateSchemaCorruptionE2ETestCase( - description="postgres migrate blocks cleanly when required state column has wrong type", - mutation_sql_template=( - 'ALTER TABLE {state_versions} ALTER COLUMN "schema_version" ' - 'TYPE text USING "schema_version"::text' - ), - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ), - ), - ids=lambda case: case.description, -) -def test_given_corrupt_postgres_state_schema_when_migrating_then_cli_blocks_cleanly( - test_case: PostgresStateSchemaCorruptionE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_schema_corruption", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_schema_corruption", - config=postgres_e2e_config, - state_schema=state_schema, - ) - }, - ) - - try: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - execute_postgres_sql( - sql=test_case.mutation_sql_template.format( - state_versions=quoted_relation_name( - schema_name=state_schema, - name="state_versions", - ), - virtual_environment_node_refs=quoted_relation_name( - schema_name=state_schema, - name="virtual_environment_node_refs", - ), - ), - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateResetInvalidE2ETestCase( - description="postgres explicit rollback blocks cleanly when backup schema is deleted", - expected_exit_code=1, - expected_error_fragment="backup_", - ) - ], - ids=lambda case: case.description, -) -def test_given_deleted_postgres_state_backup_when_rolling_back_then_it_blocks_cleanly( - test_case: PostgresStateResetInvalidE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_deleted_backup", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_deleted_backup", - config=postgres_e2e_config, - state_schema=state_schema, - ) - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb(command=("--no-color", "state", "migrate"), project_dir=project_dir).returncode - == 0 - ) - events_relation: str = quoted_relation_name( - schema_name=state_schema, - name="state_migration_events", - ) - backup_id: str = str( - fetch_postgres_rows( - sql=( - f"SELECT backup_id FROM {events_relation} " - "WHERE action = 'backup' ORDER BY created_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0][0] - ) - execute_postgres_sql( - sql=f"DROP SCHEMA {quote_identifier(f'{state_schema}__backup_{backup_id}')} CASCADE", - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback", "--backup-id", backup_id), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateResetInvalidE2ETestCase( - description="postgres latest rollback blocks cleanly when backup schema is deleted", - expected_exit_code=1, - expected_error_fragment="No state backup is available for rollback", - ) - ], - ids=lambda case: case.description, -) -def test_given_deleted_latest_postgres_state_backup_when_rolling_back_then_it_blocks_cleanly( - test_case: PostgresStateResetInvalidE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_deleted_latest_backup", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_deleted_latest_backup", - config=postgres_e2e_config, - state_schema=state_schema, - ) - }, - ) - - try: - assert ( - run_sqb(command=("--no-color", "state", "init"), project_dir=project_dir).returncode - == 0 - ) - assert ( - run_sqb(command=("--no-color", "state", "migrate"), project_dir=project_dir).returncode - == 0 - ) - events_relation: str = quoted_relation_name( - schema_name=state_schema, - name="state_migration_events", - ) - backup_id: str = str( - fetch_postgres_rows( - sql=( - f"SELECT backup_id FROM {events_relation} " - "WHERE action = 'backup' ORDER BY created_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0][0] - ) - execute_postgres_sql( - sql=f"DROP SCHEMA {quote_identifier(f'{state_schema}__backup_{backup_id}')} CASCADE", - config=postgres_e2e_config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateExplicitRollbackE2ETestCase( - description="postgres rollback accepts explicit backup id", - expected_exit_code=0, - expected_schema_version=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backup_when_rolling_back_explicit_backup_id_then_restores_backup( - test_case: PostgresStateExplicitRollbackE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_project", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_project", - config=postgres_e2e_config, - state_schema=state_schema, - ) - }, - ) - - try: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == test_case.expected_exit_code, ( - init_result.stdout + init_result.stderr - ) - migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - assert migrate_result.returncode == test_case.expected_exit_code, ( - migrate_result.stdout + migrate_result.stderr - ) - events_table: str = quoted_relation_name( - schema_name=state_schema, - name="state_migration_events", - ) - backup_id: str = str( - fetch_postgres_rows( - sql=( - f"SELECT backup_id FROM {events_table} " - "WHERE action = 'backup' ORDER BY created_at DESC LIMIT 1" - ), - config=postgres_e2e_config, - )[0][0] - ) - execute_postgres_sql( - sql=( - "UPDATE " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')} " - "SET schema_version = 2" - ), - config=postgres_e2e_config, - ) - second_migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - assert second_migrate_result.returncode == test_case.expected_exit_code, ( - second_migrate_result.stdout + second_migrate_result.stderr - ) - execute_postgres_sql( - sql=( - "DELETE FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')}" - ), - config=postgres_e2e_config, - ) - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback", "--backup-id", backup_id), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code, ( - rollback_result.stdout + rollback_result.stderr - ) - assert test_case.expected_rollback_fragment in rollback_result.stdout - assert fetch_postgres_rows( - sql=( - "SELECT schema_version FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_schema_version,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateLocalOverrideE2ETestCase( - description="local state connection override controls postgres state database", - expected_exit_code=0, - expected_schema_version=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_local_state_override_when_running_state_init_then_cli_uses_local_config( - test_case: PostgresStateLocalOverrideE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - bad_state_config: dict[str, object] = {**postgres_e2e_config, "port": 1} - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_project", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_project", - config=postgres_e2e_config, - state_config=bad_state_config, - state_schema=state_schema, - ), - "sqlbuild_local.toml": build_postgres_local_state_connection_toml( - config=postgres_e2e_config - ), - }, - ) - - try: - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert fetch_postgres_rows( - sql=( - "SELECT schema_version FROM " - f"{quoted_relation_name(schema_name=state_schema, name='state_versions')}" - ), - config=postgres_e2e_config, - ) == ((test_case.expected_schema_version,),) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateConnectionErrorE2ETestCase( - description="postgres state connection failure renders coded cli error", - expected_exit_code=1, - expected_error_fragment="Could not connect to Postgres state backend", - ) - ], - ids=lambda case: case.description, -) -def test_given_bad_postgres_state_connection_when_running_state_init_then_cli_reports_error( - test_case: PostgresStateConnectionErrorE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - bad_state_config: dict[str, object] = {**postgres_e2e_config, "port": 1} - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_project", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_project", - config=postgres_e2e_config, - state_config=bad_state_config, - state_schema=state_schema, - ) - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateResetInvalidE2ETestCase( - description="postgres reset makes migrate block until state is initialized again", - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_reset_when_running_migrate_then_cli_reports_invalid_state( - test_case: PostgresStateResetInvalidE2ETestCase, - tmp_path: Path, - postgres_e2e_config: dict[str, object], -) -> None: - state_schema: str = build_unique_schema_name(prefix="sqb_state_e2e") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="postgres_state_project", - repo_files={ - "sqlbuild_project.toml": build_postgres_state_project_toml( - project_name="postgres_state_project", - config=postgres_e2e_config, - state_schema=state_schema, - allow_reset=True, - ) - }, - ) - - try: - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - reset_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "reset", "--auto-approve"), - project_dir=project_dir, - ) - assert reset_result.returncode == 0, reset_result.stdout + reset_result.stderr - - migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=migrate_result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - finally: - cleanup_postgres_state_schemas(schema_name=state_schema, config=postgres_e2e_config) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index 7b5fb5d0aa..0ec54603f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -11,9 +11,6 @@ import pytest -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) from tests.e2e.src.sqlbuild.cli.commands.main.providers._test_types import ( ProviderCommandConcurrencyE2ETestCase, ProviderCommandDiagnosticE2ETestCase, @@ -668,64 +665,6 @@ def alias_task(ctx, marker_provider: AliasMarkerProvider): assert test_case.expected_error_fragment in result.stdout + result.stderr -@pytest.mark.parametrize( - "test_case", - [ - ProviderCommandFailureE2ETestCase( - description="virtual build provider tears down when command fails after setup", - command=("--no-color", "build", "--select", "+fact_orders"), - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "providers/marker.py": PROVIDER_MARKER_FILE, - "python/tasks/failing_task.py": dedent( - """ - from sqlbuild.refs import model - from providers.marker import MarkerProvider - from sqlbuild.tasks import task - - - @task(depends_on=model("fact_orders")) - def failing_task(ctx, marker_provider: MarkerProvider): - marker_provider.mark("virtual_task") - raise RuntimeError("intentional provider failure") - """ - ).strip() - + "\n", - "models/fact_orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - }, - expected_marker_entries=("setup", "virtual_task", "teardown"), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_build_provider_failure_when_provider_was_setup_then_provider_tears_down( - test_case: ProviderCommandFailureE2ETestCase, - tmp_path: Path, -) -> None: - marker_path: Path = tmp_path / "provider-virtual-marker.log" - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="provider_virtual_project", - repo_files=test_case.repo_files, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - env={"MARKER_PATH": str(marker_path)}, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - env={"MARKER_PATH": str(marker_path)}, - ) - - assert result.returncode != 0, result.stdout + result.stderr - marker_entries: tuple[str, ...] = tuple(marker_path.read_text(encoding="utf-8").splitlines()) - assert marker_entries == test_case.expected_marker_entries - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/_test_types.py deleted file mode 100644 index 62ca7f7518..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/_test_types.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Test types for reconcile e2e tests.""" - -from __future__ import annotations - -from dataclasses import dataclass, field - - -@dataclass(frozen=True) -class ReconcileE2ETestCase: - description: str - command: tuple[str, ...] - expected_fragments: tuple[str, ...] - expected_exit_code: int = 0 - expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] = field( - default_factory=tuple - ) - expected_report_fragments: tuple[str, ...] = field(default_factory=tuple) - unexpected_fragments: tuple[str, ...] = field(default_factory=tuple) - input_text: str | None = None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/helpers.py deleted file mode 100644 index 752bab05b2..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/helpers.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, -) - - -def build_virtual_seed_reconcile_repo_files() -> dict[str, str]: - return build_virtual_plan_repo_files( - stg_orders_sql='SELECT id, amount_cents FROM __seed("order_amounts")' - ) | { - "seeds/orders.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " path: order_amounts.csv\n" - " columns:\n" - " - name: id\n" - " type: integer\n" - " - name: amount_cents\n" - " type: integer\n" - ), - "seeds/order_amounts.csv": "id,amount_cents\n1,100\n", - } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/test_reconcile.py b/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/test_reconcile.py deleted file mode 100644 index eb11c02c35..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/reconcile/test_reconcile.py +++ /dev/null @@ -1,952 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.main.reconcile._test_types import ( - ReconcileE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.reconcile.helpers import ( - build_virtual_seed_reconcile_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="report shows no issues for healthy vde", - command=("reconcile", "--virtual-env", "dev"), - expected_fragments=("Reconcile report for dev: no issues.",), - ) - ], - ids=lambda case: case.description, -) -def test_given_healthy_virtual_environment_when_reconciling_then_report_is_clean( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_report", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="report detects missing logical seed view", - command=("reconcile", "--virtual-env", "dev"), - expected_fragments=( - "Reconcile report for dev:", - "missing logical seed target: order_amounts", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_seed_view_when_reconciling_then_report_flags_seed_issue( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_seed_report", - repo_files=build_virtual_seed_reconcile_repo_files(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql='DROP VIEW "dev__dev"."order_amounts"', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="repair-view recreates missing logical seed view", - command=( - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--seed", - "order_amounts", - ), - expected_fragments=( - "Repair", - "seed order_amounts", - "VDE dev", - "action recreate logical seed view from state", - "result repaired", - ), - expected_query_results=( - ("SELECT id, amount_cents FROM dev__dev.order_amounts ORDER BY id", ((1, 100),)), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_seed_view_when_repairing_then_it_is_recreated( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_seed_repair", - repo_files=build_virtual_seed_reconcile_repo_files(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql='DROP VIEW "dev__dev"."order_amounts"', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="report detects missing physical seed relation", - command=("reconcile", "--virtual-env", "dev"), - expected_fragments=( - "Reconcile report for dev:", - "missing physical seed relation: order_amounts", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_physical_seed_relation_when_reconciling_then_report_flags_seed_issue( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_seed_physical_report", - repo_files=build_virtual_seed_reconcile_repo_files(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'seed' AND artifact_name = 'order_amounts' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name = physical_rows[0] - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=f'DROP TABLE "{schema_name}"."{relation_name}"', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="report detects missing tracked physical seed state", - command=("reconcile", "--virtual-env", "dev"), - expected_fragments=( - "Reconcile report for dev:", - "missing tracked physical seed relation: order_amounts", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_seed_physical_state_when_reconciling_then_report_flags_seed_issue( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_seed_physical_state_report", - repo_files=build_virtual_seed_reconcile_repo_files(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "DELETE FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'seed' AND artifact_name = 'order_amounts'" - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="report detects and repair-view recreates missing logical vde view", - command=( - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "fact_orders", - ), - expected_fragments=( - "Repair", - "model fact_orders", - "VDE dev", - "action recreate logical view from state", - "result repaired", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((1,),)),), - expected_report_fragments=( - "Virtual reconcile", - "Reconcile report for dev:", - "missing logical target: fact_orders", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_view_when_reconciling_and_repairing_then_it_is_recreated( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_repair_view", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql='DROP VIEW "dev__dev"."fact_orders"', - ) - - report_result: subprocess.CompletedProcess[str] = run_sqb( - command=("reconcile", "--virtual-env", "dev"), - project_dir=project_dir, - ) - - assert report_result.returncode == 0, report_result.stdout + report_result.stderr - for fragment in test_case.expected_report_fragments: - assert fragment in report_result.stdout + report_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="report detects and repair-view blocks missing physical relation", - command=( - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "fact_orders", - ), - expected_exit_code=1, - expected_fragments=("error[C252]: missing physical relation for 'fact_orders'",), - expected_report_fragments=( - "Virtual reconcile", - "Reconcile report for dev:", - "missing physical relation: fact_orders", - ), - unexpected_fragments=("Traceback",), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_physical_relation_when_reconciling_and_repairing_then_it_blocks( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_missing_physical", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'fact_orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name = physical_rows[0] - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=f'DROP TABLE "{schema_name}"."{relation_name}"', - ) - - report_result: subprocess.CompletedProcess[str] = run_sqb( - command=("reconcile", "--virtual-env", "dev"), - project_dir=project_dir, - ) - - assert report_result.returncode == 0, report_result.stdout + report_result.stderr - for fragment in test_case.expected_report_fragments: - assert fragment in report_result.stdout + report_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - combined_output: str = result.stdout + result.stderr - assert result.returncode == test_case.expected_exit_code, combined_output - for fragment in test_case.expected_fragments: - assert fragment in combined_output - for fragment in test_case.unexpected_fragments: - assert fragment not in combined_output - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="attach rebinds logical ref to tracked physical relation", - command=(), - expected_fragments=( - "Attach", - "model fact_orders", - "VDE dev", - "physical", - "result attached", - ), - expected_query_results=(("SELECT id FROM dev__dev.fact_orders ORDER BY id", ((2,),)),), - input_text="attach fact_orders\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_tracked_physical_relation_when_attaching_then_logical_ref_is_rebound( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_attach", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - attach_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name, version_hash " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'fact_orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name, _version_hash = attach_relation_rows[0] - attach_relation: str = f'"{schema_name}"."{relation_name}"' - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "fact_orders", - "--physical-relation", - attach_relation, - ), - project_dir=project_dir, - input_text=test_case.input_text, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - for query_sql, expected_rows in test_case.expected_query_results: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="attach blocks untracked physical relation", - command=(), - expected_exit_code=1, - expected_fragments=("is not a tracked relation",), - ) - ], - ids=lambda case: case.description, -) -def test_given_untracked_physical_relation_when_attaching_then_it_blocks( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_attach_untracked", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql='CREATE TABLE "dev__sqb_physical"."untracked_orders" AS SELECT 9 AS id', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "fact_orders", - "--physical-relation", - '"dev__sqb_physical"."untracked_orders"', - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="attach blocks physical relation tracked for another model", - command=(), - expected_exit_code=1, - expected_fragments=("is not a tracked relation for 'fact_orders'",), - ) - ], - ids=lambda case: case.description, -) -def test_given_wrong_model_physical_relation_when_attaching_then_it_blocks_before_ref_update( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_attach_wrong_model", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - original_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - (project_dir / "models" / "dim_customers.sql").write_text( - "MODEL ();\n\nSELECT 2 AS customer_id\n", - encoding="utf-8", - ) - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - wrong_model_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'dim_customers' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name = wrong_model_relation_rows[0] - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "fact_orders", - "--physical-relation", - f'"{schema_name}"."{relation_name}"', - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - assert ( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - == original_ref_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="attach cancellation blocks before ref mutation", - command=(), - expected_exit_code=1, - expected_fragments=("reconcile attach cancelled",), - input_text="nope\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_wrong_confirmation_when_attaching_then_it_cancels( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_attach_cancelled", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - original_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - attach_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'fact_orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name = attach_relation_rows[0] - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "fact_orders", - "--physical-relation", - f'"{schema_name}"."{relation_name}"', - ), - project_dir=project_dir, - input_text=test_case.input_text, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - assert ( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - == original_ref_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="repair-view blocks logical target table", - command=( - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "fact_orders", - ), - expected_exit_code=1, - expected_fragments=("logical target for 'fact_orders' is a table",), - ) - ], - ids=lambda case: case.description, -) -def test_given_logical_target_table_when_repairing_view_then_it_blocks( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_repair_table_block", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - 'DROP VIEW "dev__dev"."fact_orders"; ' - 'CREATE TABLE "dev__dev"."fact_orders" AS SELECT 1 AS id' - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="attach blocks logical target table before ref mutation", - command=(), - expected_exit_code=1, - expected_fragments=("logical target for 'fact_orders' is a table",), - ) - ], - ids=lambda case: case.description, -) -def test_given_logical_target_table_when_attaching_then_it_blocks_before_ref_update( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_attach_table_block", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - original_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ).returncode - == 0 - ) - attach_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'fact_orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name = attach_relation_rows[0] - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - 'DROP VIEW "dev__dev"."fact_orders"; ' - 'CREATE TABLE "dev__dev"."fact_orders" AS SELECT 1 AS id' - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "fact_orders", - "--physical-relation", - f'"{schema_name}"."{relation_name}"', - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - assert ( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - == original_ref_rows - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="repair-view blocks when target vde is locked", - command=( - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "fact_orders", - ), - expected_exit_code=1, - expected_fragments=("virtual environment 'dev' is locked",), - ) - ], - ids=lambda case: case.description, -) -def test_given_target_virtual_environment_lock_when_repairing_view_then_it_blocks( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_repair_locked", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql='DROP VIEW "dev__dev"."fact_orders"', - ) - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) " - "VALUES ('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - assert ( - query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_name FROM information_schema.tables " - "WHERE table_schema = 'dev__dev' AND table_name = 'fact_orders'" - ), - ) - == [] - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileE2ETestCase( - description="attach blocks when target vde is locked", - command=(), - expected_exit_code=1, - expected_fragments=("virtual environment 'dev' is locked",), - ) - ], - ids=lambda case: case.description, -) -def test_given_target_virtual_environment_lock_when_attaching_then_it_blocks_before_ref_update( - test_case: ReconcileE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_reconcile_attach_locked", - repo_files=build_virtual_plan_repo_files(stg_orders_sql="SELECT 1 AS id"), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - original_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - (project_dir / "models" / "stg_orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), project_dir=project_dir - ).returncode - == 0 - ) - attach_relation_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT schema_name, relation_name FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'fact_orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - ) - schema_name, relation_name = attach_relation_rows[0] - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.locks " - "(lock_key, owner_id, expires_at, created_at, updated_at) " - "VALUES ('virtual_env:dev', 'test-owner', TIMESTAMP '2999-01-01', " - "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "reconcile", - "attach", - "--auto-approve", - "--virtual-env", - "dev", - "--model", - "fact_orders", - "--physical-relation", - f'"{schema_name}"."{relation_name}"', - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - for fragment in test_case.expected_fragments: - assert fragment in result.stdout + result.stderr - assert ( - query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT version_hash FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model' " - "AND node_name = 'fact_orders'" - ), - ) - == original_ref_rows - ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 23120ea596..5a066f3a23 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -3,6 +3,40 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class EvaluatedPredicateCase: + description: str + sql: str + expected_evaluated_models: int = 1 + + +@dataclass(frozen=True) +class GroupedRulesCase: + description: str + arguments: tuple[str, ...] + result_key: str + expected_rules: tuple[str, ...] + configuration: str = "" + + +@dataclass(frozen=True) +class UnevaluatedRuleCase: + description: str + model_options: str = "" + configuration: str = "" + expected_exit: int = 1 + + +@dataclass(frozen=True) +class UnevaluatedResourceCase: + description: str + path: str + template: str + expected_evaluated_models: int = 1 + expected_reason: str = "E_GUARD_FUNCTION_NESTING_DEPTH_EXCEEDED" + adapter: str = "duckdb" + + @dataclass(frozen=True) class RulesPerformanceGuardTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index bfefb25a65..d127c1bfc3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -1,5 +1,49 @@ """Compiler Rules end-to-end fixture helpers.""" +import json +import subprocess +import sys +from pathlib import Path + from scripts.rules_benchmark._helpers.custom_rules import write_custom_rules __all__ = ("write_custom_rules",) + + +def write_unevaluated_rules_project( + *, + project_dir: Path, + model_options: str = "", + configuration: str = "", + adapter: str = "duckdb", + selected_rules: tuple[str, ...] = ("SQBRSQL035",), + resource_path: str = "models/staging/orders.sql", + resource_template: str = "MODEL ({options});\nSELECT {expression} AS order_id", +) -> None: + (project_dir / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{adapter}"\n[rules]\nselect = {json.dumps(selected_rules)}\n' + + configuration + ) + models: Path = project_dir / "models/staging" + models.mkdir(parents=True) + (models / "orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id\n") + resource: Path = project_dir / resource_path + resource.parent.mkdir(parents=True, exist_ok=True) + resource.write_text( + resource_template.format(options=model_options, expression="ABS(" * 130 + "1" + ")" * 130) + ) + + +def run_unevaluated_rules_cli(project_dir: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + *args, + ], + capture_output=True, + text=True, + check=False, + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py new file mode 100644 index 0000000000..726ad9555a --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_unevaluated.py @@ -0,0 +1,223 @@ +"""Real CLI coverage of compiler-accepted SQL beyond the Rules parser budget.""" + +import json +import subprocess +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( + EvaluatedPredicateCase, + GroupedRulesCase, + UnevaluatedResourceCase, + UnevaluatedRuleCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + run_unevaluated_rules_cli, + write_unevaluated_rules_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + UnevaluatedResourceCase( + "model guard", + "models/staging/orders.sql", + "MODEL ();\nSELECT {expression} AS order_id", + 0, + ), + UnevaluatedResourceCase( + "audit guard", "audits/orders.sql", "AUDIT ();\nSELECT {expression} AS order_id" + ), + UnevaluatedResourceCase( + "SQL-test guard", + "tests/unit/test_orders.sql", + "TEST ();\nWITH __ref__orders AS (SELECT {expression} AS order_id), __expected__orders AS (SELECT 1 AS order_id) SELECT 1", + ), + ], + ids=lambda case: case.description, +) +def test_given_guarded_resource_when_running_rules_then_reports_failure_and_coverage( + test_case: UnevaluatedResourceCase, + tmp_path: Path, +) -> None: + write_unevaluated_rules_project( + project_dir=tmp_path, + resource_path=test_case.path, + resource_template=test_case.template, + adapter=test_case.adapter, + ) + for _ in range(2): + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "--json", "run", "SQBRSQL035" + ) + assert result.returncode == 1, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["unevaluated_resources"] == 1 + assert payload["evaluated_models"] == test_case.expected_evaluated_models + (finding,) = payload["findings"] + assert finding["code"] == "rules-unevaluated" + assert finding["path"] == test_case.path + assert finding["affected_rules"] == ["SQBRSQL035"] + assert test_case.expected_reason in finding["message"] + human: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "run", "SQBRSQL035" + ) + assert human.returncode == 1 + assert ( + f"{test_case.expected_evaluated_models} models evaluated, 1 resources could not be evaluated" + in human.stdout + ) + assert "passed" not in human.stdout + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--no-cache", "--json" + ) + assert compiled.returncode == 1, compiled.stdout + compiled.stderr + diagnostics: list[dict[str, Any]] = json.loads(compiled.stdout)["diagnostics"] + assert any( + item["code"] == "rules-unevaluated" and item["path"] == test_case.path + for item in diagnostics + ) + + +@pytest.mark.parametrize( + "test_case", + [ + EvaluatedPredicateCase( + "prefix NOT LIKE", + "SELECT 1 AS order_id WHERE NOT SPLIT_PART('orders:pending', ':', 2) LIKE 'pending%'", + ), + EvaluatedPredicateCase( + "prefix NOT LIKE inside CASE", + "SELECT CASE WHEN NOT SPLIT_PART('orders:pending', ':', 2) LIKE 'pending%' THEN 1 ELSE 0 END AS order_id", + ), + ], + ids=lambda case: case.description, +) +def test_given_prefix_not_like_when_running_rules_then_evaluates_the_complete_model( + test_case: EvaluatedPredicateCase, + tmp_path: Path, +) -> None: + write_unevaluated_rules_project( + project_dir=tmp_path, + resource_path="models/staging/orders.sql", + resource_template="MODEL (database warehouse, schema analytics);\n" + test_case.sql, + adapter="snowflake", + ) + for _ in range(2): + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, + "rules", + "--json", + "run", + "SQBRSQL035", + ) + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["unevaluated_resources"] == 0 + assert payload["evaluated_models"] == test_case.expected_evaluated_models + assert payload["findings"] == [] + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, + "compile", + "--no-cache", + "--json", + ) + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + assert json.loads(compiled.stdout)["diagnostics"] == [] + + +@pytest.mark.parametrize( + "test_case", + [ + UnevaluatedRuleCase( + "model analysis opt out", model_options="sql_analysis false", expected_exit=0 + ), + UnevaluatedRuleCase( + "path analysis opt out", + configuration="\n[path_defaults.staging]\nsql_analysis = false\n", + expected_exit=0, + ), + UnevaluatedRuleCase( + "explicit scoped rule ignore", + configuration='\n[[rules.rule_ignores]]\nrules = ["SQBRSQL035"]\npaths = ["models/staging/orders.sql"]\nreason = "External SQL syntax"\n', + expected_exit=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_explicit_escape_hatch_when_running_rules_then_honors_policy( + test_case: UnevaluatedRuleCase, + tmp_path: Path, +) -> None: + write_unevaluated_rules_project( + project_dir=tmp_path, + model_options=test_case.model_options, + configuration=test_case.configuration, + ) + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "rules", "--json", "run", "SQBRSQL035" + ) + assert result.returncode == test_case.expected_exit, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["unevaluated_resources"] == 0 + assert payload["findings"] == [] + compiled: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, "compile", "--no-cache", "--json" + ) + assert compiled.returncode == test_case.expected_exit, compiled.stdout + compiled.stderr + assert json.loads(compiled.stdout)["diagnostics"] == [] + + +@pytest.mark.parametrize( + "test_case", + [ + GroupedRulesCase( + "compile groups two Rules", + ("compile", "--json", "--no-cache"), + "diagnostics", + ("SQBRSQL034", "SQBRSQL035"), + ), + GroupedRulesCase( + "Rules CLI groups selected family", + ("rules", "--json", "run", "SQBRSQL"), + "findings", + tuple(f"SQBRSQL{number:03}" for number in range(1, 42)), + ), + GroupedRulesCase( + "suppression precedes grouping", + ("compile", "--json", "--no-cache"), + "diagnostics", + ("SQBRSQL034",), + '\n[[rules.rule_ignores]]\nrules = ["SQBRSQL035"]\npaths = ["models/staging/orders.sql"]\nreason = "External SQL syntax"\n', + ), + ], + ids=lambda case: case.description, +) +def test_given_one_failed_resource_and_multiple_rules_when_running_cli_then_groups_cause( + test_case: GroupedRulesCase, + tmp_path: Path, +) -> None: + write_unevaluated_rules_project( + project_dir=tmp_path, + selected_rules=("SQBRSQL034", "SQBRSQL035"), + configuration=test_case.configuration, + ) + for _ in range(2): + result: subprocess.CompletedProcess[str] = run_unevaluated_rules_cli( + tmp_path, *test_case.arguments + ) + assert result.returncode == 1, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + (finding,) = payload[test_case.result_key] + assert finding["code"] == "rules-unevaluated" + assert finding["affected_rules"] == list(test_case.expected_rules) + assert finding["message"].startswith( + f"{len(test_case.expected_rules)} selected Rules could not evaluate this resource:" + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py index a275dc7e36..685ddac907 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/_test_types.py @@ -6,6 +6,29 @@ from pathlib import Path +@dataclass(frozen=True) +class ScenarioNamespaceE2ETestCase: + description: str + namespace_a: str = "job-a" + namespace_b: str = "job-b" + expected_total: int = 15 + expected_other_total: int = 27 + expected_exit_code: int = 0 + expected_relation_count: int = 3 + expected_error: str = "C458" + + +@dataclass(frozen=True) +class ScenarioNamespaceSourceE2ETestCase: + description: str + expected_namespace: str | None + expected_source: str + project_config: str = "" + local_config: str = "" + environment: tuple[tuple[str, str], ...] = () + cli_args: tuple[str, ...] = () + + @dataclass(frozen=True) class ScenarioCliE2ETestCase: """Test case for sqb scenario command e2e verification.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py index 15d83abd71..ca9fce031c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/helpers.py @@ -8,7 +8,55 @@ from pathlib import Path from typing import cast -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + query_duckdb, + run_sqb, +) + + +def prepare_namespace_project(tmp_path: Path) -> Path: + """Create the shared-schema namespace test project.""" + return prepare_inline_project( + tmp_path=tmp_path, + project_name="scenario_namespaces", + repo_files=build_scenario_project_files(), + ) + + +def change_namespace_fixture(project: Path) -> None: + """Give the second job different inputs and an independently correct expectation.""" + scenario: Path = project / "tests/scenarios/order_totals_pass.sql" + scenario.write_text( + scenario.read_text() + .replace("10 AS amount", "22 AS amount") + .replace("15 AS total_amount", "27 AS total_amount") + ) + + +def run_namespaced_scenario(project: Path, namespace: str, *, retain: bool = False) -> None: + """Run the real CLI and verify machine output and progress attribution.""" + retain_args: tuple[str, ...] = {True: ("--retain",), False: ()}[retain] + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, + command=( + "--no-color", + "scenario", + "test", + "order_totals_pass", + "--scenario-namespace", + namespace, + "--json", + *retain_args, + ), + ) + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, object] = json.loads(result.stdout) + assert payload["execution"] == { + "scenario_namespace": namespace, + "scenario_namespace_source": "cli", + } + assert f"Scenario namespace: {namespace} (source: cli)" in result.stderr def build_scenario_project_files() -> dict[str, str]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py new file mode 100644 index 0000000000..d6c89a3c98 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py @@ -0,0 +1,283 @@ +"""Real CLI namespace isolation on a shared DuckDB schema.""" + +from __future__ import annotations + +import contextvars +import json +import subprocess +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.cli.entry.main.entry import main +from sqlbuild.compiler.planner._helpers.scenario.artifacts import compute_scenario_hash_prefix +from sqlbuild.executor.scenario._helpers.execution import run as scenario_execution +from tests.e2e.src.sqlbuild.cli.commands.main.scenario._test_types import ( + ScenarioNamespaceE2ETestCase, + ScenarioNamespaceSourceE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.scenario.helpers import ( + change_namespace_fixture, + list_scenario_relation_names, + prepare_namespace_project, + run_namespaced_scenario, + scenario_relation_name_by_suffix, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import query_duckdb, run_sqb + + +@pytest.mark.parametrize( + "test_case", + (ScenarioNamespaceE2ETestCase("different inputs and isolated success and failure teardown"),), + ids=lambda case: case.description, +) +def test_given_two_namespaces_when_running_and_tearing_down_then_results_are_independent( + test_case: ScenarioNamespaceE2ETestCase, tmp_path: Path +) -> None: + project: Path = prepare_namespace_project(tmp_path) + database: Path = project / "scenario_demo.duckdb" + run_namespaced_scenario(project, test_case.namespace_a, retain=True) + names_a: set[str] = set(list_scenario_relation_names(db_path=database)) + assert len(names_a) == test_case.expected_relation_count + change_namespace_fixture(project) + run_namespaced_scenario(project, test_case.namespace_b, retain=True) + names_b: set[str] = set(list_scenario_relation_names(db_path=database)) - names_a + assert len(names_b) == test_case.expected_relation_count + for namespace, expected in ( + (test_case.namespace_a, test_case.expected_total), + (test_case.namespace_b, test_case.expected_other_total), + ): + prefix: str = compute_scenario_hash_prefix( + project_name="scenario_demo", scenario_name="order_totals_pass", run_namespace=namespace + ) + assert query_duckdb( + db_path=database, sql=f'SELECT total_amount FROM "__sqb_{prefix}__model__order_totals"' + ) == [(expected,)] + run_namespaced_scenario(project, test_case.namespace_b) + assert set(list_scenario_relation_names(db_path=database)) == names_a + scenario: Path = project / "tests/scenarios/order_totals_pass.sql" + scenario.write_text(scenario.read_text().replace("27 AS total_amount", "28 AS total_amount")) + failed: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, + command=( + "scenario", + "test", + "order_totals_pass", + "--scenario-namespace", + test_case.namespace_b, + ), + ) + assert failed.returncode == 1 + assert set(list_scenario_relation_names(db_path=database)) == names_a + + +@pytest.mark.parametrize( + "test_case", + (ScenarioNamespaceE2ETestCase("A fixtures survive B complete execution"),), + ids=lambda case: case.description, +) +def test_given_interleaved_cli_runs_when_b_tears_down_then_a_finishes_correctly( + test_case: ScenarioNamespaceE2ETestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + project: Path = prepare_namespace_project(tmp_path) + original: Callable[..., Any] = scenario_execution.execute_scenario_models + interleaved = False + + def execute_models(**kwargs: Any) -> Any: + nonlocal interleaved + interleaved = True + monkeypatch.setattr(scenario_execution, "execute_scenario_models", original) + change_namespace_fixture(project) + # DuckDB allows simultaneous connections in one process. A fresh context keeps + # CLI observability separate, while both runs use the real shared warehouse. + result: int = contextvars.Context().run( + main, + [ + "--project-dir", + str(project), + "--no-color", + "scenario", + "test", + "order_totals_pass", + "--scenario-namespace", + test_case.namespace_b, + ], + ) + assert result == test_case.expected_exit_code + relations: list[tuple[str]] = ( + kwargs["connection"] + .execute( + "SELECT table_name FROM information_schema.tables WHERE table_name LIKE '__sqb_%'" + ) + .fetchall() + ) + assert len(relations) == 1 # A's fixture survives B's prepare and teardown. + return original(**kwargs) + + monkeypatch.setattr(scenario_execution, "execute_scenario_models", execute_models) + result: int = main( + [ + "--project-dir", + str(project), + "--no-color", + "scenario", + "test", + "order_totals_pass", + "--scenario-namespace", + test_case.namespace_a, + "--retain", + ] + ) + assert interleaved + assert result == test_case.expected_exit_code + database: Path = project / "scenario_demo.duckdb" + total: str = scenario_relation_name_by_suffix(db_path=database, suffix="__model__order_totals") + assert query_duckdb(db_path=database, sql=f'SELECT total_amount FROM "{total}"') == [ + (test_case.expected_total,) + ] + + +@pytest.mark.parametrize( + "test_case", + ( + ScenarioNamespaceE2ETestCase( + "invalid namespace fails before connecting", + namespace_a="invalid/value", + expected_exit_code=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_namespace_when_running_cli_then_no_warehouse_is_created( + test_case: ScenarioNamespaceE2ETestCase, tmp_path: Path +) -> None: + project: Path = prepare_namespace_project(tmp_path) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, + command=("scenario", "test", "--scenario-namespace", test_case.namespace_a), + ) + assert result.returncode == test_case.expected_exit_code + assert test_case.expected_error in result.stderr + assert not (project / "scenario_demo.duckdb").exists() + + +@pytest.mark.parametrize( + "test_case", + (ScenarioNamespaceE2ETestCase("janitor recognizes all namespace hashes"),), + ids=lambda case: case.description, +) +def test_given_namespaced_leftovers_when_running_janitor_then_all_namespaces_are_cleaned( + test_case: ScenarioNamespaceE2ETestCase, tmp_path: Path +) -> None: + project: Path = prepare_namespace_project(tmp_path) + source: Path = project / "sources/raw.yml" + source.write_text(source.read_text().replace("schema: main", "schema: raw")) + config: Path = project / "sqlbuild_project.toml" + config.write_text(config.read_text() + "\n[janitor]\nenabled = true\nretention_days = 0\n") + run_namespaced_scenario(project, test_case.namespace_a, retain=True) + run_namespaced_scenario(project, test_case.namespace_b, retain=True) + database: Path = project / "scenario_demo.duckdb" + assert ( + len(list_scenario_relation_names(db_path=database)) == 2 * test_case.expected_relation_count + ) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, command=("--no-color", "janitor", "--auto-approve") + ) + assert result.returncode == test_case.expected_exit_code + assert not list_scenario_relation_names(db_path=database) + + +@pytest.mark.parametrize( + "test_case", + ( + ScenarioNamespaceSourceE2ETestCase("unset", None, "unset"), + ScenarioNamespaceSourceE2ETestCase( + "project", + "project", + "project config", + project_config='[scenario]\nrun_namespace = "project"\n', + ), + ScenarioNamespaceSourceE2ETestCase( + "local overrides project", + "local", + "local config", + project_config='[scenario]\nrun_namespace = "project"\n', + local_config='[scenario]\nrun_namespace = "local"\n', + ), + ScenarioNamespaceSourceE2ETestCase( + "environment overrides config", + "env-job", + "env", + project_config='[scenario]\nrun_namespace = "project"\n', + local_config='[scenario]\nrun_namespace = "local"\n', + environment=(("SQLBUILD_SCENARIO_NAMESPACE", "env-job"),), + ), + ScenarioNamespaceSourceE2ETestCase( + "CLI overrides environment", + "cli-job", + "cli", + project_config='[scenario]\nrun_namespace = "project"\n', + environment=(("SQLBUILD_SCENARIO_NAMESPACE", "env-job"),), + cli_args=("--scenario-namespace", "cli-job"), + ), + ), + ids=lambda case: case.description, +) +def test_given_namespace_sources_when_running_cli_then_reports_effective_value( + test_case: ScenarioNamespaceSourceE2ETestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("SQLBUILD_SCENARIO_NAMESPACE", raising=False) + project: Path = prepare_namespace_project(tmp_path) + config: Path = project / "sqlbuild_project.toml" + config.write_text(config.read_text() + "\n" + test_case.project_config) + (project / "sqlbuild_local.toml").write_text(test_case.local_config) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, + command=( + "--no-color", + "scenario", + "test", + "order_totals_pass", + "--json", + *test_case.cli_args, + ), + env=dict(test_case.environment), + ) + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, Any] = json.loads(result.stdout) + assert payload["execution"]["scenario_namespace"] == test_case.expected_namespace + assert payload["execution"]["scenario_namespace_source"] == test_case.expected_source + + +@pytest.mark.parametrize( + "test_case", + (ScenarioNamespaceE2ETestCase("capture cleanup leaves another run intact"),), + ids=lambda case: case.description, +) +def test_given_retained_run_when_capturing_another_namespace_then_cleanup_is_isolated( + test_case: ScenarioNamespaceE2ETestCase, tmp_path: Path +) -> None: + project: Path = prepare_namespace_project(tmp_path) + database: Path = project / "scenario_demo.duckdb" + run_namespaced_scenario(project, test_case.namespace_a, retain=True) + names_a: tuple[str, ...] = list_scenario_relation_names(db_path=database) + result: subprocess.CompletedProcess[str] = run_sqb( + project_dir=project, + command=( + "--no-color", + "scenario", + "capture", + "order_totals_pass", + "--scenario-namespace", + test_case.namespace_b, + ), + ) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert f"Scenario namespace: {test_case.namespace_b} (source: cli)" in result.stdout + assert list_scenario_relation_names(db_path=database) == names_a + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py index 2907149c44..042beb31ae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py @@ -183,7 +183,7 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n "local_model_error", "ERROR", "error[X608]:", - "missing_function", + "not_a_number_", "PASS=0 FAIL=0 ERROR=1 SKIP=0 TOTAL=1", ), retained_duckdb_relative_path=Path("target/run/scenarios/local_model_error/local.duckdb"), @@ -193,7 +193,7 @@ def test_given_partial_scenario_fixture_when_required_type_is_known_then_typed_n ( "models/local_model_error.sql", "MODEL (materialized table);\n\n" - "SELECT missing_function(amount) AS bad_value\n" + "SELECT CAST('not_a_number_' || amount AS INTEGER) AS bad_value\n" 'FROM __source("raw_orders")\n', ), ( @@ -889,7 +889,7 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte "local_model_error", "ERROR", "error[X608]:", - "missing_function", + "not_a_number_", "PASS=0 FAIL=0 ERROR=1 SKIP=0 TOTAL=1", ), retained_duckdb_relative_path=Path( @@ -901,7 +901,7 @@ def test_given_missing_snapshot_when_running_local_scenario_then_reports_expecte ( "models/local_model_error.sql", "MODEL (materialized table);\n\n" - "SELECT missing_function(amount) AS bad_value\n" + "SELECT CAST('not_a_number_' || amount AS INTEGER) AS bad_value\n" 'FROM __source("raw_orders")\n', ), ( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py index 8467925e28..30e69d645b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/_test_types.py @@ -25,17 +25,3 @@ class SeedJsonOutputTestCase: expected_status: str expected_summary: dict[str, int] expected_assets: tuple[tuple[str, str], ...] - - -@dataclass(frozen=True) -class VirtualSeedE2ETestCase: - """Test case for virtual sqb seed e2e verification.""" - - description: str - expected_seed_rows: tuple[tuple[object, ...], ...] - expected_seed_fragments: tuple[str, ...] - expected_current_seed_fragments: tuple[str, ...] - expected_build_fragments: tuple[str, ...] - unexpected_build_fragments: tuple[str, ...] = field(default_factory=tuple) - expected_json_command: str = "seed" - expected_json_reason: str = "no_change" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py index 3f0bf9053a..5e1b5ac518 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed.py @@ -4,22 +4,16 @@ import json import subprocess -from collections import defaultdict from pathlib import Path from typing import Any import pytest -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) from tests.e2e.src.sqlbuild.cli.commands.main.seed._test_types import ( SeedE2ETestCase, SeedJsonOutputTestCase, - VirtualSeedE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, prepare_inline_project, prepare_waffle_shop, query_duckdb, @@ -254,142 +248,3 @@ def test_given_failing_standalone_seed_when_writing_json_file_then_failure_is_ho assert tuple((asset["name"], asset["status"]) for asset in assets) == ( test_case.expected_assets ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSeedE2ETestCase( - description="virtual seed command persists VDE seed state", - expected_seed_rows=((1, 100),), - expected_seed_fragments=( - "Plan ready 1 selected", - "order_amounts", - "Execution sqb seed (concurrency:", - "\u2713 Completed successfully", - ), - expected_current_seed_fragments=( - "Plan ready 1 selected", - "order_amounts (current)", - ), - expected_build_fragments=("Plan ready 1 selected", "fact_orders"), - unexpected_build_fragments=("order_amounts",), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_project_when_running_seed_then_persists_seed_state( - test_case: VirtualSeedE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_seed_command", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "seeds/schema.yml": ( - "seeds:\n" - " - name: order_amounts\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: amount_cents\n" - " type: INTEGER\n" - ), - "seeds/order_amounts.csv": "order_id,amount_cents\n1,100\n", - "models/fact_orders.sql": ( - "MODEL (materialized table);\n\n" - 'SELECT order_id, amount_cents FROM __seed("order_amounts")\n' - ), - }, - ) - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stderr - - seed_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "seed"), - project_dir=project_dir, - ) - - assert seed_result.returncode == 0, seed_result.stdout + seed_result.stderr - fragment: str - for fragment in test_case.expected_seed_fragments: - assert fragment in seed_result.stdout, seed_result.stdout - seed_ref_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'seed' ORDER BY node_name" - ), - ) - assert seed_ref_rows == [("order_amounts",)] - seed_physical_rows: list[tuple[object, ...]] = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT artifact_type, artifact_name, schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'seed' AND artifact_name = 'order_amounts'" - ), - ) - assert len(seed_physical_rows) == 1 - assert seed_physical_rows[0][0:2] == ("seed", "order_amounts") - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.order_amounts ORDER BY order_id", - ) == list(test_case.expected_seed_rows) - - current_seed_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "seed", "--select", "order_amounts"), - project_dir=project_dir, - ) - assert current_seed_result.returncode == 0, ( - current_seed_result.stdout + current_seed_result.stderr - ) - for fragment in test_case.expected_current_seed_fragments: - assert fragment in current_seed_result.stdout, current_seed_result.stdout - - json_seed_result: subprocess.CompletedProcess[str] = run_sqb( - command=("seed", "--select", "order_amounts", "--json"), - project_dir=project_dir, - ) - assert json_seed_result.returncode == 0, json_seed_result.stderr - json_payload: dict[str, object] = json.loads(json_seed_result.stdout) - assert json_payload["command"] == test_case.expected_json_command - assets_by_kind: defaultdict[object, list[dict[str, object]]] = defaultdict(list) - for asset in json_payload["assets"]: - typed_asset: dict[str, object] = dict(asset) - assets_by_kind[typed_asset.get("kind")].append(typed_asset) - seed_assets: list[dict[str, object]] = assets_by_kind["seed"] - assert len(seed_assets) == 1 - assert seed_assets[0]["name"] == "order_amounts" - assert seed_assets[0]["reason"] == test_case.expected_json_reason - - physical_schema_name, physical_relation_name = seed_physical_rows[0][2:4] - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=f'DROP TABLE "{physical_schema_name}"."{physical_relation_name}"', - ) - reload_seed_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "seed", "--select", "order_amounts"), - project_dir=project_dir, - ) - assert reload_seed_result.returncode == 0, reload_seed_result.stdout + reload_seed_result.stderr - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT order_id, amount_cents FROM dev__dev.order_amounts ORDER BY order_id", - ) == list(test_case.expected_seed_rows) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changes-only"), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stdout + build_result.stderr - for fragment in test_case.expected_build_fragments: - assert fragment in build_result.stdout, build_result.stdout - for fragment in test_case.unexpected_build_fragments: - assert fragment not in build_result.stdout, build_result.stdout diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py index fd1daf8ea2..7b52fbd292 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/_test_types.py @@ -152,35 +152,3 @@ class SnowflakeSourceDeferralE2ETestCase: expected_loader_rows: tuple[tuple[object, ...], ...] command: tuple[str, ...] = ("--no-color", "build", "--select", "stg_orders") expected_return_code: int = 0 - - -@dataclass(frozen=True) -class SnowflakeVirtualSeedE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_seed_strategy: str - - -@dataclass(frozen=True) -class SnowflakeVirtualLifecycleE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class SnowflakeReconcileE2ETestCase: - description: str - expected_rows: tuple[tuple[object, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class SnowflakeJanitorDetachedVdeE2ETestCase: - description: str - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - expected_ref_count_after: int - expected_return_code: int = 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py index 64bdc23382..21fbabdc1d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/helpers.py @@ -62,59 +62,6 @@ def build_snowflake_project_toml(*, project_name: str, schema_name: str) -> str: ) -def build_snowflake_virtual_seed_project_toml( - *, database_name: str, schema_name: str, unsuffixed_virtual_env: str | None = None -) -> str: - unsuffixed_line: str = {None: ""}.get( - unsuffixed_virtual_env, - f'unsuffixed_virtual_env = "{unsuffixed_virtual_env}"\n', - ) - return ( - 'name = "snowflake_virtual_seed"\n' - 'adapter = "snowflake"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "\n" - "[connection]\n" - 'account = "${ENV:SQB_TEST_SNOWFLAKE_ACCOUNT}"\n' - 'user = "${ENV:SQB_TEST_SNOWFLAKE_USER}"\n' - 'authenticator = "${ENV:SQB_TEST_SNOWFLAKE_AUTHENTICATOR}"\n' - 'token = "${ENV:SQB_TEST_SNOWFLAKE_PAT}"\n' - 'role = "${ENV:SQB_TEST_SNOWFLAKE_ROLE}"\n' - 'warehouse = "${ENV:SQB_TEST_SNOWFLAKE_WAREHOUSE}"\n' - 'database = "${ENV:SQB_TEST_SNOWFLAKE_DATABASE}"\n\n' - "[targets.dev]\n" - f'database = "{database_name}"\n' - f'schema = "{schema_name}"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - f"{unsuffixed_line}\n" - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ) - - -def virtual_seed_source_yml(*, schema_name: str) -> str: - return f"sources:\n - name: raw_orders\n schema: {schema_name}\n table: raw_orders\n" - - -def virtual_seed_orders_model(*, amount_expression: str) -> str: - return ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - f"SELECT id, ordered_at, {amount_expression} AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ) - - def build_snowflake_source_deferral_project_toml( *, project_name: str, dev_schema_name: str, prod_schema_name: str ) -> str: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 13fbb876be..07c685357e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -4,7 +4,6 @@ import subprocess from collections import defaultdict from pathlib import Path -from textwrap import dedent from typing import Any import pytest @@ -30,9 +29,7 @@ SnowflakeDbtScenarioLocalReplayE2ETestCase, SnowflakeDiffE2ETestCase, SnowflakeIntermediateDagStrategyE2ETestCase, - SnowflakeJanitorDetachedVdeE2ETestCase, SnowflakeNodeResultE2ETestCase, - SnowflakeReconcileE2ETestCase, SnowflakeScenarioLocalReplayE2ETestCase, SnowflakeScenarioRemoteE2ETestCase, SnowflakeSnapshotApplyE2ETestCase, @@ -40,8 +37,6 @@ SnowflakeSourceDeferralE2ETestCase, SnowflakeSourceLoaderSchemaEvolutionE2ETestCase, SnowflakeSourceLoaderStrategiesE2ETestCase, - SnowflakeVirtualLifecycleE2ETestCase, - SnowflakeVirtualSeedE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.snowflake.helpers import ( assert_current_snowflake_snapshot_rows, @@ -51,7 +46,6 @@ build_snowflake_local_config, build_snowflake_project_toml, build_snowflake_source_deferral_project_toml, - build_snowflake_virtual_seed_project_toml, cleanup_snowflake_schema, ensure_query_schema_ready, execute_snowflake_sql, @@ -62,8 +56,6 @@ prepare_snowflake_waffle_shop, relation_name, snowflake_relation_row_count, - virtual_seed_orders_model, - virtual_seed_source_yml, write_local_environment_override, write_snowflake_dbt_scenario_date_trunc_model, ) @@ -78,7 +70,6 @@ build_real_warehouse_snapshot_project_files, prepare_dbt_profile_workspace, prepare_inline_project, - query_duckdb, run_sqb, stringify_warehouse_rows, ) @@ -301,463 +292,6 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ cleanup_snowflake_schema(schema_name=schema_name) -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeVirtualSeedE2ETestCase( - description="virtual seeded incremental build uses clone on snowflake", - expected_rows=(("1", "10"), ("2", "21"), ("3", "31")), - expected_seed_strategy="durable_clone", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_incremental_change_when_building_on_snowflake_then_seeds_with_clone( - test_case: SnowflakeVirtualSeedE2ETestCase, - tmp_path: Path, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_seed") - database_name: str = str(build_snowflake_connection_config(schema=schema_name)["database"]) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="snowflake_virtual_seed", - repo_files={ - "sqlbuild_project.toml": build_snowflake_virtual_seed_project_toml( - database_name=database_name, - schema_name=schema_name, - ), - "sources/raw.yml": virtual_seed_source_yml(schema_name=schema_name), - "models/orders.sql": virtual_seed_orders_model(amount_expression="amount_cents + 0"), - }, - ) - try: - execute_snowflake_sql( - schema_name=schema_name, - sql=f"CREATE SCHEMA IF NOT EXISTS {database_name}.{schema_name}", - ) - execute_snowflake_sql( - schema_name=schema_name, - sql=dedent( - f""" - CREATE OR REPLACE TABLE - {relation_name(schema_name=schema_name, name="raw_orders")} ( - id INTEGER, - ordered_at TIMESTAMP_NTZ, - amount_cents INTEGER - ) - """ - ).strip(), - ) - execute_snowflake_sql( - schema_name=schema_name, - sql=dedent( - f""" - INSERT INTO {relation_name(schema_name=schema_name, name="raw_orders")} VALUES - (1, '2026-01-01 00:00:00', 10), - (2, '2026-01-02 00:00:00', 20) - """ - ).strip(), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - virtual_seed_orders_model(amount_expression="amount_cents + 1"), - encoding="utf-8", - ) - execute_snowflake_sql( - schema_name=schema_name, - sql=( - f"INSERT INTO {relation_name(schema_name=schema_name, name='raw_orders')} " - "VALUES (3, '2026-01-03 00:00:00', 30)" - ), - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == 0, build_result.stderr - assert ( - stringify_warehouse_rows( - fetch_snowflake_rows( - schema_name=schema_name, - sql=( - f"SELECT id, amount_cents FROM {database_name}.{schema_name}__dev.orders " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT seed_strategy FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) == [(test_case.expected_seed_strategy,)] - finally: - cleanup_snowflake_schema(schema_name=schema_name) - - -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeReconcileE2ETestCase( - description="reconcile repair-view recreates snowflake logical view", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Repair", - "model orders", - "VDE dev", - "action recreate logical view from state", - "result repaired", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_view_when_repairing_on_snowflake_then_view_is_recreated( - test_case: SnowflakeReconcileE2ETestCase, - tmp_path: Path, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_reconcile") - database_name: str = str(build_snowflake_connection_config(schema=schema_name)["database"]) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="snowflake_virtual_reconcile", - repo_files={ - "sqlbuild_project.toml": build_snowflake_virtual_seed_project_toml( - database_name=database_name, - schema_name=schema_name, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_snowflake_sql( - schema_name=schema_name, - sql=f"DROP VIEW {relation_name(schema_name=f'{schema_name}__dev', name='orders')}", - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - stringify_warehouse_rows( - fetch_snowflake_rows( - schema_name=schema_name, - sql=( - "SELECT id FROM " - f"{relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - finally: - cleanup_snowflake_schema(schema_name=schema_name) - cleanup_snowflake_schema(schema_name=f"{schema_name}__dev") - cleanup_snowflake_schema(schema_name=f"{schema_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeReconcileE2ETestCase( - description="reconcile attach rebinds snowflake logical view", - expected_rows=(("2",),), - expected_stdout_fragments=("Attach", "model orders", "result attached"), - ) - ], - ids=lambda case: case.description, -) -def test_given_tracked_physical_relation_when_attaching_on_snowflake_then_view_is_rebound( - test_case: SnowflakeReconcileE2ETestCase, - tmp_path: Path, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_attach") - database_name: str = str(build_snowflake_connection_config(schema=schema_name)["database"]) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="snowflake_virtual_attach", - repo_files={ - "sqlbuild_project.toml": build_snowflake_virtual_seed_project_toml( - database_name=database_name, - schema_name=schema_name, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ).returncode - == 0 - ) - database_name, physical_schema_name, physical_relation_name = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT database_name, schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - )[0] - physical_relation: str = f"{database_name}.{physical_schema_name}.{physical_relation_name}" - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - physical_relation, - "--auto-approve", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - assert ( - stringify_warehouse_rows( - fetch_snowflake_rows( - schema_name=schema_name, - sql=( - "SELECT id FROM " - f"{relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - finally: - cleanup_snowflake_schema(schema_name=schema_name) - cleanup_snowflake_schema(schema_name=f"{schema_name}__dev") - cleanup_snowflake_schema(schema_name=f"{schema_name}__pr") - cleanup_snowflake_schema(schema_name=f"{schema_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeVirtualLifecycleE2ETestCase( - description="adopt and detach preserve snowflake logical table", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Adopted 1 models into virtual environment dev.", - "Detached 1 models from virtual environment dev.", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_stateless_table_when_adopting_and_detaching_on_snowflake_then_table_is_preserved( - test_case: SnowflakeVirtualLifecycleE2ETestCase, - tmp_path: Path, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_lifecycle") - database_name: str = str(build_snowflake_connection_config(schema=schema_name)["database"]) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="snowflake_virtual_lifecycle", - repo_files={ - "sqlbuild_project.toml": ( - build_snowflake_virtual_seed_project_toml( - database_name=database_name, - schema_name=schema_name, - unsuffixed_virtual_env="dev", - ) - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - execute_snowflake_sql( - schema_name=schema_name, - sql=f"CREATE SCHEMA IF NOT EXISTS {database_name}.{schema_name}", - ) - execute_snowflake_sql( - schema_name=schema_name, - sql=( - f"CREATE OR REPLACE TABLE {relation_name(schema_name=schema_name, name='orders')} " - "AS SELECT 1 AS id" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert adopt_result.returncode == test_case.expected_return_code, ( - adopt_result.stdout + adopt_result.stderr - ) - assert detach_result.returncode == test_case.expected_return_code, ( - detach_result.stdout + detach_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in adopt_result.stdout + detach_result.stdout - assert ( - stringify_warehouse_rows( - fetch_snowflake_rows( - schema_name=schema_name, - sql=( - f"SELECT id FROM {relation_name(schema_name=schema_name, name='orders')} " - "ORDER BY id" - ), - ) - ) - == test_case.expected_rows - ) - finally: - cleanup_snowflake_schema(schema_name=schema_name) - cleanup_snowflake_schema(schema_name=f"{schema_name}__sqb_physical") - - -@pytest.mark.parametrize( - "test_case", - [ - SnowflakeJanitorDetachedVdeE2ETestCase( - description="janitor prunes snowflake detached VDE refs and physical versions", - expected_stdout_fragments=( - "eligible for deletion", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "state item", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_detached_vde_when_running_janitor_on_snowflake_then_refs_are_pruned( - test_case: SnowflakeJanitorDetachedVdeE2ETestCase, - tmp_path: Path, -) -> None: - schema_name: str = build_unique_schema_name(prefix="sqlbuild_virtual_janitor") - database_name: str = str(build_snowflake_connection_config(schema=schema_name)["database"]) - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="snowflake_virtual_janitor", - repo_files={ - "sqlbuild_project.toml": ( - build_snowflake_virtual_seed_project_toml( - database_name=database_name, - schema_name=schema_name, - unsuffixed_virtual_env="dev", - ) - + "\n[janitor]\n" - + "enabled = true\n" - + "retention_days = 0\n" - + "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - try: - execute_snowflake_sql( - schema_name=schema_name, - sql=f"CREATE SCHEMA IF NOT EXISTS {database_name}.{schema_name}", - ) - execute_snowflake_sql( - schema_name=schema_name, - sql=( - f"CREATE OR REPLACE TABLE {relation_name(schema_name=schema_name, name='orders')} " - "AS SELECT 1 AS id" - ), - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ).returncode - == 0 - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_return_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environments", - ) == [(test_case.expected_virtual_environment_count_after,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model'" - ), - ) == [(test_case.expected_ref_count_after,)] - finally: - cleanup_snowflake_schema(schema_name=schema_name) - cleanup_snowflake_schema(schema_name=f"{schema_name}__sqb_physical") - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py index 8326cbb15e..5faf5172ac 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/_test_types.py @@ -133,36 +133,3 @@ class SqlServerSnapshotApplyE2ETestCase: expected_current_delete_rows: tuple[tuple[object, ...], ...] expected_historical_timestamp_rows: tuple[tuple[object, ...], ...] expected_historical_check_rows: tuple[tuple[object, ...], ...] - - -@dataclass(frozen=True) -class SqlServerVirtualLifecycleE2ETestCase: - description: str - expected_rows: tuple[tuple[str, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class SqlServerReconcileE2ETestCase: - description: str - expected_rows: tuple[tuple[str, ...], ...] - expected_stdout_fragments: tuple[str, ...] - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class SqlServerJanitorDetachedVdeE2ETestCase: - description: str - expected_stdout_fragments: tuple[str, ...] - expected_virtual_environment_count_after: int - expected_ref_count_after: int - expected_return_code: int = 0 - - -@dataclass(frozen=True) -class SqlServerVirtualSeedE2ETestCase: - description: str - expected_rows: tuple[tuple[str, ...], ...] - expected_seed_strategy: str - expected_return_code: int = 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py index 26421ea52c..d15a8cbef7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py @@ -49,40 +49,6 @@ def build_sqlserver_project_toml( ) -def build_sqlserver_virtual_project_toml( - *, - project_name: str, - schema_name: str, - config: dict[str, object], - unsuffixed_virtual_env: str | None = None, -) -> str: - unsuffixed_line: str = {None: ""}.get( - unsuffixed_virtual_env, - f'unsuffixed_virtual_env = "{unsuffixed_virtual_env}"\n', - ) - return ( - f'name = "{project_name}"\n' - 'adapter = "sqlserver"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'database = "{config["database"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n\n' - "[targets.dev]\n" - f'schema = "{schema_name}"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - f"{unsuffixed_line}\n" - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ) - - def build_sqlserver_source_deferral_project_toml( *, project_name: str, dev_schema_name: str, prod_schema_name: str, config: dict[str, object] ) -> str: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index f9780db95d..054236995a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -20,10 +20,8 @@ SqlServerDbtProfileE2ETestCase, SqlServerDiffE2ETestCase, SqlServerIntermediateDagStrategyE2ETestCase, - SqlServerJanitorDetachedVdeE2ETestCase, SqlServerLoaderWaffleShopE2ETestCase, SqlServerNodeResultE2ETestCase, - SqlServerReconcileE2ETestCase, SqlServerScenarioLocalReplayE2ETestCase, SqlServerSnapshotApplyE2ETestCase, SqlServerSnapshotE2ETestCase, @@ -31,8 +29,6 @@ SqlServerSourceLoaderDagE2ETestCase, SqlServerSourceLoaderE2ETestCase, SqlServerSourceLoaderStrategiesE2ETestCase, - SqlServerVirtualLifecycleE2ETestCase, - SqlServerVirtualSeedE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.sqlserver.helpers import ( adapt_sqlserver_project_files, @@ -43,7 +39,6 @@ build_sqlserver_config, build_sqlserver_project_toml, build_sqlserver_source_deferral_project_toml, - build_sqlserver_virtual_project_toml, build_unique_schema_name, cleanup_sqlserver_schema, ensure_sqlserver_schema_ready, @@ -63,7 +58,6 @@ build_real_warehouse_existing_snapshot_project_files, build_real_warehouse_snapshot_project_files, prepare_inline_project, - query_duckdb, run_sqb, stringify_warehouse_rows, ) @@ -559,472 +553,6 @@ def test_given_waffle_shop_when_running_full_build_on_sqlserver_then_expected_ta cleanup_sqlserver_schema(schema_name=schema_name, config=config) -@pytest.mark.parametrize( - "test_case", - [ - SqlServerVirtualSeedE2ETestCase( - description="virtual seeded incremental build uses copy on sqlserver", - expected_rows=(("1", "10"), ("2", "21"), ("3", "31")), - expected_seed_strategy="copy", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_incremental_change_when_building_on_sqlserver_then_seeds_with_copy( - tmp_path: Path, - test_case: SqlServerVirtualSeedE2ETestCase, -) -> None: - config: dict[str, object] = build_sqlserver_config() - schema_name: str = build_unique_schema_name(prefix="sqb_virtual_seed") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="sqlserver_virtual_seed", - repo_files={ - "sqlbuild_project.toml": build_sqlserver_virtual_project_toml( - project_name="sqlserver_virtual_seed", - schema_name=schema_name, - config=config, - ), - "sources/raw.yml": ( - "sources:\n" - " - name: raw_orders\n" - f" schema: {schema_name}\n" - " table: raw_orders\n" - ), - "models/orders.sql": ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - "SELECT id, ordered_at, amount_cents + 0 AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ), - }, - ) - ensure_sqlserver_schema_ready(schema_name=schema_name, config=config) - - try: - execute_sqlserver_sql( - sql=( - f"CREATE TABLE {relation_name(schema_name=schema_name, name='raw_orders')} (" - "id INT, ordered_at DATETIME2, amount_cents INT)" - ), - config=config, - ) - execute_sqlserver_sql( - sql=( - f"INSERT INTO {relation_name(schema_name=schema_name, name='raw_orders')} VALUES " - "(1, '2026-01-01T00:00:00', 10), " - "(2, '2026-01-02T00:00:00', 20)" - ), - config=config, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - ( - "MODEL (\n" - " materialized incremental,\n" - " incremental_strategy delete_insert,\n" - " cursor ordered_at,\n" - " cursor_type timestamp,\n" - " cursor_grain day,\n" - " replay_on_change bounded-7d\n" - ");\n\n" - "SELECT id, ordered_at, amount_cents + 1 AS amount_cents\n" - 'FROM __source("raw_orders")\n' - ), - encoding="utf-8", - ) - execute_sqlserver_sql( - sql=( - f"INSERT INTO {relation_name(schema_name=schema_name, name='raw_orders')} " - "VALUES (3, '2026-01-03T00:00:00', 30)" - ), - config=config, - ) - - build_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--start-cursor-ts", - "2026-01-02T00:00:00", - "--end-cursor-ts", - "2026-01-04T00:00:00", - ), - project_dir=project_dir, - ) - - assert build_result.returncode == test_case.expected_return_code, ( - build_result.stdout + build_result.stderr - ) - rows: tuple[tuple[object, ...], ...] = fetch_sqlserver_rows( - sql=( - f"SELECT id, amount_cents FROM " - f"{relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - config=config, - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT seed_strategy FROM sqlbuild_state.physical_relation_ancestry " - "WHERE model_name = 'orders'" - ), - ) == [(test_case.expected_seed_strategy,)] - finally: - cleanup_sqlserver_schema(schema_name=schema_name, config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__dev", config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__sqb_physical", config=config) - - -@pytest.mark.parametrize( - "test_case", - [ - SqlServerReconcileE2ETestCase( - description="reconcile repair-view recreates sqlserver logical view", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Repair", - "model orders", - "VDE dev", - "action recreate logical view from state", - "result repaired", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_logical_view_when_repairing_on_sqlserver_then_view_is_recreated( - tmp_path: Path, - test_case: SqlServerReconcileE2ETestCase, -) -> None: - config: dict[str, object] = build_sqlserver_config() - schema_name: str = build_unique_schema_name(prefix="sqb_virtual_reconcile") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="sqlserver_virtual_reconcile", - repo_files={ - "sqlbuild_project.toml": build_sqlserver_virtual_project_toml( - project_name="sqlserver_virtual_reconcile", - schema_name=schema_name, - config=config, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - ensure_sqlserver_schema_ready(schema_name=schema_name, config=config) - - try: - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - execute_sqlserver_sql( - sql=f"DROP VIEW {relation_name(schema_name=f'{schema_name}__dev', name='orders')}", - config=config, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "repair-view", - "--virtual-env", - "dev", - "--model", - "orders", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - rows: tuple[tuple[object, ...], ...] = fetch_sqlserver_rows( - sql=( - f"SELECT id FROM {relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - config=config, - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - finally: - cleanup_sqlserver_schema(schema_name=schema_name, config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__dev", config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__sqb_physical", config=config) - - -@pytest.mark.parametrize( - "test_case", - [ - SqlServerReconcileE2ETestCase( - description="reconcile attach rebinds sqlserver logical view", - expected_rows=(("2",),), - expected_stdout_fragments=("Attach", "model orders", "result attached"), - ) - ], - ids=lambda case: case.description, -) -def test_given_tracked_physical_relation_when_attaching_on_sqlserver_then_view_is_rebound( - tmp_path: Path, - test_case: SqlServerReconcileE2ETestCase, -) -> None: - config: dict[str, object] = build_sqlserver_config() - schema_name: str = build_unique_schema_name(prefix="sqb_virtual_attach") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="sqlserver_virtual_attach", - repo_files={ - "sqlbuild_project.toml": build_sqlserver_virtual_project_toml( - project_name="sqlserver_virtual_attach", - schema_name=schema_name, - config=config, - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - ensure_sqlserver_schema_ready(schema_name=schema_name, config=config) - - try: - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("--no-color", "build"), project_dir=project_dir).returncode == 0 - (project_dir / "models" / "orders.sql").write_text( - "MODEL ();\n\nSELECT 2 AS id\n", - encoding="utf-8", - ) - assert ( - run_sqb( - command=("--no-color", "build", "--virtual-env", "pr"), - project_dir=project_dir, - ).returncode - == 0 - ) - _database_name, physical_schema_name, physical_relation_name = query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT database_name, schema_name, relation_name " - "FROM sqlbuild_state.physical_relations " - "WHERE artifact_type = 'model' AND artifact_name = 'orders' " - "ORDER BY updated_at DESC LIMIT 1" - ), - )[0] - physical_relation: str = f'"{physical_schema_name}"."{physical_relation_name}"' - - result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "reconcile", - "attach", - "--virtual-env", - "dev", - "--model", - "orders", - "--physical-relation", - physical_relation, - "--auto-approve", - ), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_return_code, result.stdout + result.stderr - for fragment in test_case.expected_stdout_fragments: - assert fragment in result.stdout - rows: tuple[tuple[object, ...], ...] = fetch_sqlserver_rows( - sql=( - f"SELECT id FROM {relation_name(schema_name=f'{schema_name}__dev', name='orders')} " - "ORDER BY id" - ), - config=config, - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - finally: - cleanup_sqlserver_schema(schema_name=schema_name, config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__dev", config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__pr", config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__sqb_physical", config=config) - - -@pytest.mark.parametrize( - "test_case", - [ - SqlServerVirtualLifecycleE2ETestCase( - description="adopt and detach preserve sqlserver logical table", - expected_rows=(("1",),), - expected_stdout_fragments=( - "Adopted 1 models into virtual environment dev.", - "Detached 1 models from virtual environment dev.", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_stateless_table_when_adopting_and_detaching_on_sqlserver_then_table_is_preserved( - tmp_path: Path, - test_case: SqlServerVirtualLifecycleE2ETestCase, -) -> None: - config: dict[str, object] = build_sqlserver_config() - schema_name: str = build_unique_schema_name(prefix="sqb_virtual_lifecycle") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="sqlserver_virtual_lifecycle", - repo_files={ - "sqlbuild_project.toml": build_sqlserver_virtual_project_toml( - project_name="sqlserver_virtual_lifecycle", - schema_name=schema_name, - config=config, - unsuffixed_virtual_env="dev", - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - ensure_sqlserver_schema_ready(schema_name=schema_name, config=config) - - try: - execute_sqlserver_sql( - sql=f"CREATE TABLE {relation_name(schema_name=schema_name, name='orders')} (id INT)", - config=config, - ) - execute_sqlserver_sql( - sql=f"INSERT INTO {relation_name(schema_name=schema_name, name='orders')} VALUES (1)", - config=config, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert adopt_result.returncode == test_case.expected_return_code, ( - adopt_result.stdout + adopt_result.stderr - ) - assert detach_result.returncode == test_case.expected_return_code, ( - detach_result.stdout + detach_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in adopt_result.stdout + detach_result.stdout - rows: tuple[tuple[object, ...], ...] = fetch_sqlserver_rows( - sql=( - f"SELECT id FROM {relation_name(schema_name=schema_name, name='orders')} " - "ORDER BY id" - ), - config=config, - ) - assert stringify_warehouse_rows(rows) == test_case.expected_rows - finally: - cleanup_sqlserver_schema(schema_name=schema_name, config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__sqb_physical", config=config) - - -@pytest.mark.parametrize( - "test_case", - [ - SqlServerJanitorDetachedVdeE2ETestCase( - description="janitor prunes sqlserver detached VDE refs and physical versions", - expected_stdout_fragments=( - "eligible for deletion", - "detached VDEs pruned", - "Eligible detached VDEs", - "dev detached virtual environment", - "state item", - ), - expected_virtual_environment_count_after=0, - expected_ref_count_after=0, - ) - ], - ids=lambda case: case.description, -) -def test_given_detached_vde_when_running_janitor_on_sqlserver_then_refs_are_pruned( - tmp_path: Path, - test_case: SqlServerJanitorDetachedVdeE2ETestCase, -) -> None: - config: dict[str, object] = build_sqlserver_config() - schema_name: str = build_unique_schema_name(prefix="sqb_virtual_janitor") - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="sqlserver_virtual_janitor", - repo_files={ - "sqlbuild_project.toml": ( - build_sqlserver_virtual_project_toml( - project_name="sqlserver_virtual_janitor", - schema_name=schema_name, - config=config, - unsuffixed_virtual_env="dev", - ) - + "\n[janitor]\n" - + "enabled = true\n" - + "retention_days = 0\n" - + "delete_tracked_only = false\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - ensure_sqlserver_schema_ready(schema_name=schema_name, config=config) - - try: - execute_sqlserver_sql( - sql=f"CREATE TABLE {relation_name(schema_name=schema_name, name='orders')} (id INT)", - config=config, - ) - execute_sqlserver_sql( - sql=f"INSERT INTO {relation_name(schema_name=schema_name, name='orders')} VALUES (1)", - config=config, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert ( - run_sqb( - command=("--no-color", "state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - assert ( - run_sqb( - command=("--no-color", "state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ).returncode - == 0 - ) - - janitor_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "janitor", "--auto-approve"), - project_dir=project_dir, - ) - - assert janitor_result.returncode == test_case.expected_return_code, ( - janitor_result.stdout + janitor_result.stderr - ) - for fragment in test_case.expected_stdout_fragments: - assert fragment in janitor_result.stdout - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT COUNT(*) FROM sqlbuild_state.virtual_environments", - ) == [(test_case.expected_virtual_environment_count_after,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model'" - ), - ) == [(test_case.expected_ref_count_after,)] - finally: - cleanup_sqlserver_schema(schema_name=schema_name, config=config) - cleanup_sqlserver_schema(schema_name=f"{schema_name}__sqb_physical", config=config) - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/state/__init__.py b/tests/e2e/src/sqlbuild/cli/commands/main/state/__init__.py deleted file mode 100644 index 77a35b19b8..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/state/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""State command e2e tests.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/state/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/state/_test_types.py deleted file mode 100644 index a402e0bc6c..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/state/_test_types.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class StateLifecycleE2ETestCase: - description: str - expected_exit_code: int - expected_init_fragment: str - expected_migrate_fragment: str - expected_rollback_fragment: str - expected_reset_fragment: str - expected_schema_version: int - expected_init_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class StateLifecycleErrorE2ETestCase: - description: str - project_toml: str - command: tuple[str, ...] - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class StateLocalOverrideE2ETestCase: - description: str - expected_exit_code: int - expected_schema_version: int - - -@dataclass(frozen=True) -class StateExplicitRollbackE2ETestCase: - description: str - expected_exit_code: int - expected_rollback_fragment: str - expected_schema_version: int - - -@dataclass(frozen=True) -class StateModeGuardE2ETestCase: - description: str - project_toml: str - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class StateSchemaCorruptionE2ETestCase: - description: str - mutation_sql: str - expected_exit_code: int - expected_error_fragment: str - - -@dataclass(frozen=True) -class StateAdoptDetachE2ETestCase: - description: str - expected_adopt_fragment: str - expected_detach_fragment: str - expected_detached_status: str - expected_operation_rows: tuple[tuple[object, ...], ...] - expected_operation_event_rows: tuple[tuple[object, ...], ...] - expected_detached_error_fragment: str - expected_query_results_after_adopt: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] - expected_query_results_after_detach: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/state/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/state/helpers.py deleted file mode 100644 index 2dc746fe93..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/state/helpers.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -import subprocess - - -def assert_state_cli_error( - *, - result: subprocess.CompletedProcess[str], - expected_exit_code: int, - expected_error_fragment: str, -) -> None: - assert result.returncode == expected_exit_code - assert "error[E001]" in result.stderr - assert expected_error_fragment in result.stderr - assert "Traceback" not in result.stderr - - -def build_postgres_state_project_toml( - *, - project_name: str, - config: dict[str, object], - state_schema: str, - allow_reset: bool = False, - state_config: dict[str, object] | None = None, -) -> str: - allow_reset_value: str = str(allow_reset).lower() - effective_state_config: dict[str, object] = state_config or config - return ( - f'name = "{project_name}"\n' - 'adapter = "postgres"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n" - "[connection]\n" - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'dbname = "{config["dbname"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n\n' - "[targets.dev]\n" - 'schema = "preserve"\n\n' - "[targets.dev.state]\n" - 'backend = "postgres"\n' - f'schema = "{state_schema}"\n' - f"allow_reset = {allow_reset_value}\n\n" - "[targets.dev.state.connection]\n" - f'host = "{effective_state_config["host"]}"\n' - f"port = {effective_state_config['port']}\n" - f'dbname = "{effective_state_config["dbname"]}"\n' - f'user = "{effective_state_config["user"]}"\n' - f'password = "{effective_state_config["password"]}"\n' - ) - - -def build_postgres_local_state_connection_toml(*, config: dict[str, object]) -> str: - return ( - "[targets.dev.state.connection]\n" - f'host = "{config["host"]}"\n' - f"port = {config['port']}\n" - f'dbname = "{config["dbname"]}"\n' - f'user = "{config["user"]}"\n' - f'password = "{config["password"]}"\n' - ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/state/test_state.py b/tests/e2e/src/sqlbuild/cli/commands/main/state/test_state.py deleted file mode 100644 index b0b713765c..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/state/test_state.py +++ /dev/null @@ -1,1386 +0,0 @@ -from __future__ import annotations - -import subprocess -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.state._test_types import ( - StateAdoptDetachE2ETestCase, - StateExplicitRollbackE2ETestCase, - StateLifecycleE2ETestCase, - StateLifecycleErrorE2ETestCase, - StateLocalOverrideE2ETestCase, - StateModeGuardE2ETestCase, - StateSchemaCorruptionE2ETestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.state.helpers import assert_state_cli_error -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - execute_duckdb, - prepare_inline_project, - query_duckdb, - run_sqb, -) - - -@pytest.mark.parametrize( - "test_case", - [ - StateLifecycleE2ETestCase( - description="duckdb state lifecycle commands manage state tables", - expected_exit_code=0, - expected_init_fragment="Virtual State Initialized", - expected_migrate_fragment="Virtual State Migrated", - expected_rollback_fragment="Virtual State Rolled Back", - expected_reset_fragment="Virtual State Reset", - expected_schema_version=1, - expected_init_fragments=( - "physical_relation_ancestry", - "function_versions", - "virtual_environment_node_refs", - "virtual_environment_checkpoint_function_refs", - "microbatch_events", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_config_when_running_state_lifecycle_then_state_store_is_updated( - test_case: StateLifecycleE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="versioned_state_project", - repo_files={ - "sqlbuild_project.toml": """ -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" -allow_reset = true - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == test_case.expected_exit_code, ( - init_result.stdout + init_result.stderr - ) - assert test_case.expected_init_fragment in init_result.stdout - for fragment in test_case.expected_init_fragments: - assert fragment in init_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql="SELECT schema_version FROM sqlbuild_state.state_versions", - ) == [(test_case.expected_schema_version,)] - - migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - assert migrate_result.returncode == test_case.expected_exit_code, ( - migrate_result.stdout + migrate_result.stderr - ) - assert test_case.expected_migrate_fragment in migrate_result.stdout - - execute_duckdb( - db_path=state_db_path, - sql="DELETE FROM sqlbuild_state.state_versions", - ) - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback"), - project_dir=project_dir, - ) - assert rollback_result.returncode == test_case.expected_exit_code, ( - rollback_result.stdout + rollback_result.stderr - ) - assert test_case.expected_rollback_fragment in rollback_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql="SELECT schema_version FROM sqlbuild_state.state_versions", - ) == [(test_case.expected_schema_version,)] - - reset_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "reset", "--auto-approve"), - project_dir=project_dir, - ) - assert reset_result.returncode == test_case.expected_exit_code, ( - reset_result.stdout + reset_result.stderr - ) - assert test_case.expected_reset_fragment in reset_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_schema = 'sqlbuild_state' AND table_name = 'state_versions'" - ), - ) == [(0,)] - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="explicit rollback blocks cleanly when backup schema is deleted", - project_toml="", - expected_exit_code=1, - expected_error_fragment="sqlbuild_state__backup_", - ) - ], - ids=lambda case: case.description, -) -def test_given_deleted_duckdb_state_backup_when_rolling_back_then_it_blocks_cleanly( - test_case: StateModeGuardE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="duckdb_state_deleted_backup", - repo_files={ - "sqlbuild_project.toml": """ -name = "duckdb_state_deleted_backup" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip() - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("state", "migrate"), project_dir=project_dir).returncode == 0 - backup_id: str = str( - query_duckdb( - db_path=state_db_path, - sql=( - "SELECT backup_id FROM sqlbuild_state.state_migration_events " - "WHERE action = 'backup' ORDER BY created_at DESC LIMIT 1" - ), - )[0][0] - ) - execute_duckdb( - db_path=state_db_path, - sql=f'DROP SCHEMA "sqlbuild_state__backup_{backup_id}" CASCADE', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback", "--backup-id", backup_id), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="latest rollback blocks cleanly when backup schema is deleted", - project_toml="", - expected_exit_code=1, - expected_error_fragment="No state backup is available for rollback", - ) - ], - ids=lambda case: case.description, -) -def test_given_deleted_latest_duckdb_state_backup_when_rolling_back_then_it_blocks_cleanly( - test_case: StateModeGuardE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="duckdb_state_deleted_latest_backup", - repo_files={ - "sqlbuild_project.toml": """ -name = "duckdb_state_deleted_latest_backup" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip() - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert run_sqb(command=("state", "migrate"), project_dir=project_dir).returncode == 0 - backup_id: str = str( - query_duckdb( - db_path=state_db_path, - sql=( - "SELECT backup_id FROM sqlbuild_state.state_migration_events " - "WHERE action = 'backup' ORDER BY created_at DESC LIMIT 1" - ), - )[0][0] - ) - execute_duckdb( - db_path=state_db_path, - sql=f'DROP SCHEMA "sqlbuild_state__backup_{backup_id}" CASCADE', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="migrate blocks cleanly when required state table is missing", - project_toml="", - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_duckdb_state_table_when_migrating_then_cli_blocks_cleanly( - test_case: StateModeGuardE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="duckdb_state_missing_table", - repo_files={ - "sqlbuild_project.toml": """ -name = "duckdb_state_missing_table" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip() - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=state_db_path, - sql='DROP TABLE "sqlbuild_state"."virtual_environment_node_refs"', - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - ( - StateSchemaCorruptionE2ETestCase( - description="migrate blocks cleanly when required state column is missing", - mutation_sql='ALTER TABLE "sqlbuild_state"."state_versions" DROP COLUMN "updated_at"', - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ), - StateSchemaCorruptionE2ETestCase( - description="migrate blocks cleanly when required state column has wrong type", - mutation_sql=( - 'ALTER TABLE "sqlbuild_state"."state_versions" ' - 'ALTER COLUMN "schema_version" SET DATA TYPE TEXT' - ), - expected_exit_code=1, - expected_error_fragment="Cannot backup invalid state schema", - ), - ), - ids=lambda case: case.description, -) -def test_given_corrupt_duckdb_state_schema_when_migrating_then_cli_blocks_cleanly( - test_case: StateSchemaCorruptionE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="duckdb_state_schema_corruption", - repo_files={ - "sqlbuild_project.toml": """ -name = "duckdb_state_schema_corruption" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip() - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - execute_duckdb(db_path=state_db_path, sql=test_case.mutation_sql) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - ( - StateLifecycleErrorE2ETestCase( - description="reset blocks when allow reset is false", - project_toml=""" -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" -allow_reset = false - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - command=("--no-color", "state", "reset", "--auto-approve"), - expected_exit_code=1, - expected_error_fragment=("set `allow_reset = true` under `[targets..state]`"), - ), - StateLifecycleErrorE2ETestCase( - description="reset blocks without auto approve", - project_toml=""" -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" -allow_reset = true - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - command=("--no-color", "state", "reset"), - expected_exit_code=1, - expected_error_fragment="state reset requires --auto-approve", - ), - StateLifecycleErrorE2ETestCase( - description="rollback blocks before backup exists", - project_toml=""" -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - command=("--no-color", "state", "rollback"), - expected_exit_code=1, - expected_error_fragment="No state backup is available for rollback", - ), - ), - ids=lambda case: case.description, -) -def test_given_duckdb_state_config_when_running_blocked_state_command_then_cli_reports_error( - test_case: StateLifecycleErrorE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="versioned_state_project", - repo_files={"sqlbuild_project.toml": test_case.project_toml}, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - result: subprocess.CompletedProcess[str] = run_sqb( - command=test_case.command, - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="state init blocks outside virtual mode", - project_toml=""" -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - expected_exit_code=1, - expected_error_fragment="State commands require virtual_environments = true", - ) - ], - ids=lambda case: case.description, -) -def test_given_direct_mode_project_when_running_state_init_then_cli_blocks_cleanly( - test_case: StateModeGuardE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="versioned_state_project", - repo_files={"sqlbuild_project.toml": test_case.project_toml}, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - - assert_state_cli_error( - result=result, - expected_exit_code=test_case.expected_exit_code, - expected_error_fragment=test_case.expected_error_fragment, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - StateLocalOverrideE2ETestCase( - description="local state connection override controls CLI state database", - expected_exit_code=0, - expected_schema_version=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_local_state_override_when_running_state_init_then_cli_uses_local_state_database( - test_case: StateLocalOverrideE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="versioned_state_project", - repo_files={ - "sqlbuild_project.toml": """ -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "project-state.duckdb" -""".lstrip(), - "sqlbuild_local.toml": """ -[targets.dev.state.connection] -database = "local-state.duckdb" -""".lstrip(), - }, - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert query_duckdb( - db_path=project_dir / "local-state.duckdb", - sql="SELECT schema_version FROM sqlbuild_state.state_versions", - ) == [(test_case.expected_schema_version,)] - assert not (project_dir / "project-state.duckdb").exists() - - -@pytest.mark.parametrize( - "test_case", - [ - StateAdoptDetachE2ETestCase( - description="adopt and detach preserve unsuffixed logical names interactively", - expected_adopt_fragment="Adopted 1 models into virtual environment dev.", - expected_detach_fragment="Detached 1 models from virtual environment dev.", - expected_detached_status="detached", - expected_operation_rows=( - ("adopt:dev", "adopt", "succeeded", "dev"), - ("detach:dev", "detach", "succeeded", "dev"), - ), - expected_operation_event_rows=( - ("adopt:dev", "start", "running", "starting adopt"), - ("adopt:dev", "finish", "succeeded", "adopted 1 models"), - ("detach:dev", "start", "running", "starting detach"), - ("detach:dev", "finish", "succeeded", "detached 1 models"), - ), - expected_detached_error_fragment="detached", - expected_query_results_after_adopt=(("SELECT id FROM dev.orders", ((1,),)),), - expected_query_results_after_detach=(("SELECT id FROM dev.orders", ((1,),)),), - ) - ], - ids=lambda case: case.description, -) -def test_given_unsuffixed_virtual_environment_when_adopting_and_detaching_then_names_are_preserved( - test_case: StateAdoptDetachE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_adopt_detach", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_adopt_detach"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - - adopt_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - assert adopt_result.returncode == 0, adopt_result.stdout + adopt_result.stderr - assert test_case.expected_adopt_fragment in adopt_result.stdout + adopt_result.stderr - for query_sql, expected_rows in test_case.expected_query_results_after_adopt: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - assert detach_result.returncode == 0, detach_result.stdout + detach_result.stderr - assert test_case.expected_detach_fragment in detach_result.stdout + detach_result.stderr - for query_sql, expected_rows in test_case.expected_query_results_after_detach: - assert query_duckdb(db_path=project_dir / "warehouse.duckdb", sql=query_sql) == list( - expected_rows - ) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT status FROM sqlbuild_state.virtual_environments " - "WHERE virtual_environment_name = 'dev'" - ), - ) == [(test_case.expected_detached_status,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT node_name FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE virtual_environment_name = 'dev' AND node_type = 'model'" - ), - ) == [("orders",)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT operation_id, operation_type, status, virtual_environment_name " - "FROM sqlbuild_state.state_operations ORDER BY operation_id" - ), - ) == list(test_case.expected_operation_rows) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT operation_id, action, status, message " - "FROM sqlbuild_state.state_operation_events ORDER BY operation_id, created_at" - ), - ) == list(test_case.expected_operation_event_rows) - - build_after_detach: subprocess.CompletedProcess[str] = run_sqb( - command=("build",), - project_dir=project_dir, - ) - assert build_after_detach.returncode == 1, build_after_detach.stdout + build_after_detach.stderr - assert test_case.expected_detached_error_fragment in ( - build_after_detach.stdout + build_after_detach.stderr - ) - - promote_after_detach: subprocess.CompletedProcess[str] = run_sqb( - command=("promote", "--from", "dev", "--to", "prod"), - project_dir=project_dir, - ) - assert promote_after_detach.returncode == 1, ( - promote_after_detach.stdout + promote_after_detach.stderr - ) - assert test_case.expected_detached_error_fragment in ( - promote_after_detach.stdout + promote_after_detach.stderr - ) - - rollback_after_detach: subprocess.CompletedProcess[str] = run_sqb( - command=("rollback",), - project_dir=project_dir, - ) - assert rollback_after_detach.returncode == 1, ( - rollback_after_detach.stdout + rollback_after_detach.stderr - ) - assert test_case.expected_detached_error_fragment in ( - rollback_after_detach.stdout + rollback_after_detach.stderr - ) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="adopt blocks without unsuffixed virtual env", - project_toml="", - expected_exit_code=1, - expected_error_fragment="unsuffixed_virtual_env", - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_unsuffixed_virtual_env_when_adopting_then_it_blocks_with_config_error( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_adopt_missing_unsuffixed", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_adopt_missing_unsuffixed"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="adopt blocks copy fallback without explicit permission", - project_toml="", - expected_exit_code=1, - expected_error_fragment="requires --allow-copy", - ) - ], - ids=lambda case: case.description, -) -def test_given_copy_fallback_required_when_adopting_without_allow_copy_then_it_blocks( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_adopt_allow_copy_required", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_adopt_allow_copy_required"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="interrupted detach records failed operation", - project_toml="", - expected_exit_code=1, - expected_error_fragment="simulated detach copy failure", - ) - ], - ids=lambda case: case.description, -) -def test_given_detach_copy_failure_when_detaching_then_operation_is_marked_failed( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_detach_failed_operation", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_detach_failed_operation"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "adapters/failing_duckdb.py": ( - "from typing import Any\n" - "from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder\n" - "from sqlbuild.adapter.contract.exceptions import AdapterUserError\n" - "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n" - "class FailingDuckDbAdapter(DuckDbAdapter):\n" - " adapter_name = 'failing_duckdb'\n\n" - " def move_or_copy_relation(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " origin: str,\n" - " destination: str,\n" - " remove_origin: bool,\n" - " allow_copy_fallback: bool,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " raise AdapterUserError('simulated detach copy failure')\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert ( - run_sqb( - command=("state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == 0 - ) - (project_dir / "sqlbuild_local.toml").write_text( - 'adapter = "failing_duckdb"\n\n[connection]\ndatabase = "warehouse.duckdb"\n' - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT status FROM sqlbuild_state.state_operations WHERE operation_id = 'detach:dev'" - ), - ) == [("failed",)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT action, status, message FROM sqlbuild_state.state_operation_events " - "WHERE operation_id = 'detach:dev' ORDER BY created_at" - ), - ) == [ - ("start", "running", "starting detach"), - ("fail", "failed", test_case.expected_error_fragment), - ] - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_type FROM information_schema.tables " - "WHERE table_schema = 'dev' AND table_name = 'orders'" - ), - ) == [("VIEW",)] - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_type FROM information_schema.tables " - "WHERE table_schema = 'dev__sqb_physical' AND table_name = 'orders__v_orders'" - ), - ) == [("BASE TABLE",)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT status FROM sqlbuild_state.virtual_environments " - "WHERE virtual_environment_name = 'dev'" - ), - ) == [("finalized",)] - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="interrupted adopt records failed operation and leaves recoverable residue", - project_toml="", - expected_exit_code=1, - expected_error_fragment="simulated adopt failure after move", - ) - ], - ids=lambda case: case.description, -) -def test_given_adopt_move_failure_when_adopting_then_operation_is_marked_failed( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_adopt_failed_operation", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_adopt_failed_operation"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "adapters/failing_duckdb.py": ( - "from typing import Any\n" - "from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder\n" - "from sqlbuild.adapter.contract.exceptions import AdapterUserError\n" - "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n" - "class FailingDuckDbAdapter(DuckDbAdapter):\n" - " adapter_name = 'failing_duckdb'\n\n" - " def move_or_copy_relation(\n" - " self,\n" - " connection: Any,\n" - " *,\n" - " origin: str,\n" - " destination: str,\n" - " remove_origin: bool,\n" - " allow_copy_fallback: bool,\n" - " statement_recorder: StatementRecorder,\n" - " ) -> None:\n" - " super().move_or_copy_relation(\n" - " connection=connection,\n" - " origin=origin,\n" - " destination=destination,\n" - " remove_origin=remove_origin,\n" - " allow_copy_fallback=allow_copy_fallback,\n" - " statement_recorder=statement_recorder,\n" - " )\n" - " raise AdapterUserError('simulated adopt failure after move')\n" - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="CREATE SCHEMA dev; CREATE TABLE dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - (project_dir / "sqlbuild_local.toml").write_text( - 'adapter = "failing_duckdb"\n\n' - "[connection]\n" - f'database = "{project_dir / "warehouse.duckdb"}"\n' - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql="SELECT status FROM sqlbuild_state.state_operations WHERE operation_id = 'adopt:dev'", - ) == [("failed",)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT action, status, message FROM sqlbuild_state.state_operation_events " - "WHERE operation_id = 'adopt:dev' ORDER BY created_at" - ), - ) == [ - ("start", "running", "starting adopt"), - ("fail", "failed", test_case.expected_error_fragment), - ] - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_type FROM information_schema.tables " - "WHERE table_schema = 'dev__sqb_physical' AND table_name = 'orders__v_orders'" - ), - ) == [("BASE TABLE",)] - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT COUNT(*) FROM information_schema.tables " - "WHERE table_schema = 'dev' AND table_name = 'orders'" - ), - ) == [(0,)] - assert query_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "SELECT COUNT(*) FROM sqlbuild_state.virtual_environment_node_refs " - "WHERE node_type = 'model'" - ), - ) == [(0,)] - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="detach recreates view models as views", - project_toml="", - expected_exit_code=0, - expected_error_fragment="VIEW", - ) - ], - ids=lambda case: case.description, -) -def test_given_view_model_when_detaching_then_stateless_target_remains_a_view( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_detach_view_model", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_detach_view_model"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/orders.sql": "MODEL (materialized view);\n\nSELECT 1 AS id\n", - }, - ) - execute_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="CREATE SCHEMA dev; CREATE VIEW dev.orders AS SELECT 1 AS id", - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - assert ( - run_sqb( - command=("state", "adopt", "--allow-copy"), - project_dir=project_dir, - input_text="adopt dev\n", - ).returncode - == test_case.expected_exit_code - ) - - detach_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "detach", "--allow-copy"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert detach_result.returncode == test_case.expected_exit_code, ( - detach_result.stdout + detach_result.stderr - ) - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql="SELECT id FROM dev.orders", - ) == [(1,)] - assert query_duckdb( - db_path=project_dir / "warehouse.duckdb", - sql=( - "SELECT table_type FROM information_schema.tables " - "WHERE table_schema = 'dev' AND table_name = 'orders'" - ), - ) == [(test_case.expected_error_fragment,)] - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="adopt cancels on wrong confirmation", - project_toml="", - expected_exit_code=1, - expected_error_fragment="cancelled", - ) - ], - ids=lambda case: case.description, -) -def test_given_wrong_confirmation_when_adopting_then_it_cancels( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_adopt_cancelled", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_adopt_cancelled"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "adopt"), - project_dir=project_dir, - input_text="nope\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - - -@pytest.mark.parametrize( - "test_case", - [ - StateModeGuardE2ETestCase( - description="detach blocks non-finalized virtual environment", - project_toml="", - expected_exit_code=1, - expected_error_fragment="requires a finalized virtual environment", - ) - ], - ids=lambda case: case.description, -) -def test_given_non_finalized_virtual_environment_when_detaching_then_it_blocks( - tmp_path: Path, - test_case: StateModeGuardE2ETestCase, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_state_detach_not_finalized", - repo_files={ - "sqlbuild_project.toml": ( - 'name = "virtual_state_detach_not_finalized"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n' - 'unsuffixed_virtual_env = "dev"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' - ), - "models/orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - }, - ) - assert run_sqb(command=("state", "init"), project_dir=project_dir).returncode == 0 - execute_duckdb( - db_path=project_dir / "state.duckdb", - sql=( - "INSERT INTO sqlbuild_state.virtual_environments " - "(virtual_environment_name, status, created_at, updated_at) " - "VALUES ('dev', 'active', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)" - ), - ) - - result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "detach"), - project_dir=project_dir, - input_text="detach dev\n", - ) - - assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr - assert test_case.expected_error_fragment in (result.stdout + result.stderr) - - -@pytest.mark.parametrize( - "test_case", - [ - StateExplicitRollbackE2ETestCase( - description="rollback accepts explicit backup id", - expected_exit_code=0, - expected_rollback_fragment="Virtual State Rolled Back", - expected_schema_version=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backup_when_rolling_back_explicit_backup_id_then_restores_backup( - test_case: StateExplicitRollbackE2ETestCase, - tmp_path: Path, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="versioned_state_project", - repo_files={ - "sqlbuild_project.toml": """ -name = "versioned_state_project" -adapter = "duckdb" -default_target = "dev" - -[settings] -virtual_environments = true - -[connection] -database = "warehouse.duckdb" - -[targets.dev.state] -backend = "duckdb" -schema = "sqlbuild_state" - -[targets.dev.state.connection] -database = "state.duckdb" -""".lstrip(), - }, - ) - state_db_path: Path = project_dir / "state.duckdb" - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == test_case.expected_exit_code, ( - init_result.stdout + init_result.stderr - ) - migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - assert migrate_result.returncode == test_case.expected_exit_code, ( - migrate_result.stdout + migrate_result.stderr - ) - backup_id: str = query_duckdb( - db_path=state_db_path, - sql=( - "SELECT backup_id FROM sqlbuild_state.state_migration_events " - "WHERE action = 'backup' ORDER BY created_at DESC LIMIT 1" - ), - )[0][0] - execute_duckdb( - db_path=state_db_path, - sql="UPDATE sqlbuild_state.state_versions SET schema_version = 2", - ) - second_migrate_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "migrate"), - project_dir=project_dir, - ) - assert second_migrate_result.returncode == test_case.expected_exit_code, ( - second_migrate_result.stdout + second_migrate_result.stderr - ) - execute_duckdb(db_path=state_db_path, sql="DELETE FROM sqlbuild_state.state_versions") - - rollback_result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "state", "rollback", "--backup-id", backup_id), - project_dir=project_dir, - ) - - assert rollback_result.returncode == test_case.expected_exit_code, ( - rollback_result.stdout + rollback_result.stderr - ) - assert test_case.expected_rollback_fragment in rollback_result.stdout - assert query_duckdb( - db_path=state_db_path, - sql="SELECT schema_version FROM sqlbuild_state.state_versions", - ) == [(test_case.expected_schema_version,)] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py index 6822bce3da..4fd75dc99c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/_test_types.py @@ -197,3 +197,11 @@ class TerminalTestProgressE2ETestCase: expected_test_row_count: int expected_screen_fragments: tuple[str, ...] expected_summary: str + + +@dataclass(frozen=True) +class ChainCteScopeTestCase: + description: str + queries: tuple[str, ...] + expected_sql: str + expected_exit_code: int = 0 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py index 45639506f2..777eb95cc7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/helpers.py @@ -361,7 +361,7 @@ def build_partial_ref_fixture_project_files() -> dict[str, str]: def build_open_schema_ref_fixture_project_files() -> dict[str, str]: - """Build a fixture with a required column beyond a partial star-model contract.""" + """Build a fixture with a required column beyond a partial star-model schema.""" return { "sqlbuild_project.toml": ( @@ -372,7 +372,6 @@ def build_open_schema_ref_fixture_project_files() -> dict[str, str]: ), "models/stg_orders.sql": ( "MODEL (\n" - " contract enforced,\n" " columns (\n" " order_id (type INTEGER),\n" " ),\n" @@ -1457,7 +1456,7 @@ def build_cursor_window_project_files(*, tests: dict[str, str]) -> dict[str, str " cursor_inputs (raw_orders (column order_date, roles [filter, watermark]),),\n" " batch_size 1h,\n" ");\n\n" - "SELECT COUNT(*) AS order_count\n" + "SELECT MAX(DATE_TRUNC('hour', order_date)) AS order_hour, COUNT(*) AS order_count\n" 'FROM __source("raw_orders")\n' "WHERE order_date >= __cursor_start() AND order_date < __cursor_end()\n" ), @@ -1479,3 +1478,24 @@ def build_declared_window_test(*, window: str, model_name: str) -> dict[str, str "SELECT 1\n" ) } + + +def build_cte_scope_project_files(*, queries: tuple[str, ...], expected: str) -> dict[str, str]: + """Build a chain with deliberately overlapping model-local CTE names.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "cte_scope"\nadapter = "duckdb"\n[connection]\ndatabase = "orders.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n - name: raw_orders\n schema: main\n table: raw_orders\n" + ), + "tests/unit/chain.sql": ( + 'TEST (name "order_chain");\nWITH ' + "__source__raw_orders AS (SELECT 1 AS order_id), " + f"__expected__orders_{len(queries) - 1} AS ({expected}) SELECT 1" + ), + } + for index, query in enumerate(queries): + files[f"models/orders_{index}.sql"] = f"MODEL ();\n{query}" + return files diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_chain_cte_scope.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_chain_cte_scope.py new file mode 100644 index 0000000000..e2a6712a56 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_chain_cte_scope.py @@ -0,0 +1,113 @@ +"""Chained model CTEs retain their lexical scope through the real DuckDB CLI.""" + +from pathlib import Path +from subprocess import CompletedProcess + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import ChainCteScopeTestCase +from tests.e2e.src.sqlbuild.cli.commands.main.test.helpers import build_cte_scope_project_files +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + +_FIRST: str = ( + 'WITH final AS (SELECT order_id FROM __source("raw_orders")) SELECT final.order_id FROM final' +) +_SECOND: str = ( + 'WITH final AS (SELECT order_id + 1 AS order_id FROM __ref("orders_0")) ' + "SELECT final.order_id FROM final" +) + + +@pytest.mark.parametrize( + "test_case", + ( + ChainCteScopeTestCase("two final CTEs", (_FIRST, _SECOND), "SELECT 2 AS order_id"), + ChainCteScopeTestCase( + "three final CTEs", + ( + _FIRST, + _SECOND, + 'WITH final AS (SELECT order_id + 1 AS order_id FROM __ref("orders_1")) ' + "SELECT final.order_id FROM final", + ), + "SELECT 3 AS order_id", + ), + ChainCteScopeTestCase( + "shared non-final CTEs with different shapes", + ( + 'WITH staged AS (SELECT order_id FROM __source("raw_orders")), ' + "final AS (SELECT order_id FROM staged) SELECT order_id FROM final", + 'WITH staged AS (SELECT order_id + 10 AS customer_id FROM __ref("orders_0")), ' + "final AS (SELECT customer_id FROM staged) SELECT customer_id FROM final", + ), + "SELECT 11 AS customer_id", + ), + ChainCteScopeTestCase( + "nested shadowing and qualified stars", + ( + _FIRST, + 'WITH "Final" AS (SELECT order_id + 1 AS order_id FROM __ref("orders_0")), ' + 'nested AS (WITH "Final" AS (SELECT 7 AS extra) SELECT "Final".* FROM "Final") ' + 'SELECT f.order_id + n.extra AS order_id FROM "Final" f CROSS JOIN nested n', + ), + "SELECT 9 AS order_id", + ), + ), + ids=lambda case: case.description, +) +def test_given_overlapping_ctes_when_testing_chain_then_each_model_uses_its_own_rows( + tmp_path: Path, test_case: ChainCteScopeTestCase +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="cte_scope", + repo_files=build_cte_scope_project_files( + queries=test_case.queries, expected=test_case.expected_sql + ), + ) + result: CompletedProcess[str] = run_sqb(command=("--no-color", "test"), project_dir=project) + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + compiled: CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project + ) + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + sql: str = next( + (project / "target" / "compiled" / "tests").rglob("order_chain.sql") + ).read_text() + assert "__sqb_cte_" in sql + + +@pytest.mark.parametrize( + "test_case", + ( + ChainCteScopeTestCase( + "upstream rows must not pass downstream expectation", + (_FIRST, _SECOND), + "SELECT 1 AS order_id", + 1, + ), + ), + ids=lambda case: case.description, +) +def test_given_same_shape_wrong_upstream_rows_when_testing_chain_then_reports_authored_difference( + tmp_path: Path, + test_case: ChainCteScopeTestCase, +) -> None: + project: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="wrong_scope", + repo_files=build_cte_scope_project_files( + queries=test_case.queries, expected=test_case.expected_sql + ), + ) + result: CompletedProcess[str] = run_sqb(command=("--no-color", "test"), project_dir=project) + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_exit_code, output + assert "FAIL=1" in output + assert "orders_1" in output + assert "unexpected sample 1: order_id=2; missing sample 1: order_id=1" in output + assert "__sqb_cte_" not in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py new file mode 100644 index 0000000000..6066ef65c6 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_statement_size.py @@ -0,0 +1,55 @@ +"""Large SQL tests execute through the real DuckDB CLI without an arbitrary cap.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.test._test_types import SqlTestE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb + + +@pytest.mark.parametrize( + "test_case", + [SqlTestE2ETestCase("large combined SQL executes", 0, "PASS=1")], + ids=lambda case: case.description, +) +def test_given_combined_sql_over_256k_when_testing_on_duckdb_then_passes( + tmp_path: Path, test_case: SqlTestE2ETestCase +) -> None: + # A literal survives compiler formatting, unlike padding with whitespace/comments. + payload: str = "orders" * 45_000 + assert len(payload) > 256_000 + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="large_sql_test", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "large_sql_test"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "orders.duckdb"\n' + ), + "models/raw_orders.sql": "MODEL ();\nSELECT 'orders' AS details\n", + "models/orders.sql": 'MODEL ();\nSELECT details FROM __ref("raw_orders")\n', + "tests/unit/test_orders.sql": ( + 'TEST (name "large_orders");\nWITH __ref__raw_orders AS (\n' + f"SELECT '{payload}' AS details\n), __expected__orders AS (\n" + f"SELECT '{payload}' AS details\n) SELECT 1\n" + ), + }, + ) + + tested: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "test"), project_dir=project_dir + ) + + output: str = tested.stdout + tested.stderr + assert tested.returncode == test_case.expected_exit_code, output + assert test_case.expected_stdout_fragment in output + assert "FAIL=0" in output + assert "T001" not in output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py index 3c4320c7fb..8704acfdc0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/test/test_test.py @@ -328,7 +328,7 @@ def test_given_inspection_when_requesting_json_file_then_parser_rejects_conflict expected_stdout_fragment="PASS=1", ), FixtureCompatibilityE2ETestCase( - description="open star-model contract accepts required fixture payload columns", + description="open star-model schema accepts required fixture payload columns", repo_files=build_open_schema_ref_fixture_project_files(), expected_stdout_fragment="PASS=1", ), @@ -437,8 +437,8 @@ def test_given_complex_strings_in_values_when_processing_then_literals_remain_in supplied_column="order_identifer", ), expected_stderr_fragments=( - "supplies unknown columns: order_identifer", - "must provide required non-nullable columns: order_id", + "[B302] tests/unit/test_orders.sql:4:38:", + "SQL test '__source__raw_orders' names unknown column 'order_identifer'", ), ), SqlTestFixtureValidationE2ETestCase( @@ -1115,8 +1115,8 @@ def test_given_deep_shared_graph_missing_mock_when_testing_then_error_is_reporte expected_tests=_UNKNOWN_EXPECTED_COLUMN, expected_exit_code=1, expected_output_fragments=( - "SQL test 'unknown_column': tests/unit/unknown_column.sql:8:", - _UNKNOWN_COLUMN_MESSAGE, + "[B302] tests/unit/unknown_column.sql:9:36:", + "SQL test '__expected__orders' names unknown column 'region'", ), unexpected_output_fragments=("Connecting to", "Binder Error"), ), diff --git a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py index 18e689e97d..8255870910 100644 --- a/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/bigquery/test_client.py @@ -26,9 +26,13 @@ from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind -from sqlbuild.virtual.freshness.main._state_record import source_freshness_record_from_observation -from sqlbuild.virtual.freshness.models import SourceFreshnessObservation +from sqlbuild.compiler.source_freshness.main.data_version_hash import ( + source_freshness_data_version_hash, +) +from sqlbuild.compiler.source_freshness.main.normalization import ( + normalize_source_freshness_data_version, +) +from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.bigquery._test_types import ( BigQueryBuildFlowTestCase, BigQueryDeleteInsertCursorTestCase, @@ -350,36 +354,33 @@ def test_given_table_dml_when_getting_freshness_metadata_then_modified_time_adva assert isinstance(changed_data_version, test_case.expected_data_version_type) assert isinstance(batch_metadata.data_version, test_case.expected_data_version_type) assert changed_data_version > initial_data_version - initial_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=initial_data_version, + initial_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=initial_data_version, value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), - observed_at=datetime.now(tz=UTC), ), - virtual_environment_name="dev", - ).data_version_hash - repeated_initial_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=initial_data_version, + value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), + ) + repeated_initial_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=initial_data_version, value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), - observed_at=datetime.now(tz=UTC), ), - virtual_environment_name="dev", - ).data_version_hash - changed_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=changed_data_version, + value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), + ) + changed_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=changed_data_version, value_kind=SourceFreshnessValueKind(changed_metadata.value_kind), - observed_at=datetime.now(tz=UTC), ), - virtual_environment_name="dev", - ).data_version_hash + value_kind=SourceFreshnessValueKind(changed_metadata.value_kind), + ) assert repeated_initial_hash == initial_hash assert changed_hash != initial_hash diff --git a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py index 9fc31dc606..803775bd04 100644 --- a/tests/integration/src/sqlbuild/adapters/databricks/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/databricks/test_client.py @@ -25,9 +25,13 @@ from sqlbuild.compiler.fingerprints.main.write import write_fingerprint from sqlbuild.compiler.fingerprints.models import Fingerprint, FingerprintSet from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind -from sqlbuild.virtual.freshness.main._state_record import source_freshness_record_from_observation -from sqlbuild.virtual.freshness.models import SourceFreshnessObservation +from sqlbuild.compiler.source_freshness.main.data_version_hash import ( + source_freshness_data_version_hash, +) +from sqlbuild.compiler.source_freshness.main.normalization import ( + normalize_source_freshness_data_version, +) +from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.databricks._test_types import ( DatabricksBuildFlowTestCase, DatabricksExpressionNullabilityRuleTestCase, @@ -332,36 +336,33 @@ def test_given_delta_table_dml_when_getting_freshness_metadata_then_version_adva assert isinstance(initial_data_version, datetime) assert isinstance(changed_data_version, datetime) assert changed_data_version > initial_data_version - initial_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=initial_data_version, + initial_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=initial_data_version, value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), - observed_at=datetime.now(tz=UTC), ), - virtual_environment_name="dev", - ).data_version_hash - repeated_initial_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=initial_data_version, + value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), + ) + repeated_initial_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=initial_data_version, value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), - observed_at=datetime.now(tz=UTC), ), - virtual_environment_name="dev", - ).data_version_hash - changed_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=changed_data_version, + value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), + ) + changed_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=changed_data_version, value_kind=SourceFreshnessValueKind(changed_metadata.value_kind), - observed_at=datetime.now(tz=UTC), ), - virtual_environment_name="dev", - ).data_version_hash + value_kind=SourceFreshnessValueKind(changed_metadata.value_kind), + ) assert repeated_initial_hash == initial_hash assert changed_hash != initial_hash diff --git a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py index a93459311a..26c87ee52e 100644 --- a/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/snowflake/test_client.py @@ -22,9 +22,13 @@ from sqlbuild.adapter.contract.types import FunctionNullabilityRule from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind -from sqlbuild.virtual.freshness.main._state_record import source_freshness_record_from_observation -from sqlbuild.virtual.freshness.models import SourceFreshnessObservation +from sqlbuild.compiler.source_freshness.main.data_version_hash import ( + source_freshness_data_version_hash, +) +from sqlbuild.compiler.source_freshness.main.normalization import ( + normalize_source_freshness_data_version, +) +from sqlbuild.spec.contracts.types import SourceFreshnessValueKind from tests.integration.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeBuildFlowTestCase, SnowflakeExpressionNullabilityRuleTestCase, @@ -359,36 +363,33 @@ def test_given_table_dml_when_getting_freshness_metadata_then_last_altered_advan assert isinstance(changed_metadata.data_version, test_case.expected_data_version_type) assert isinstance(batch_metadata.data_version, test_case.expected_data_version_type) assert changed_metadata.data_version > initial_metadata.data_version - initial_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=initial_metadata.data_version, + initial_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=initial_metadata.data_version, value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), - observed_at=datetime.now(), ), - virtual_environment_name="dev", - ).data_version_hash - repeated_initial_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=initial_metadata.data_version, + value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), + ) + repeated_initial_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=initial_metadata.data_version, value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), - observed_at=datetime.now(), ), - virtual_environment_name="dev", - ).data_version_hash - changed_hash: str = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name="raw_orders", - strategy=SourceFreshnessStrategy.ADAPTER, - data_version=changed_metadata.data_version, + value_kind=SourceFreshnessValueKind(initial_metadata.value_kind), + ) + changed_hash: str = source_freshness_data_version_hash( + source_name="raw_orders", + strategy="adapter", + data_version=normalize_source_freshness_data_version( + value=changed_metadata.data_version, value_kind=SourceFreshnessValueKind(changed_metadata.value_kind), - observed_at=datetime.now(), ), - virtual_environment_name="dev", - ).data_version_hash + value_kind=SourceFreshnessValueKind(changed_metadata.value_kind), + ) assert repeated_initial_hash == initial_hash assert changed_hash != initial_hash diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 522599eb7f..7012831de2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -52,6 +52,47 @@ class DerivedNativeCompileTestCase: expected_rows: tuple[tuple[int, ...], ...] +@dataclass(frozen=True) +class KeywordFunctionCompileTestCase: + """Expected compile result for a DuckDB SQL keyword function projection.""" + + description: str + projection: str + expected_exit_code: int + expected_diagnostics: tuple[str, ...] + + +@dataclass(frozen=True) +class StarExpansionCompileTestCase: + """Expected diagnostics for a model reading a star over an inferred upstream.""" + + description: str + star_model_sql: str + downstream_sql: str + expected_exit_code: int + expected_diagnostics: tuple[str, ...] + + +@dataclass(frozen=True) +class StarLineageCompileTestCase: + """Expected compile-report lineage for a star over a derived table.""" + + description: str + star_model_sql: str + expected_edge_count: int + + +@dataclass(frozen=True) +class SnapshotValidityCompileTestCase: + """Expected diagnostics for a model reading a snapshot relation.""" + + description: str + snapshot_config: str + downstream_sql: str + expected_exit_code: int + expected_diagnostics: tuple[str, ...] + + @dataclass(frozen=True) class CombinedCompilationTestCase: """Expected CLI behavior for schema-bound CTE compilation.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 0c46702399..4be88f914d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -47,7 +47,8 @@ def write_from_values_format_project(*, tmp_path: Path, adapter: str) -> tuple[P models: Path = project_dir / "models" models.mkdir() (models / "customers.sql").write_text( - 'MODEL (description "Customers.");\nSELECT 1 AS customer_key\n', encoding="utf-8" + 'MODEL (description "Customers.");\nSELECT 1 AS customer_key, 1 AS order_count\n', + encoding="utf-8", ) (models / "orders.sql").write_text( 'MODEL (description "Orders.");\nSELECT customer_key, 1 AS order_count ' @@ -380,3 +381,18 @@ def dropped_relation_microbatch_sql(*, batch_concurrency: int) -> str: 'SELECT id, ordered_at FROM __source("raw_orders")\n' "WHERE id >= __cursor_start() AND id < __cursor_end()\n" ) + + +def compile_duckdb_projection( + *, tmp_path: Path, capsys: pytest.CaptureFixture[str], projection: str +) -> tuple[int, tuple[object, ...]]: + """Compile one DuckDB model projection and return the exit and diagnostic codes.""" + + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + (models / "orders.sql").write_text(f"MODEL ();\nSELECT {projection}") + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + result: dict[str, object] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) + return exit_code, tuple(item["code"] for item in diagnostics) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py index 034efd68b6..30c5a84dd9 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/_test_types.py @@ -73,13 +73,6 @@ class PlanAsTargetTestCase: expected_fragment: str = "" -@dataclass(frozen=True) -class VirtualModeMigrationTestCase: - description: str - expected_exit_code: int - expected_fragment: str - - @dataclass(frozen=True) class StagedReplaceTestCase: description: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py index 1a3364ef7b..f024e3bd51 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/helpers.py @@ -45,29 +45,6 @@ schema = "prod" """ ).lstrip() -VIRTUAL_PROJECT_TOML: str = dedent( - f""" - name = "orders_project" - adapter = "duckdb" - default_target = "dev" - - [settings] - virtual_environments = true - - [connection] - database = "{DATABASE_FILE}" - - [targets.dev] - schema = "dev" - - [targets.dev.state] - backend = "duckdb" - schema = "sqlbuild_state" - - [targets.dev.state.connection] - database = "state.duckdb" - """ -).lstrip() JANITOR_PROJECT_TOML: str = dedent( f""" name = "orders_project" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py b/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py deleted file mode 100644 index 149f232b6c..0000000000 --- a/tests/integration/src/sqlbuild/cli/commands/main/model_migrations/test_virtual_mode.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Integration coverage for rejecting model migrations in virtual environments.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.integration.src.sqlbuild.cli.commands.main.model_migrations._test_types import ( - VirtualModeMigrationTestCase, -) -from tests.integration.src.sqlbuild.cli.commands.main.model_migrations.helpers import ( - VIRTUAL_PROJECT_TOML, - CliRun, - incremental_orders_sql, - load_raw_orders, - run_sqb, - write_project, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualModeMigrationTestCase( - description="virtual build rejects migrate_from", - expected_exit_code=1, - expected_fragment="migrate_from is supported only in direct mode", - ) - ], - ids=lambda case: case.description, -) -def test_given_migrate_from_when_building_virtually_then_rejects_before_building( - test_case: VirtualModeMigrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - write_project( - project_dir=tmp_path, - models={"stg_customer_orders": incremental_orders_sql(migrate_from="stg_orders")}, - project_toml=VIRTUAL_PROJECT_TOML, - ) - load_raw_orders(project_dir=tmp_path, first_day=1, last_day=3) - initialized: CliRun = run_sqb(project_dir=tmp_path, args=("state", "init"), capsys=capsys) - - result: CliRun = run_sqb( - project_dir=tmp_path, args=("build", "--virtual-env", "feature"), capsys=capsys - ) - - assert initialized.exit_code == 0, initialized.output - assert result.exit_code == test_case.expected_exit_code, result.output - assert test_case.expected_fragment in result.output - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py b/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py index b6f7e6f962..ffb7d41daf 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_build_test_planning.py @@ -35,15 +35,6 @@ ), } -_VIRTUAL_PROJECT_TOML: str = ( - 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' - "[settings]\nvirtual_environments = true\n\n" - '[connection]\ndatabase = "warehouse.duckdb"\n\n' - '[targets.dev]\nschema = "dev"\n\n' - '[targets.dev.state]\nbackend = "duckdb"\nschema = "sqlbuild_state"\n\n' - '[targets.dev.state.connection]\ndatabase = "state.duckdb"\n' -) - @pytest.mark.parametrize( "test_case", @@ -52,7 +43,7 @@ description="build with tests rejects an invalid SQL test fixture", build_flags=(), expected_exit_code=1, - expected_fragment="Invalid SQL test fixtures", + expected_fragment="error[B302]", ), BuildTestPlanningTestCase( description="build without tests does not plan SQL tests", @@ -82,48 +73,5 @@ def test_given_invalid_test_fixture_when_building_then_only_test_runs_plan_sql_t assert test_case.expected_fragment in captured -@pytest.mark.parametrize( - "test_case", - [ - BuildTestPlanningTestCase( - description="virtual build with tests rejects an invalid SQL test fixture", - build_flags=("--virtual-env", "feature"), - expected_exit_code=1, - expected_fragment="Invalid SQL test fixtures", - ), - BuildTestPlanningTestCase( - description="virtual build without tests does not plan SQL tests", - build_flags=("--virtual-env", "feature", "--no-tests"), - expected_exit_code=0, - expected_fragment="Completed successfully", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_test_fixture_when_building_virtually_then_only_test_runs_plan_sql_tests( - test_case: BuildTestPlanningTestCase, - tmp_path: Path, - capsys: pytest.CaptureFixture[str], -) -> None: - project_files: dict[str, str] = { - **_PROJECT_FILES, - "sqlbuild_project.toml": _VIRTUAL_PROJECT_TOML, - } - for relative_path, contents in project_files.items(): - destination: Path = tmp_path / relative_path - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_text(contents, encoding="utf-8") - assert main(["--project-dir", str(tmp_path), "--no-color", "state", "init"]) == 0 - _ = capsys.readouterr() - - exit_code: int = main( - ["--project-dir", str(tmp_path), "--no-color", "build", *test_case.build_flags] - ) - captured: str = "".join(capsys.readouterr()) - - assert exit_code == test_case.expected_exit_code, captured - assert test_case.expected_fragment in captured - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py index 29ee9e7177..5ce7a0bf1e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_native_facts.py @@ -14,7 +14,12 @@ AliasSourceCompileTestCase, BoundProjectionCompileTestCase, DerivedNativeCompileTestCase, + KeywordFunctionCompileTestCase, + SnapshotValidityCompileTestCase, + StarExpansionCompileTestCase, + StarLineageCompileTestCase, ) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import compile_duckdb_projection @pytest.mark.parametrize( @@ -79,6 +84,227 @@ def test_given_derived_native_facts_when_compiling_then_contract_and_execution_a assert rows == test_case.expected_rows +@pytest.mark.parametrize( + "test_case", + ( + StarExpansionCompileTestCase( + description="qualified star beside a joined column expands for downstream readers", + star_model_sql=( + 'SELECT staged.*, extra.status FROM __ref("staged_orders") AS staged ' + "CROSS JOIN (SELECT 'open' AS status) AS extra" + ), + downstream_sql='SELECT order_id, status FROM __ref("wide_orders")', + expected_exit_code=0, + expected_diagnostics=(), + ), + StarExpansionCompileTestCase( + description="bare star over an inferred upstream still rejects unknown columns", + star_model_sql='SELECT * FROM __ref("staged_orders")', + downstream_sql='SELECT order_id, missing_column FROM __ref("wide_orders")', + expected_exit_code=1, + expected_diagnostics=("B002",), + ), + ), + ids=lambda case: case.description, +) +def test_given_star_over_inferred_upstream_when_compiling_then_downstream_binding_is_exact( + test_case: StarExpansionCompileTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + (models / "staged_orders.sql").write_text("MODEL ();\nSELECT 1 AS order_id") + (models / "wide_orders.sql").write_text(f"MODEL ();\n{test_case.star_model_sql}") + (models / "order_readers.sql").write_text(f"MODEL ();\n{test_case.downstream_sql}") + + for _ in ("cold", "warm"): + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) + result: dict[str, object] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) + + assert exit_code == test_case.expected_exit_code + assert tuple(item["code"] for item in diagnostics) == test_case.expected_diagnostics + + +@pytest.mark.parametrize( + "test_case", + ( + StarLineageCompileTestCase( + description="star over a joined derived table omits unprojected seed columns", + star_model_sql=( + "SELECT * FROM (SELECT input.order_id, amount " + 'FROM __ref("staged_orders") AS input ' + 'LEFT JOIN __seed("order_labels") AS labels ' + "ON labels.order_id = input.order_id) AS nested_orders" + ), + expected_edge_count=2, + ), + ), + ids=lambda case: case.description, +) +def test_given_star_over_derived_table_when_compiling_then_lineage_matches_projection( + test_case: StarLineageCompileTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + seeds: Path = tmp_path / "seeds" + seeds.mkdir() + (seeds / "order_labels.csv").write_text("order_id,label\n1,first\n") + (seeds / "order_labels.yml").write_text( + "seeds:\n - name: order_labels\n columns:\n" + " - name: order_id\n type: INTEGER\n" + " - name: label\n type: VARCHAR\n" + ) + models: Path = tmp_path / "models" + models.mkdir() + (models / "staged_orders.sql").write_text( + "MODEL ();\nSELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount" + ) + (models / "wide_orders.sql").write_text(f"MODEL ();\n{test_case.star_model_sql}") + + for _ in ("cold", "warm"): + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json"]) + result: dict[str, object] = json.loads(capsys.readouterr().out) + resources: dict[str, object] = cast(dict[str, object], result["resources"]) + compiled_models: list[dict[str, object]] = cast( + list[dict[str, object]], resources["models"] + ) + lineage_by_model: dict[object, dict[str, object]] = { + model["name"]: cast(dict[str, object], model["lineage"]) for model in compiled_models + } + lineage: dict[str, object] = lineage_by_model["wide_orders"] + + assert exit_code == 0 + assert result["diagnostics"] == [] + # order_id and amount each trace to staged_orders; the joined seed's label is + # not projected by the derived table, so it contributes no edge. + assert lineage["edge_count"] == test_case.expected_edge_count + + +@pytest.mark.parametrize( + "test_case", + ( + SnapshotValidityCompileTestCase( + description="default validity columns are readable downstream", + snapshot_config="", + downstream_sql=( + 'SELECT customer_id, valid_from FROM __ref("customer_snapshot") ' + "WHERE valid_to IS NULL" + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + SnapshotValidityCompileTestCase( + description="renamed validity columns are readable downstream", + snapshot_config=", valid_from_column active_from, valid_to_column active_to", + downstream_sql=( + 'SELECT customer_id, active_from FROM __ref("customer_snapshot") ' + "WHERE active_to IS NULL" + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + SnapshotValidityCompileTestCase( + description="unknown snapshot columns are still rejected", + snapshot_config="", + downstream_sql='SELECT customer_id, missing_column FROM __ref("customer_snapshot")', + expected_exit_code=1, + expected_diagnostics=("B002",), + ), + ), + ids=lambda case: case.description, +) +def test_given_snapshot_reader_when_compiling_then_validity_columns_are_known( + test_case: SnapshotValidityCompileTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + models: Path = tmp_path / "models" + models.mkdir() + (models / "customer_snapshot.sql").write_text( + "MODEL (materialized snapshot, unique_key [customer_id], snapshot_strategy timestamp, " + f"updated_at updated_at{test_case.snapshot_config});\n" + "SELECT 1 AS customer_id, 'pro' AS plan, TIMESTAMP '2026-01-01' AS updated_at" + ) + (models / "current_customers.sql").write_text(f"MODEL ();\n{test_case.downstream_sql}") + + exit_code: int = main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + result: dict[str, object] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], result["diagnostics"]) + + assert exit_code == test_case.expected_exit_code + assert tuple(item["code"] for item in diagnostics) == test_case.expected_diagnostics + + +@pytest.mark.parametrize( + "test_case", + ( + KeywordFunctionCompileTestCase( + description="keyword timestamps compile and execute", + projection=( + "CURRENT_TIMESTAMP AS created_at, LOCALTIMESTAMP AS local_created_at, " + "CURRENT_TIME AS created_time, LOCALTIME AS local_created_time" + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + KeywordFunctionCompileTestCase( + description="keyword timestamp in a CTE arithmetic filter", + projection=( + "created_at FROM (SELECT CURRENT_TIMESTAMP AS created_at) AS recent " + "WHERE created_at > CURRENT_TIMESTAMP - INTERVAL 1 DAY" + ), + expected_exit_code=0, + expected_diagnostics=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_duckdb_keyword_function_when_compiling_then_compiled_sql_executes( + test_case: KeywordFunctionCompileTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code, codes = compile_duckdb_projection( + tmp_path=tmp_path, capsys=capsys, projection=test_case.projection + ) + + assert exit_code == test_case.expected_exit_code + assert codes == test_case.expected_diagnostics + compiled_sql: str = (tmp_path / "target" / "compiled" / "models" / "orders.sql").read_text() + with duckdb.connect() as connection: + assert len(connection.execute(compiled_sql).fetchall()) == 1 + + +@pytest.mark.parametrize( + "test_case", + ( + KeywordFunctionCompileTestCase( + description="parenthesized keyword timestamp is still unknown", + projection="CURRENT_TIMESTAMP() AS created_at", + expected_exit_code=1, + expected_diagnostics=("B101",), + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_duckdb_keyword_call_when_compiling_then_function_is_rejected( + test_case: KeywordFunctionCompileTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + exit_code, codes = compile_duckdb_projection( + tmp_path=tmp_path, capsys=capsys, projection=test_case.projection + ) + + assert exit_code == test_case.expected_exit_code + assert codes == test_case.expected_diagnostics + + @pytest.mark.parametrize( "test_case", ( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_dollar_quoted_literals.py b/tests/integration/src/sqlbuild/cli/commands/main/test_dollar_quoted_literals.py index 3eae8372df..2bbc00db44 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_dollar_quoted_literals.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_dollar_quoted_literals.py @@ -30,7 +30,16 @@ 1, ), ), - ) + ), + DollarQuotedLiteralBuildTestCase( + description="tagged dollar-quoted literal with an apostrophe stays opaque", + projection=( + "$note$ customer's note $note$ AS order_label,\n" + " $$plain$$ AS order_note,\n" + " o.order_id" + ), + expected_rows=((" customer's note ", "plain", 1),), + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py index 35d87a189d..e66a8b7067 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_format_preservation.py @@ -99,14 +99,14 @@ def test_given_authored_syntax_when_formatting_then_spelling_and_idempotence_are "test_case", [ FormatterSyntaxTestCase( - "computed variant key needs source-preserving parser support", + "computed variant key retains source-preserving parser support", "SELECT payload:customers[TO_VARCHAR(order_id)] AS customer FROM orders", - ("format-unsafe", "Expected RBracket"), + ("payload:customers[TO_VARCHAR(order_id)]",), ) ], ids=lambda case: case.description, ) -def test_given_unsupported_variant_key_when_formatting_then_compilable_file_stays_untouched( +def test_given_computed_variant_key_when_formatting_then_preserves_syntax_and_compiles( test_case: FormatterSyntaxTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') @@ -120,13 +120,14 @@ def test_given_unsupported_variant_key_when_formatting_then_compilable_file_stay model.write_text(original) assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == 0 capsys.readouterr() - assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 1 + assert main(["--project-dir", str(tmp_path), "format", "--json"]) == 0 payload: dict[str, object] = json.loads(capsys.readouterr().out) - assert all(fragment in json.dumps(payload) for fragment in test_case.expected_fragments) - assert payload["formatted_files"] == [] - assert model.read_text() == original - assert main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) == 1 - assert model.read_text() == original + assert payload["formatted_files"] + formatted: str = model.read_text() + assert all(fragment in formatted for fragment in test_case.expected_fragments) + assert main(["--project-dir", str(tmp_path), "format", "--check", "--json"]) == 0 + assert model.read_text() == formatted + assert main(["--project-dir", str(tmp_path), "compile", "--no-cache"]) == 0 @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index efe84796b8..5b2ad5b97e 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -6,6 +6,110 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +@dataclass(frozen=True) +class ExpressionMemoCase: + description: str + expression: str + expected_analysis_calls: int = 1 + + +@dataclass(frozen=True) +class ExpressionBatchCase: + description: str + orders_expression: str + customers_expression: str + expected_batches: tuple[tuple[str, ...], ...] + + +@dataclass(frozen=True) +class ShapeCacheCase: + description: str + sql: str + expected_columns: int = 2 + expected_star: bool = True + + +@dataclass(frozen=True) +class IdentifierBindingCase: + description: str + dialect: str + projection: str + reference: str + expected_codes: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ValidationCacheCase: + description: str + enabled: bool + expected_hits: int + expected_bypasses: int + + +@dataclass(frozen=True) +class NativeCatalogCase: + description: str + sql: str + expected_codes: tuple[str, ...] = () + spelling: str = "MiSsPeLlEd" + expected_sql: str = "SELECT * FROM __sqlbuild_table_function_table_fn__orders" + + +@dataclass(frozen=True) +class SemanticCompileCase: + description: str + upstream: str + downstream: str + expected_code: str | None + header: str = "MODEL (materialized view);\n" + expected_diagnostic_count: int = 1 + expected_column: int | None = None + + +@dataclass(frozen=True) +class DiagnosticUxCase: + description: str + edits: tuple[tuple[str, str, str], ...] + extra_files: tuple[tuple[str, str], ...] = () + expected_errors: int = 1 + expected_fragments: tuple[str, ...] = () + expected_partial: bool = True + + +@dataclass(frozen=True) +class ColumnSuggestionCase: + description: str + name: str + columns: tuple[str, ...] + expected_match: str | None + + +@dataclass(frozen=True) +class SemanticTriageCase: + description: str + column_type: str = "DATE" + cursor_type: str = "timestamp" + setting: str = "" + expected_codes: tuple[str, ...] = () + independent_sql: str = "" + root_contract: str = "" + + +@dataclass(frozen=True) +class SnowflakeSemanticReleaseCase: + description: str + sql: str + expected_codes: tuple[str, ...] = () + expected_exit_code: int = 0 + + +@dataclass(frozen=True) +class SnowflakeOutputInferenceCase: + description: str + sql: str + expected_type: str | None = None + + @dataclass(frozen=True) class RulesPipelineIntegrationTestCase: """One configured Rules failure in the shared planning compiler.""" @@ -249,3 +353,15 @@ class ReshapedStarLineageIntegrationTestCase: description: str query_sql: str expected_sources: tuple[frozenset[tuple[str, str]], ...] + + +@dataclass(frozen=True) +class SharedBindingQueryCase: + description: str + orders_summary_sql: str + customers_summary_sql: str + expected_shared_queries: int + expected_codes: tuple[str, ...] = () + later_models: tuple[tuple[str, str], ...] = () + expected_lineage: tuple[tuple[str, str], ...] = () + expected_findings: tuple[tuple[str, str], ...] = () diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py index 68b7497597..55fc3feca7 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/helpers.py @@ -5,10 +5,14 @@ from pathlib import Path from typing import Any +import pytest + from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile._helpers.analysis import compact from sqlbuild.compiler.compile.main._build_compile_inputs import build_compile_inputs from sqlbuild.compiler.compile.models import ( + CompactBatchPreparation, CompileAdapterContext, CompileAuditInput, CompiledLineageColumnFact, @@ -23,6 +27,7 @@ from sqlbuild.compiler.pipeline.models import CompilePipelineOptions, CompilePipelineResult from sqlbuild.runtime.contracts.models import ConnectionHooks from sqlbuild.sql_values.types import CollectionRendering +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import SharedBindingQueryCase _SCHEMA_FIXTURE_PATH: Path = ( Path(__file__).resolve().parents[5] / "fixtures" / "dbt_manifest_v12_schema.json" @@ -168,3 +173,32 @@ def lineage_source_pairs(column: CompiledLineageColumnFact) -> frozenset[tuple[s return frozenset( (source.resource_name, source.column_name) for source in column.upstream_columns ) + + +_SHARED_BINDING_UPSTREAM_SQL: str = "MODEL (materialized view); SELECT 1 AS id, 2.5 AS amount" + + +def write_shared_binding_project(*, project_dir: Path, test_case: SharedBindingQueryCase) -> None: + (project_dir / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + models: Path = project_dir / "models" + models.mkdir() + (models / "orders.sql").write_text(_SHARED_BINDING_UPSTREAM_SQL) + (models / "customers.sql").write_text(_SHARED_BINDING_UPSTREAM_SQL) + (models / "orders_summary.sql").write_text(test_case.orders_summary_sql) + (models / "customers_summary.sql").write_text(test_case.customers_summary_sql) + for name, sql in test_case.later_models: + (models / f"{name}.sql").write_text(sql) + + +def trace_native_compact_batches(monkeypatch: pytest.MonkeyPatch) -> list[CompactBatchPreparation]: + original: Callable[..., object] = compact._run_compact_analysis_batch + preparations: list[CompactBatchPreparation] = [] + + def traced(*, preparation: CompactBatchPreparation) -> object: + preparations.append(preparation) + return original(preparation=preparation) + + monkeypatch.setattr(compact, "_run_compact_analysis_batch", traced) + return preparations diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py new file mode 100644 index 0000000000..ce365a20bb --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_binding_waves.py @@ -0,0 +1,69 @@ +"""Dependency-ready models bind against producer outputs in their first analysis.""" + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile._helpers.assembly import project +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import NativeCatalogCase + + +@pytest.mark.parametrize( + "test_case", + [NativeCatalogCase("closed dependency waves", 'SELECT id FROM __ref("orders")')], + ids=lambda case: case.description, +) +def test_given_inferred_producer_when_binding_consumers_then_avoids_deferred_requests_and_invalidates_changed_shapes( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: raw_orders\n expression: SELECT 1 AS id, 2 AS quantity\n" + ) + (tmp_path / "models").mkdir() + producer: Path = tmp_path / "models/orders.sql" + producer.write_text('MODEL (materialized view); SELECT id FROM __source("raw_orders")') + (tmp_path / "models/report.sql").write_text("MODEL (materialized view); " + test_case.sql) + original: Callable[..., Any] = project.get_schema_validations + requests: list[Any] = [] + + def traced(**kwargs: Any) -> Any: + requests.extend(kwargs["requests"]) + return original(**kwargs) + + monkeypatch.setattr(project, "get_schema_validations", traced) + args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] + assert main(args) == 0 + cold: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in cold["diagnostics"]) == test_case.expected_codes + assert requests == [] + assert main(args) == 0 + warm: dict[str, Any] = json.loads(capsys.readouterr().out) + assert warm["resources"] == cold["resources"] + producer.write_text(producer.read_text().replace("SELECT id", "SELECT quantity")) + assert main(args) == 1 + changed: dict[str, Any] = json.loads(capsys.readouterr().out) + assert [item["code"] for item in changed["diagnostics"]] == ["B002"] + assert requests == [] + assert main([*args, "--no-cache"]) == 1 + oracle: dict[str, Any] = json.loads(capsys.readouterr().out) + assert oracle["diagnostics"] == changed["diagnostics"] + producer.write_text(producer.read_text().replace("SELECT quantity", "SELECT id")) + assert main(args) == 0 + restored: dict[str, Any] = json.loads(capsys.readouterr().out) + assert restored["resources"] == cold["resources"] + assert restored["diagnostics"] == cold["diagnostics"] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py new file mode 100644 index 0000000000..592026618f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_completed_shape_cache.py @@ -0,0 +1,71 @@ +"""Completed star shapes survive partial compact-cache reuse.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ShapeCacheCase + + +@pytest.mark.parametrize( + "test_case", + [ + ShapeCacheCase( + "derived star", 'SELECT * FROM (SELECT id, quantity FROM __source("raw_orders")) q' + ), + ShapeCacheCase( + "CTE star", + 'WITH q AS (SELECT id, quantity FROM __source("raw_orders")) SELECT * FROM q', + ), + ], + ids=lambda case: case.description, +) +def test_given_completed_shape_when_an_unrelated_model_changes_then_cached_columns_and_lineage_survive( + test_case: ShapeCacheCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: raw_orders\n table: orders\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n - name: quantity\n type: INTEGER\n" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/order_inputs.sql").write_text( + 'MODEL (materialized view); SELECT id, quantity FROM __source("raw_orders")' + ) + (tmp_path / "models/orders.sql").write_text( + "MODEL (materialized view);\n" + + test_case.sql.replace('__source("raw_orders")', '__ref("order_inputs")') + ) + edited: Path = tmp_path / "models/customers.sql" + edited.write_text("MODEL (materialized view); SELECT 1 AS id") + args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] + assert main(args) == 0 + cold: dict[str, Any] = json.loads(capsys.readouterr().out) + cold_models: dict[str, dict[str, Any]] = { + model["name"]: model for model in cold["resources"]["models"] + } + expected: dict[str, Any] = cold_models["orders"] + assert expected["column_count"] == test_case.expected_columns + assert expected["lineage"]["has_star"] is test_case.expected_star + edited.write_text(edited.read_text() + "\n-- unrelated edit\n") + assert main(args) == 0 + changed: dict[str, Any] = json.loads(capsys.readouterr().out) + changed_models: dict[str, dict[str, Any]] = { + model["name"]: model for model in changed["resources"]["models"] + } + actual: dict[str, Any] = changed_models["orders"] + assert actual == expected + assert main([*args, "--no-cache"]) == 0 + oracle: dict[str, Any] = json.loads(capsys.readouterr().out) + assert changed["resources"] == oracle["resources"] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py new file mode 100644 index 0000000000..f72f6980c9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_expression_catalog_cache.py @@ -0,0 +1,107 @@ +"""Expression schemas are memoized only within their owning compiler catalog.""" + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile._helpers.assembly import semantic_shapes +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + ExpressionBatchCase, + ExpressionMemoCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ExpressionMemoCase("shared closed expression", "SELECT 1 AS id")], + ids=lambda case: case.description, +) +def test_given_equal_source_expressions_when_compiling_then_infers_once_and_invalidates_changed_content( + test_case: ExpressionMemoCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + 'MODEL (materialized view); SELECT id FROM __source("raw_orders")' + ) + (tmp_path / "sources").mkdir() + source: Path = tmp_path / "sources/orders.yml" + source.write_text( + f"sources:\n - name: raw_orders\n expression: {test_case.expression}\n - name: raw_customers\n expression: {test_case.expression}\n" + ) + original: Callable[..., Any] = semantic_shapes.analyze_queries_with_compact_polyglot_batch + calls: list[object] = [] + + def traced(**kwargs: Any) -> Any: + calls.append(kwargs["query_sqls"]) + return original(**kwargs) + + monkeypatch.setattr(semantic_shapes, "analyze_queries_with_compact_polyglot_batch", traced) + args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] + assert main(args) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert payload["diagnostics"] == [] + assert len(calls) == test_case.expected_analysis_calls + source.write_text(source.read_text().replace("AS id", "AS quantity")) + assert main(args) == 1 + changed: dict[str, Any] = json.loads(capsys.readouterr().out) + assert [item["code"] for item in changed["diagnostics"]] == ["B002"] + assert len(calls) == test_case.expected_analysis_calls * 2 + + +@pytest.mark.parametrize( + "test_case", + [ + ExpressionBatchCase( + description="distinct expressions share one native batch and later passes reuse it", + orders_expression="SELECT 1 AS id", + customers_expression="SELECT 2 AS customer_id", + expected_batches=(("SELECT 1 AS id", "SELECT 2 AS customer_id"),), + ) + ], + ids=lambda case: case.description, +) +def test_given_distinct_source_expressions_when_compiling_then_infers_them_in_one_batch( + test_case: ExpressionBatchCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + 'MODEL (materialized view); SELECT id FROM __source("raw_orders")' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n" + f" - name: raw_orders\n expression: {test_case.orders_expression}\n" + f" - name: raw_customers\n expression: {test_case.customers_expression}\n" + ) + original: Callable[..., Any] = semantic_shapes.analyze_queries_with_compact_polyglot_batch + batches: list[tuple[str, ...]] = [] + + def traced(**kwargs: Any) -> Any: + batches.append(kwargs["query_sqls"]) + return original(**kwargs) + + monkeypatch.setattr(semantic_shapes, "analyze_queries_with_compact_polyglot_batch", traced) + assert main(["--project-dir", str(tmp_path), "compile", "--json"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert payload["diagnostics"] == [] + assert tuple(batches) == test_case.expected_batches + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index f2a9dd39a0..e99faeedc4 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -713,7 +713,7 @@ def test_given_project_with_defer_to_when_compiling_then_resolves_refs_to_deferr "WITH __source__raw AS (", "__ref__stg_orders AS (", "FROM __source__raw", - "local_helper AS (", + "__sqb_cte_0 AS (", "__actual__fact_orders AS (", "FROM __ref__stg_orders", "'stg_orders' AS model_name", diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py new file mode 100644 index 0000000000..504f4cc9d9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_native_binding_catalog.py @@ -0,0 +1,292 @@ +"""Native catalog semantics through the compiler and physical rebinding path.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile._helpers.assembly import project as assembly +from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics +from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + NativeCatalogCase, + ValidationCacheCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase("derived alias", 'SELECT "orderid" FROM (SELECT 1 AS "OrderId") q'), + NativeCatalogCase( + "CTE and alias", + 'WITH "OrderValues" AS (SELECT 1 AS "OrderId") SELECT "orderid" FROM "ordervalues"', + ), + NativeCatalogCase( + "binding preserves type checks", + 'SELECT SUM("active") FROM (SELECT TRUE AS "Active") q', + ("B213",), + ), + ], + ids=lambda case: case.description, +) +def test_given_quoted_case_semantics_when_compiling_then_resolves_native_aliases( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + "MODEL (database warehouse, schema analytics);\n" + test_case.sql + ) + main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "authored quoted typo", 'SELECT "MiSsPeLlEd" FROM (SELECT 1 AS "OrderId") q', ("B002",) + ), + NativeCatalogCase( + "uppercase quoted typo", + 'SELECT "MISSPELLED" FROM (SELECT 1 AS "OrderId") q', + ("B002",), + "MISSPELLED", + ), + ], + ids=lambda case: case.description, +) +def test_given_quoted_typo_when_validating_then_keeps_authored_spelling_and_span( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n' + ) + (tmp_path / "models").mkdir() + sql: str = "MODEL (database warehouse, schema analytics);\n" + test_case.sql + (tmp_path / "models/orders.sql").write_text(sql) + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + (diagnostic,) = payload["diagnostics"] + assert (diagnostic["code"],) == test_case.expected_codes + assert test_case.spelling in diagnostic["message"] + assert diagnostic["line"] == 2 + assert f'"{test_case.spelling}"' in payload["resources"]["models"][0]["query_sql"] + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "physical schema refresh", 'SELECT "orderid" FROM __source("orders")', ("B002",) + ) + ], + ids=lambda case: case.description, +) +def test_given_catalog_when_rebinding_physical_shapes_then_does_not_reuse_stale_columns( + test_case: NativeCatalogCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text("sources:\n - name: orders\n table: orders\n") + (tmp_path / "models/report.sql").write_text( + "MODEL (database warehouse, schema analytics);\n" + test_case.sql + ) + project: CompiledProject = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=SnowflakeAdapter() + ) + assert project.binding_catalog is not None + keys: frozenset[CompiledObjectKey] = frozenset(model.key for model in project.models) + profile: ExpressionInferenceProfile = ExpressionInferenceProfile( + sql_analysis_dialect="snowflake" + ) + missing: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + project=project, + columns={"orders": (ColumnInfo("customer", "VARCHAR"),)}, + profile=profile, + selected_keys=keys, + ) + present: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + project=project, + columns={"orders": (ColumnInfo("OrderId", "INTEGER"),)}, + profile=profile, + selected_keys=keys, + ) + repeated: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + project=project, + columns={"orders": (ColumnInfo("customer", "VARCHAR"),)}, + profile=profile, + selected_keys=keys, + ) + assert tuple(item.code for item in missing) == test_case.expected_codes + assert present == () + assert repeated == missing + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "unquoted type error with quoted-case policy", "SELECT SUM(TRUE) AS total", ("B213",) + ), + NativeCatalogCase( + "quoted type error with quoted-case policy", 'SELECT SUM(TRUE) AS "total"', ("B213",) + ), + ], + ids=lambda case: case.description, +) +def test_given_sql_when_case_policy_enabled_then_fused_validation_is_reused( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + "[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + "MODEL (database warehouse, schema analytics);\n" + test_case.sql + ) + + def repeated_binding(**kwargs: Any) -> None: + raise AssertionError("already validated SQL was normalized and bound again") + + monkeypatch.setattr(assembly, "get_complete_schema_binding_request", repeated_binding) + assert ( + main( + [ + "--project-dir", + str(tmp_path), + "compile", + "--lineage-mode", + "rich", + "--no-cache", + "--json", + ] + ) + == 1 + ) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [ + ValidationCacheCase("enabled target reuses semantic failures", True, 1, 0), + ValidationCacheCase("disabled target revalidates unchanged SQL", False, 0, 1), + ], + ids=lambda case: case.description, +) +def test_given_target_cache_policy_when_compiling_twice_then_reuses_only_enabled_validation( + test_case: ValidationCacheCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\ndefault_target = "local"\n' + '[rules]\nselect = []\n[targets.local]\nschema = "preserve"\n' + f"compile_cache = {str(test_case.enabled).lower()}\n" + "[targets.local.connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n" + ) + (tmp_path / "models").mkdir() + model: Path = tmp_path / "models/orders.sql" + model.write_text( + 'MODEL (database warehouse, schema analytics);\nSELECT SUM("Active") FROM (SELECT TRUE AS "Active") q' + ) + args: list[str] = [ + "--project-dir", + str(tmp_path), + "compile", + "--lineage-mode", + "rich", + "--json", + ] + assert main(args) == 1 + cold: dict[str, Any] = json.loads(capsys.readouterr().out) + assert main(args) == 1 + warm: dict[str, Any] = json.loads(capsys.readouterr().out) + assert [item["code"] for item in cold["diagnostics"]] == ["B213"] + assert warm["diagnostics"] == cold["diagnostics"] + assert ( + warm["compile_timings"]["analysis_batch_cache_hits"] + + warm["compile_timings"]["analysis_entry_cache_hits"] + ) == test_case.expected_hits + assert warm["compile_timings"]["analysis_cache_bypasses"] == test_case.expected_bypasses + assert warm["compile_timings"]["analysis_cache_misses"] == 0 + model.write_text(model.read_text().replace("TRUE", "1")) + assert main(args) == 0 + changed: dict[str, Any] = json.loads(capsys.readouterr().out) + assert changed["diagnostics"] == [] + assert changed["compile_timings"]["analysis_batch_cache_hits"] == 0 + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "quoted bound CTE lineage", + 'WITH q AS (SELECT id, CASE WHEN id > 0 THEN amount ELSE 0 END AS total FROM __source("raw_orders")) SELECT id AS "ORDER_ID", total FROM q', + ) + ], + ids=lambda case: case.description, +) +def test_given_quoted_catalog_binding_when_compiling_then_preserves_bound_expression_lineage( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + "[connection]\nsession_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n" + ) + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: raw_orders\n expression: SELECT 1 AS id, 2 AS amount\n" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + "MODEL (database warehouse, schema analytics); " + test_case.sql + ) + assert ( + main( + [ + "--project-dir", + str(tmp_path), + "compile", + "--lineage-mode", + "rich", + "--no-cache", + "--json", + ] + ) + == 0 + ) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + (model,) = payload["resources"]["models"] + assert model["column_count"] == 2 + assert model["lineage"]["edge_count"] == 3 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py new file mode 100644 index 0000000000..6a293153d9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_review_bindings.py @@ -0,0 +1,201 @@ +"""Real compiler regressions for binding boundary review findings.""" + +import json +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile._helpers.assembly import project +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + IdentifierBindingCase, + NativeCatalogCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "open USING", + 'SELECT * FROM __source("orders") o JOIN __source("customers") c USING (id)', + ), + NativeCatalogCase( + "open QUALIFY", + 'SELECT o.id, ROW_NUMBER() OVER () AS n FROM __source("orders") o QUALIFY o.quantity > 1', + ), + NativeCatalogCase( + "unregistered relation remains open", + "SELECT o.id, ROW_NUMBER() OVER () AS n FROM external_orders o QUALIFY o.quantity > 1", + ), + NativeCatalogCase( + "closed QUALIFY", + 'SELECT o.id, ROW_NUMBER() OVER () AS n FROM __source("known") o QUALIFY o.quantity > 1', + ("B002",), + ), + ], + ids=lambda case: case.description, +) +def test_given_open_sources_when_validating_clauses_then_only_closed_shapes_prove_missing_columns( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n table: orders\n - name: customers\n table: customers\n" + " - name: known\n table: known\n contract: enforced\n columns:\n - name: id\n type: INTEGER\n" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/report.sql").write_text("MODEL (materialized view);\n" + test_case.sql) + result: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + assert result == int(bool(test_case.expected_codes)) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + f"nested markers depth {depth}", + 'SELECT * FROM __table_fn("table_fn__orders")(' + + '__udf("udf__quantity")(' * depth + + "1" + + ")" * depth + + ")", + ) + for depth in (1, 2, 4, 8, 16, 32) + ], + ids=lambda case: case.description, +) +def test_given_nested_interpolations_when_normalizing_then_outer_edits_own_their_arguments( + test_case: NativeCatalogCase, +) -> None: + assert normalize_analysis_sql(sql=test_case.sql, dialect="duckdb") == test_case.expected_sql + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "nested function compile", + 'SELECT * FROM __table_fn("table_fn__orders")(__udf("udf__quantity")(1))', + ) + ], + ids=lambda case: case.description, +) +def test_given_nested_function_calls_when_compiling_then_native_normalization_does_not_panic( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + functions: Path = tmp_path / "functions/sql" + functions.mkdir(parents=True) + (functions / "table_fn__orders.sql").write_text( + "FUNCTION (arguments (x INTEGER), returns table (id INTEGER)); SELECT x AS id" + ) + (functions / "udf__quantity.sql").write_text( + "FUNCTION (arguments (x INTEGER), returns INTEGER); x + 1" + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text("MODEL (materialized view); " + test_case.sql) + assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [ + IdentifierBindingCase("Snowflake exact quoted", "snowflake", '"customer"', '"customer"'), + IdentifierBindingCase( + "Snowflake quoted mismatch", "snowflake", '"customer"', '"CUSTOMER"', ("B002",) + ), + IdentifierBindingCase( + "Snowflake unquoted uppercase", "snowflake", "customer", '"CUSTOMER"' + ), + IdentifierBindingCase("DuckDB quoted insensitive", "duckdb", '"Customer"', '"CUSTOMER"'), + IdentifierBindingCase("Postgres quoted exact", "postgres", '"Customer"', '"Customer"'), + IdentifierBindingCase( + "Postgres quoted mismatch", "postgres", '"Customer"', '"customer"', ("B002",) + ), + IdentifierBindingCase("Postgres unquoted lowercase", "postgres", "Customer", '"customer"'), + ], + ids=lambda case: case.description, +) +def test_given_inferred_output_when_binding_downstream_then_preserves_dialect_identifier_identity( + test_case: IdentifierBindingCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + original: Callable[..., Any] = project.get_schema_validations + + def without_deferred_requests(**kwargs: Any) -> Any: + assert kwargs["requests"] == () + return original(**kwargs) + + monkeypatch.setattr(project, "get_schema_validations", without_deferred_requests) + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "{test_case.dialect}"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + header: str = "MODEL (materialized view, database warehouse, schema analytics);\n" + (tmp_path / "models/orders.sql").write_text(header + f"SELECT 1 AS {test_case.projection}") + (tmp_path / "models/report.sql").write_text( + header + f'SELECT {test_case.reference} FROM __ref("orders")' + ) + result: int = main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + assert result == int(bool(test_case.expected_codes)) + + +@pytest.mark.parametrize( + "test_case", + [ + NativeCatalogCase( + "accepted numeric strings", + 'MODEL (materialized view, columns (quantity (audits [accepted_values (values ["1", "2"])]))); SELECT 1 AS quantity', + ), + NativeCatalogCase( + "relationship case", + 'MODEL (materialized view, columns (customer_id (audits [relationships (to __ref("customers"), field customer_id)]))); SELECT 1 AS customer_id', + ), + NativeCatalogCase( + "relationship coercion", + "MODEL (materialized view, columns (customer_id (audits [relationships (to __ref(\"customers\"), field customer_id)]))); SELECT '1' AS customer_id", + ), + ], + ids=lambda case: case.description, +) +def test_given_metadata_predicates_when_compiling_then_allows_dialect_coercion_and_case_resolution( + test_case: NativeCatalogCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text(test_case.sql) + (tmp_path / "models/customers.sql").write_text( + "MODEL (materialized view); SELECT 1 AS CUSTOMER_ID" + ) + assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in payload["diagnostics"]) == test_case.expected_codes + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py index 24cbc0e024..034fd936a3 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_binding.py @@ -121,13 +121,14 @@ def test_given_complete_contract_when_clause_references_missing_column_then_comp output: str = capsys.readouterr().out assert exit_code == test_case.expected_exit_code + assert f"error[{test_case.expected_error_code}]" in output + assert f"'{test_case.missing_column}'" in output + assert "(context:" not in output + assert "model: downstream" in output assert ( - f"error[{test_case.expected_error_code}]: Unknown column '{test_case.missing_column}'" + f"--> models/downstream.sql:2:{test_case.query_sql.index(test_case.missing_column) + 1}" in output ) - assert "(context:" in output - assert "model: downstream" in output - assert "--> models/downstream.sql:2:" in output @pytest.mark.parametrize( @@ -170,13 +171,13 @@ def test_given_two_complete_contracts_when_unqualified_column_is_ambiguous_then_ "test_case", [ SemanticBindingIntegrationTestCase( - description="given partial upstream when column absence is unproven then compile succeeds", - expected_exit_code=0, + description="explicit projection closes output names over an open input", + expected_exit_code=1, ) ], ids=lambda case: case.description, ) -def test_given_partial_upstream_when_column_absence_is_unproven_then_compile_succeeds( +def test_given_explicit_projection_when_output_column_is_missing_then_compile_fails( test_case: SemanticBindingIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -193,7 +194,7 @@ def test_given_partial_upstream_when_column_absence_is_unproven_then_compile_suc output: str = capsys.readouterr().out assert exit_code == test_case.expected_exit_code - assert "error[B" not in output + assert "error[B002]" in output @pytest.mark.parametrize( @@ -203,7 +204,7 @@ def test_given_partial_upstream_when_column_absence_is_unproven_then_compile_suc description="given default enforced source contract when column is missing then compile fails", source_contract_yaml="", expected_exit_code=1, - expected_output_fragment="error[B002]: Unknown column 'missing' in table 'raw_orders'", + expected_output_fragment="error[B002]: Unknown column 'missing' in raw_orders", expected_absent_output_fragment="0 errors, 0 warnings", ), SourceContractDefaultIntegrationTestCase( diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py new file mode 100644 index 0000000000..ab772d325c --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_diagnostic_ux.py @@ -0,0 +1,230 @@ +"""Real compiler diagnostics, cache recovery, and SQL-test snippets.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +from _pytest.capture import CaptureResult + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile._helpers.diagnostics.details import closest_column +from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + ColumnSuggestionCase, + DiagnosticUxCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DiagnosticUxCase( + "root typo", + (("models/marts/fact_orders.sql", " o.quantity,", " o.qty,"),), + expected_fragments=( + "Unknown column 'qty' in stg_orders (as o)", + "^^^", + "did you mean 'quantity'?", + "stg_orders has: quantity", + "2 downstream uses of fact_orders.quantity", + ), + ), + DiagnosticUxCase( + "both root errors", + ( + ("models/marts/fact_orders.sql", " o.quantity,", " o.qty,"), + ( + "models/marts/fact_orders.sql", + "p ON o.order_id = p.order_id", + "p ON o.order_id = p.order_id\nWHERE o.ordered_at > 5", + ), + ), + expected_errors=2, + expected_fragments=( + "between TIMESTAMP and INTEGER", + "o.ordered_at is TIMESTAMP, 5 is INTEGER", + "TIMESTAMP '2026-04-01'", + "^^^^^^^^^^^^^^^^", + "2 downstream uses", + ), + ), + DiagnosticUxCase( + "independent same input error", + ( + ("models/marts/fact_orders.sql", " o.quantity,", " o.qty,"), + ("models/marts/hourly_order_activity.sql", "SUM(o.quantity)", "SUM(o.missing)"), + ), + expected_errors=2, + expected_fragments=( + "Unknown column 'missing' in fact_orders (as o)", + "1 downstream uses", + ), + ), + DiagnosticUxCase( + "independent closed input error", + (("models/marts/fact_orders.sql", " o.quantity,", " o.qty,"),), + extra_files=( + ( + "models/independent.sql", + 'MODEL (materialized view);\nSELECT w.missing FROM __seed("waffle_types") w CROSS JOIN __ref("fact_orders") o\n', + ), + ), + expected_errors=2, + expected_fragments=( + "Unknown column 'missing' in waffle_types (as w)", + "2 downstream uses", + ), + ), + DiagnosticUxCase( + "explicit output alias remains authoritative", + (("models/marts/fact_orders.sql", " o.quantity,", " o.qty AS qty,"),), + expected_errors=3, + expected_fragments=("Unknown column 'quantity' in fact_orders", "error[B302]"), + ), + DiagnosticUxCase( + "predicate error does not poison another input projection", + (), + extra_files=( + ("models/available.sql", "MODEL (materialized view); SELECT 1 AS qty"), + ( + "models/predicate_bad.sql", + 'MODEL (materialized view); SELECT a.qty FROM __ref("available") a CROSS JOIN __ref("stg_orders") o WHERE o.qty > 5', + ), + ( + "models/independent_use.sql", + 'MODEL (materialized view); SELECT quantity FROM __ref("predicate_bad")', + ), + ), + expected_errors=2, + expected_fragments=( + "Unknown column 'qty' in stg_orders", + "Unknown column 'quantity' in predicate_bad", + ), + ), + DiagnosticUxCase( + "open source detail", + (("models/marts/fact_orders.sql", " o.quantity,", " o.qty,"),), + extra_files=( + ( + "sources/optional.yml", + "sources:\n - name: external_orders\n table: external_orders\n", + ), + ( + "models/open_read.sql", + 'MODEL (materialized view);\nSELECT id FROM __source("external_orders")\n', + ), + ), + expected_fragments=( + "open_read: reads open source external_orders", + "sqb contract generate --from --select source:external_orders --write", + ), + ), + DiagnosticUxCase( + "SQL test snippet", + (("tests/unit/test_fact_orders.sql", "quantity", "quantitty"),), + expected_fragments=( + "error[B302]", + "sql test: test_fact_orders", + "quantitty", + "^^^^^^^^^", + ), + expected_partial=False, + ), + DiagnosticUxCase( + "wide schema evidence is bounded", + (), + extra_files=( + ( + "models/wide.sql", + "MODEL (materialized view);\nSELECT 1 AS quantity, " + + ", ".join(f"1 AS field_{index}" for index in range(20)), + ), + ( + "models/wide_bad.sql", + 'MODEL (materialized view);\nSELECT w.qty FROM __ref("wide") w', + ), + ), + expected_fragments=("wide has: quantity", "and 11 more", "did you mean 'quantity'?"), + ), + ], + ids=lambda case: case.description, +) +def test_given_semantic_failure_when_compiling_cold_and_warm_then_reports_root_evidence( + test_case: DiagnosticUxCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project: Path = tmp_path / "orders" + assert main(["playground", str(project)]) == 0 + capsys.readouterr() + for relative, old, new in test_case.edits: + path: Path = project / relative + text: str = path.read_text() + assert old in text + path.write_text(text.replace(old, new, 1)) + for relative, text in test_case.extra_files: + (project / relative).write_text(text) + args: list[str] = ["--no-color", "--project-dir", str(project), "compile"] + assert main([*args, "--json"]) == 1 + cold_output: CaptureResult[str] = capsys.readouterr() + cold: dict[str, Any] = json.loads(cold_output.out) + assert main([*args, "--json"]) == 1 + warm_output: CaptureResult[str] = capsys.readouterr() + warm: dict[str, Any] = json.loads(warm_output.out) + assert cold["summary"]["errors"] == test_case.expected_errors + assert warm["diagnostics"] == cold["diagnostics"] + assert warm["semantic_checks_partial"] == cold["semantic_checks_partial"] + assert bool(cold["semantic_checks_partial"]) == test_case.expected_partial + assert ( + cold_output.err.split("Semantic checks were partial")[-1] + == warm_output.err.split("Semantic checks were partial")[-1] + or not test_case.expected_partial + ) + assert main(args) == 1 + human: str = capsys.readouterr().out + assert all(fragment in human for fragment in test_case.expected_fragments), human + assert "review the SQL expression, input types, and authoritative schema" not in human + assert "resource: sql_test:" not in human + assert "(context:" not in human + + +@pytest.mark.parametrize( + "test_case", + [DiagnosticUxCase("unique help", (), expected_errors=0)], + ids=lambda case: case.description, +) +def test_given_semantic_codes_when_explaining_then_each_code_owns_distinct_help( + test_case: DiagnosticUxCase, +) -> None: + help_by_code: dict[str, str] = {**semantic_help_catalogue(), "B002": "did you mean 'quantity'?"} + assert len(help_by_code) - len(set(help_by_code.values())) == test_case.expected_errors + assert all( + "review the SQL expression, input types, and authoritative schema" not in value + for value in help_by_code.values() + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ColumnSuggestionCase("abbreviation", "qty", ("quantity", "id", "status"), "quantity"), + ColumnSuggestionCase("typo", "quantitty", ("quantity", "status"), "quantity"), + ColumnSuggestionCase("no close match", "unrelated_field", ("quantity", "id"), None), + ColumnSuggestionCase("unicode case folding", "straße", ("STRASSE",), "STRASSE"), + ], + ids=lambda case: case.description, +) +def test_given_column_spelling_when_suggesting_then_uses_bounded_edit_distance( + test_case: ColumnSuggestionCase, +) -> None: + assert ( + closest_column(name=test_case.name, columns=dict.fromkeys(test_case.columns, "INTEGER")) + == test_case.expected_match + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py new file mode 100644 index 0000000000..cc8fdd53d2 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_scopes.py @@ -0,0 +1,223 @@ +"""Real CLI coverage of mixed, empty, expression and projected SQL scopes.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest +from _pytest.capture import CaptureResult + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import SemanticCompileCase +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import write_semantic_binding_project + +_UPSTREAM: str = "SELECT 1 AS id, 'placed' AS status" + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticCompileCase( + "mixed closed and open", + _UPSTREAM, + 'SELECT u.missing FROM __ref("upstream") u CROSS JOIN __source("open_orders") o', + "B002", + ), + SemanticCompileCase( + "unqualified might belong to open", + _UPSTREAM, + 'SELECT missing FROM __ref("upstream") u CROSS JOIN __source("open_orders") o', + None, + ), + SemanticCompileCase( + "no relation references", + _UPSTREAM, + "WITH orders AS (SELECT 1 AS id) SELECT missing FROM orders", + "B002", + ), + SemanticCompileCase( + "explicit names over open", + 'SELECT id FROM __source("open_orders")', + 'SELECT missing FROM __ref("upstream")', + "B002", + ), + SemanticCompileCase( + "open star", + 'SELECT * FROM __source("open_orders")', + 'SELECT missing FROM __ref("upstream")', + None, + ), + SemanticCompileCase( + "closed star", + 'SELECT * FROM __source("closed_orders")', + 'SELECT missing FROM __ref("upstream")', + "B002", + ), + SemanticCompileCase( + "expression source", + _UPSTREAM, + 'SELECT missing FROM __source("expression_orders")', + "B002", + ), + SemanticCompileCase( + "grouping", _UPSTREAM, 'SELECT id, COUNT(*) AS n FROM __ref("upstream")', "B230" + ), + SemanticCompileCase( + "aggregate placement", + _UPSTREAM, + 'SELECT id FROM __ref("upstream") WHERE COUNT(*) > 0', + "B231", + ), + SemanticCompileCase( + "unique key", + _UPSTREAM, + "SELECT 1 AS id", + "B300", + "MODEL (materialized table, unique_key [missing]);\n", + ), + SemanticCompileCase( + "date literal valid control", + _UPSTREAM, + "SELECT 1 AS id WHERE TIMESTAMP '2026-01-02' > '2026-01-01'", + None, + ), + SemanticCompileCase( + "timestamp numeric comparison rejected", + _UPSTREAM, + "SELECT 1 AS id WHERE TIMESTAMP '2026-01-02' > 5", + "B217", + ), + SemanticCompileCase( + "excess derived relation aliases", + _UPSTREAM, + "SELECT * FROM (SELECT 1 AS id) AS orders(first_id, second_id)", + "B005", + ), + SemanticCompileCase( + "fewer derived relation aliases remain valid", + _UPSTREAM, + "SELECT first_id, quantity FROM (SELECT 1 AS id, 2 AS quantity) AS orders(first_id)", + None, + ), + SemanticCompileCase( + "unknown unpivot input", + _UPSTREAM, + "SELECT * FROM (SELECT 1 AS id, 2 AS books) orders UNPIVOT (amount FOR category IN (missing))", + "B002", + ), + ], + ids=lambda case: case.description, +) +def test_given_semantic_scope_when_compiling_then_closed_shapes_are_checked( + test_case: SemanticCompileCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + write_semantic_binding_project( + project_dir=tmp_path, + upstream_sql="MODEL (materialized view);\n" + test_case.upstream, + downstream_sql=test_case.header + test_case.downstream, + ) + (tmp_path / "sources").mkdir(exist_ok=True) + (tmp_path / "sources" / "orders.yml").write_text( + "sources:\n" + " - name: open_orders\n table: orders\n" + " - name: closed_orders\n table: orders\n contract: enforced\n" + " columns:\n - name: id\n type: INTEGER\n" + " - name: expression_orders\n expression: SELECT 1 AS id\n", + encoding="utf-8", + ) + exit_code: int = main( + ["--no-color", "--project-dir", str(tmp_path), "compile", "--no-cache", "--json"] + ) + report: dict[str, Any] = json.loads(capsys.readouterr().out) + assert exit_code == int(bool(test_case.expected_code)), report + codes: str = " ".join(diagnostic["code"] for diagnostic in report["diagnostics"]) + assert (test_case.expected_code or "") in codes, report + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticCompileCase( + "aggregate after macro", + "SELECT 1 AS id", + 'SELECT id FROM __ref("upstream") WHERE SUM(id) > 0', + "B231", + expected_column=44, + ), + SemanticCompileCase( + "count star after macro", + "SELECT 1 AS id", + 'SELECT id FROM __ref("upstream") WHERE COUNT(*) > 0', + "B231", + expected_column=40, + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_before_invalid_aggregate_when_compiling_then_reports_authored_position( + test_case: SemanticCompileCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_semantic_binding_project( + project_dir=tmp_path, + upstream_sql=test_case.header + test_case.upstream, + downstream_sql=test_case.header + test_case.downstream, + ) + exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "compile", "--no-cache"]) + output: CaptureResult[str] = capsys.readouterr() + assert exit_code == 1 + assert f"downstream.sql:2:{test_case.expected_column}" in output.out + output.err + assert test_case.expected_code is not None + assert test_case.expected_code in output.out + output.err + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticCompileCase( + "runtime equality", _UPSTREAM, "SELECT 1 = 'not-a-number' AS result", "B218" + ), + SemanticCompileCase( + "runtime cast", + _UPSTREAM, + "SELECT CAST(TIMESTAMP '2026-01-01' AS INTEGER) AS result", + "B218", + ), + SemanticCompileCase( + "identical span-less conversions are reported once", + _UPSTREAM, + "SELECT 1 = 'not-a-number' AS first_result, 2 = 'not-a-number' AS second_result", + "B218", + ), + ], + ids=lambda case: case.description, +) +def test_given_implicit_conversion_when_compiling_cold_and_warm_then_errors_block( + test_case: SemanticCompileCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + write_semantic_binding_project( + project_dir=tmp_path, + upstream_sql=test_case.header + test_case.upstream, + downstream_sql=test_case.header + test_case.downstream, + ) + arguments: list[str] = ["--no-color", "--project-dir", str(tmp_path), "compile", "--json"] + assert main(arguments) == 1 + cold: dict[str, Any] = json.loads(capsys.readouterr().out) + assert main(arguments) == 1 + warm: dict[str, Any] = json.loads(capsys.readouterr().out) + assert cold["diagnostics"] == warm["diagnostics"] + assert cold["summary"]["warnings"] == 0 + errors: list[dict[str, Any]] = cold["diagnostics"] + assert cold["summary"]["errors"] == len(errors) == test_case.expected_diagnostic_count + assert {error["code"] for error in errors} == {test_case.expected_code} + assert all(error["severity"] == "error" for error in errors) + assert main(arguments[:-1]) == 1 + human: str = capsys.readouterr().out + assert f"error[{test_case.expected_code}]" in human + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py new file mode 100644 index 0000000000..342e9704bd --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_snowflake_release.py @@ -0,0 +1,151 @@ +"""Synthetic Snowflake semantic regressions across native dependency upgrades.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + SnowflakeOutputInferenceCase, + SnowflakeSemanticReleaseCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeSemanticReleaseCase( + "hierarchical pseudo-column", + 'SELECT LEVEL AS depth FROM __source("orders") CONNECT BY LEVEL <= 3', + ), + SnowflakeSemanticReleaseCase( + "multi-argument count", + 'SELECT COUNT(order_id, quantity) AS present FROM __source("orders")', + ), + SnowflakeSemanticReleaseCase( + "flatten argument input scope", + 'SELECT f.value AS item FROM __source("orders") o, LATERAL FLATTEN(input => PARSE_JSON(value):items) f', + ), + SnowflakeSemanticReleaseCase( + "lateral subquery", + 'SELECT g.item FROM __source("orders") o, LATERAL FLATTEN(input => PARSE_JSON(o.value)) f, LATERAL (SELECT f.value AS item) g', + ), + SnowflakeSemanticReleaseCase( + "timestamp subtypes", + 'SELECT CAST(ordered_at AS TIMESTAMP_NTZ) AS ntz, CAST(ordered_at AS TIMESTAMP_LTZ) AS ltz, CAST(ordered_at AS TIMESTAMP_TZ) AS tz FROM __source("orders")', + ), + SnowflakeSemanticReleaseCase( + "comment-wrapped projection", + 'SELECT (/* output identity */ quantity) AS quantity FROM __source("orders")', + ), + SnowflakeSemanticReleaseCase( + "implicit set conversion", + "SELECT 'pending'::VARCHAR AS value UNION ALL SELECT 1", + expected_codes=("B215",), + expected_exit_code=1, + ), + SnowflakeSemanticReleaseCase( + "directional accepted conversion", "SELECT TRUE AS value UNION ALL SELECT 1" + ), + SnowflakeSemanticReleaseCase( + "directional rejected conversion", + "SELECT 1 AS value UNION ALL SELECT TRUE", + expected_codes=("B215",), + expected_exit_code=1, + ), + SnowflakeSemanticReleaseCase( + "left-folded chain", "SELECT TRUE AS value UNION ALL SELECT 1 UNION ALL SELECT 1.5" + ), + SnowflakeSemanticReleaseCase( + "left-folded rejected chain", + "SELECT 1 AS value UNION ALL SELECT TRUE UNION ALL SELECT 1.5", + expected_codes=("B215",), + expected_exit_code=1, + ), + SnowflakeSemanticReleaseCase( + "intersect binds tighter", + "SELECT TRUE AS value UNION SELECT 1 INTERSECT SELECT FALSE", + expected_codes=("B215",), + expected_exit_code=1, + ), + SnowflakeSemanticReleaseCase( + "date difference return type", + 'SELECT DATEDIFF(day, ordered_on, ordered_on) > quantity AS later FROM __source("orders")', + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_semantics_when_compiling_then_preserves_native_verdicts( + test_case: SnowflakeSemanticReleaseCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "sources").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n table: orders\n contract: enforced\n columns:\n" + " - {name: order_id, type: INTEGER}\n" + " - {name: quantity, type: INTEGER}\n" + " - {name: value, type: VARCHAR}\n" + " - {name: ordered_on, type: DATE}\n" + " - {name: ordered_at, type: TIMESTAMP}\n" + ) + (tmp_path / "models/report.sql").write_text( + "MODEL (materialized view, database warehouse, schema analytics);\n" + test_case.sql + ) + assert ( + main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) + == test_case.expected_exit_code + ) + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + assert ( + tuple(sorted({item["code"] for item in payload["diagnostics"]})) == test_case.expected_codes + ) + for item in payload["diagnostics"]: + assert item.get("help") + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeOutputInferenceCase( + "numeric argument does not imply numeric return", "SELECT HASH(1) AS value" + ), + SnowflakeOutputInferenceCase( + "text argument does not imply text return", + "SELECT REGEXP_COUNT('orders', 'o') AS value", + ), + SnowflakeOutputInferenceCase( + "directional accumulated output", "SELECT TRUE AS value UNION ALL SELECT 1", "BOOLEAN" + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_query_when_inferring_rich_types_then_preserves_proven_or_unknown_types( + test_case: SnowflakeOutputInferenceCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "snowflake"\n') + (tmp_path / "models").mkdir() + (tmp_path / "models/report.sql").write_text( + "MODEL (materialized view, database warehouse, schema analytics);\n" + test_case.sql + ) + project: CompiledProject = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=SnowflakeAdapter(), + column_lineage_mode=ColumnLineageMode.RICH, + ) + assert not project.diagnostics + assert project.models[0].inferred_columns is not None + assert project.models[0].inferred_columns[0].type == test_case.expected_type + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py new file mode 100644 index 0000000000..2bdd90eef7 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_semantic_triage.py @@ -0,0 +1,276 @@ +"""Synthetic regressions for target semantics and error-local recovery.""" + +import json +from pathlib import Path +from typing import Any +from unittest.mock import Mock + +import pytest + +import sqlbuild._native as native +from sqlbuild.adapter.contract.models import ColumnInfo, ExpressionInferenceProfile +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile.main._source_bindings import get_source_binding_diagnostics +from sqlbuild.compiler.compile.models import CompiledProject, CompilerDiagnostic +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import SemanticTriageCase + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticTriageCase("date"), + SemanticTriageCase("timestamp", column_type="TIMESTAMP"), + SemanticTriageCase("timestamp ntz", column_type="TIMESTAMP_NTZ"), + SemanticTriageCase("timestamp ltz", column_type="TIMESTAMP_LTZ"), + SemanticTriageCase("timestamp tz", column_type="TIMESTAMP_TZ"), + SemanticTriageCase("varchar incompatible", column_type="VARCHAR", expected_codes=("B301",)), + SemanticTriageCase("number incompatible", column_type="NUMBER", expected_codes=("B301",)), + SemanticTriageCase("boolean incompatible", column_type="BOOLEAN", expected_codes=("B301",)), + ], + ids=lambda case: case.description, +) +def test_given_cursor_families_when_compiling_then_accepts_temporal_pairs_only( + test_case: SemanticTriageCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "models").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n table: orders\n contract: enforced\n" + f" columns:\n - name: ordered_at\n type: {test_case.column_type}\n" + ) + (tmp_path / "models/recent_orders.sql").write_text( + "MODEL (materialized incremental, database warehouse, schema analytics, " + f"incremental_strategy append, cursor ordered_at, cursor_type {test_case.cursor_type}, " + 'cursor_grain day);\nSELECT ordered_at FROM __source("orders")' + ) + main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) + result: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticTriageCase( + "enabled", setting="session_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true }\n" + ), + SemanticTriageCase( + "disabled", + setting="session_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = false }\n", + expected_codes=("B002",), + ), + SemanticTriageCase("absent", expected_codes=("B002",)), + ], + ids=lambda case: case.description, +) +def test_given_effective_target_when_binding_quotes_then_honours_session_setting( + test_case: SemanticTriageCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "snowflake"\n[rules]\nselect = []\n' + '[targets.selected]\nschema = "preserve"\n[targets.selected.connection]\n' + + test_case.setting + ) + (tmp_path / "sources").mkdir() + (tmp_path / "models").mkdir() + (tmp_path / "sources/orders.yml").write_text( + "sources:\n - name: orders\n table: orders\n contract: enforced\n" + " columns:\n - name: customer\n type: VARCHAR\n" + ) + (tmp_path / "models/quoted_orders.sql").write_text( + "MODEL (materialized view, database warehouse, schema analytics);\n" + 'SELECT o."customer" FROM __source("orders") o' + ) + for _ in range(2): + main(["--project-dir", str(tmp_path), "compile", "--target", "selected", "--json"]) + result: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes + project: CompiledProject = build_compiled_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=SnowflakeAdapter(), + selected_target="selected", + ) + physical: tuple[CompilerDiagnostic, ...] = get_source_binding_diagnostics( + project=project, + columns={"orders": (ColumnInfo(name="customer", type="VARCHAR"),)}, + profile=ExpressionInferenceProfile(sql_analysis_dialect="snowflake"), + selected_keys=frozenset(model.key for model in project.models), + ) + assert physical == () + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticTriageCase("type cascade", expected_codes=("B212",)), + SemanticTriageCase( + "declared type cascade", + root_contract=", contract enforced, columns (total (type INTEGER), ordered_at (type TIMESTAMP))", + expected_codes=("B212",), + ), + SemanticTriageCase( + "independent column remains checked", + independent_sql=", missing", + expected_codes=("B212", "B002"), + ), + SemanticTriageCase( + "independent type error remains checked", + independent_sql=", ordered_at + 1 AS independent", + expected_codes=("B212", "B212"), + ), + ], + ids=lambda case: case.description, +) +def test_given_type_error_root_when_binding_dependants_then_recovers_only_poisoned_outputs( + test_case: SemanticTriageCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + "MODEL (materialized view, contract enforced, columns (quantity (type INTEGER), " + "ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, " + "CAST('2026-04-01' AS TIMESTAMP) AS ordered_at" + ) + (tmp_path / "models/broken.sql").write_text( + f"MODEL (materialized view{test_case.root_contract});\nSELECT quantity + ordered_at AS total, " + 'ordered_at FROM __ref("orders")' + ) + (tmp_path / "models/downstream.sql").write_text( + "MODEL (materialized view);\nSELECT total > ordered_at AS compared" + + test_case.independent_sql + + ' FROM __ref("broken")' + ) + (tmp_path / "models/final_orders.sql").write_text( + 'MODEL (materialized view);\nSELECT compared AS final_value FROM __ref("downstream")' + ) + main(["--project-dir", str(tmp_path), "compile", "--json"]) + result: dict[str, Any] = json.loads(capsys.readouterr().out) + main(["--project-dir", str(tmp_path), "compile", "--json"]) + warm: dict[str, Any] = json.loads(capsys.readouterr().out) + assert warm["diagnostics"] == result["diagnostics"] + assert warm["semantic_checks_partial"] == result["semantic_checks_partial"] + assert sorted(item["code"] for item in result["diagnostics"]) == sorted( + test_case.expected_codes + ), result["diagnostics"] + assert "downstream" in result["semantic_checks_partial"] + notes: list[str] = [] + for item in result["diagnostics"]: + notes.extend(item.get("notes", [])) + assert any("2 downstream output uses" in note for note in notes) + + +@pytest.mark.parametrize( + "test_case", + [SemanticTriageCase("repeated projections", expected_codes=("B217",))], + ids=lambda case: case.description, +) +def test_given_repetitive_sql_when_mapping_diagnostic_then_preserves_authored_span( + test_case: SemanticTriageCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + (tmp_path / "models").mkdir() + (tmp_path / "models/orders.sql").write_text( + "MODEL (materialized view);\nSELECT CAST(1 AS INTEGER) AS quantity" + ) + projections: str = ",\n".join(f"quantity + {index} AS amount_{index}" for index in range(300)) + sql: str = ( + "MODEL (materialized view);\nSELECT\n" + + projections + + "\nFROM __ref(\"orders\")\nWHERE quantity > TIMESTAMP '2026-04-01'" + ) + (tmp_path / "models/report.sql").write_text(sql) + assert main(["--project-dir", str(tmp_path), "compile", "--no-cache", "--json"]) == 1 + result: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes + location: dict[str, Any] = result["diagnostics"][0]["location"] + assert location["line"] == len(sql.splitlines()) + assert location["column"] == len("WHERE ") + 1 + assert location["end_column"] == len(sql.splitlines()[-1]) + 1 + + +@pytest.mark.parametrize( + "test_case", + [SemanticTriageCase("warm semantic cache", expected_codes=("B212",))], + ids=lambda case: case.description, +) +def test_given_cached_validation_when_warm_or_schema_changed_then_reuses_only_matching_inputs( + test_case: SemanticTriageCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "sources").mkdir() + (tmp_path / "models").mkdir() + source: Path = tmp_path / "sources/orders.yml" + declaration: str = "sources:\n - name: orders\n table: orders\n contract: enforced\n columns:\n - name: quantity\n type: INTEGER\n" + source.write_text(declaration) + (tmp_path / "models/totals.sql").write_text( + 'MODEL (materialized view);\nSELECT quantity + 1 AS total FROM __source("orders")' + ) + args: list[str] = ["--project-dir", str(tmp_path), "compile", "--json"] + with monkeypatch.context() as patch: + repeated_validation: Mock = Mock( + side_effect=AssertionError("cold compile repeated fused validation") + ) + patch.setattr(native, "validate_sql_with_schemas_json", repeated_validation) + assert main(args) == 0 + repeated_validation.assert_not_called() + capsys.readouterr() + with monkeypatch.context() as patch: + forbidden: Mock = Mock( + side_effect=AssertionError("warm compile repeated native semantic work") + ) + patch.setattr(native, "validate_sql_with_schemas_json", forbidden) + patch.setattr(native, "analyze_project_queries_compact_json", forbidden) + assert main(args) == 0 + forbidden.assert_not_called() + capsys.readouterr() + source.write_text(declaration.replace("INTEGER", "TIMESTAMP")) + assert main(args) == 1 + result: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes + + +@pytest.mark.parametrize( + "test_case", + [SemanticTriageCase("distinct spans with identical messages", expected_codes=("B212", "B212"))], + ids=lambda case: case.description, +) +def test_given_repeated_root_messages_when_recovering_then_preserves_each_output_cause( + test_case: SemanticTriageCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' + ) + (tmp_path / "models").mkdir() + files: dict[str, str] = { + "orders": "MODEL (materialized view, contract enforced, columns (quantity (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT CAST(1 AS INTEGER) AS quantity, CAST('2026-04-01' AS TIMESTAMP) AS ordered_at", + "broken": 'MODEL (materialized view, contract enforced, columns (total_a (type INTEGER), total_b (type INTEGER), ordered_at (type TIMESTAMP)));\nSELECT quantity + ordered_at AS total_a, quantity + ordered_at AS total_b, ordered_at FROM __ref("orders")', + "downstream": 'MODEL (materialized view);\nSELECT total_a > ordered_at AS a, total_b > ordered_at AS b FROM __ref("broken")', + "final_orders": 'MODEL (materialized view);\nSELECT a, b FROM __ref("downstream")', + } + for name, sql in files.items(): + (tmp_path / "models" / f"{name}.sql").write_text(sql) + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 + result: dict[str, Any] = json.loads(capsys.readouterr().out) + assert tuple(item["code"] for item in result["diagnostics"]) == test_case.expected_codes + for diagnostic in result["diagnostics"]: + assert ( + "2 downstream output uses were not type-checked because of this error" + in diagnostic["notes"] + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py new file mode 100644 index 0000000000..08d36c33c2 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_shared_binding_queries.py @@ -0,0 +1,294 @@ +"""Binding-mode models with identical SQL shapes and inputs share one native query.""" + +import json +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.compiler.compile.models import CompactBatchPreparation +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import SharedBindingQueryCase +from tests.integration.src.sqlbuild.compiler.pipeline.helpers import ( + trace_native_compact_batches, + write_shared_binding_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SharedBindingQueryCase( + description="identical shapes and inputs share one query", + orders_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("orders")', + customers_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("customers")', + expected_shared_queries=1, + ), + SharedBindingQueryCase( + description="a relation name used as a qualifier is not shared", + orders_summary_sql=( + 'MODEL (materialized view); SELECT orders.id FROM __ref("orders") AS orders' + ), + customers_summary_sql=( + 'MODEL (materialized view); SELECT customers.id FROM __ref("customers") AS customers' + ), + expected_shared_queries=0, + ), + SharedBindingQueryCase( + description="an authored lowercase stub-named CTE is not shared", + orders_summary_sql=( + "MODEL (materialized view); WITH __sqlbuild_project_input_0 AS " + '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("orders")' + ), + customers_summary_sql=( + "MODEL (materialized view); WITH __sqlbuild_project_input_0 AS " + '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("customers")' + ), + expected_shared_queries=0, + later_models=( + ( + "next_orders", + 'MODEL (materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', + ), + ), + ), + SharedBindingQueryCase( + description="an authored uppercase stub-named CTE is not shared", + orders_summary_sql=( + "MODEL (materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " + '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("orders")' + ), + customers_summary_sql=( + "MODEL (materialized view); WITH __SQLBUILD_PROJECT_INPUT_0 AS " + '(SELECT CAST(2 AS VARCHAR) AS id) SELECT id FROM __ref("customers")' + ), + expected_shared_queries=0, + later_models=( + ( + "next_orders", + 'MODEL (materialized view); SELECT id + 1 AS next_id FROM __ref("orders_summary")', + ), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_models_with_equal_shapes_when_compiling_then_shares_only_lossless_queries( + test_case: SharedBindingQueryCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_shared_binding_project(project_dir=tmp_path, test_case=test_case) + preparations: list[CompactBatchPreparation] = trace_native_compact_batches(monkeypatch) + + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert payload["diagnostics"] == [] + assert ( + sum(len(preparation.shared_query_indexes) for preparation in preparations) + == test_case.expected_shared_queries + ) + for model, upstream in (("orders_summary", "orders"), ("customers_summary", "customers")): + assert ( + main( + [ + "--project-dir", + str(tmp_path), + "lineage", + f"{model}.id", + "--format", + "json", + "--direction", + "upstream", + ] + ) + == 0 + ) + lineage: dict[str, Any] = json.loads(capsys.readouterr().out) + assert [step["source"]["resource_name"] for step in lineage["trace"]] == [upstream] + + +@pytest.mark.parametrize( + "test_case", + [ + SharedBindingQueryCase( + description="findings after the relation keep each model's names and positions", + orders_summary_sql=( + 'MODEL (materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' + ), + customers_summary_sql=( + 'MODEL (materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' + ), + expected_shared_queries=1, + expected_codes=("B002",), + ), + ], + ids=lambda case: case.description, +) +def test_given_shared_query_with_finding_when_compiling_then_reports_each_model_exactly( + test_case: SharedBindingQueryCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_shared_binding_project(project_dir=tmp_path, test_case=test_case) + preparations: list[CompactBatchPreparation] = trace_native_compact_batches(monkeypatch) + + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert ( + sum(len(preparation.shared_query_indexes) for preparation in preparations) + == test_case.expected_shared_queries + ) + findings: dict[str, dict[str, Any]] = { + diagnostic["resource_name"]: diagnostic for diagnostic in payload["diagnostics"] + } + assert sorted(findings) == ["customers_summary", "orders_summary"] + for model, upstream, sql in ( + ("orders_summary", "orders", test_case.orders_summary_sql), + ("customers_summary", "customers", test_case.customers_summary_sql), + ): + finding: dict[str, Any] = findings[model] + assert (finding["code"],) == test_case.expected_codes + assert finding["message"] == f"Unknown column 'missing' in {upstream}" + assert finding["location"]["column"] == sql.index("missing") + 1 + assert finding["location"]["end_column"] == sql.index("missing") + len("missing") + 1 + + +@pytest.mark.parametrize( + "test_case", + [ + SharedBindingQueryCase( + description="later-wave members reuse an exact shared result", + orders_summary_sql='MODEL (materialized view); SELECT id, amount FROM __ref("orders")', + customers_summary_sql=( + 'MODEL (materialized view); SELECT id, amount FROM __ref("customers")' + ), + expected_shared_queries=1, + later_models=( + ( + "orders_rollup", + 'MODEL (materialized view); SELECT id, amount FROM __ref("orders_summary")', + ), + ( + "customers_rollup", + 'MODEL (materialized view); SELECT id, amount FROM __ref("customers_summary")', + ), + ), + expected_lineage=( + ("orders_rollup", "orders_summary"), + ("customers_rollup", "customers_summary"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_later_wave_members_with_equal_keys_when_compiling_then_reuses_shared_result( + test_case: SharedBindingQueryCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_shared_binding_project(project_dir=tmp_path, test_case=test_case) + preparations: list[CompactBatchPreparation] = trace_native_compact_batches(monkeypatch) + + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 0 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert payload["diagnostics"] == [] + assert ( + sum(len(preparation.shared_query_indexes) for preparation in preparations) + == test_case.expected_shared_queries + ) + analyzed_sql: list[str] = [] + for preparation in preparations: + analyzed_sql.extend(str(query["sql"]) for query in preparation.queries) + assert not any("_summary" in sql for sql in analyzed_sql) + for model, upstream in test_case.expected_lineage: + assert ( + main( + [ + "--project-dir", + str(tmp_path), + "lineage", + f"{model}.amount", + "--format", + "json", + "--direction", + "upstream", + "--depth", + "1", + ] + ) + == 0 + ) + lineage: dict[str, Any] = json.loads(capsys.readouterr().out) + assert [step["source"]["resource_name"] for step in lineage["trace"]] == [upstream] + + +@pytest.mark.parametrize( + "test_case", + [ + SharedBindingQueryCase( + description="a later-wave member of a group with findings keeps its own names", + orders_summary_sql=( + 'MODEL (materialized view); SELECT id FROM __ref("orders") WHERE missing > 0' + ), + customers_summary_sql=( + 'MODEL (materialized view); SELECT id FROM __ref("customers") WHERE missing > 0' + ), + expected_shared_queries=2, + expected_codes=("B002",), + later_models=( + ("orders_mid", 'MODEL (materialized view); SELECT id, amount FROM __ref("orders")'), + ( + "orders_late", + 'MODEL (materialized view); SELECT id FROM __ref("orders_mid") WHERE missing > 0', + ), + ), + expected_findings=( + ("orders_summary", "orders"), + ("customers_summary", "customers"), + ("orders_late", "orders_mid"), + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_later_wave_member_of_group_with_findings_when_compiling_then_reports_it_exactly( + test_case: SharedBindingQueryCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + write_shared_binding_project(project_dir=tmp_path, test_case=test_case) + preparations: list[CompactBatchPreparation] = trace_native_compact_batches(monkeypatch) + + assert main(["--project-dir", str(tmp_path), "compile", "--json", "--no-cache"]) == 1 + payload: dict[str, Any] = json.loads(capsys.readouterr().out) + + assert ( + sum(len(preparation.shared_query_indexes) for preparation in preparations) + == test_case.expected_shared_queries + ) + findings: dict[str, dict[str, Any]] = { + diagnostic["resource_name"]: diagnostic for diagnostic in payload["diagnostics"] + } + assert sorted(findings) == sorted(model for model, _ in test_case.expected_findings) + sql_by_model: dict[str, str] = { + "orders_summary": test_case.orders_summary_sql, + "customers_summary": test_case.customers_summary_sql, + **dict(test_case.later_models), + } + for model, upstream in test_case.expected_findings: + finding: dict[str, Any] = findings[model] + assert (finding["code"],) == test_case.expected_codes + assert finding["message"] == f"Unknown column 'missing' in {upstream}" + assert finding["location"]["column"] == sql_by_model[model].index("missing") + 1 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py index 222f641d1b..1bad0a901d 100644 --- a/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py +++ b/tests/integration/src/sqlbuild/executor/testing/test_sql_test_execution.py @@ -7,6 +7,7 @@ import pytest +from sqlbuild.adapter.contract.types import StatementSizeLimit from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter from sqlbuild.compiler.planner.models import ChainStep, SqlTestPlanEntry @@ -39,8 +40,8 @@ class TinySqlLimitDuckDbAdapter(DuckDbAdapter): - def recommended_max_sql_length(self) -> int | None: - return 80 + def max_statement_size(self) -> StatementSizeLimit | None: + return (80, "bytes") class FailingDifferenceSampleDuckDbAdapter(DuckDbAdapter): @@ -692,6 +693,8 @@ def test_given_oversized_unit_test_sql_when_executing_then_it_returns_clear_erro assert result.outcome == test_case.expected_outcome assert result.error_message is not None - assert "recommended maximum" in result.error_message + assert result.error_code == "T001" + assert result.step_results[0].error_code == "T001" + assert "maximum of 80 bytes" in result.error_message assert "scenario test" in result.error_message verify_test_result(result=result, test_case=test_case) diff --git a/tests/integration/src/sqlbuild/microbatches/classes/_test_types.py b/tests/integration/src/sqlbuild/microbatches/classes/_test_types.py index 4a8d674228..3452faff37 100644 --- a/tests/integration/src/sqlbuild/microbatches/classes/_test_types.py +++ b/tests/integration/src/sqlbuild/microbatches/classes/_test_types.py @@ -26,3 +26,9 @@ class RetiredDirectStoreRecordTestCase: description: str retired_record_type: str expected_event_count: int + + +@dataclass(frozen=True) +class RetiredColumnsTestCase: + description: str + expected_existing_count: int diff --git a/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py b/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py new file mode 100644 index 0000000000..359c5c12e5 --- /dev/null +++ b/tests/integration/src/sqlbuild/microbatches/classes/test_retired_columns.py @@ -0,0 +1,112 @@ +"""Direct event publication remains compatible with older tables' extra columns.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any + +import pytest + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.microbatches.classes.direct_store import DirectMicrobatchEventStore +from sqlbuild.microbatches.main.deterministic_event_id import deterministic_microbatch_event_id +from sqlbuild.microbatches.main.project_coverage import project_microbatch_coverage +from sqlbuild.microbatches.main.project_replay import project_replay_requirement +from sqlbuild.microbatches.models import ( + MicrobatchCoverageProjection, + MicrobatchEvent, + MicrobatchInterval, + ReplayRequirementProjection, +) +from sqlbuild.microbatches.types import MicrobatchRecordType, ReplayRequirementState +from tests.integration.src.sqlbuild.microbatches.classes._test_types import RetiredColumnsTestCase +from tests.integration.src.sqlbuild.microbatches.classes.helpers import build_events + +# The previous release's physical schema, intentionally independent of current constants. +OLD_TABLE_DDL: str = """ +CREATE TABLE main._sqlbuild_microbatches ( + event_id VARCHAR, record_type VARCHAR, scope_kind VARCHAR, scope_key VARCHAR, + model_name VARCHAR, target_database VARCHAR, target_schema VARCHAR, target_name VARCHAR, + physical_generation_id VARCHAR, virtual_environment_name VARCHAR, + virtual_model_version_hash VARCHAR, origin_run_id VARCHAR, origin_run_started_at TIMESTAMP, + execution_run_id VARCHAR, execution_run_started_at TIMESTAMP, run_type VARCHAR, + completion_type VARCHAR, run_start VARCHAR, run_end VARCHAR, partition_start VARCHAR, + partition_end VARCHAR, batch_size VARCHAR, cursor_column VARCHAR, cursor_type VARCHAR, + cursor_grain VARCHAR, model_version_hash VARCHAR, definition_hash VARCHAR, + fingerprint_status VARCHAR, replay_requirement_id VARCHAR, required_model_version_hash VARCHAR, + previous_model_version_hash VARCHAR, replay_policy VARCHAR, rows_affected BIGINT, + completed_at TIMESTAMP, coverage_source VARCHAR, observed_row_count BIGINT, + observed_at TIMESTAMP, synthetic_reason VARCHAR, unaccounted_policy VARCHAR, created_at TIMESTAMP +) +""" + + +@pytest.mark.parametrize( + "test_case", + [ + RetiredColumnsTestCase( + description="old nullable columns preserve event publication", expected_existing_count=2 + ) + ], + ids=lambda case: case.description, +) +def test_given_old_table_when_writing_requirements_and_completions_then_projection_succeeds( + test_case: RetiredColumnsTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + try: + connection.execute(OLD_TABLE_DDL) + store: DirectMicrobatchEventStore = DirectMicrobatchEventStore( + adapter=adapter, connection=connection + ) + completion: MicrobatchEvent = build_events(count=1)[0] + requirement_id: str = deterministic_microbatch_event_id( + scope=completion.scope, + record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, + partition_start=None, + partition_end=None, + completion_reason="F2", + ) + requirement: MicrobatchEvent = replace( + completion, + event_id=requirement_id, + record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, + completion_type=None, + partition_start=None, + partition_end=None, + required_model_version_hash="F2", + ) + store.write(requirement) + assert tuple(event.event_id for event in store.read_scope_history(completion.scope)) == ( + requirement.event_id, + ) + store.write(completion) + assert ( + store.write_many((requirement, completion)).already_existing + == test_case.expected_existing_count + ) + history: tuple[MicrobatchEvent, ...] = store.read_scope_history(completion.scope) + assert {event.event_id for event in history} == {requirement.event_id, completion.event_id} + intervals: tuple[MicrobatchInterval, ...] = (MicrobatchInterval(start="0", end="1"),) + coverage: MicrobatchCoverageProjection = project_microbatch_coverage( + events=history, expected_intervals=intervals, cursor_type="integer" + ) + projection: ReplayRequirementProjection = project_replay_requirement( + requirement=requirement, + current_model_version_hash="F2", + expected_intervals=intervals, + coverage=coverage, + cursor_type="integer", + ) + assert projection.state == ReplayRequirementState.VERIFIED_COMPLETE + assert connection.execute( + "SELECT virtual_environment_name, virtual_model_version_hash " + "FROM main._sqlbuild_microbatches" + ).fetchall() == [(None, None), (None, None)] + finally: + adapter.close(connection) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/virtual/executor/_helpers/_test_types.py b/tests/integration/src/sqlbuild/virtual/executor/_helpers/_test_types.py deleted file mode 100644 index bc36c703d2..0000000000 --- a/tests/integration/src/sqlbuild/virtual/executor/_helpers/_test_types.py +++ /dev/null @@ -1,26 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class VirtualBuildPipelineTestCase: - description: str - expected_schema: str - expected_model_names: tuple[str, ...] - expected_target: str - - -@dataclass(frozen=True) -class VirtualLeaseAcquireBoundaryTestCase: - """Expected cleanup when interruption happens immediately after lease acquisition.""" - - description: str - expected_error_type: type[BaseException] - expected_active_lock_count: int - - -@dataclass(frozen=True) -class VirtualMicrobatchResolverTestCase: - description: str - expected_environment_name: str - expected_scope_kind: str - expected_selected_model_names: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/virtual/executor/_helpers/test_build.py b/tests/integration/src/sqlbuild/virtual/executor/_helpers/test_build.py deleted file mode 100644 index f0e3f8cb5a..0000000000 --- a/tests/integration/src/sqlbuild/virtual/executor/_helpers/test_build.py +++ /dev/null @@ -1,326 +0,0 @@ -from __future__ import annotations - -import subprocess -from collections.abc import Callable -from pathlib import Path -from typing import Any -from unittest.mock import Mock - -import pytest - -import sqlbuild.virtual.executor._helpers.build as virtual_build_module -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.main.discover import discover_project_inputs -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.planner.models import ModelPlanEntry, PlanOutput -from sqlbuild.compiler.planner.types import MaterializationType, PlanAction, PlanReason -from sqlbuild.executor.build.models import BuildExecutionResult, BuildRuntimeParams -from sqlbuild.executor.build.types import BuildStatus, ExecutionStatus -from sqlbuild.executor.run.models import ModelExecutionResult -from sqlbuild.microbatches.models import MicrobatchScope -from sqlbuild.microbatches.types import MicrobatchEventStore -from sqlbuild.virtual.executor.classes.microbatch_lease_manager import ( - VirtualMicrobatchLeaseManager, -) -from sqlbuild.virtual.executor.models import ( - VirtualBuildHooks, - VirtualBuildOptions, - VirtualBuildPipelineResult, - VirtualEnvironmentNames, -) -from sqlbuild.virtual.state.classes.duckdb import DuckDbStateBackend -from sqlbuild.virtual.state.classes.microbatch_store import VirtualMicrobatchEventStore -from sqlbuild.virtual.state.models import StateBackendConfig, StateLockRecord -from sqlbuild.virtual.state.types import StateBackendName -from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( - build_virtual_wide_dag_repo_files, -) -from tests.e2e.src.sqlbuild.cli.commands.main.plan.helpers import ( - build_virtual_plan_project_toml, -) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( - prepare_inline_project, - run_sqb, -) -from tests.integration.src.sqlbuild.virtual.executor._helpers._test_types import ( - VirtualBuildPipelineTestCase, - VirtualLeaseAcquireBoundaryTestCase, - VirtualMicrobatchResolverTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildPipelineTestCase( - description="physical destinations and runtime are scoped before pipeline starts", - expected_schema="dev__sqb_physical", - expected_model_names=("model_01", "model_02", "model_03", "model_04"), - expected_target="dev", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_build_when_pipeline_starts_then_schema_and_runtime_are_project_scoped( - test_case: VirtualBuildPipelineTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_schema_preflight", - repo_files=build_virtual_wide_dag_repo_files( - model_count=len(test_case.expected_model_names), - ), - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - - adapter: DuckDbAdapter = DuckDbAdapter() - connection_config: dict[str, object] = { - "database": str(project_dir / "warehouse.duckdb"), - } - observed_model_names: list[str] = [] - - def fake_run_build_pipeline( - *, plan: PlanOutput, runtime: BuildRuntimeParams, **kwargs: Any - ) -> BuildExecutionResult: - assert runtime.runtime_dir == project_dir / "target" - assert runtime.target == test_case.expected_target - assert all( - entry.destination.schema == test_case.expected_schema for entry in plan.model_entries - ) - observed_model_names.extend(entry.name for entry in plan.model_entries) - return BuildExecutionResult( - status=BuildStatus.SUCCESS, - model_results=tuple( - ModelExecutionResult(model_name=entry.name, status=ExecutionStatus.SUCCESS) - for entry in plan.model_entries - ), - ) - - monkeypatch.setattr(virtual_build_module, "run_build_pipeline", fake_run_build_pipeline) - monkeypatch.setattr( - virtual_build_module, - "_persist_successful_virtual_build", - lambda **_: None, - ) - - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - result: VirtualBuildPipelineResult = virtual_build_module.run_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=adapter, - connection_config=connection_config, - options=VirtualBuildOptions(concurrency=8), - hooks=VirtualBuildHooks(), - ) - - assert result.execution_result.status == BuildStatus.SUCCESS - assert tuple(sorted(observed_model_names)) == test_case.expected_model_names - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualLeaseAcquireBoundaryTestCase( - description="interrupt after acquire still releases model version lease", - expected_error_type=KeyboardInterrupt, - expected_active_lock_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_interrupt_after_lease_acquire_when_virtual_build_unwinds_then_lease_is_released( - test_case: VirtualLeaseAcquireBoundaryTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - project_dir: Path = prepare_inline_project( - tmp_path=tmp_path, - project_name="virtual_acquire_boundary", - repo_files={ - "sqlbuild_project.toml": build_virtual_plan_project_toml(), - "models/orders.sql": ( - "MODEL (materialized incremental, incremental_strategy merge, " - "unique_key [id], full_refresh true);\n\nSELECT 1 AS id\n" - ), - }, - ) - init_result: subprocess.CompletedProcess[str] = run_sqb( - command=("state", "init"), - project_dir=project_dir, - ) - assert init_result.returncode == 0, init_result.stdout + init_result.stderr - original_acquire: Callable[..., None] = VirtualMicrobatchLeaseManager.acquire - - def _acquire_then_interrupt( - self: VirtualMicrobatchLeaseManager, - **kwargs: Any, - ) -> None: - original_acquire(self, **kwargs) - raise KeyboardInterrupt - - monkeypatch.setattr(VirtualMicrobatchLeaseManager, "acquire", _acquire_then_interrupt) - discovered_inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=project_dir) - with pytest.raises(test_case.expected_error_type): - virtual_build_module.run_virtual_build( - project_dir=project_dir, - discovered_inputs=discovered_inputs, - adapter=DuckDbAdapter(), - connection_config={"database": str(project_dir / "warehouse.duckdb")}, - options=VirtualBuildOptions(), - hooks=VirtualBuildHooks(), - ) - backend: DuckDbStateBackend = DuckDbStateBackend() - connection: Any = backend.connect({"database": str(project_dir / "state.duckdb")}) - try: - active_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema="sqlbuild_state", - ) - finally: - backend.close(connection) - assert len(active_locks) == test_case.expected_active_lock_count - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualMicrobatchResolverTestCase( - description="downstream-only producer uses target virtual state realm", - expected_environment_name="feature_env", - expected_scope_kind="virtual_physical", - expected_selected_model_names=("daily_events",), - ) - ], - ids=lambda case: case.description, -) -def test_given_downstream_only_virtual_build_when_resolving_producer_then_state_stays_in_target_environment( - test_case: VirtualMicrobatchResolverTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - warehouse_config: dict[str, object] = {"database": str(tmp_path / "warehouse.duckdb")} - connection: Any = adapter.connect(warehouse_config) - connection.execute("CREATE SCHEMA physical") - connection.execute("CREATE TABLE physical.upstream_events (event_time TIMESTAMP)") - connection.execute("CREATE TABLE physical.daily_events (event_time TIMESTAMP)") - consumer: ModelPlanEntry = ModelPlanEntry( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="daily_events"), - name="daily_events", - relative_path=Path("models/daily_events.sql"), - materialization_type=MaterializationType.INCREMENTAL, - action=PlanAction.INCREMENTAL_DELETE_INSERT, - reason=PlanReason.NORMAL_INCREMENTAL, - destination=CompiledRelationLocation( - None, "physical", "daily_events", "physical.daily_events" - ), - fingerprint_query_sql="SELECT event_time FROM physical.upstream_events", - fingerprint_version_hash="consumer-v1", - resolved_sql="SELECT event_time FROM physical.upstream_events", - logical_ddl="", - ) - producer_location: CompiledRelationLocation = CompiledRelationLocation( - None, "physical", "upstream_events", "physical.upstream_events" - ) - captured_scopes: list[MicrobatchScope] = [] - - def capture_pipeline( - *, plan: PlanOutput, runtime: BuildRuntimeParams, **_kwargs: object - ) -> BuildExecutionResult: - state_resolver: ( - Callable[[ModelPlanEntry, object], tuple[MicrobatchEventStore, MicrobatchScope]] | None - ) = runtime.microbatch_state_resolver - location_resolver: ( - Callable[ - [str, CompiledRelationLocation, str | None, object], - tuple[MicrobatchEventStore, MicrobatchScope], - ] - | None - ) = runtime.microbatch_location_state_resolver - assert state_resolver is not None - assert location_resolver is not None - selected_store, selected_scope = state_resolver(consumer, connection) - producer_store, producer_scope = location_resolver( - "upstream_events", producer_location, "producer-v1", connection - ) - assert isinstance(selected_store, VirtualMicrobatchEventStore) - assert isinstance(producer_store, VirtualMicrobatchEventStore) - assert tuple(entry.name for entry in plan.model_entries) == ( - test_case.expected_selected_model_names - ) - captured_scopes.extend((selected_scope, producer_scope)) - return BuildExecutionResult(status=BuildStatus.SUCCESS) - - monkeypatch.setattr(virtual_build_module, "load_custom_materializations", Mock(return_value={})) - monkeypatch.setattr( - virtual_build_module, "load_custom_prepare_version_functions", Mock(return_value={}) - ) - monkeypatch.setattr(virtual_build_module, "run_build_pipeline", capture_pipeline) - state_config: StateBackendConfig = StateBackendConfig( - backend=StateBackendName.DUCKDB, - schema="state_schema", - connection={"database": str(tmp_path / "state.duckdb")}, - ) - runtime: Any = virtual_build_module._VirtualBuildRuntime( - project_dir=tmp_path, - discovered_inputs=Mock(loader_functions=()), - adapter=adapter, - connection_config=warehouse_config, - options=VirtualBuildOptions(), - hooks=VirtualBuildHooks(), - backend=DuckDbStateBackend(), - config=state_config, - names=VirtualEnvironmentNames(target_vde_name=test_case.expected_environment_name), - ) - python_plan: Mock = Mock() - python_plan.lifecycle_plan.ingress_loader_names = frozenset() - try: - downstream_only_plan: PlanOutput = PlanOutput( - model_entries=(consumer,), - model_locations={"upstream_events": producer_location}, - ) - virtual_build_module._execute_virtual_build_plan( - runtime=runtime, - plan=Mock(executor_plan_output=downstream_only_plan), - project=CompiledProject( - run_id="virtual-run", - effective_target_name="test", - effective_connection=warehouse_config, - effective_vars={}, - ), - python_plan=python_plan, - reads=Mock( - semantics=Mock(expected_version_hashes={"daily_events": "consumer-v1"}), - available_seed_physical_relations={}, - ), - exec_hooks=Mock( - on_node_start=None, - on_node_complete=None, - on_sub_progress=None, - on_scheduler_state=None, - on_statement_complete=None, - ), - ingress_load_results=(), - microbatch_lease_check=Mock(), - ) - finally: - adapter.close(connection) - - selected_scope, producer_scope = captured_scopes - assert selected_scope.scope_kind == test_case.expected_scope_kind - assert producer_scope.scope_kind == test_case.expected_scope_kind - assert selected_scope.virtual_environment_name == test_case.expected_environment_name - assert producer_scope.virtual_environment_name == test_case.expected_environment_name - assert selected_scope.scope_key.split(":", 3)[:3] == producer_scope.scope_key.split(":", 3)[:3] diff --git a/tests/integration/src/sqlbuild/virtual/executor/classes/__init__.py b/tests/integration/src/sqlbuild/virtual/executor/classes/__init__.py deleted file mode 100644 index 5fe252e380..0000000000 --- a/tests/integration/src/sqlbuild/virtual/executor/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Integration tests for virtual executor classes.""" diff --git a/tests/integration/src/sqlbuild/virtual/executor/classes/_test_types.py b/tests/integration/src/sqlbuild/virtual/executor/classes/_test_types.py deleted file mode 100644 index e9e45657fd..0000000000 --- a/tests/integration/src/sqlbuild/virtual/executor/classes/_test_types.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Test types for virtual executor class integration tests.""" - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class VirtualMicrobatchLeaseManagerTestCase: - """Expected renewable lease behavior for one manager lifecycle.""" - - description: str - renew_interval_seconds: float - expected_loss_fragment: str - - -@dataclass(frozen=True) -class VirtualSharedFullRefreshTestCase: - """Incremental mode that must preserve a referenced physical version.""" - - description: str - incremental_strategy: str - incremental_mode: str | None = None - expected_error_fragment: str = "cannot replace a shared physical version" - - -@dataclass(frozen=True) -class VirtualConcurrentLeaseTestCase: - """Incremental mode whose full refresh must serialize by physical version.""" - - description: str - incremental_strategy: str - incremental_mode: str | None - expected_conflict_fragment: str = "physical version is already being mutated" - expected_shared_fragment: str = "cannot replace a shared physical version" - - -@dataclass(frozen=True) -class VirtualLeaseCancellationTestCase: - """Expected lock cleanup when acquisition is cancelled after fencing.""" - - description: str - expected_error_type: type[BaseException] - expected_remaining_lock_count: int diff --git a/tests/integration/src/sqlbuild/virtual/executor/classes/helpers.py b/tests/integration/src/sqlbuild/virtual/executor/classes/helpers.py deleted file mode 100644 index b78bdf4073..0000000000 --- a/tests/integration/src/sqlbuild/virtual/executor/classes/helpers.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Helpers for virtual executor class integration tests.""" - -from pathlib import Path - -from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledRelationLocation -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import ( - IncrementalMode, - MaterializationType, - PlanAction, - PlanReason, -) - - -def build_virtual_microbatch_lease_entry( - *, - action: PlanAction = PlanAction.INCREMENTAL_DELETE_INSERT, - incremental_strategy: str = "delete_insert", - incremental_mode: str | None = IncrementalMode.MICROBATCH, -) -> ModelPlanEntry: - """Build a virtual microbatch entry for lease-manager tests.""" - - return ModelPlanEntry( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), - name="orders", - relative_path=Path("models/orders.sql"), - materialization_type=MaterializationType.INCREMENTAL, - action=action, - reason=PlanReason.NORMAL_INCREMENTAL, - destination=CompiledRelationLocation( - database=None, - schema="dev__sqb_physical", - name="orders__v_f2", - qualified_name="dev__sqb_physical.orders__v_f2", - ), - fingerprint_query_sql="SELECT 1 AS id", - resolved_sql="SELECT 1 AS id", - logical_ddl="", - incremental_strategy=incremental_strategy, - incremental_mode=incremental_mode, - ) diff --git a/tests/integration/src/sqlbuild/virtual/executor/classes/test_microbatch_lease_manager.py b/tests/integration/src/sqlbuild/virtual/executor/classes/test_microbatch_lease_manager.py deleted file mode 100644 index deb0815356..0000000000 --- a/tests/integration/src/sqlbuild/virtual/executor/classes/test_microbatch_lease_manager.py +++ /dev/null @@ -1,446 +0,0 @@ -"""Integration coverage for renewable virtual microbatch leases.""" - -from __future__ import annotations - -import time -from collections.abc import Callable -from concurrent.futures import Future, ThreadPoolExecutor -from datetime import datetime, timedelta -from pathlib import Path -from threading import Event -from typing import Any - -import pytest - -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import IncrementalMode, PlanAction -from sqlbuild.errors.contracts.exceptions import ExecutorInputError -from sqlbuild.virtual.executor.classes import microbatch_lease_manager as lease_manager_module -from sqlbuild.virtual.executor.classes.microbatch_lease_manager import ( - VirtualMicrobatchLeaseManager, -) -from sqlbuild.virtual.state.classes.duckdb import DuckDbStateBackend -from sqlbuild.virtual.state.models import ( - PhysicalRelationRecord, - StateLockRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import PhysicalArtifactType, VirtualEnvironmentStatus -from tests.integration.src.sqlbuild.virtual.executor.classes._test_types import ( - VirtualConcurrentLeaseTestCase, - VirtualLeaseCancellationTestCase, - VirtualMicrobatchLeaseManagerTestCase, - VirtualSharedFullRefreshTestCase, -) -from tests.integration.src.sqlbuild.virtual.executor.classes.helpers import ( - build_virtual_microbatch_lease_entry, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualMicrobatchLeaseManagerTestCase( - description="renewal followed by owner loss fences target mutation", - renew_interval_seconds=0.02, - expected_loss_fragment="incremental physical-version lease was lost", - ) - ], - ids=lambda case: case.description, -) -def test_given_renewable_lease_when_owner_is_replaced_then_manager_fences_and_preserves_new_owner( - test_case: VirtualMicrobatchLeaseManagerTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - state_path: Path = tmp_path / "state.duckdb" - connection_config: dict[str, object] = {"database": str(state_path)} - backend: DuckDbStateBackend = DuckDbStateBackend() - connection: Any = backend.connect(connection_config) - backend.initialize( - connection=connection, - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - ) - backend.close(connection) - monkeypatch.setattr( - lease_manager_module, - "_LEASE_RENEW_INTERVAL_SECONDS", - test_case.renew_interval_seconds, - ) - manager: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=backend, - connection_config=connection_config, - warehouse_connection_config={"database": "warehouse.duckdb"}, - schema="sqlbuild_state", - run_id="lease-renewal-test", - ) - manager.acquire( - entries=(build_virtual_microbatch_lease_entry(),), - expected_version_hashes={"orders": "F2"}, - ) - - connection = backend.connect(connection_config) - try: - initial_lock: StateLockRecord = backend.list_active_locks( - connection=connection, - schema="sqlbuild_state", - )[0] - finally: - backend.close(connection) - renewal_deadline: float = time.monotonic() + 2.0 - renewed_lock: StateLockRecord = initial_lock - while ( - renewed_lock.expires_at <= initial_lock.expires_at and time.monotonic() < renewal_deadline - ): - time.sleep(test_case.renew_interval_seconds) - connection = backend.connect(connection_config) - try: - renewed_lock = backend.list_active_locks( - connection=connection, - schema="sqlbuild_state", - )[0] - finally: - backend.close(connection) - assert renewed_lock.expires_at > initial_lock.expires_at - - monkeypatch.setattr(backend, "renew_lock", lambda **_kwargs: False) - - loss_deadline: float = time.monotonic() + 2.0 - loss_error: ExecutorInputError | None = None - while loss_error is None and time.monotonic() < loss_deadline: - time.sleep(test_case.renew_interval_seconds) - try: - manager.assert_active() - except ExecutorInputError as error: - loss_error = error - assert loss_error is not None - assert test_case.expected_loss_fragment in str(loss_error) - - connection = backend.connect(connection_config) - try: - assert backend.release_lock( - connection=connection, - schema="sqlbuild_state", - lock_key=renewed_lock.lock_key, - owner_id=renewed_lock.owner_id, - ) - assert backend.acquire_lock( - connection=connection, - schema="sqlbuild_state", - lock_key=renewed_lock.lock_key, - owner_id="replacement-owner", - expires_at=datetime.now() + timedelta(hours=1), - ) - finally: - backend.close(connection) - - manager.close() - connection = backend.connect(connection_config) - try: - remaining_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema="sqlbuild_state", - ) - finally: - backend.close(connection) - assert len(remaining_locks) == 1 - assert remaining_locks[0].owner_id == "replacement-owner" - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSharedFullRefreshTestCase( - description="append full refresh rejects shared version", - incremental_strategy="append", - ), - VirtualSharedFullRefreshTestCase( - description="delete insert full refresh rejects shared version", - incremental_strategy="delete_insert", - ), - VirtualSharedFullRefreshTestCase( - description="merge full refresh rejects shared version", - incremental_strategy="merge", - ), - VirtualSharedFullRefreshTestCase( - description="microbatch full refresh rejects shared version", - incremental_strategy="delete_insert", - incremental_mode=IncrementalMode.MICROBATCH, - ), - ], - ids=lambda case: case.description, -) -def test_given_shared_incremental_version_when_full_refresh_then_rejects_immutable_overwrite( - test_case: VirtualSharedFullRefreshTestCase, - tmp_path: Path, -) -> None: - connection_config: dict[str, object] = {"database": str(tmp_path / "state.duckdb")} - backend: DuckDbStateBackend = DuckDbStateBackend() - connection: Any = backend.connect(connection_config) - backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="0.0.test") - backend.upsert_physical_relation( - connection=connection, - schema="sqlbuild_state", - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="F2", - database_name=None, - schema_name="dev__sqb_physical", - relation_name="orders__v_f2", - relation_type="table", - ), - ) - backend.upsert_virtual_environment( - connection=connection, - schema="sqlbuild_state", - record=VirtualEnvironmentRecord( - virtual_environment_name="shared", - status=VirtualEnvironmentStatus.FINALIZED, - ), - ) - backend.replace_virtual_environment_model_refs( - connection=connection, - schema="sqlbuild_state", - virtual_environment_name="shared", - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name="shared", - model_name="orders", - version_hash="F2", - ), - ), - ) - backend.close(connection) - manager: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=backend, - connection_config=connection_config, - warehouse_connection_config={"database": "warehouse.duckdb"}, - schema="sqlbuild_state", - run_id="shared-full-refresh-test", - ) - - with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): - manager.acquire( - entries=( - build_virtual_microbatch_lease_entry( - action=PlanAction.CREATE_TABLE, - incremental_strategy=test_case.incremental_strategy, - incremental_mode=test_case.incremental_mode, - ), - ), - expected_version_hashes={"orders": "F2"}, - ) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualConcurrentLeaseTestCase( - description="merge full refresh serializes physical version mutation", - incremental_strategy="merge", - incremental_mode=None, - ), - VirtualConcurrentLeaseTestCase( - description="microbatch full refresh preserves physical version serialization", - incremental_strategy="delete_insert", - incremental_mode=IncrementalMode.MICROBATCH, - ), - ], - ids=lambda case: case.description, -) -def test_given_concurrent_full_refreshes_when_reference_check_blocks_then_version_lease_serializes( - test_case: VirtualConcurrentLeaseTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - connection_config: dict[str, object] = {"database": str(tmp_path / "state.duckdb")} - backend: DuckDbStateBackend = DuckDbStateBackend() - connection: Any = backend.connect(connection_config) - backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="0.0.test") - backend.close(connection) - manager_a: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=backend, - connection_config=connection_config, - warehouse_connection_config={"database": "warehouse.duckdb"}, - schema="sqlbuild_state", - run_id="concurrent-a", - ) - manager_b: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=backend, - connection_config=connection_config, - warehouse_connection_config={"database": "warehouse.duckdb"}, - schema="sqlbuild_state", - run_id="concurrent-b", - ) - manager_c: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=backend, - connection_config=connection_config, - warehouse_connection_config={"database": "warehouse.duckdb"}, - schema="sqlbuild_state", - run_id="concurrent-c", - ) - entry: ModelPlanEntry = build_virtual_microbatch_lease_entry( - action=PlanAction.CREATE_TABLE, - incremental_strategy=test_case.incremental_strategy, - incremental_mode=test_case.incremental_mode, - ) - reference_check_started: Event = Event() - allow_reference_check: Event = Event() - original_reference_check: Callable[..., None] = manager_a._reject_shared_full_refresh - - def _blocked_reference_check(**kwargs: Any) -> None: - reference_check_started.set() - assert allow_reference_check.wait(timeout=5.0) - original_reference_check(**kwargs) - - monkeypatch.setattr(manager_a, "_reject_shared_full_refresh", _blocked_reference_check) - with ThreadPoolExecutor(max_workers=1) as executor: - acquire_a: Future[None] = executor.submit( - manager_a.acquire, - entries=(entry,), - expected_version_hashes={"orders": "F2"}, - ) - assert reference_check_started.wait(timeout=5.0) - with pytest.raises(PlannerInputError, match=test_case.expected_conflict_fragment): - manager_b.acquire(entries=(entry,), expected_version_hashes={"orders": "F2"}) - allow_reference_check.set() - acquire_a.result(timeout=5.0) - - manager_a.close() - manager_b.acquire(entries=(entry,), expected_version_hashes={"orders": "F2"}) - connection = backend.connect(connection_config) - try: - backend.upsert_physical_relation( - connection=connection, - schema="sqlbuild_state", - record=PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="F2", - database_name=None, - schema_name="dev__sqb_physical", - relation_name="orders__v_f2", - relation_type="table", - ), - ) - finally: - backend.close(connection) - publication_fenced: Event = Event() - allow_publication: Event = Event() - original_record_publish: Callable[..., None] = backend._upsert_virtual_environment_record - - def _blocked_record_publish(**kwargs: Any) -> None: - publication_fenced.set() - assert allow_publication.wait(timeout=5.0) - original_record_publish(**kwargs) - - def _publish_refs() -> bool: - publish_connection: Any = backend.connect(connection_config) - try: - return backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=publish_connection, - schema="sqlbuild_state", - record=VirtualEnvironmentRecord( - virtual_environment_name="published", - status=VirtualEnvironmentStatus.ACTIVE, - ), - refs_by_node_type={ - "model": ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name="published", - node_type="model", - node_name="orders", - version_hash="F2", - ), - ), - }, - leases=manager_b.active_leases, - ) - finally: - backend.close(publish_connection) - - monkeypatch.setattr(backend, "_upsert_virtual_environment_record", _blocked_record_publish) - with ThreadPoolExecutor(max_workers=1) as executor: - publish_refs: Future[bool] = executor.submit(_publish_refs) - assert publication_fenced.wait(timeout=5.0) - with pytest.raises(PlannerInputError, match=test_case.expected_conflict_fragment): - manager_c.acquire(entries=(entry,), expected_version_hashes={"orders": "F2"}) - allow_publication.set() - assert publish_refs.result(timeout=5.0) - manager_b.close() - with pytest.raises(PlannerInputError, match=test_case.expected_shared_fragment): - manager_c.acquire(entries=(entry,), expected_version_hashes={"orders": "F2"}) - manager_c.close() - connection = backend.connect(connection_config) - try: - remaining_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema="sqlbuild_state", - ) - finally: - backend.close(connection) - assert remaining_locks == () - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualLeaseCancellationTestCase( - description="cancellation after fencing releases merge full refresh lease", - expected_error_type=KeyboardInterrupt, - expected_remaining_lock_count=0, - ), - ], - ids=lambda case: case.description, -) -def test_given_fenced_full_refresh_when_acquisition_is_cancelled_then_releases_version_lease( - test_case: VirtualLeaseCancellationTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - connection_config: dict[str, object] = {"database": str(tmp_path / "state.duckdb")} - backend: DuckDbStateBackend = DuckDbStateBackend() - connection: Any = backend.connect(connection_config) - backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="0.0.test") - backend.close(connection) - manager: VirtualMicrobatchLeaseManager = VirtualMicrobatchLeaseManager( - backend=backend, - connection_config=connection_config, - warehouse_connection_config={"database": "warehouse.duckdb"}, - schema="sqlbuild_state", - run_id="cancelled-full-refresh", - ) - - def _cancel_reference_check(**_kwargs: Any) -> None: - raise KeyboardInterrupt - - monkeypatch.setattr(manager, "_reject_shared_full_refresh", _cancel_reference_check) - with pytest.raises(test_case.expected_error_type): - manager.acquire( - entries=( - build_virtual_microbatch_lease_entry( - action=PlanAction.CREATE_TABLE, - incremental_strategy="merge", - incremental_mode=None, - ), - ), - expected_version_hashes={"orders": "F2"}, - ) - connection = backend.connect(connection_config) - try: - remaining_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema="sqlbuild_state", - ) - finally: - backend.close(connection) - assert len(remaining_locks) == test_case.expected_remaining_lock_count - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py deleted file mode 100644 index 27c66108f4..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/_test_types.py +++ /dev/null @@ -1,284 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.microbatches.types import MicrobatchRecordType -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus -from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( - ConditionalPublicationPayload, - StateRefContractObservation, -) - - -@dataclass(frozen=True) -class ConditionalVirtualRefPublishTestCase: - """Expected result when virtual refs are conditionally published under a lease.""" - - description: str - expected_stale_publish: bool - expected_owned_publish: bool - expected_model_version_hash: str - - -@dataclass(frozen=True) -class AtomicFinalizedVirtualPublishTestCase: - """Expected rollback and success behavior for finalized conditional publication.""" - - description: str - checkpoint_id: str - expected_error_fragment: str - expected_checkpoint_count_after_failure: int - expected_checkpoint_count_after_success: int - - -@dataclass(frozen=True) -class ConditionalPublicationPayloadContractTestCase: - """One payload and its expected conditional publication outcome.""" - - description: str - payload: ConditionalPublicationPayload - expected_valid: bool - expected_error_fragment: str | None = None - - -@dataclass(frozen=True) -class DuckDbStateBackendLifecycleTestCase: - description: str - schema: str - sqlbuild_version: str - expected_schema_version: int - expected_backup_prefix: str - - -@dataclass(frozen=True) -class DuckDbStateBackendValidationTestCase: - description: str - schema: str - expected_issue_count: int - - -@dataclass(frozen=True) -class DuckDbStateBackendRollbackTestCase: - description: str - schema: str - sqlbuild_version: str - expected_restored_sqlbuild_version: str - expected_index_names: tuple[str, ...] - - -@dataclass(frozen=True) -class DuckDbStateBackendErrorTestCase: - description: str - schema: str - expected_error_type: type[Exception] - expected_message_fragment: str - - -@dataclass(frozen=True) -class DuckDbStateBackendEventTestCase: - description: str - schema: str - sqlbuild_version: str - expected_actions_after_backup: tuple[str, ...] - expected_actions_after_rollback: tuple[str, ...] - expected_backup_actions: tuple[str, ...] - - -@dataclass(frozen=True) -class DuckDbStateBackendIdempotencyTestCase: - description: str - schema: str - first_sqlbuild_version: str - second_sqlbuild_version: str - expected_schema_version_rows: int - expected_latest_sqlbuild_version: str - - -@dataclass(frozen=True) -class DuckDbStateBackendTableCreationTestCase: - description: str - schema: str - sqlbuild_version: str - expected_table_names: tuple[str, ...] - expected_index_names: tuple[str, ...] - - -@dataclass(frozen=True) -class DuckDbStateBackendCoreRecordsTestCase: - description: str - schema: str - sqlbuild_version: str - expected_model_name: str - expected_version_hash: str - expected_virtual_environment_name: str - expected_ref_count: int - expected_ref_count_after_replace: int - expected_relation_name: str - expected_replaced_relation_name: str - - -@dataclass(frozen=True) -class DuckDbStateBackendAtomicRefUpdateTestCase: - description: str - schema: str - sqlbuild_version: str - virtual_environment_name: str - model_name: str - seed_name: str - expected_original_model_hash: str - expected_original_seed_hash: str - expected_updated_model_hash: str - expected_updated_seed_hash: str - expected_duplicate_seed_hash: str - - -@dataclass(frozen=True) -class DuckDbStateBackendSourceFreshnessTestCase: - description: str - schema: str - sqlbuild_version: str - virtual_environment_name: str - expected_source_names: tuple[str, ...] - expected_source_names_after_replace: tuple[str, ...] - - -@dataclass(frozen=True) -class DuckDbStateBackendSeedRefTestCase: - description: str - schema: str - sqlbuild_version: str - virtual_environment_name: str - seed_name: str - version_hash: str - identity_metadata_hash: str - identity_metadata_json_b64: str - replacement_version_hash: str - expected_ref_count_after_replace: int - - -@dataclass(frozen=True) -class DuckDbStateBackendPythonNodeIdentityTestCase: - description: str - schema: str - sqlbuild_version: str - first_virtual_environment_name: str - second_virtual_environment_name: str - node_type: str - node_name: str - first_version_hash: str - second_version_hash: str - expected_ref_versions: tuple[str, str] - orphan_version_hash: str - expected_pruned_count: int - - -@dataclass(frozen=True) -class DuckDbStateBackendNodeResultTestCase: - description: str - schema: str - sqlbuild_version: str - virtual_environment_name: str - isolated_virtual_environment_name: str - expected_latest_payload: dict[str, object] - expected_failed_status: str - expected_history_count: int - expected_target_isolated_payload: dict[str, object] - expected_rollback_row_count: int - - -@dataclass(frozen=True) -class DuckDbStateBackendIndexValidationTestCase: - description: str - schema: str - sqlbuild_version: str - dropped_index_name: str - expected_issue_kind: str - - -@dataclass(frozen=True) -class DuckDbStateBackendColumnValidationTestCase: - description: str - schema: str - sqlbuild_version: str - dropped_table_name: str - dropped_column_name: str - expected_issue_kind: str - - -@dataclass(frozen=True) -class DuckDbStateBackendTransactionRollbackTestCase: - description: str - schema: str - sqlbuild_version: str - virtual_environment_name: str - model_name: str - original_version_hash: str - duplicate_version_hash: str - expected_ref_count: int - - -@dataclass(frozen=True) -class DuckDbStateBackendLockTestCase: - description: str - schema: str - sqlbuild_version: str - lock_key: str - first_owner: str - second_owner: str - expected_active_lock_count: int - - -@dataclass(frozen=True) -class DuckDbStateBackendConcurrentLockTestCase: - description: str - schema: str - sqlbuild_version: str - lock_key: str - first_owner: str - second_owner: str - expected_success_count: int - expected_active_lock_count: int - - -@dataclass(frozen=True) -class DuckDbStateBackendOperationEventTestCase: - description: str - schema: str - sqlbuild_version: str - expected_operation_id: str - expected_virtual_environment_name: str - - -@dataclass(frozen=True) -class MicrobatchStateRoundTripTestCase: - """Expected append-only microbatch event persistence behavior.""" - - description: str - expected_event_count: int - - -@dataclass(frozen=True) -class StateReadContractTestCase: - description: str - schema: str - sqlbuild_version: str - expected_environments: tuple[tuple[str, VirtualEnvironmentStatus], ...] - expected_active_lock_keys: tuple[str, ...] - expected_expired_lock_keys: tuple[str, ...] - - -@dataclass(frozen=True) -class DuckDbStateBackendRefContractTestCase: - description: str - expected_observation: StateRefContractObservation - schema: str - sqlbuild_version: str - - -@dataclass(frozen=True) -class MicrobatchBulkAppendTestCase: - """One production-shaped microbatch event batch written through the bulk append.""" - - description: str - record_type: MicrobatchRecordType - expected_write_counts: tuple[int, int] = (1, 1) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py b/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py deleted file mode 100644 index 1d0ba81788..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/helpers.py +++ /dev/null @@ -1,511 +0,0 @@ -from __future__ import annotations - -from dataclasses import replace -from datetime import datetime, timedelta -from pathlib import Path -from typing import Any, NamedTuple - -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.microbatches.types import ( - MicrobatchCompletionType, - MicrobatchFingerprintStatus, - MicrobatchRecordType, - MicrobatchRunType, -) -from sqlbuild.virtual.state.classes.duckdb import DuckDbStateBackend -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - PhysicalRelationRecord, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - VirtualEnvironmentStatus, -) - -_ENVIRONMENT: str = "dev" -_CHECKPOINT_ID: str = "checkpoint-1" -_CHECKPOINT: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - _CHECKPOINT_ID, _ENVIRONMENT -) - - -class ConditionalPublicationPayload(NamedTuple): - """Arguments for one conditional environment publication call.""" - - record: VirtualEnvironmentRecord - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] - checkpoint: VirtualEnvironmentCheckpointRecord | None - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] - - -VALID_PAYLOAD: ConditionalPublicationPayload = ConditionalPublicationPayload( - record=VirtualEnvironmentRecord(_ENVIRONMENT, VirtualEnvironmentStatus.FINALIZED), - refs_by_node_type={ - "model": (VirtualEnvironmentNodeRefRecord(_ENVIRONMENT, "model", "orders", "model-v1"),), - "udf": (VirtualEnvironmentNodeRefRecord(_ENVIRONMENT, "udf", "normalize", "function-v1"),), - "seed": (VirtualEnvironmentNodeRefRecord(_ENVIRONMENT, "seed", "countries", "seed-v1"),), - }, - checkpoint=_CHECKPOINT, - checkpoint_refs=( - VirtualEnvironmentCheckpointModelRefRecord(_CHECKPOINT_ID, "orders", "model-v1"), - ), - checkpoint_function_refs=( - VirtualEnvironmentCheckpointFunctionRefRecord(_CHECKPOINT_ID, "normalize", "function-v1"), - ), - checkpoint_seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord(_CHECKPOINT_ID, "countries", "seed-v1"), - ), -) -ACTIVE_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - record=replace(VALID_PAYLOAD.record, status=VirtualEnvironmentStatus.ACTIVE), - checkpoint=None, - checkpoint_refs=(), - checkpoint_function_refs=(), - checkpoint_seed_refs=(), -) -MISSING_CHECKPOINT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint=None, - checkpoint_refs=(), - checkpoint_function_refs=(), - checkpoint_seed_refs=(), -) -ACTIVE_CHECKPOINT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - record=replace(VALID_PAYLOAD.record, status=VirtualEnvironmentStatus.ACTIVE) -) -FINALIZING_CHECKPOINT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - record=replace(VALID_PAYLOAD.record, status=VirtualEnvironmentStatus.FINALIZING) -) -DETACHED_CHECKPOINT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - record=replace(VALID_PAYLOAD.record, status=VirtualEnvironmentStatus.DETACHED) -) -FAILED_CHECKPOINT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - record=replace(VALID_PAYLOAD.record, status=VirtualEnvironmentStatus.FAILED) -) -CHECKPOINT_ENVIRONMENT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint=replace(_CHECKPOINT, virtual_environment_name="other") -) -CHECKPOINT_ID_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint_function_refs=( - replace(VALID_PAYLOAD.checkpoint_function_refs[0], checkpoint_id="other"), - ) -) -MODEL_VERSION_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint_refs=(replace(VALID_PAYLOAD.checkpoint_refs[0], version_hash="other"),) -) -FUNCTION_OMISSION_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint_function_refs=() -) -SEED_EXTRA_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint_seed_refs=( - *VALID_PAYLOAD.checkpoint_seed_refs, - VirtualEnvironmentCheckpointSeedRefRecord(_CHECKPOINT_ID, "extra", "seed-v2"), - ) -) -CHECKPOINT_DUPLICATE_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - checkpoint_refs=(*VALID_PAYLOAD.checkpoint_refs, *VALID_PAYLOAD.checkpoint_refs) -) -PUBLISHED_DUPLICATE_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - refs_by_node_type={ - **VALID_PAYLOAD.refs_by_node_type, - "model": ( - *VALID_PAYLOAD.refs_by_node_type["model"], - *VALID_PAYLOAD.refs_by_node_type["model"], - ), - } -) -REF_ENVIRONMENT_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - refs_by_node_type={ - **VALID_PAYLOAD.refs_by_node_type, - "model": ( - replace( - VALID_PAYLOAD.refs_by_node_type["model"][0], - virtual_environment_name="other", - ), - ), - } -) -REF_NODE_TYPE_PAYLOAD: ConditionalPublicationPayload = VALID_PAYLOAD._replace( - refs_by_node_type={ - **VALID_PAYLOAD.refs_by_node_type, - "model": (replace(VALID_PAYLOAD.refs_by_node_type["model"][0], node_type="seed"),), - } -) - - -def open_duckdb_state_backend(*, db_path: Path) -> tuple[DuckDbStateBackend, Any]: - backend: DuckDbStateBackend = DuckDbStateBackend() - connection: Any = backend.connect({"database": str(db_path)}) - return backend, connection - - -def fetch_all(connection: Any, sql: str) -> list[tuple[Any, ...]]: - return connection.execute(sql).fetchall() - - -STATE_READ_CONTRACT_FUNCTION: FunctionVersionRecord = FunctionVersionRecord( - function_name="is_large_order", - version_hash="function-v1", - language="sql", - returns="BOOLEAN", - arguments_json_b64="W3sibmFtZSI6ImFtb3VudCIsInR5cGUiOiJJTlRFR0VSIn1d", - return_columns_json_b64="W10=", - packages_json_b64="W10=", - runtime_version=None, - entry_point=None, - body_sql_b64="YW1vdW50ID4gOQ==", - definition_text_b64="YW1vdW50ID4gOQ==", - status=ModelVersionStatus.READY, -) -STATE_READ_CONTRACT_RELATION_V1: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="orders-v1", - database_name=None, - schema_name="sqb_orders", - relation_name="orders__v1", - relation_type="table", -) -STATE_READ_CONTRACT_RELATION_V2: PhysicalRelationRecord = replace( - STATE_READ_CONTRACT_RELATION_V1, version_hash="orders-v2", relation_name="orders__v2" -) - - -class StateReadContractObservation(NamedTuple): - """Values read back after writing the shared state read contract fixture.""" - - function_before_upsert: FunctionVersionRecord | None - function_after_upsert: FunctionVersionRecord | None - function_after_second_upsert: FunctionVersionRecord | None - relation_v1: PhysicalRelationRecord | None - relation_missing: PhysicalRelationRecord | None - relations: tuple[PhysicalRelationRecord, ...] - environments: tuple[tuple[str, VirtualEnvironmentStatus], ...] - active_lock_keys: tuple[str, ...] - expired_lock_keys: tuple[str, ...] - - -def exercise_state_read_contract( - *, backend: StateBackend, connection: Any, schema: str -) -> StateReadContractObservation: - function_before_upsert: FunctionVersionRecord | None = backend.get_function_version( - connection=connection, - schema=schema, - function_name=STATE_READ_CONTRACT_FUNCTION.function_name, - version_hash=STATE_READ_CONTRACT_FUNCTION.version_hash, - ) - backend.upsert_function_version( - connection=connection, schema=schema, record=STATE_READ_CONTRACT_FUNCTION - ) - function_after_upsert: FunctionVersionRecord | None = backend.get_function_version( - connection=connection, - schema=schema, - function_name=STATE_READ_CONTRACT_FUNCTION.function_name, - version_hash=STATE_READ_CONTRACT_FUNCTION.version_hash, - ) - backend.upsert_function_version( - connection=connection, - schema=schema, - record=replace(STATE_READ_CONTRACT_FUNCTION, status=ModelVersionStatus.FAILED), - ) - function_after_second_upsert: FunctionVersionRecord | None = backend.get_function_version( - connection=connection, - schema=schema, - function_name=STATE_READ_CONTRACT_FUNCTION.function_name, - version_hash=STATE_READ_CONTRACT_FUNCTION.version_hash, - ) - backend.upsert_physical_relation( - connection=connection, schema=schema, record=STATE_READ_CONTRACT_RELATION_V1 - ) - backend.upsert_physical_relation( - connection=connection, schema=schema, record=STATE_READ_CONTRACT_RELATION_V2 - ) - relation_v1: PhysicalRelationRecord | None = backend.get_physical_relation_for_artifact( - connection=connection, - schema=schema, - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="orders-v1", - ) - relation_missing: PhysicalRelationRecord | None = backend.get_physical_relation_for_artifact( - connection=connection, - schema=schema, - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="orders-v3", - ) - relations: tuple[PhysicalRelationRecord, ...] = backend.list_physical_relations_for_artifact( - connection=connection, - schema=schema, - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - ) - backend.upsert_virtual_environment( - connection=connection, - schema=schema, - record=VirtualEnvironmentRecord("dev", VirtualEnvironmentStatus.ACTIVE), - ) - backend.upsert_virtual_environment( - connection=connection, - schema=schema, - record=VirtualEnvironmentRecord("prod", VirtualEnvironmentStatus.FINALIZED), - ) - environments: tuple[tuple[str, VirtualEnvironmentStatus], ...] = tuple( - (record.virtual_environment_name, record.status) - for record in backend.list_virtual_environments(connection=connection, schema=schema) - ) - backend.acquire_lock( - connection=connection, - schema=schema, - lock_key="lock_expired", - owner_id="run-1", - expires_at=datetime.now() - timedelta(days=2), - ) - backend.acquire_lock( - connection=connection, - schema=schema, - lock_key="lock_active", - owner_id="run-1", - expires_at=datetime.now() + timedelta(days=2), - ) - active_lock_keys: tuple[str, ...] = tuple( - lock.lock_key for lock in backend.list_active_locks(connection=connection, schema=schema) - ) - expired_lock_keys: tuple[str, ...] = tuple( - lock.lock_key for lock in backend.list_expired_locks(connection=connection, schema=schema) - ) - return StateReadContractObservation( - function_before_upsert=function_before_upsert, - function_after_upsert=function_after_upsert, - function_after_second_upsert=function_after_second_upsert, - relation_v1=relation_v1, - relation_missing=relation_missing, - relations=relations, - environments=environments, - active_lock_keys=active_lock_keys, - expired_lock_keys=expired_lock_keys, - ) - - -class StateRefContractObservation(NamedTuple): - """Values read back after writing the shared state ref contract fixture.""" - - node_refs_after_first_replace: tuple[VirtualEnvironmentNodeRefRecord, ...] - node_refs_after_second_replace: tuple[VirtualEnvironmentNodeRefRecord, ...] - untouched_seed_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] - grouped_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - grouped_seed_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - published_environment: VirtualEnvironmentRecord | None - published_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - deleted_environment: VirtualEnvironmentRecord | None - deleted_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - surviving_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] - - -def exercise_state_ref_contract( - *, backend: StateBackend, connection: Any, schema: str -) -> StateRefContractObservation: - backend.replace_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name="dev", - node_type="seed", - refs=(VirtualEnvironmentNodeRefRecord("dev", "seed", "countries", "seed-v1"),), - ) - backend.replace_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name="dev", - node_type="model", - refs=( - VirtualEnvironmentNodeRefRecord("dev", "model", "orders", "orders-v1"), - VirtualEnvironmentNodeRefRecord("dev", "model", "customers", "customers-v1"), - ), - ) - node_refs_after_first_replace: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="dev", node_type="model" - ) - ) - backend.replace_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name="dev", - node_type="model", - refs=(VirtualEnvironmentNodeRefRecord("dev", "model", "orders", "orders-v2"),), - ) - node_refs_after_second_replace: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="dev", node_type="model" - ) - ) - untouched_seed_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="dev", node_type="seed" - ) - ) - backend.replace_virtual_environment_node_ref_groups( - connection=connection, - schema=schema, - virtual_environment_name="dev", - refs_by_node_type={ - "table_fn": ( - VirtualEnvironmentNodeRefRecord("dev", "table_fn", "customer_orders", "fn-v1"), - ), - "udf": (VirtualEnvironmentNodeRefRecord("dev", "udf", "normalize_email", "fn-v2"),), - }, - ) - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( - backend.get_virtual_environment_function_refs( - connection=connection, schema=schema, virtual_environment_name="dev" - ) - ) - backend.replace_virtual_environment_node_ref_groups( - connection=connection, - schema=schema, - virtual_environment_name="qa", - refs_by_node_type={ - "model": (VirtualEnvironmentNodeRefRecord("qa", "model", "orders", "orders-v3"),), - "seed": (VirtualEnvironmentNodeRefRecord("qa", "seed", "countries", "seed-v2"),), - }, - ) - grouped_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="qa", node_type="model" - ) - ) - grouped_seed_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="qa", node_type="seed" - ) - ) - backend.upsert_virtual_environment_and_replace_node_ref_groups( - connection=connection, - schema=schema, - record=VirtualEnvironmentRecord("prod", VirtualEnvironmentStatus.ACTIVE), - refs_by_node_type={ - "model": (VirtualEnvironmentNodeRefRecord("prod", "model", "orders", "orders-v1"),) - }, - ) - published_environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=connection, schema=schema, virtual_environment_name="prod" - ) - published_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="prod", node_type="model" - ) - ) - backend.delete_virtual_environment( - connection=connection, schema=schema, virtual_environment_name="prod" - ) - deleted_environment: VirtualEnvironmentRecord | None = backend.get_virtual_environment( - connection=connection, schema=schema, virtual_environment_name="prod" - ) - deleted_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="prod", node_type="model" - ) - ) - surviving_model_refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - backend.get_virtual_environment_node_refs( - connection=connection, schema=schema, virtual_environment_name="qa", node_type="model" - ) - ) - return StateRefContractObservation( - node_refs_after_first_replace=node_refs_after_first_replace, - node_refs_after_second_replace=node_refs_after_second_replace, - untouched_seed_refs=untouched_seed_refs, - function_refs=function_refs, - grouped_model_refs=grouped_model_refs, - grouped_seed_refs=grouped_seed_refs, - published_environment=published_environment, - published_model_refs=published_model_refs, - deleted_environment=deleted_environment, - deleted_model_refs=deleted_model_refs, - surviving_model_refs=surviving_model_refs, - ) - - -EXPECTED_STATE_REF_CONTRACT_OBSERVATION: StateRefContractObservation = StateRefContractObservation( - node_refs_after_first_replace=( - VirtualEnvironmentNodeRefRecord("dev", "model", "customers", "customers-v1"), - VirtualEnvironmentNodeRefRecord("dev", "model", "orders", "orders-v1"), - ), - node_refs_after_second_replace=( - VirtualEnvironmentNodeRefRecord("dev", "model", "orders", "orders-v2"), - ), - untouched_seed_refs=(VirtualEnvironmentNodeRefRecord("dev", "seed", "countries", "seed-v1"),), - function_refs=( - VirtualEnvironmentFunctionRefRecord("dev", "table_fn", "customer_orders", "fn-v1"), - VirtualEnvironmentFunctionRefRecord("dev", "udf", "normalize_email", "fn-v2"), - ), - grouped_model_refs=(VirtualEnvironmentNodeRefRecord("qa", "model", "orders", "orders-v3"),), - grouped_seed_refs=(VirtualEnvironmentNodeRefRecord("qa", "seed", "countries", "seed-v2"),), - published_environment=VirtualEnvironmentRecord("prod", VirtualEnvironmentStatus.ACTIVE), - published_model_refs=(VirtualEnvironmentNodeRefRecord("prod", "model", "orders", "orders-v1"),), - deleted_environment=None, - deleted_model_refs=(), - surviving_model_refs=(VirtualEnvironmentNodeRefRecord("qa", "model", "orders", "orders-v3"),), -) - - -def build_production_shaped_microbatch_event( - *, scope: MicrobatchScope, record_type: MicrobatchRecordType -) -> MicrobatchEvent: - """Build one event with the nullable columns the executor leaves unset for its kind. - - Replay requirements carry no completion or observation facts, and executed partition - completions carry no observation facts, so each single-event batch has timestamp and - integer columns that are NULL in every row. - """ - - run_started_at: datetime = datetime(2026, 1, 1, 9, 0, 0) - common: dict[str, Any] = { - "scope": scope, - "origin_run_id": "run-1", - "execution_run_id": "run-1", - "run_type": MicrobatchRunType.NORMAL, - "run_start": "0", - "run_end": "2", - "batch_size": "1", - "cursor_column": "batch_id", - "cursor_type": "integer", - "model_version_hash": "F2", - "definition_hash": "definition", - "fingerprint_status": MicrobatchFingerprintStatus.KNOWN, - "origin_run_started_at": run_started_at, - "execution_run_started_at": run_started_at, - "created_at": datetime(2026, 1, 1, 9, 0, 1), - } - events: dict[MicrobatchRecordType, MicrobatchEvent] = { - MicrobatchRecordType.REPLAY_REQUIREMENT: MicrobatchEvent( - event_id="requirement-1", - record_type=record_type, - replay_requirement_id="requirement-1", - required_model_version_hash="F2", - replay_policy="full", - **common, - ), - MicrobatchRecordType.PARTITION_COMPLETION: MicrobatchEvent( - event_id="completion-1", - record_type=record_type, - completion_type=MicrobatchCompletionType.INITIAL, - partition_start="0", - partition_end="1", - rows_affected=3, - completed_at=datetime(2026, 1, 1, 9, 0, 2), - **common, - ), - } - return events[record_type] diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py deleted file mode 100644 index a77e8bf730..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/_test_types.py +++ /dev/null @@ -1,239 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.microbatches.types import MicrobatchRecordType -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus -from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( - ConditionalPublicationPayload, - StateRefContractObservation, -) - - -@dataclass(frozen=True) -class MicrobatchBulkAppendTestCase: - """One production-shaped event batch and its idempotent write counts.""" - - description: str - record_type: MicrobatchRecordType - expected_write_counts: tuple[int, int] = (1, 1) - - -@dataclass(frozen=True) -class PostgresConditionalVirtualRefPublishTestCase: - """Expected conditional virtual-ref publication results.""" - - description: str - expected_stale_publish: bool - expected_owned_publish: bool - expected_model_version_hash: str - - -@dataclass(frozen=True) -class PostgresAtomicFinalizedVirtualPublishTestCase: - """Expected rollback and success behavior for finalized conditional publication.""" - - description: str - checkpoint_id: str - expected_error_fragment: str - expected_checkpoint_count_after_failure: int - expected_checkpoint_count_after_success: int - - -@dataclass(frozen=True) -class PostgresConditionalPublicationPayloadContractTestCase: - """One payload and its expected PostgreSQL conditional publication outcome.""" - - description: str - payload: ConditionalPublicationPayload - expected_valid: bool - expected_error_fragment: str | None = None - - -@dataclass(frozen=True) -class PostgresStateBackendLifecycleTestCase: - description: str - sqlbuild_version: str - expected_schema_version: int - expected_actions_after_backup: tuple[str, ...] - expected_actions_after_rollback: tuple[str, ...] - expected_backup_actions: tuple[str, ...] - - -@dataclass(frozen=True) -class PostgresStateBackendValidationTestCase: - description: str - expected_issue_count: int - - -@dataclass(frozen=True) -class PostgresStateBackendErrorTestCase: - description: str - expected_error_type: type[Exception] - expected_message_fragment: str - - -@dataclass(frozen=True) -class PostgresStateBackendExplicitRollbackTestCase: - description: str - sqlbuild_version: str - expected_restored_schema_version: int - expected_index_names: tuple[str, ...] - - -@dataclass(frozen=True) -class PostgresStateBackendTableCreationTestCase: - description: str - sqlbuild_version: str - expected_table_names: tuple[str, ...] - expected_index_names: tuple[str, ...] - - -@dataclass(frozen=True) -class PostgresStateBackendCoreRecordsTestCase: - description: str - sqlbuild_version: str - expected_model_name: str - expected_version_hash: str - expected_virtual_environment_name: str - expected_ref_count: int - expected_ref_count_after_replace: int - expected_relation_name: str - expected_replaced_relation_name: str - - -@dataclass(frozen=True) -class PostgresStateBackendSourceFreshnessTestCase: - description: str - sqlbuild_version: str - virtual_environment_name: str - expected_source_names: tuple[str, ...] - expected_source_names_after_replace: tuple[str, ...] - - -@dataclass(frozen=True) -class PostgresStateBackendSeedRefTestCase: - description: str - sqlbuild_version: str - virtual_environment_name: str - seed_name: str - version_hash: str - identity_metadata_hash: str - identity_metadata_json_b64: str - expected_ref_count_after_replace: int - - -@dataclass(frozen=True) -class PostgresStateBackendPythonNodeIdentityTestCase: - description: str - sqlbuild_version: str - first_virtual_environment_name: str - second_virtual_environment_name: str - node_type: str - node_name: str - first_version_hash: str - second_version_hash: str - expected_ref_versions: tuple[str, str] - orphan_version_hash: str - expected_pruned_count: int - - -@dataclass(frozen=True) -class PostgresStateBackendNodeResultTestCase: - description: str - sqlbuild_version: str - virtual_environment_name: str - isolated_virtual_environment_name: str - expected_latest_payload: dict[str, object] - expected_failed_status: str - expected_history_count: int - expected_target_isolated_payload: dict[str, object] - expected_rollback_row_count: int - - -@dataclass(frozen=True) -class PostgresStateBackendIndexValidationTestCase: - description: str - sqlbuild_version: str - dropped_index_name: str - expected_issue_kind: str - - -@dataclass(frozen=True) -class PostgresStateBackendColumnValidationTestCase: - description: str - sqlbuild_version: str - dropped_table_name: str - dropped_column_name: str - expected_issue_kind: str - - -@dataclass(frozen=True) -class PostgresStateBackendTransactionRollbackTestCase: - description: str - sqlbuild_version: str - virtual_environment_name: str - model_name: str - original_version_hash: str - duplicate_version_hash: str - expected_ref_count: int - - -@dataclass(frozen=True) -class PostgresStateBackendLockTestCase: - description: str - sqlbuild_version: str - lock_key: str - first_owner: str - second_owner: str - expected_active_lock_count: int - - -@dataclass(frozen=True) -class PostgresStateBackendConcurrentLockTestCase: - description: str - sqlbuild_version: str - lock_key: str - first_owner: str - second_owner: str - expected_success_count: int - expected_active_lock_count: int - - -@dataclass(frozen=True) -class PostgresStateBackendOperationEventTestCase: - description: str - sqlbuild_version: str - expected_operation_id: str - expected_virtual_environment_name: str - - -@dataclass(frozen=True) -class PostgresMicrobatchStateRoundTripTestCase: - """Expected append-only Postgres microbatch event behavior.""" - - description: str - expected_event_count: int - - -@dataclass(frozen=True) -class PostgresRetiredMicrobatchRecordTestCase: - description: str - retired_record_type: str - expected_event_count: int - - -@dataclass(frozen=True) -class PostgresStateReadContractTestCase: - description: str - sqlbuild_version: str - expected_environments: tuple[tuple[str, VirtualEnvironmentStatus], ...] - expected_active_lock_keys: tuple[str, ...] - expected_expired_lock_keys: tuple[str, ...] - - -@dataclass(frozen=True) -class PostgresStateBackendRefContractTestCase: - description: str - expected_observation: StateRefContractObservation - sqlbuild_version: str diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/conftest.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/conftest.py deleted file mode 100644 index 907cd4f20e..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/conftest.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterator -from pathlib import Path -from typing import Any - -import pytest - -from sqlbuild.virtual.state.classes.postgres import PostgresStateBackend -from tests.integration.src.sqlbuild.virtual.state.classes.postgres.helpers import ( - build_unique_schema_name, -) - - -def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: - test_dir: Path = Path(__file__).resolve().parent - for item in items: - if not Path(str(item.path)).resolve().is_relative_to(test_dir): - continue - item.add_marker(pytest.mark.real_warehouse) - item.add_marker(pytest.mark.postgres) - - -@pytest.fixture(scope="session") -def postgres_state_container() -> Iterator[Any]: - from testcontainers.postgres import PostgresContainer - - try: - with PostgresContainer("postgres:17") as container: - yield container - except Exception as exc: - pytest.skip(f"Docker not available for Postgres testcontainer: {exc}") - - -@pytest.fixture(scope="session") -def postgres_state_config(postgres_state_container: Any) -> dict[str, object]: - return { - "host": postgres_state_container.get_container_host_ip(), - "port": int(postgres_state_container.get_exposed_port(5432)), - "dbname": postgres_state_container.dbname, - "user": postgres_state_container.username, - "password": postgres_state_container.password, - } - - -@pytest.fixture -def postgres_state_schema() -> str: - return build_unique_schema_name(prefix="sqb_state") - - -@pytest.fixture -def postgres_state_backend() -> PostgresStateBackend: - return PostgresStateBackend() - - -@pytest.fixture -def postgres_state_connection( - postgres_state_backend: PostgresStateBackend, - postgres_state_config: dict[str, object], - postgres_state_schema: str, -) -> Iterator[Any]: - connection: Any = postgres_state_backend.connect(postgres_state_config) - try: - yield connection - finally: - with connection.cursor() as cursor: - cursor.execute(f'DROP SCHEMA IF EXISTS "{postgres_state_schema}" CASCADE') - cursor.execute( - "SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE %s", - [f"{postgres_state_schema}__backup_%"], - ) - backup_schemas: list[str] = [row[0] for row in cursor.fetchall()] - backup_schema: str - for backup_schema in backup_schemas: - cursor.execute(f'DROP SCHEMA IF EXISTS "{backup_schema}" CASCADE') - postgres_state_backend.close(connection) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/helpers.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/helpers.py deleted file mode 100644 index b3dd62009c..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/helpers.py +++ /dev/null @@ -1,99 +0,0 @@ -from __future__ import annotations - -import uuid -from datetime import datetime -from typing import Any - -from sqlbuild.microbatches.classes.event_codec import MicrobatchEventCodec -from sqlbuild.microbatches.constants import MICROBATCH_COLUMNS -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.microbatches.types import ( - MicrobatchCompletionType, - MicrobatchFingerprintStatus, - MicrobatchRecordType, - MicrobatchRunType, -) -from sqlbuild.virtual.state.constants import MICROBATCH_EVENT_TABLE - - -def build_unique_schema_name(*, prefix: str) -> str: - suffix: str = uuid.uuid4().hex[:10] - return f"{prefix}_{suffix}" - - -def fetch_all(connection: Any, sql: str) -> list[tuple[Any, ...]]: - with connection.cursor() as cursor: - cursor.execute(sql) - return [tuple(row) for row in cursor.fetchall()] - - -def qualified_name(*, schema: str, table: str) -> str: - return f"{quote_identifier(schema)}.{quote_identifier(table)}" - - -def quote_identifier(identifier: str) -> str: - return '"' + identifier.replace('"', '""') + '"' - - -def postgres_microbatch_scope() -> MicrobatchScope: - """Build the shared virtual scope used by Postgres history tests.""" - - return MicrobatchScope( - scope_kind="virtual_physical", - scope_key="postgres:state:orders:F2:analytics.orders__F2", - model_name="orders", - target_database="warehouse", - target_schema="analytics", - target_name="orders__F2", - physical_generation_id="F2:analytics.orders__F2", - virtual_environment_name="dev", - virtual_model_version_hash="F2", - ) - - -def postgres_microbatch_event(*, scope: MicrobatchScope, event_id: str) -> MicrobatchEvent: - """Build one active Postgres microbatch event.""" - - return MicrobatchEvent( - event_id=event_id, - record_type=MicrobatchRecordType.PARTITION_COMPLETION, - scope=scope, - origin_run_id="run-1", - execution_run_id="run-1", - run_type=MicrobatchRunType.NORMAL, - completion_type=MicrobatchCompletionType.INITIAL, - run_start="0", - run_end="1", - partition_start="0", - partition_end="1", - batch_size="1", - cursor_column="batch_id", - cursor_type="integer", - model_version_hash="F2", - definition_hash="definition", - fingerprint_status=MicrobatchFingerprintStatus.KNOWN, - rows_affected=0, - created_at=datetime(2026, 1, 1), - ) - - -def insert_raw_postgres_microbatch_record( - *, - connection: Any, - schema: str, - event: MicrobatchEvent, - event_id: str, - record_type: str, -) -> None: - """Insert one persisted Postgres event with a raw record type.""" - - values: list[object | None] = list(MicrobatchEventCodec.values(event)) - values[0] = event_id - values[1] = record_type - placeholders: str = ", ".join("%s" for _ in values) - with connection.cursor() as cursor: - cursor.execute( - f"INSERT INTO {qualified_name(schema=schema, table=MICROBATCH_EVENT_TABLE)} " - f"({', '.join(MICROBATCH_COLUMNS)}) VALUES ({placeholders})", - values, - ) diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py b/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py deleted file mode 100644 index 492c20ad0c..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/postgres/test_postgres.py +++ /dev/null @@ -1,2471 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import replace -from datetime import UTC, datetime, timedelta -from typing import Any - -import pytest - -from sqlbuild.executor.node_results.models import ( - NodeResultEnvelope, - NodeResultQuery, - NodeResultRecord, -) -from sqlbuild.executor.node_results.types import NodeResultStatus -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -from sqlbuild.microbatches.types import ( - MicrobatchCompletionType, - MicrobatchFingerprintStatus, - MicrobatchRecordType, - MicrobatchRunType, -) -from sqlbuild.virtual.state.classes.postgres import PostgresStateBackend -from sqlbuild.virtual.state.constants import ( - SOURCE_FRESHNESS_OBSERVATION_TABLE, - STATE_TABLE_INDEXES, - STATE_TABLES, -) -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - PhysicalRelationAncestryRecord, - PhysicalRelationRecord, - PythonNodeVersionRecord, - ReconcileEventRecord, - SourceFreshnessRecord, - StateLockLease, - StateLockRecord, - StateOperationEventRecord, - StateOperationRecord, - StateSchemaValidationResult, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentPythonNodeRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - ReconcileAction, - StateMigrationAction, - StateOperationStatus, - StateOperationType, - StateSchemaValidationIssueKind, - VirtualEnvironmentStatus, -) -from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( - ACTIVE_CHECKPOINT_PAYLOAD, - ACTIVE_PAYLOAD, - CHECKPOINT_DUPLICATE_PAYLOAD, - CHECKPOINT_ENVIRONMENT_PAYLOAD, - CHECKPOINT_ID_PAYLOAD, - DETACHED_CHECKPOINT_PAYLOAD, - EXPECTED_STATE_REF_CONTRACT_OBSERVATION, - FAILED_CHECKPOINT_PAYLOAD, - FINALIZING_CHECKPOINT_PAYLOAD, - FUNCTION_OMISSION_PAYLOAD, - MISSING_CHECKPOINT_PAYLOAD, - MODEL_VERSION_PAYLOAD, - PUBLISHED_DUPLICATE_PAYLOAD, - REF_ENVIRONMENT_PAYLOAD, - REF_NODE_TYPE_PAYLOAD, - SEED_EXTRA_PAYLOAD, - STATE_READ_CONTRACT_FUNCTION, - STATE_READ_CONTRACT_RELATION_V1, - STATE_READ_CONTRACT_RELATION_V2, - VALID_PAYLOAD, - StateReadContractObservation, - StateRefContractObservation, - build_production_shaped_microbatch_event, - exercise_state_read_contract, - exercise_state_ref_contract, -) -from tests.integration.src.sqlbuild.virtual.state.classes.postgres._test_types import ( - MicrobatchBulkAppendTestCase, - PostgresAtomicFinalizedVirtualPublishTestCase, - PostgresConditionalPublicationPayloadContractTestCase, - PostgresConditionalVirtualRefPublishTestCase, - PostgresMicrobatchStateRoundTripTestCase, - PostgresRetiredMicrobatchRecordTestCase, - PostgresStateBackendColumnValidationTestCase, - PostgresStateBackendConcurrentLockTestCase, - PostgresStateBackendCoreRecordsTestCase, - PostgresStateBackendErrorTestCase, - PostgresStateBackendExplicitRollbackTestCase, - PostgresStateBackendIndexValidationTestCase, - PostgresStateBackendLifecycleTestCase, - PostgresStateBackendLockTestCase, - PostgresStateBackendNodeResultTestCase, - PostgresStateBackendOperationEventTestCase, - PostgresStateBackendPythonNodeIdentityTestCase, - PostgresStateBackendRefContractTestCase, - PostgresStateBackendSourceFreshnessTestCase, - PostgresStateBackendTableCreationTestCase, - PostgresStateBackendTransactionRollbackTestCase, - PostgresStateBackendValidationTestCase, - PostgresStateReadContractTestCase, -) -from tests.integration.src.sqlbuild.virtual.state.classes.postgres.helpers import ( - fetch_all, - insert_raw_postgres_microbatch_record, - postgres_microbatch_event, - postgres_microbatch_scope, - qualified_name, - quote_identifier, -) - - -@pytest.mark.parametrize( - "test_case", - ( - PostgresRetiredMicrobatchRecordTestCase( - description="retired producer completion", - retired_record_type="producer_completion", - expected_event_count=1, - ), - PostgresRetiredMicrobatchRecordTestCase( - description="retired consumer frontier", - retired_record_type="consumer_frontier", - expected_event_count=1, - ), - ), - ids=lambda case: case.description, -) -def test_given_retired_postgres_rows_when_reading_history_then_only_active_events_returned( - test_case: PostgresRetiredMicrobatchRecordTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="test", - ) - scope: MicrobatchScope = postgres_microbatch_scope() - event: MicrobatchEvent = postgres_microbatch_event(scope=scope, event_id="active-event") - postgres_state_backend.append_microbatch_events( - connection=postgres_state_connection, - schema=postgres_state_schema, - events=(event,), - ) - insert_raw_postgres_microbatch_record( - connection=postgres_state_connection, - schema=postgres_state_schema, - event=event, - event_id="retired-event", - record_type=test_case.retired_record_type, - ) - - scope_history: tuple[MicrobatchEvent, ...] = ( - postgres_state_backend.read_microbatch_scope_history( - connection=postgres_state_connection, - schema=postgres_state_schema, - scope=scope, - ) - ) - retention_history: tuple[MicrobatchEvent, ...] = ( - postgres_state_backend.read_microbatch_retention_history( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - ) - model_history: tuple[MicrobatchEvent, ...] = ( - postgres_state_backend.read_microbatch_model_history( - connection=postgres_state_connection, - schema=postgres_state_schema, - scope=scope, - ) - ) - - assert len(scope_history) == test_case.expected_event_count - assert tuple(item.event_id for item in scope_history) == (event.event_id,) - assert retention_history == scope_history - assert model_history == scope_history - - -EXPECTED_STATE_INDEX_NAMES: list[str] = [] -for state_indexes in STATE_TABLE_INDEXES.values(): - for state_index_name in state_indexes: - EXPECTED_STATE_INDEX_NAMES.append(state_index_name) -EXPECTED_STATE_INDEX_NAMES.sort() - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresConditionalPublicationPayloadContractTestCase( - description="postgres exact finalized payload succeeds", - payload=VALID_PAYLOAD, - expected_valid=True, - ), - PostgresConditionalPublicationPayloadContractTestCase( - description="postgres active payload without checkpoint succeeds", - payload=ACTIVE_PAYLOAD, - expected_valid=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_valid_conditional_payload_when_postgres_publishes_then_contract_succeeds( - test_case: PostgresConditionalPublicationPayloadContractTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="test", - ) - assert ( - postgres_state_backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=test_case.payload.record, - refs_by_node_type=test_case.payload.refs_by_node_type, - leases=(), - checkpoint=test_case.payload.checkpoint, - checkpoint_refs=test_case.payload.checkpoint_refs, - checkpoint_function_refs=test_case.payload.checkpoint_function_refs, - checkpoint_seed_refs=test_case.payload.checkpoint_seed_refs, - ) - is test_case.expected_valid - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresConditionalPublicationPayloadContractTestCase( - "postgres finalized requires checkpoint", - MISSING_CHECKPOINT_PAYLOAD, - False, - "requires a checkpoint", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres active forbids complete checkpoint payload", - ACTIVE_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres finalizing forbids complete checkpoint payload", - FINALIZING_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres detached forbids complete checkpoint payload", - DETACHED_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres failed forbids complete checkpoint payload", - FAILED_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres checkpoint environment matches", - CHECKPOINT_ENVIRONMENT_PAYLOAD, - False, - "match the published environment", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres checkpoint ids match", - CHECKPOINT_ID_PAYLOAD, - False, - "checkpoint_id must match", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres model refs correspond", - MODEL_VERSION_PAYLOAD, - False, - "model refs must exactly match", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres function refs have no omissions", - FUNCTION_OMISSION_PAYLOAD, - False, - "function refs must exactly match", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres seed refs have no extras", - SEED_EXTRA_PAYLOAD, - False, - "seed refs must exactly match", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres checkpoint refs have no duplicates", - CHECKPOINT_DUPLICATE_PAYLOAD, - False, - "duplicate identities", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres current refs have no duplicates", - PUBLISHED_DUPLICATE_PAYLOAD, - False, - "duplicate identities", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres ref environment matches", - REF_ENVIRONMENT_PAYLOAD, - False, - "virtual_environment_name", - ), - PostgresConditionalPublicationPayloadContractTestCase( - "postgres ref group matches node type", - REF_NODE_TYPE_PAYLOAD, - False, - "node_type must match", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_conditional_payload_when_postgres_publishes_then_contract_rejects_before_write( - test_case: PostgresConditionalPublicationPayloadContractTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="test", - ) - with pytest.raises(StateBackendConfigError, match=test_case.expected_error_fragment or ""): - postgres_state_backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=test_case.payload.record, - refs_by_node_type=test_case.payload.refs_by_node_type, - leases=(), - checkpoint=test_case.payload.checkpoint, - checkpoint_refs=test_case.payload.checkpoint_refs, - checkpoint_function_refs=test_case.payload.checkpoint_function_refs, - checkpoint_seed_refs=test_case.payload.checkpoint_seed_refs, - ) - - assert test_case.expected_valid is False - assert ( - postgres_state_backend.get_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.payload.record.virtual_environment_name, - ) - is None - ) - - -@pytest.mark.parametrize( - "test_case", - [ - MicrobatchBulkAppendTestCase( - description="replay requirement with no completion or observation facts", - record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, - ), - MicrobatchBulkAppendTestCase( - description="executed partition completion with no observation facts", - record_type=MicrobatchRecordType.PARTITION_COMPLETION, - ), - ], - ids=lambda case: case.description, -) -def test_given_all_null_nullable_columns_when_bulk_appending_then_postgres_persists_idempotently( - test_case: MicrobatchBulkAppendTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="test", - ) - scope: MicrobatchScope = MicrobatchScope( - scope_kind="virtual_physical", - scope_key="postgres:state:orders:F2:analytics.orders__F2", - model_name="orders", - target_database="warehouse", - target_schema="analytics", - target_name="orders__F2", - physical_generation_id="F2:analytics.orders__F2", - virtual_environment_name="dev", - virtual_model_version_hash="F2", - ) - event: MicrobatchEvent = build_production_shaped_microbatch_event( - scope=scope, record_type=test_case.record_type - ) - - first: MicrobatchWriteResult = postgres_state_backend.append_microbatch_events( - connection=postgres_state_connection, schema=postgres_state_schema, events=(event,) - ) - second: MicrobatchWriteResult = postgres_state_backend.append_microbatch_events( - connection=postgres_state_connection, schema=postgres_state_schema, events=(event,) - ) - - history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( - connection=postgres_state_connection, schema=postgres_state_schema, scope=scope - ) - assert (first.inserted, second.already_existing) == test_case.expected_write_counts - assert history == (event,) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresMicrobatchStateRoundTripTestCase( - description="postgres events are idempotent", - expected_event_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_microbatch_event_when_appending_then_scope_history_round_trips( - test_case: PostgresMicrobatchStateRoundTripTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="test", - ) - scope: MicrobatchScope = MicrobatchScope( - scope_kind="virtual_physical", - scope_key="postgres:state:orders:F2:analytics.orders__F2", - model_name="orders", - target_database="warehouse", - target_schema="analytics", - target_name="orders__F2", - physical_generation_id="F2:analytics.orders__F2", - virtual_environment_name="dev", - virtual_model_version_hash="F2", - ) - event: MicrobatchEvent = MicrobatchEvent( - event_id="event-1", - record_type=MicrobatchRecordType.PARTITION_COMPLETION, - scope=scope, - origin_run_id="run-1", - execution_run_id="run-1", - run_type=MicrobatchRunType.NORMAL, - completion_type=MicrobatchCompletionType.INITIAL, - run_start="0", - run_end="1", - partition_start="0", - partition_end="1", - batch_size="1", - cursor_column="batch_id", - cursor_type="integer", - model_version_hash="F2", - definition_hash="definition", - fingerprint_status=MicrobatchFingerprintStatus.KNOWN, - rows_affected=0, - created_at=datetime(2026, 1, 1), - ) - - postgres_state_backend.append_microbatch_events( - connection=postgres_state_connection, - schema=postgres_state_schema, - events=(event,), - ) - postgres_state_backend.append_microbatch_events( - connection=postgres_state_connection, - schema=postgres_state_schema, - events=(event,), - ) - - history: tuple[MicrobatchEvent, ...] = postgres_state_backend.read_microbatch_scope_history( - connection=postgres_state_connection, - schema=postgres_state_schema, - scope=scope, - ) - retention_history: tuple[MicrobatchEvent, ...] = ( - postgres_state_backend.read_microbatch_retention_history( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - ) - model_history: tuple[MicrobatchEvent, ...] = ( - postgres_state_backend.read_microbatch_model_history( - connection=postgres_state_connection, - schema=postgres_state_schema, - scope=scope, - ) - ) - assert len(history) == test_case.expected_event_count - assert history == (event,) - assert retention_history == history - assert model_history == history - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendLifecycleTestCase( - description="initializes backs up rolls back and resets state tables", - sqlbuild_version="0.0.test", - expected_schema_version=1, - expected_actions_after_backup=( - StateMigrationAction.INIT.value, - StateMigrationAction.BACKUP.value, - ), - expected_actions_after_rollback=( - StateMigrationAction.INIT.value, - StateMigrationAction.ROLLBACK.value, - ), - expected_backup_actions=(StateMigrationAction.INIT.value,), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_running_lifecycle_then_state_tables_are_managed( - test_case: PostgresStateBackendLifecycleTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - assert fetch_all( - postgres_state_connection, - "SELECT schema_version FROM " - f"{qualified_name(schema=postgres_state_schema, table='state_versions')}", - ) == [(test_case.expected_schema_version,)] - assert postgres_state_backend.inspect_schema( - connection=postgres_state_connection, - schema=postgres_state_schema, - ).valid - - backup_id: str = postgres_state_backend.create_backup( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - backup_schema: str = f"{postgres_state_schema}__backup_{backup_id}" - assert fetch_all( - postgres_state_connection, - "SELECT action FROM " - f"{qualified_name(schema=postgres_state_schema, table='state_migration_events')} " - "ORDER BY created_at", - ) == [(action,) for action in test_case.expected_actions_after_backup] - assert fetch_all( - postgres_state_connection, - "SELECT action FROM " - f"{qualified_name(schema=backup_schema, table='state_migration_events')} " - "ORDER BY created_at", - ) == [(action,) for action in test_case.expected_backup_actions] - - with postgres_state_connection.cursor() as cursor: - cursor.execute( - f"DELETE FROM {qualified_name(schema=postgres_state_schema, table='state_versions')}" - ) - assert ( - fetch_all( - postgres_state_connection, - f"SELECT * FROM {qualified_name(schema=postgres_state_schema, table='state_versions')}", - ) - == [] - ) - - rolled_back_backup_id: str = postgres_state_backend.rollback( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - assert rolled_back_backup_id == backup_id - assert fetch_all( - postgres_state_connection, - "SELECT action FROM " - f"{qualified_name(schema=postgres_state_schema, table='state_migration_events')} " - "ORDER BY created_at", - ) == [(action,) for action in test_case.expected_actions_after_rollback] - - postgres_state_backend.reset(connection=postgres_state_connection, schema=postgres_state_schema) - validation_after_reset: StateSchemaValidationResult = postgres_state_backend.inspect_schema( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - assert not validation_after_reset.valid - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendValidationTestCase( - description="reports invalid manually-created state schema", - expected_issue_count=24, - ) - ], - ids=lambda case: case.description, -) -def test_given_broken_postgres_state_tables_when_validating_then_reports_schema_issues( - test_case: PostgresStateBackendValidationTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - with postgres_state_connection.cursor() as cursor: - cursor.execute(f"CREATE SCHEMA {quote_identifier(postgres_state_schema)}") - cursor.execute( - f"CREATE TABLE {qualified_name(schema=postgres_state_schema, table='state_versions')} (" - "schema_version TEXT, updated_at TIMESTAMP)" - ) - - result: StateSchemaValidationResult = postgres_state_backend.inspect_schema( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - - assert len(result.issues) == test_case.expected_issue_count - assert not result.valid - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendErrorTestCase( - description="blocks rollback when no backup exists", - expected_error_type=Exception, - expected_message_fragment="No state backup is available", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_without_backup_when_rolling_back_then_blocks_cleanly( - test_case: PostgresStateBackendErrorTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="0.0.test", - ) - - with pytest.raises(test_case.expected_error_type) as exc_info: - postgres_state_backend.rollback( - connection=postgres_state_connection, schema=postgres_state_schema - ) - - assert test_case.expected_message_fragment in str(exc_info.value) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendErrorTestCase( - description="blocks backup when state schema is invalid", - expected_error_type=Exception, - expected_message_fragment="Cannot backup invalid state schema", - ) - ], - ids=lambda case: case.description, -) -def test_given_invalid_postgres_state_schema_when_creating_backup_then_blocks_cleanly( - test_case: PostgresStateBackendErrorTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - with postgres_state_connection.cursor() as cursor: - cursor.execute(f"CREATE SCHEMA {quote_identifier(postgres_state_schema)}") - cursor.execute( - f"CREATE TABLE {qualified_name(schema=postgres_state_schema, table='state_versions')} " - "(schema_version INTEGER)" - ) - - with pytest.raises(test_case.expected_error_type) as exc_info: - postgres_state_backend.create_backup( - connection=postgres_state_connection, schema=postgres_state_schema - ) - - assert test_case.expected_message_fragment in str(exc_info.value) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendExplicitRollbackTestCase( - description="rolls back to explicitly selected backup", - sqlbuild_version="0.0.test", - expected_restored_schema_version=1, - expected_index_names=tuple(EXPECTED_STATE_INDEX_NAMES), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backups_when_rolling_back_explicit_id_then_restores_backup( - test_case: PostgresStateBackendExplicitRollbackTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - first_backup_id: str = postgres_state_backend.create_backup( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - with postgres_state_connection.cursor() as cursor: - cursor.execute( - "UPDATE " - f"{qualified_name(schema=postgres_state_schema, table='state_versions')} " - "SET schema_version = 2" - ) - postgres_state_backend.create_backup( - connection=postgres_state_connection, schema=postgres_state_schema - ) - with postgres_state_connection.cursor() as cursor: - cursor.execute( - f"DELETE FROM {qualified_name(schema=postgres_state_schema, table='state_versions')}" - ) - - postgres_state_backend.rollback( - connection=postgres_state_connection, - schema=postgres_state_schema, - backup_id=first_backup_id, - ) - - assert fetch_all( - postgres_state_connection, - "SELECT schema_version FROM " - f"{qualified_name(schema=postgres_state_schema, table='state_versions')}", - ) == [(test_case.expected_restored_schema_version,)] - assert ( - tuple( - row[0] - for row in fetch_all( - postgres_state_connection, - "SELECT indexname FROM pg_indexes " - f"WHERE schemaname = '{postgres_state_schema}' ORDER BY indexname", - ) - ) - == test_case.expected_index_names - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendTableCreationTestCase( - description="initializes all phase two state tables", - sqlbuild_version="0.0.test", - expected_table_names=tuple(sorted(STATE_TABLES)), - expected_index_names=tuple(EXPECTED_STATE_INDEX_NAMES), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_initializing_then_creates_all_state_tables( - test_case: PostgresStateBackendTableCreationTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - - rows: list[tuple[Any, ...]] = fetch_all( - postgres_state_connection, - "SELECT table_name FROM information_schema.tables " - f"WHERE table_schema = '{postgres_state_schema}' ORDER BY table_name", - ) - index_rows: list[tuple[Any, ...]] = fetch_all( - postgres_state_connection, - "SELECT indexname FROM pg_indexes " - f"WHERE schemaname = '{postgres_state_schema}' ORDER BY indexname", - ) - - assert tuple(row[0] for row in rows) == test_case.expected_table_names - assert tuple(row[0] for row in index_rows) == test_case.expected_index_names - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendIndexValidationTestCase( - description="reports missing node results latest index", - sqlbuild_version="0.0.test", - dropped_index_name="idx_sqb_node_results_latest", - expected_issue_kind=StateSchemaValidationIssueKind.MISSING_INDEX.value, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_required_index_is_missing_then_validation_reports_it( - test_case: PostgresStateBackendIndexValidationTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - - with postgres_state_connection.cursor() as cursor: - cursor.execute( - "DROP INDEX " - f"{qualified_name(schema=postgres_state_schema, table=test_case.dropped_index_name)}" - ) - - validation_result: StateSchemaValidationResult = postgres_state_backend.inspect_schema( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - - assert test_case.expected_issue_kind in tuple( - issue.kind.value for issue in validation_result.issues - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendColumnValidationTestCase( - description="reports missing node results payload column", - sqlbuild_version="0.0.test", - dropped_table_name="node_results", - dropped_column_name="payload_json_b64", - expected_issue_kind=StateSchemaValidationIssueKind.MISSING_COLUMN.value, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_node_results_column_missing_then_validation_reports_it( - test_case: PostgresStateBackendColumnValidationTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - - with postgres_state_connection.cursor() as cursor: - cursor.execute( - "ALTER TABLE " - f"{qualified_name(schema=postgres_state_schema, table=test_case.dropped_table_name)} " - f"DROP COLUMN {quote_identifier(test_case.dropped_column_name)}" - ) - - validation_result: StateSchemaValidationResult = postgres_state_backend.inspect_schema( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - - assert test_case.expected_issue_kind in tuple( - issue.kind.value for issue in validation_result.issues - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendCoreRecordsTestCase( - description="persists model physical relation and virtual environment refs", - sqlbuild_version="0.0.test", - expected_model_name="fact_orders", - expected_version_hash="abc123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="fact_orders__v_abc123", - expected_replaced_relation_name="fact_orders__v_abc123_replaced", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_upserting_core_records_then_round_trips_state( - test_case: PostgresStateBackendCoreRecordsTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - assert ( - postgres_state_backend.get_model_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - is None - ) - assert ( - postgres_state_backend.get_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - is None - ) - assert ( - postgres_state_backend.get_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - is None - ) - model_record: ModelVersionRecord = ModelVersionRecord( - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - definition_identity_hash="definition-identity-hash", - identity_metadata_hash="identity-metadata-hash", - status=ModelVersionStatus.READY, - definition_text_b64="U0VMRUNUIDEgQVMgaWQ=", - identity_metadata_json_b64="e30=", - compiled_sql_b64="U0VMRUNUIDEgQVMgaWQ=", - ) - postgres_state_backend.upsert_model_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=model_record, - ) - relation_record: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=test_case.expected_relation_name, - relation_type="table", - ) - postgres_state_backend.upsert_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=relation_record, - ) - replaced_relation_record: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=test_case.expected_replaced_relation_name, - relation_type="table", - ) - postgres_state_backend.upsert_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=replaced_relation_record, - ) - ancestry_record: PhysicalRelationAncestryRecord = PhysicalRelationAncestryRecord( - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - parent_model_name=test_case.expected_model_name, - parent_version_hash="parent123", - seed_strategy="copy", - ) - postgres_state_backend.upsert_physical_relation_ancestry( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=ancestry_record, - ) - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - status=VirtualEnvironmentStatus.FINALIZED, - baseline_virtual_environment_name=None, - ) - postgres_state_backend.upsert_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=virtual_environment_record, - ) - postgres_state_backend.replace_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ), - ), - ) - - assert ( - postgres_state_backend.get_model_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == model_record - ) - assert ( - postgres_state_backend.get_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == replaced_relation_record - ) - assert ( - postgres_state_backend.get_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - == virtual_environment_record - ) - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert len(refs) == test_case.expected_ref_count - assert refs[0].model_name == test_case.expected_model_name - assert refs[0].version_hash == test_case.expected_version_hash - postgres_state_backend.replace_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - refs=(), - ) - replaced_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert len(replaced_refs) == test_case.expected_ref_count_after_replace - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendSourceFreshnessTestCase( - description="persists and replaces source freshness observations", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - expected_source_names=("raw.customers", "raw.orders"), - expected_source_names_after_replace=("raw.orders",), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_backend_when_replacing_source_freshness_then_round_trips_records( - test_case: PostgresStateBackendSourceFreshnessTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - with postgres_state_connection.cursor() as cursor: - cursor.execute("SET TIME ZONE 'Asia/Singapore'") - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - postgres_state_backend.upsert_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=VirtualEnvironmentRecord( - virtual_environment_name=test_case.virtual_environment_name, - status=VirtualEnvironmentStatus.ACTIVE, - ), - ) - first_observed_at: datetime = datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC) - second_observed_at: datetime = datetime(2026, 1, 2, 12, 0, 0, tzinfo=UTC) - postgres_state_backend.replace_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - records=( - SourceFreshnessRecord( - virtual_environment_name=test_case.virtual_environment_name, - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="1", - data_version_hash="hash-1", - observed_at=first_observed_at, - ), - SourceFreshnessRecord( - virtual_environment_name=test_case.virtual_environment_name, - source_name="raw.customers", - strategy="sql", - value_kind="string", - data_version="batch-1", - data_version_hash="hash-2", - observed_at=first_observed_at, - ), - ), - ) - - records: tuple[SourceFreshnessRecord, ...] = ( - postgres_state_backend.get_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert tuple(record.source_name for record in records) == test_case.expected_source_names - - postgres_state_backend.replace_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - records=( - SourceFreshnessRecord( - virtual_environment_name=test_case.virtual_environment_name, - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="2", - data_version_hash="hash-3", - observed_at=second_observed_at, - ), - ), - ) - - replaced_records: tuple[SourceFreshnessRecord, ...] = ( - postgres_state_backend.get_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert tuple(record.source_name for record in replaced_records) == ( - test_case.expected_source_names_after_replace - ) - assert replaced_records[0].data_version == "2" - assert replaced_records[0].data_version_hash == "hash-3" - assert replaced_records[0].observed_at == second_observed_at - with postgres_state_connection.cursor() as cursor: - cursor.execute( - "SELECT observed_at FROM " - f"{qualified_name(schema=postgres_state_schema, table=SOURCE_FRESHNESS_OBSERVATION_TABLE)}" - ) - stored_observed_at: datetime = cursor.fetchone()[0] - assert stored_observed_at == second_observed_at.replace(tzinfo=None) - - postgres_state_backend.delete_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - postgres_state_backend.get_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - == () - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendPythonNodeIdentityTestCase( - description="persists python node versions and separate VDE refs", - sqlbuild_version="0.0.test", - first_virtual_environment_name="dev_alice", - second_virtual_environment_name="dev_bob", - node_type="task", - node_name="prepare_orders", - first_version_hash="version-one", - second_version_hash="version-two", - expected_ref_versions=("version-one", "version-two"), - orphan_version_hash="version-orphan", - expected_pruned_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_upserting_python_node_identity_then_round_trips_refs( - test_case: PostgresStateBackendPythonNodeIdentityTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - first_record: PythonNodeVersionRecord = PythonNodeVersionRecord( - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.first_version_hash, - definition_hash="definition-one", - identity_metadata_hash="metadata-one", - definition_json_b64="e30=", - identity_metadata_json_b64="e30=", - status=ModelVersionStatus.READY, - ) - second_record: PythonNodeVersionRecord = PythonNodeVersionRecord( - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.second_version_hash, - definition_hash="definition-two", - identity_metadata_hash="metadata-two", - definition_json_b64="e30=", - identity_metadata_json_b64="e30=", - status=ModelVersionStatus.READY, - ) - orphan_record: PythonNodeVersionRecord = PythonNodeVersionRecord( - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.orphan_version_hash, - definition_hash="definition-orphan", - identity_metadata_hash="metadata-orphan", - definition_json_b64="e30=", - identity_metadata_json_b64="e30=", - status=ModelVersionStatus.READY, - ) - - postgres_state_backend.upsert_python_node_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=first_record, - ) - postgres_state_backend.upsert_python_node_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=second_record, - ) - postgres_state_backend.upsert_python_node_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=orphan_record, - ) - postgres_state_backend.upsert_virtual_environment_python_node_ref( - connection=postgres_state_connection, - schema=postgres_state_schema, - ref=VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=test_case.first_virtual_environment_name, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.first_version_hash, - ), - ) - postgres_state_backend.upsert_virtual_environment_python_node_ref( - connection=postgres_state_connection, - schema=postgres_state_schema, - ref=VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=test_case.second_virtual_environment_name, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.second_version_hash, - ), - ) - - first_refs: tuple[VirtualEnvironmentPythonNodeRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_python_node_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.first_virtual_environment_name, - ) - ) - second_refs: tuple[VirtualEnvironmentPythonNodeRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_python_node_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.second_virtual_environment_name, - ) - ) - - assert ( - postgres_state_backend.get_python_node_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.first_version_hash, - ) - == first_record - ) - assert tuple(ref.version_hash for ref in (*first_refs, *second_refs)) == ( - test_case.expected_ref_versions - ) - assert ( - postgres_state_backend.count_unreferenced_python_node_versions( - connection=postgres_state_connection, schema=postgres_state_schema - ) - == test_case.expected_pruned_count - ) - assert ( - postgres_state_backend.prune_unreferenced_python_node_versions( - connection=postgres_state_connection, schema=postgres_state_schema - ) - == test_case.expected_pruned_count - ) - assert ( - postgres_state_backend.get_python_node_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.orphan_version_hash, - ) - is None - ) - assert ( - postgres_state_backend.get_python_node_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.second_version_hash, - ) - == second_record - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendNodeResultTestCase( - description="stores and reads VDE node result history by environment", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - isolated_virtual_environment_name="pr_1", - expected_latest_payload={"value": 2}, - expected_failed_status="failed", - expected_history_count=2, - expected_target_isolated_payload={"value": 99}, - expected_rollback_row_count=5, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_node_results_when_reading_then_scopes_by_environment_and_status( - test_case: PostgresStateBackendNodeResultTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - first_record: NodeResultRecord = NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - run_id="run_1", - status=NodeResultStatus.SUCCESS.value, - payload={"value": 1}, - metadata={"source": "first"}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1, 12, 0, 0), - ) - postgres_state_backend.insert_node_result( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - record=first_record, - ) - postgres_state_backend.insert_node_result( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - record=NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - run_id="run_2", - status=NodeResultStatus.SUCCESS.value, - payload=test_case.expected_latest_payload, - metadata={"source": "second"}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1, 12, 1, 0), - ), - ) - postgres_state_backend.insert_node_result( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - record=NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - run_id="run_3", - status=NodeResultStatus.FAILED.value, - payload=None, - metadata={}, - error_message="boom", - materialized=None, - ts=datetime(2026, 1, 1, 12, 2, 0), - ), - ) - postgres_state_backend.insert_node_result( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.isolated_virtual_environment_name, - record=first_record, - ) - postgres_state_backend.insert_node_result( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - record=NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="prod", - target_name=None, - run_id="run_4", - status=NodeResultStatus.SUCCESS.value, - payload=test_case.expected_target_isolated_payload, - metadata={"source": "target"}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1, 12, 3, 0), - ), - ) - backup_id: str = postgres_state_backend.create_backup( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - with postgres_state_connection.cursor() as cursor: - cursor.execute( - f"DELETE FROM {qualified_name(schema=postgres_state_schema, table='node_results')}" - ) - postgres_state_backend.rollback( - connection=postgres_state_connection, - schema=postgres_state_schema, - backup_id=backup_id, - ) - - latest_success: tuple[NodeResultEnvelope, ...] = postgres_state_backend.read_node_results( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=1, - ), - ) - explicit_failed: tuple[NodeResultEnvelope, ...] = postgres_state_backend.read_node_results( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=None, - run_id="run_3", - limit=1, - ), - ) - isolated_results: tuple[NodeResultEnvelope, ...] = postgres_state_backend.read_node_results( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.isolated_virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=5, - ), - ) - history_results: tuple[NodeResultEnvelope, ...] = postgres_state_backend.read_node_results( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=5, - ), - ) - target_isolated_results: tuple[NodeResultEnvelope, ...] = ( - postgres_state_backend.read_node_results( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="prod", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=1, - ), - ) - ) - rollback_rows: list[tuple[object, ...]] = fetch_all( - postgres_state_connection, - "SELECT COUNT(*) FROM " - f"{qualified_name(schema=postgres_state_schema, table='node_results')}", - ) - - assert latest_success[0].payload == test_case.expected_latest_payload - assert explicit_failed[0].status == test_case.expected_failed_status - assert len(isolated_results) == 1 - assert len(history_results) == test_case.expected_history_count - assert target_isolated_results[0].payload == test_case.expected_target_isolated_payload - assert rollback_rows == [(test_case.expected_rollback_row_count,)] - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendErrorTestCase( - description="blocks mismatched source freshness virtual environment", - expected_error_type=Exception, - expected_message_fragment="must match replacement virtual_environment_name", - ) - ], - ids=lambda case: case.description, -) -def test_given_mismatched_postgres_source_freshness_record_when_replacing_then_blocks_cleanly( - test_case: PostgresStateBackendErrorTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="0.0.test", - ) - record: SourceFreshnessRecord = SourceFreshnessRecord( - virtual_environment_name="other", - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="1", - data_version_hash="hash-1", - observed_at=datetime(2026, 1, 1, 12, 0, 0), - ) - with pytest.raises(test_case.expected_error_type) as exc_info: - postgres_state_backend.replace_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - records=(record,), - ) - - assert test_case.expected_message_fragment in str(exc_info.value) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendErrorTestCase( - description="blocks duplicate source freshness records", - expected_error_type=Exception, - expected_message_fragment="Duplicate source freshness record", - ) - ], - ids=lambda case: case.description, -) -def test_given_duplicate_postgres_source_freshness_records_when_replacing_then_blocks_cleanly( - test_case: PostgresStateBackendErrorTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="0.0.test", - ) - records: tuple[SourceFreshnessRecord, ...] = ( - SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="1", - data_version_hash="hash-1", - observed_at=datetime(2026, 1, 1, 12, 0, 0), - ), - SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="sql", - value_kind="string", - data_version="batch-1", - data_version_hash="hash-2", - observed_at=datetime(2026, 1, 1, 12, 0, 0), - ), - ) - with pytest.raises(test_case.expected_error_type) as exc_info: - postgres_state_backend.replace_virtual_environment_source_freshness( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - records=records, - ) - - assert test_case.expected_message_fragment in str(exc_info.value) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendLockTestCase( - description="acquires blocks releases and replaces expired locks", - sqlbuild_version="0.0.test", - lock_key="virtual_env:dev", - first_owner="run-1", - second_owner="run-2", - expected_active_lock_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_managing_locks_then_enforces_active_owner( - test_case: PostgresStateBackendLockTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - future_expiry: datetime = datetime.now() + timedelta(hours=1) - expired_at: datetime = datetime.now() - timedelta(hours=1) - - assert postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=future_expiry, - ) - assert not postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=future_expiry, - ) - active_locks: tuple[StateLockRecord, ...] = postgres_state_backend.list_active_locks( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - assert len(active_locks) == test_case.expected_active_lock_count - assert active_locks[0].owner_id == test_case.first_owner - assert not postgres_state_backend.release_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - ) - assert postgres_state_backend.release_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - ) - assert ( - postgres_state_backend.list_active_locks( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - == () - ) - assert postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=expired_at, - ) - assert postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=future_expiry, - ) - replacement_locks: tuple[StateLockRecord, ...] = postgres_state_backend.list_active_locks( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - assert len(replacement_locks) == test_case.expected_active_lock_count - assert replacement_locks[0].owner_id == test_case.second_owner - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendTransactionRollbackTestCase( - description="rolls back VDE ref replacement when duplicate rows violate unique index", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - model_name="fact_orders", - original_version_hash="abc123", - duplicate_version_hash="def456", - expected_ref_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_ref_replace_fails_then_transaction_rolls_back( - test_case: PostgresStateBackendTransactionRollbackTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - postgres_state_backend.replace_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.original_version_hash, - ), - ), - ) - - with pytest.raises(StateBackendConfigError): - postgres_state_backend.replace_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.original_version_hash, - ), - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.duplicate_version_hash, - ), - ), - ) - - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - postgres_state_backend.get_virtual_environment_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert len(refs) == test_case.expected_ref_count - assert refs[0].version_hash == test_case.original_version_hash - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendCoreRecordsTestCase( - description="preserves created_at across current-state replacements", - sqlbuild_version="0.0.test", - expected_model_name="fact_orders", - expected_version_hash="abc123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="fact_orders__v_abc123", - expected_replaced_relation_name="fact_orders__v_abc123_replaced", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_upserting_same_identity_then_created_at_is_preserved( - test_case: PostgresStateBackendCoreRecordsTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - model_record: ModelVersionRecord = ModelVersionRecord( - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - definition_identity_hash="definition-identity-hash", - identity_metadata_hash="identity-metadata-hash", - status=ModelVersionStatus.READY, - ) - postgres_state_backend.upsert_model_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=model_record, - ) - original_created_at: datetime = fetch_all( - postgres_state_connection, - "SELECT created_at FROM " - f"{qualified_name(schema=postgres_state_schema, table='model_versions')} " - f"WHERE model_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - - postgres_state_backend.upsert_model_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=model_record, - ) - - replaced_created_at: datetime = fetch_all( - postgres_state_connection, - "SELECT created_at FROM " - f"{qualified_name(schema=postgres_state_schema, table='model_versions')} " - f"WHERE model_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - assert ( - postgres_state_backend.get_model_version( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == model_record - ) - assert replaced_created_at == original_created_at - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendCoreRecordsTestCase( - description="preserves created_at for physical relations and virtual environments", - sqlbuild_version="0.0.test", - expected_model_name="fact_orders", - expected_version_hash="abc123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="fact_orders__v_abc123", - expected_replaced_relation_name="fact_orders__v_abc123_replaced", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_replacing_rows_then_preserves_created_at( - test_case: PostgresStateBackendCoreRecordsTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - relation_record: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=test_case.expected_relation_name, - relation_type="table", - ) - postgres_state_backend.upsert_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=relation_record, - ) - physical_created_at: datetime = fetch_all( - postgres_state_connection, - "SELECT created_at FROM " - f"{qualified_name(schema=postgres_state_schema, table='physical_relations')} " - "WHERE artifact_type = 'model' " - f"AND artifact_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - postgres_state_backend.upsert_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=relation_record, - ) - replaced_physical_created_at: datetime = fetch_all( - postgres_state_connection, - "SELECT created_at FROM " - f"{qualified_name(schema=postgres_state_schema, table='physical_relations')} " - "WHERE artifact_type = 'model' " - f"AND artifact_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - status=VirtualEnvironmentStatus.FINALIZED, - baseline_virtual_environment_name=None, - ) - postgres_state_backend.upsert_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=virtual_environment_record, - ) - virtual_created_at: datetime = fetch_all( - postgres_state_connection, - "SELECT created_at FROM " - f"{qualified_name(schema=postgres_state_schema, table='virtual_environments')} " - f"WHERE virtual_environment_name = '{test_case.expected_virtual_environment_name}'", - )[0][0] - postgres_state_backend.upsert_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=virtual_environment_record, - ) - replaced_virtual_created_at: datetime = fetch_all( - postgres_state_connection, - "SELECT created_at FROM " - f"{qualified_name(schema=postgres_state_schema, table='virtual_environments')} " - f"WHERE virtual_environment_name = '{test_case.expected_virtual_environment_name}'", - )[0][0] - - assert ( - postgres_state_backend.get_physical_relation( - connection=postgres_state_connection, - schema=postgres_state_schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == relation_record - ) - assert ( - postgres_state_backend.get_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - == virtual_environment_record - ) - assert replaced_physical_created_at == physical_created_at - assert replaced_virtual_created_at == virtual_created_at - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendOperationEventTestCase( - description="records operation and reconcile events", - sqlbuild_version="0.0.test", - expected_operation_id="op-1", - expected_virtual_environment_name="dev", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_recording_operation_events_then_they_round_trip( - test_case: PostgresStateBackendOperationEventTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - postgres_state_backend.upsert_state_operation( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=StateOperationRecord( - operation_id=test_case.expected_operation_id, - operation_type=StateOperationType.PROMOTE, - status=StateOperationStatus.RUNNING, - virtual_environment_name=test_case.expected_virtual_environment_name, - ), - ) - postgres_state_backend.create_state_operation_event( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=StateOperationEventRecord( - event_id="event-1", - operation_id=test_case.expected_operation_id, - action="start", - status=StateOperationStatus.RUNNING, - message="started", - ), - ) - postgres_state_backend.create_reconcile_event( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=ReconcileEventRecord( - event_id="event-2", - action=ReconcileAction.REPORT, - status=StateOperationStatus.SUCCEEDED, - message="clean", - ), - ) - - assert postgres_state_backend.get_state_operation( - connection=postgres_state_connection, - schema=postgres_state_schema, - operation_id=test_case.expected_operation_id, - ) == StateOperationRecord( - operation_id=test_case.expected_operation_id, - operation_type=StateOperationType.PROMOTE, - status=StateOperationStatus.RUNNING, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - assert fetch_all( - postgres_state_connection, - "SELECT action, status, message FROM " - f"{qualified_name(schema=postgres_state_schema, table='state_operation_events')} " - f"WHERE operation_id = '{test_case.expected_operation_id}'", - ) == [("start", "running", "started")] - assert fetch_all( - postgres_state_connection, - "SELECT action, status, message FROM " - f"{qualified_name(schema=postgres_state_schema, table='reconcile_events')} " - "WHERE event_id = 'event-2'", - ) == [("report", "succeeded", "clean")] - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendConcurrentLockTestCase( - description="allows only one winner across two Postgres connections", - sqlbuild_version="0.0.test", - lock_key="virtual_env:dev", - first_owner="run-1", - second_owner="run-2", - expected_success_count=1, - expected_active_lock_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_two_connections_acquire_same_lock_then_only_one_succeeds( - test_case: PostgresStateBackendConcurrentLockTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_config: dict[str, object], - postgres_state_schema: str, -) -> None: - second_connection: Any = postgres_state_backend.connect(postgres_state_config) - try: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - future_expiry: datetime = datetime.now() + timedelta(hours=1) - - with ThreadPoolExecutor(max_workers=2) as executor: - futures: list[Future[bool]] = [ - executor.submit( - postgres_state_backend.acquire_lock, - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=future_expiry, - ), - executor.submit( - postgres_state_backend.acquire_lock, - connection=second_connection, - schema=postgres_state_schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=future_expiry, - ), - ] - results: list[bool] = [future.result() for future in futures] - - assert sum(results) == test_case.expected_success_count - active_locks: tuple[StateLockRecord, ...] = postgres_state_backend.list_active_locks( - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - assert len(active_locks) == test_case.expected_active_lock_count - assert active_locks[0].owner_id in {test_case.first_owner, test_case.second_owner} - finally: - postgres_state_backend.close(second_connection) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresConditionalVirtualRefPublishTestCase( - description="postgres stale owner cannot publish refs but current owner can", - expected_stale_publish=False, - expected_owned_publish=True, - expected_model_version_hash="version-1", - ), - ], - ids=lambda case: case.description, -) -def test_given_lease_ownership_changes_when_publishing_postgres_refs_then_publish_is_fenced( - test_case: PostgresConditionalVirtualRefPublishTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="0.0.test", - ) - lock_key: str = "model-version:warehouse:orders:version-1" - expires_at: datetime = datetime.now() + timedelta(hours=1) - assert postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=lock_key, - owner_id="owner-a", - expires_at=expires_at, - ) - stale_lease: StateLockLease = StateLockLease( - lock_key=lock_key, - owner_id="owner-a", - expires_at=expires_at, - ) - assert postgres_state_backend.release_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=lock_key, - owner_id="owner-a", - ) - assert postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=lock_key, - owner_id="owner-b", - expires_at=expires_at, - ) - owned_lease: StateLockLease = StateLockLease( - lock_key=lock_key, - owner_id="owner-b", - expires_at=expires_at, - ) - record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name="dev", - status=VirtualEnvironmentStatus.ACTIVE, - ) - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name="dev", - node_type="model", - node_name="orders", - version_hash=test_case.expected_model_version_hash, - ), - ) - - stale_published: bool = postgres_state_backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(stale_lease,), - ) - assert stale_published is test_case.expected_stale_publish - assert ( - postgres_state_backend.get_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - ) - is None - ) - - owned_published: bool = postgres_state_backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(owned_lease,), - ) - assert owned_published is test_case.expected_owned_publish - assert ( - postgres_state_backend.get_virtual_environment_node_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - node_type="model", - ) - == refs - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresAtomicFinalizedVirtualPublishTestCase( - description="postgres checkpoint failure rolls back finalized publication", - checkpoint_id="checkpoint-atomic", - expected_error_fragment="injected checkpoint failure", - expected_checkpoint_count_after_failure=0, - expected_checkpoint_count_after_success=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_checkpoint_failure_when_conditionally_publishing_postgres_then_all_rows_roll_back( - test_case: PostgresAtomicFinalizedVirtualPublishTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, - monkeypatch: pytest.MonkeyPatch, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version="0.0.test", - ) - lock_key: str = "model-version:warehouse:orders:version-1" - expires_at: datetime = datetime.now() + timedelta(hours=1) - assert postgres_state_backend.acquire_lock( - connection=postgres_state_connection, - schema=postgres_state_schema, - lock_key=lock_key, - owner_id="owner", - expires_at=expires_at, - ) - lease: StateLockLease = StateLockLease( - lock_key=lock_key, - owner_id="owner", - expires_at=expires_at, - ) - record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name="dev", - status=VirtualEnvironmentStatus.FINALIZED, - ) - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name="dev", - node_type="model", - node_name="orders", - version_hash="version-1", - ), - ) - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id=test_case.checkpoint_id, - virtual_environment_name="dev", - ) - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id=test_case.checkpoint_id, - model_name="orders", - version_hash="version-1", - ), - ) - original_insert: Callable[..., None] = ( - postgres_state_backend._insert_virtual_environment_checkpoint_rows - ) - - def _insert_then_fail(**kwargs: Any) -> None: - original_insert(**kwargs) - raise RuntimeError(test_case.expected_error_fragment) - - monkeypatch.setattr( - postgres_state_backend, - "_insert_virtual_environment_checkpoint_rows", - _insert_then_fail, - ) - with pytest.raises(RuntimeError, match=test_case.expected_error_fragment): - postgres_state_backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(lease,), - checkpoint=checkpoint, - checkpoint_refs=checkpoint_refs, - ) - assert ( - postgres_state_backend.get_virtual_environment( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - ) - is None - ) - assert ( - postgres_state_backend.get_virtual_environment_node_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - node_type="model", - ) - == () - ) - assert ( - len( - postgres_state_backend.list_virtual_environment_checkpoints( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - ) - ) - == test_case.expected_checkpoint_count_after_failure - ) - - monkeypatch.setattr( - postgres_state_backend, - "_insert_virtual_environment_checkpoint_rows", - original_insert, - ) - assert postgres_state_backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=postgres_state_connection, - schema=postgres_state_schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(lease,), - checkpoint=checkpoint, - checkpoint_refs=checkpoint_refs, - ) - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - postgres_state_backend.list_virtual_environment_checkpoints( - connection=postgres_state_connection, - schema=postgres_state_schema, - virtual_environment_name="dev", - ) - ) - assert len(checkpoints) == test_case.expected_checkpoint_count_after_success - assert ( - postgres_state_backend.get_virtual_environment_checkpoint_model_refs( - connection=postgres_state_connection, - schema=postgres_state_schema, - checkpoint_id=test_case.checkpoint_id, - ) - == checkpoint_refs - ) - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateReadContractTestCase( - description="round trips function versions relations environments and locks", - sqlbuild_version="0.0.test", - expected_environments=( - ("prod", VirtualEnvironmentStatus.FINALIZED), - ("dev", VirtualEnvironmentStatus.ACTIVE), - ), - expected_active_lock_keys=("lock_active",), - expected_expired_lock_keys=("lock_expired",), - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_reading_state_then_matches_read_contract( - test_case: PostgresStateReadContractTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - observation: StateReadContractObservation = exercise_state_read_contract( - backend=postgres_state_backend, - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - - assert observation.function_before_upsert is None - assert observation.function_after_upsert == STATE_READ_CONTRACT_FUNCTION - assert observation.function_after_second_upsert == replace( - STATE_READ_CONTRACT_FUNCTION, status=ModelVersionStatus.FAILED - ) - assert observation.relation_v1 == STATE_READ_CONTRACT_RELATION_V1 - assert observation.relation_missing is None - assert observation.relations == ( - STATE_READ_CONTRACT_RELATION_V2, - STATE_READ_CONTRACT_RELATION_V1, - ) - assert observation.environments == test_case.expected_environments - assert observation.active_lock_keys == test_case.expected_active_lock_keys - assert observation.expired_lock_keys == test_case.expected_expired_lock_keys - - -@pytest.mark.parametrize( - "test_case", - [ - PostgresStateBackendRefContractTestCase( - description="replaces node function and grouped refs per environment and type", - expected_observation=EXPECTED_STATE_REF_CONTRACT_OBSERVATION, - sqlbuild_version="0.0.test", - ) - ], - ids=lambda case: case.description, -) -def test_given_postgres_state_backend_when_replacing_refs_then_matches_ref_contract( - test_case: PostgresStateBackendRefContractTestCase, - postgres_state_backend: PostgresStateBackend, - postgres_state_connection: Any, - postgres_state_schema: str, -) -> None: - postgres_state_backend.initialize( - connection=postgres_state_connection, - schema=postgres_state_schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - observation: StateRefContractObservation = exercise_state_ref_contract( - backend=postgres_state_backend, - connection=postgres_state_connection, - schema=postgres_state_schema, - ) - - assert observation == test_case.expected_observation diff --git a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py b/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py deleted file mode 100644 index 4c8e46ecbb..0000000000 --- a/tests/integration/src/sqlbuild/virtual/state/classes/test_duckdb.py +++ /dev/null @@ -1,2782 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from concurrent.futures import Future, ThreadPoolExecutor -from dataclasses import replace -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -import pytest - -from sqlbuild.cli.commands._helpers.freshness.state import _direct_record_from_virtual_record -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, -) -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessRecord as DirectSourceFreshnessRecord, -) -from sqlbuild.executor.node_results.models import ( - NodeResultEnvelope, - NodeResultQuery, - NodeResultRecord, -) -from sqlbuild.executor.node_results.types import NodeResultStatus -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope, MicrobatchWriteResult -from sqlbuild.microbatches.types import ( - MicrobatchCompletionType, - MicrobatchFingerprintStatus, - MicrobatchRecordType, - MicrobatchRunType, -) -from sqlbuild.virtual.planner._helpers.run_despite_unchanged import ( - _direct_source_freshness_result, -) -from sqlbuild.virtual.state.constants import ( - SOURCE_FRESHNESS_OBSERVATION_TABLE, - STATE_TABLE_INDEXES, - STATE_TABLES, -) -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationAncestryRecord, - PhysicalRelationRecord, - PythonNodeVersionRecord, - ReconcileEventRecord, - SourceFreshnessRecord, - StateLockLease, - StateLockRecord, - StateOperationEventRecord, - StateOperationRecord, - StateSchemaValidationResult, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentPythonNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import ( - ModelVersionStatus, - PhysicalArtifactType, - ReconcileAction, - StateMigrationAction, - StateOperationStatus, - StateOperationType, - StateSchemaValidationIssueKind, - VirtualEnvironmentStatus, -) -from tests.integration.src.sqlbuild.virtual.state.classes._test_types import ( - AtomicFinalizedVirtualPublishTestCase, - ConditionalPublicationPayloadContractTestCase, - ConditionalVirtualRefPublishTestCase, - DuckDbStateBackendAtomicRefUpdateTestCase, - DuckDbStateBackendColumnValidationTestCase, - DuckDbStateBackendConcurrentLockTestCase, - DuckDbStateBackendCoreRecordsTestCase, - DuckDbStateBackendErrorTestCase, - DuckDbStateBackendEventTestCase, - DuckDbStateBackendIdempotencyTestCase, - DuckDbStateBackendIndexValidationTestCase, - DuckDbStateBackendLifecycleTestCase, - DuckDbStateBackendLockTestCase, - DuckDbStateBackendNodeResultTestCase, - DuckDbStateBackendOperationEventTestCase, - DuckDbStateBackendPythonNodeIdentityTestCase, - DuckDbStateBackendRefContractTestCase, - DuckDbStateBackendRollbackTestCase, - DuckDbStateBackendSourceFreshnessTestCase, - DuckDbStateBackendTableCreationTestCase, - DuckDbStateBackendTransactionRollbackTestCase, - DuckDbStateBackendValidationTestCase, - MicrobatchBulkAppendTestCase, - MicrobatchStateRoundTripTestCase, - StateReadContractTestCase, -) -from tests.integration.src.sqlbuild.virtual.state.classes.helpers import ( - ACTIVE_CHECKPOINT_PAYLOAD, - ACTIVE_PAYLOAD, - CHECKPOINT_DUPLICATE_PAYLOAD, - CHECKPOINT_ENVIRONMENT_PAYLOAD, - CHECKPOINT_ID_PAYLOAD, - DETACHED_CHECKPOINT_PAYLOAD, - EXPECTED_STATE_REF_CONTRACT_OBSERVATION, - FAILED_CHECKPOINT_PAYLOAD, - FINALIZING_CHECKPOINT_PAYLOAD, - FUNCTION_OMISSION_PAYLOAD, - MISSING_CHECKPOINT_PAYLOAD, - MODEL_VERSION_PAYLOAD, - PUBLISHED_DUPLICATE_PAYLOAD, - REF_ENVIRONMENT_PAYLOAD, - REF_NODE_TYPE_PAYLOAD, - SEED_EXTRA_PAYLOAD, - STATE_READ_CONTRACT_FUNCTION, - STATE_READ_CONTRACT_RELATION_V1, - STATE_READ_CONTRACT_RELATION_V2, - VALID_PAYLOAD, - StateReadContractObservation, - StateRefContractObservation, - build_production_shaped_microbatch_event, - exercise_state_read_contract, - exercise_state_ref_contract, - fetch_all, - open_duckdb_state_backend, -) - -EXPECTED_STATE_INDEX_NAMES: list[str] = [] -for state_indexes in STATE_TABLE_INDEXES.values(): - for state_index_name in state_indexes: - EXPECTED_STATE_INDEX_NAMES.append(state_index_name) -EXPECTED_STATE_INDEX_NAMES.sort() - - -@pytest.mark.parametrize( - "test_case", - [ - ConditionalPublicationPayloadContractTestCase( - description="duckdb exact finalized payload succeeds", - payload=VALID_PAYLOAD, - expected_valid=True, - ), - ConditionalPublicationPayloadContractTestCase( - description="duckdb active payload without checkpoint succeeds", - payload=ACTIVE_PAYLOAD, - expected_valid=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_valid_conditional_payload_when_duckdb_publishes_then_contract_succeeds( - test_case: ConditionalPublicationPayloadContractTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - backend.initialize(connection=connection, schema="sqb_state", sqlbuild_version="test") - assert ( - backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=connection, - schema="sqb_state", - record=test_case.payload.record, - refs_by_node_type=test_case.payload.refs_by_node_type, - leases=(), - checkpoint=test_case.payload.checkpoint, - checkpoint_refs=test_case.payload.checkpoint_refs, - checkpoint_function_refs=test_case.payload.checkpoint_function_refs, - checkpoint_seed_refs=test_case.payload.checkpoint_seed_refs, - ) - is test_case.expected_valid - ) - - -@pytest.mark.parametrize( - "test_case", - [ - ConditionalPublicationPayloadContractTestCase( - "duckdb finalized requires checkpoint", - MISSING_CHECKPOINT_PAYLOAD, - False, - "requires a checkpoint", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb active forbids complete checkpoint payload", - ACTIVE_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb finalizing forbids complete checkpoint payload", - FINALIZING_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb detached forbids complete checkpoint payload", - DETACHED_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb failed forbids complete checkpoint payload", - FAILED_CHECKPOINT_PAYLOAD, - False, - "requires finalized virtual environment status", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb checkpoint environment matches", - CHECKPOINT_ENVIRONMENT_PAYLOAD, - False, - "match the published environment", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb checkpoint ids match", CHECKPOINT_ID_PAYLOAD, False, "checkpoint_id must match" - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb model refs correspond", - MODEL_VERSION_PAYLOAD, - False, - "model refs must exactly match", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb function refs have no omissions", - FUNCTION_OMISSION_PAYLOAD, - False, - "function refs must exactly match", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb seed refs have no extras", - SEED_EXTRA_PAYLOAD, - False, - "seed refs must exactly match", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb checkpoint refs have no duplicates", - CHECKPOINT_DUPLICATE_PAYLOAD, - False, - "duplicate identities", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb current refs have no duplicates", - PUBLISHED_DUPLICATE_PAYLOAD, - False, - "duplicate identities", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb ref environment matches", - REF_ENVIRONMENT_PAYLOAD, - False, - "virtual_environment_name", - ), - ConditionalPublicationPayloadContractTestCase( - "duckdb ref group matches node type", - REF_NODE_TYPE_PAYLOAD, - False, - "node_type must match", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_conditional_payload_when_duckdb_publishes_then_contract_rejects_before_write( - test_case: ConditionalPublicationPayloadContractTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - backend.initialize(connection=connection, schema="sqb_state", sqlbuild_version="test") - with pytest.raises(StateBackendConfigError, match=test_case.expected_error_fragment or ""): - backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=connection, - schema="sqb_state", - record=test_case.payload.record, - refs_by_node_type=test_case.payload.refs_by_node_type, - leases=(), - checkpoint=test_case.payload.checkpoint, - checkpoint_refs=test_case.payload.checkpoint_refs, - checkpoint_function_refs=test_case.payload.checkpoint_function_refs, - checkpoint_seed_refs=test_case.payload.checkpoint_seed_refs, - ) - - assert test_case.expected_valid is False - assert ( - backend.get_virtual_environment( - connection=connection, - schema="sqb_state", - virtual_environment_name=test_case.payload.record.virtual_environment_name, - ) - is None - ) - - -@pytest.mark.parametrize( - "test_case", - [ - MicrobatchBulkAppendTestCase( - description="replay requirement with no completion or observation facts", - record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, - ), - MicrobatchBulkAppendTestCase( - description="executed partition completion with no observation facts", - record_type=MicrobatchRecordType.PARTITION_COMPLETION, - ), - ], - ids=lambda case: case.description, -) -def test_given_all_null_nullable_columns_when_bulk_appending_then_duckdb_persists_idempotently( - test_case: MicrobatchBulkAppendTestCase, tmp_path: Path -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - scope: MicrobatchScope = MicrobatchScope( - scope_kind="virtual_physical", - scope_key="duckdb:state:orders:F2:main.orders__F2", - model_name="orders", - target_database=None, - target_schema="main", - target_name="orders__F2", - physical_generation_id="F2:main.orders__F2", - virtual_environment_name="dev", - virtual_model_version_hash="F2", - ) - event: MicrobatchEvent = build_production_shaped_microbatch_event( - scope=scope, record_type=test_case.record_type - ) - try: - backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="test") - - first: MicrobatchWriteResult = backend.append_microbatch_events( - connection=connection, schema="sqlbuild_state", events=(event,) - ) - second: MicrobatchWriteResult = backend.append_microbatch_events( - connection=connection, schema="sqlbuild_state", events=(event,) - ) - history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_scope_history( - connection=connection, schema="sqlbuild_state", scope=scope - ) - finally: - backend.close(connection) - - assert (first.inserted, second.already_existing) == test_case.expected_write_counts - assert history == (event,) - - -@pytest.mark.parametrize( - "test_case", - [ - MicrobatchStateRoundTripTestCase( - description="duckdb events are idempotent and preserve concurrent timestamps", - expected_event_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_microbatch_event_when_appending_then_scope_history_round_trips( - test_case: MicrobatchStateRoundTripTestCase, tmp_path: Path -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - scope: MicrobatchScope = MicrobatchScope( - scope_kind="virtual_physical", - scope_key="duckdb:state:orders:F2:main.orders__F2", - model_name="orders", - target_database=None, - target_schema="main", - target_name="orders__F2", - physical_generation_id="F2:main.orders__F2", - virtual_environment_name="dev", - virtual_model_version_hash="F2", - ) - event: MicrobatchEvent = MicrobatchEvent( - event_id="event-1", - record_type=MicrobatchRecordType.PARTITION_COMPLETION, - scope=scope, - origin_run_id="run-1", - execution_run_id="run-1", - run_type=MicrobatchRunType.NORMAL, - completion_type=MicrobatchCompletionType.INITIAL, - run_start="0", - run_end="1", - partition_start="0", - partition_end="1", - batch_size="1", - cursor_column="batch_id", - cursor_type="integer", - model_version_hash="F2", - definition_hash="definition", - fingerprint_status=MicrobatchFingerprintStatus.KNOWN, - rows_affected=0, - created_at=datetime(2026, 1, 1), - ) - try: - backend.initialize(connection=connection, schema="sqlbuild_state", sqlbuild_version="test") - backend.append_microbatch_events( - connection=connection, schema="sqlbuild_state", events=(event,) - ) - second_event: MicrobatchEvent = replace(event, event_id="event-2") - backend.append_microbatch_events( - connection=connection, schema="sqlbuild_state", events=(second_event,) - ) - backend.append_microbatch_events( - connection=connection, schema="sqlbuild_state", events=(event,) - ) - - history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_scope_history( - connection=connection, schema="sqlbuild_state", scope=scope - ) - retention_history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_retention_history( - connection=connection, schema="sqlbuild_state" - ) - model_history: tuple[MicrobatchEvent, ...] = backend.read_microbatch_model_history( - connection=connection, schema="sqlbuild_state", scope=scope - ) - - assert len(history) == test_case.expected_event_count - assert history == (event, second_event) - assert retention_history == history - assert model_history == history - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendLifecycleTestCase( - description="initializes backs up rolls back and resets state tables", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_schema_version=1, - expected_backup_prefix="sqlbuild_state__backup_", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_running_lifecycle_then_state_tables_are_managed( - test_case: DuckDbStateBackendLifecycleTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - version_rows: list[tuple[object, ...]] = fetch_all( - connection, - f"SELECT schema_version, sqlbuild_version FROM {test_case.schema}.state_versions", - ) - assert version_rows == [(test_case.expected_schema_version, test_case.sqlbuild_version)] - assert backend.inspect_schema(connection=connection, schema=test_case.schema).valid - - backup_id: str = backend.create_backup(connection=connection, schema=test_case.schema) - assert backup_id - backup_schemas: list[tuple[object, ...]] = fetch_all( - connection, - "SELECT schema_name FROM information_schema.schemata " - f"WHERE schema_name = '{test_case.expected_backup_prefix}{backup_id}'", - ) - assert backup_schemas == [(f"{test_case.expected_backup_prefix}{backup_id}",)] - - connection.execute(f"DELETE FROM {test_case.schema}.state_versions") - assert fetch_all(connection, f"SELECT * FROM {test_case.schema}.state_versions") == [] - - rolled_back_backup_id: str = backend.rollback( - connection=connection, schema=test_case.schema - ) - assert rolled_back_backup_id == backup_id - assert fetch_all( - connection, - f"SELECT schema_version, sqlbuild_version FROM {test_case.schema}.state_versions", - ) == [(test_case.expected_schema_version, test_case.sqlbuild_version)] - - backend.reset(connection=connection, schema=test_case.schema) - validation_after_reset: StateSchemaValidationResult = backend.inspect_schema( - connection=connection, - schema=test_case.schema, - ) - assert not validation_after_reset.valid - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendNodeResultTestCase( - description="stores and reads VDE node result history by environment", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - isolated_virtual_environment_name="pr_1", - expected_latest_payload={"value": 2}, - expected_failed_status="failed", - expected_history_count=2, - expected_target_isolated_payload={"value": 99}, - expected_rollback_row_count=5, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_node_results_when_reading_then_scopes_by_environment_and_status( - test_case: DuckDbStateBackendNodeResultTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - first_record: NodeResultRecord = NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - run_id="run_1", - status=NodeResultStatus.SUCCESS.value, - payload={"value": 1}, - metadata={"source": "first"}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1, 12, 0, 0), - ) - backend.insert_node_result( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - record=first_record, - ) - backend.insert_node_result( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - record=NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - run_id="run_2", - status=NodeResultStatus.SUCCESS.value, - payload=test_case.expected_latest_payload, - metadata={"source": "second"}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1, 12, 1, 0), - ), - ) - backend.insert_node_result( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - record=NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - run_id="run_3", - status=NodeResultStatus.FAILED.value, - payload=None, - metadata={}, - error_message="boom", - materialized=None, - ts=datetime(2026, 1, 1, 12, 2, 0), - ), - ) - backend.insert_node_result( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.isolated_virtual_environment_name, - record=first_record, - ) - backend.insert_node_result( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - record=NodeResultRecord( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="prod", - target_name=None, - run_id="run_4", - status=NodeResultStatus.SUCCESS.value, - payload=test_case.expected_target_isolated_payload, - metadata={"source": "target"}, - error_message=None, - materialized=None, - ts=datetime(2026, 1, 1, 12, 3, 0), - ), - ) - backup_id: str = backend.create_backup(connection=connection, schema=test_case.schema) - connection.execute(f"DELETE FROM {test_case.schema}.node_results") - backend.rollback(connection=connection, schema=test_case.schema, backup_id=backup_id) - - latest_success: tuple[NodeResultEnvelope, ...] = backend.read_node_results( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=1, - ), - ) - explicit_failed: tuple[NodeResultEnvelope, ...] = backend.read_node_results( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=None, - run_id="run_3", - limit=1, - ), - ) - isolated_results: tuple[NodeResultEnvelope, ...] = backend.read_node_results( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.isolated_virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=5, - ), - ) - history_results: tuple[NodeResultEnvelope, ...] = backend.read_node_results( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="dev", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=5, - ), - ) - target_isolated_results: tuple[NodeResultEnvelope, ...] = backend.read_node_results( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - query=NodeResultQuery( - node_type="task", - node_name="produce_result", - target_database=None, - target_schema="prod", - target_name=None, - statuses=(NodeResultStatus.SUCCESS.value,), - run_id=None, - limit=1, - ), - ) - rollback_rows: list[tuple[object, ...]] = fetch_all( - connection, - f"SELECT COUNT(*) FROM {test_case.schema}.node_results", - ) - - assert latest_success[0].payload == test_case.expected_latest_payload - assert explicit_failed[0].status == test_case.expected_failed_status - assert len(isolated_results) == 1 - assert len(history_results) == test_case.expected_history_count - assert target_isolated_results[0].payload == test_case.expected_target_isolated_payload - assert rollback_rows == [(test_case.expected_rollback_row_count,)] - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendValidationTestCase( - description="reports invalid manually-created state schema", - schema="broken_state", - expected_issue_count=24, - ) - ], - ids=lambda case: case.description, -) -def test_given_broken_duckdb_state_tables_when_validating_then_reports_schema_issues( - test_case: DuckDbStateBackendValidationTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - connection.execute(f"CREATE SCHEMA {test_case.schema}") - connection.execute( - f"CREATE TABLE {test_case.schema}.state_versions (" - "schema_version VARCHAR, updated_at TIMESTAMP)" - ) - - result: StateSchemaValidationResult = backend.inspect_schema( - connection=connection, - schema=test_case.schema, - ) - - assert len(result.issues) == test_case.expected_issue_count - assert not result.valid - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendRollbackTestCase( - description="rolls back to explicitly selected backup", - schema="sqlbuild_state", - sqlbuild_version="0.0.before", - expected_restored_sqlbuild_version="0.0.before", - expected_index_names=tuple(EXPECTED_STATE_INDEX_NAMES), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backups_when_rolling_back_explicit_id_then_restores_backup( - test_case: DuckDbStateBackendRollbackTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - first_backup_id: str = backend.create_backup(connection=connection, schema=test_case.schema) - backend.initialize( - connection=connection, schema=test_case.schema, sqlbuild_version="0.0.after" - ) - backend.create_backup(connection=connection, schema=test_case.schema) - - backend.rollback( - connection=connection, - schema=test_case.schema, - backup_id=first_backup_id, - ) - - assert fetch_all( - connection, - f"SELECT sqlbuild_version FROM {test_case.schema}.state_versions", - ) == [(test_case.expected_restored_sqlbuild_version,)] - assert ( - tuple( - row[0] - for row in fetch_all( - connection, - "SELECT index_name FROM duckdb_indexes() " - f"WHERE schema_name = '{test_case.schema}' ORDER BY index_name", - ) - ) - == test_case.expected_index_names - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendErrorTestCase( - description="blocks rollback when no backup exists", - schema="sqlbuild_state", - expected_error_type=Exception, - expected_message_fragment="No state backup is available", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_without_backup_when_rolling_back_then_blocks_cleanly( - test_case: DuckDbStateBackendErrorTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, schema=test_case.schema, sqlbuild_version="0.0.test" - ) - with pytest.raises(test_case.expected_error_type) as exc_info: - backend.rollback(connection=connection, schema=test_case.schema) - - assert test_case.expected_message_fragment in str(exc_info.value) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendErrorTestCase( - description="blocks backup when state schema is invalid", - schema="broken_state", - expected_error_type=Exception, - expected_message_fragment="Cannot backup invalid state schema", - ) - ], - ids=lambda case: case.description, -) -def test_given_invalid_duckdb_state_schema_when_creating_backup_then_blocks_cleanly( - test_case: DuckDbStateBackendErrorTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - connection.execute(f"CREATE SCHEMA {test_case.schema}") - connection.execute( - f"CREATE TABLE {test_case.schema}.state_versions (schema_version INTEGER)" - ) - with pytest.raises(test_case.expected_error_type) as exc_info: - backend.create_backup(connection=connection, schema=test_case.schema) - - assert test_case.expected_message_fragment in str(exc_info.value) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendEventTestCase( - description="records init backup and rollback events with backed up event contents", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_actions_after_backup=( - StateMigrationAction.INIT.value, - StateMigrationAction.BACKUP.value, - ), - expected_actions_after_rollback=( - StateMigrationAction.INIT.value, - StateMigrationAction.ROLLBACK.value, - ), - expected_backup_actions=(StateMigrationAction.INIT.value,), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_lifecycle_when_events_are_recorded_then_backup_contains_event_table( - test_case: DuckDbStateBackendEventTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - backup_id: str = backend.create_backup(connection=connection, schema=test_case.schema) - backup_schema: str = f"{test_case.schema}__backup_{backup_id}" - - assert ( - tuple( - row[0] - for row in fetch_all( - connection, - "SELECT action FROM " - f"{test_case.schema}.state_migration_events ORDER BY created_at", - ) - ) - == test_case.expected_actions_after_backup - ) - assert ( - tuple( - row[0] - for row in fetch_all( - connection, - "SELECT action FROM " - f"{backup_schema}.state_migration_events ORDER BY created_at", - ) - ) - == test_case.expected_backup_actions - ) - - connection.execute(f"DELETE FROM {test_case.schema}.state_versions") - backend.rollback(connection=connection, schema=test_case.schema, backup_id=backup_id) - - assert ( - tuple( - row[0] - for row in fetch_all( - connection, - "SELECT action FROM " - f"{test_case.schema}.state_migration_events ORDER BY created_at", - ) - ) - == test_case.expected_actions_after_rollback - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendIdempotencyTestCase( - description="initialize can run sequentially without duplicate state version rows", - schema="sqlbuild_state", - first_sqlbuild_version="0.0.first", - second_sqlbuild_version="0.0.second", - expected_schema_version_rows=1, - expected_latest_sqlbuild_version="0.0.second", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_initializing_twice_then_current_version_row_is_idempotent( - test_case: DuckDbStateBackendIdempotencyTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.first_sqlbuild_version, - ) - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.second_sqlbuild_version, - ) - - assert fetch_all( - connection, - f"SELECT COUNT(*), MAX(sqlbuild_version) FROM {test_case.schema}.state_versions", - ) == [ - ( - test_case.expected_schema_version_rows, - test_case.expected_latest_sqlbuild_version, - ) - ] - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendTableCreationTestCase( - description="initializes all phase two state tables", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_table_names=tuple(sorted(STATE_TABLES)), - expected_index_names=tuple(EXPECTED_STATE_INDEX_NAMES), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_initializing_then_creates_all_state_tables( - test_case: DuckDbStateBackendTableCreationTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - - rows: list[tuple[object, ...]] = fetch_all( - connection, - "SELECT table_name FROM information_schema.tables " - f"WHERE table_schema = '{test_case.schema}' ORDER BY table_name", - ) - index_rows: list[tuple[object, ...]] = fetch_all( - connection, - "SELECT index_name FROM duckdb_indexes() " - f"WHERE schema_name = '{test_case.schema}' ORDER BY index_name", - ) - - assert tuple(row[0] for row in rows) == test_case.expected_table_names - assert tuple(row[0] for row in index_rows) == test_case.expected_index_names - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendCoreRecordsTestCase( - description="persists model physical relation and virtual environment refs", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_model_name="fact_orders", - expected_version_hash="abc123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="fact_orders__v_abc123", - expected_replaced_relation_name="fact_orders__v_abc123_replaced", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_upserting_core_records_then_round_trips_state( - test_case: DuckDbStateBackendCoreRecordsTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - assert ( - backend.get_model_version( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - is None - ) - assert ( - backend.get_physical_relation( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - is None - ) - assert ( - backend.get_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - is None - ) - - model_record: ModelVersionRecord = ModelVersionRecord( - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - definition_identity_hash="definition-identity-hash", - identity_metadata_hash="identity-metadata-hash", - status=ModelVersionStatus.READY, - definition_text_b64="U0VMRUNUIDEgQVMgaWQ=", - identity_metadata_json_b64="e30=", - compiled_sql_b64="U0VMRUNUIDEgQVMgaWQ=", - ) - backend.upsert_model_version( - connection=connection, schema=test_case.schema, record=model_record - ) - relation_record: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=test_case.expected_relation_name, - relation_type="table", - ) - backend.upsert_physical_relation( - connection=connection, schema=test_case.schema, record=relation_record - ) - replaced_relation_record: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=test_case.expected_replaced_relation_name, - relation_type="table", - ) - backend.upsert_physical_relation( - connection=connection, schema=test_case.schema, record=replaced_relation_record - ) - ancestry_record: PhysicalRelationAncestryRecord = PhysicalRelationAncestryRecord( - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - parent_model_name=test_case.expected_model_name, - parent_version_hash="parent123", - seed_strategy="copy", - ) - backend.upsert_physical_relation_ancestry( - connection=connection, schema=test_case.schema, record=ancestry_record - ) - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - status=VirtualEnvironmentStatus.FINALIZED, - baseline_virtual_environment_name=None, - ) - backend.upsert_virtual_environment( - connection=connection, - schema=test_case.schema, - record=virtual_environment_record, - ) - backend.replace_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ), - ), - ) - backend.replace_virtual_environment_node_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="seed", - refs=( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="seed", - node_name=test_case.expected_model_name, - version_hash="seed123", - ), - ), - ) - backend.upsert_virtual_environment_python_node_ref( - connection=connection, - schema=test_case.schema, - ref=VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="task", - node_name=test_case.expected_model_name, - version_hash="task123", - ), - ) - - assert ( - backend.get_model_version( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == model_record - ) - assert ( - backend.get_physical_relation( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == replaced_relation_record - ) - assert ( - backend.get_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - == virtual_environment_record - ) - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert len(refs) == test_case.expected_ref_count - assert refs[0].model_name == test_case.expected_model_name - assert refs[0].version_hash == test_case.expected_version_hash - backend.replace_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - refs=(), - ) - seed_refs: tuple[VirtualEnvironmentSeedRefRecord, ...] = ( - backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert seed_refs == ( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - seed_name=test_case.expected_model_name, - version_hash="seed123", - ), - ) - python_node_refs: tuple[VirtualEnvironmentPythonNodeRefRecord, ...] = ( - backend.get_virtual_environment_python_node_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert python_node_refs == ( - VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="task", - node_name=test_case.expected_model_name, - version_hash="task123", - ), - ) - replaced_refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert len(replaced_refs) == test_case.expected_ref_count_after_replace - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendAtomicRefUpdateTestCase( - description="rolls back VDE status and all ref groups when atomic replacement fails", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - model_name="orders", - seed_name="countries", - expected_original_model_hash="model-old", - expected_original_seed_hash="seed-old", - expected_updated_model_hash="model-new", - expected_updated_seed_hash="seed-new", - expected_duplicate_seed_hash="seed-duplicate", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_atomic_vde_ref_update_fails_then_rolls_back_all_groups( - test_case: DuckDbStateBackendAtomicRefUpdateTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - backend.upsert_virtual_environment( - connection=connection, - schema=test_case.schema, - record=VirtualEnvironmentRecord( - virtual_environment_name=test_case.virtual_environment_name, - status=VirtualEnvironmentStatus.FINALIZED, - ), - ) - backend.replace_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.expected_original_model_hash, - ), - ), - ) - backend.replace_virtual_environment_node_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - node_type="seed", - refs=( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - node_type="seed", - node_name=test_case.seed_name, - version_hash=test_case.expected_original_seed_hash, - ), - ), - ) - - with pytest.raises(StateBackendConfigError): - backend.upsert_virtual_environment_and_replace_node_ref_groups( - connection=connection, - schema=test_case.schema, - record=VirtualEnvironmentRecord( - virtual_environment_name=test_case.virtual_environment_name, - status=VirtualEnvironmentStatus.ACTIVE, - ), - refs_by_node_type={ - "model": ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - node_type="model", - node_name=test_case.model_name, - version_hash=test_case.expected_updated_model_hash, - ), - ), - "seed": ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - node_type="seed", - node_name=test_case.seed_name, - version_hash=test_case.expected_updated_seed_hash, - ), - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - node_type="seed", - node_name=test_case.seed_name, - version_hash=test_case.expected_duplicate_seed_hash, - ), - ), - }, - ) - - assert backend.get_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) == VirtualEnvironmentRecord( - virtual_environment_name=test_case.virtual_environment_name, - status=VirtualEnvironmentStatus.FINALIZED, - ) - assert backend.get_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) == ( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.expected_original_model_hash, - ), - ) - assert backend.get_virtual_environment_seed_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) == ( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - seed_name=test_case.seed_name, - version_hash=test_case.expected_original_seed_hash, - ), - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - ConditionalVirtualRefPublishTestCase( - description="stale owner cannot publish refs but current owner can", - expected_stale_publish=False, - expected_owned_publish=True, - expected_model_version_hash="version-1", - ), - ], - ids=lambda case: case.description, -) -def test_given_lease_ownership_changes_when_publishing_duckdb_refs_then_publish_is_fenced( - test_case: ConditionalVirtualRefPublishTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - schema: str = "sqlbuild_state" - lock_key: str = "model-version:warehouse:orders:version-1" - expires_at: datetime = datetime.now() + timedelta(hours=1) - backend.initialize(connection=connection, schema=schema, sqlbuild_version="0.0.test") - assert backend.acquire_lock( - connection=connection, - schema=schema, - lock_key=lock_key, - owner_id="owner-a", - expires_at=expires_at, - ) - stale_lease: StateLockLease = StateLockLease( - lock_key=lock_key, - owner_id="owner-a", - expires_at=expires_at, - ) - assert backend.release_lock( - connection=connection, - schema=schema, - lock_key=lock_key, - owner_id="owner-a", - ) - assert backend.acquire_lock( - connection=connection, - schema=schema, - lock_key=lock_key, - owner_id="owner-b", - expires_at=expires_at, - ) - owned_lease: StateLockLease = StateLockLease( - lock_key=lock_key, - owner_id="owner-b", - expires_at=expires_at, - ) - record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name="dev", - status=VirtualEnvironmentStatus.ACTIVE, - ) - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name="dev", - node_type="model", - node_name="orders", - version_hash=test_case.expected_model_version_hash, - ), - ) - - stale_published: bool = ( - backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=connection, - schema=schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(stale_lease,), - ) - ) - assert stale_published is test_case.expected_stale_publish - assert ( - backend.get_virtual_environment( - connection=connection, - schema=schema, - virtual_environment_name="dev", - ) - is None - ) - - owned_published: bool = ( - backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=connection, - schema=schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(owned_lease,), - ) - ) - assert owned_published is test_case.expected_owned_publish - assert ( - backend.get_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name="dev", - node_type="model", - ) - == refs - ) - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - AtomicFinalizedVirtualPublishTestCase( - description="duckdb checkpoint failure rolls back finalized publication", - checkpoint_id="checkpoint-atomic", - expected_error_fragment="injected checkpoint failure", - expected_checkpoint_count_after_failure=0, - expected_checkpoint_count_after_success=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_checkpoint_failure_when_conditionally_publishing_duckdb_then_all_rows_roll_back( - test_case: AtomicFinalizedVirtualPublishTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - schema: str = "sqlbuild_state" - lock_key: str = "model-version:warehouse:orders:version-1" - expires_at: datetime = datetime.now() + timedelta(hours=1) - backend.initialize(connection=connection, schema=schema, sqlbuild_version="0.0.test") - assert backend.acquire_lock( - connection=connection, - schema=schema, - lock_key=lock_key, - owner_id="owner", - expires_at=expires_at, - ) - lease: StateLockLease = StateLockLease( - lock_key=lock_key, - owner_id="owner", - expires_at=expires_at, - ) - record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name="dev", - status=VirtualEnvironmentStatus.FINALIZED, - ) - refs: tuple[VirtualEnvironmentNodeRefRecord, ...] = ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name="dev", - node_type="model", - node_name="orders", - version_hash="version-1", - ), - ) - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id=test_case.checkpoint_id, - virtual_environment_name="dev", - ) - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id=test_case.checkpoint_id, - model_name="orders", - version_hash="version-1", - ), - ) - original_insert: Callable[..., None] = backend._insert_virtual_environment_checkpoint_rows - - def _insert_then_fail(**kwargs: Any) -> None: - original_insert(**kwargs) - raise RuntimeError(test_case.expected_error_fragment) - - monkeypatch.setattr(backend, "_insert_virtual_environment_checkpoint_rows", _insert_then_fail) - with pytest.raises(RuntimeError, match=test_case.expected_error_fragment): - backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=connection, - schema=schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(lease,), - checkpoint=checkpoint, - checkpoint_refs=checkpoint_refs, - ) - assert ( - backend.get_virtual_environment( - connection=connection, - schema=schema, - virtual_environment_name="dev", - ) - is None - ) - assert ( - backend.get_virtual_environment_node_refs( - connection=connection, - schema=schema, - virtual_environment_name="dev", - node_type="model", - ) - == () - ) - assert ( - len( - backend.list_virtual_environment_checkpoints( - connection=connection, - schema=schema, - virtual_environment_name="dev", - ) - ) - == test_case.expected_checkpoint_count_after_failure - ) - - monkeypatch.setattr(backend, "_insert_virtual_environment_checkpoint_rows", original_insert) - assert backend.upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=connection, - schema=schema, - record=record, - refs_by_node_type={"model": refs}, - leases=(lease,), - checkpoint=checkpoint, - checkpoint_refs=checkpoint_refs, - ) - checkpoints: tuple[VirtualEnvironmentCheckpointRecord, ...] = ( - backend.list_virtual_environment_checkpoints( - connection=connection, - schema=schema, - virtual_environment_name="dev", - ) - ) - assert len(checkpoints) == test_case.expected_checkpoint_count_after_success - assert ( - backend.get_virtual_environment_checkpoint_model_refs( - connection=connection, - schema=schema, - checkpoint_id=test_case.checkpoint_id, - ) - == checkpoint_refs - ) - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendSourceFreshnessTestCase( - description="persists and replaces source freshness observations", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - expected_source_names=("raw.customers", "raw.orders"), - expected_source_names_after_replace=("raw.orders",), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_replacing_source_freshness_then_round_trips_latest_records( - test_case: DuckDbStateBackendSourceFreshnessTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - connection.execute("SET TimeZone = 'Asia/Singapore'") - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - backend.upsert_virtual_environment( - connection=connection, - schema=test_case.schema, - record=VirtualEnvironmentRecord( - virtual_environment_name=test_case.virtual_environment_name, - status=VirtualEnvironmentStatus.ACTIVE, - ), - ) - first_observed_at: datetime = datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC) - second_observed_at: datetime = datetime(2026, 1, 2, 12, 0, 0, tzinfo=UTC) - backend.replace_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - records=( - SourceFreshnessRecord( - virtual_environment_name=test_case.virtual_environment_name, - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="1", - data_version_hash="hash-1", - observed_at=first_observed_at, - ), - SourceFreshnessRecord( - virtual_environment_name=test_case.virtual_environment_name, - source_name="raw.customers", - strategy="sql", - value_kind="string", - data_version="batch-1", - data_version_hash="hash-2", - observed_at=first_observed_at, - ), - ), - ) - - records: tuple[SourceFreshnessRecord, ...] = ( - backend.get_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert tuple(record.source_name for record in records) == test_case.expected_source_names - - backend.replace_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - records=( - SourceFreshnessRecord( - virtual_environment_name=test_case.virtual_environment_name, - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="2", - data_version_hash="hash-3", - observed_at=second_observed_at, - ), - ), - ) - - replaced_records: tuple[SourceFreshnessRecord, ...] = ( - backend.get_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert tuple(record.source_name for record in replaced_records) == ( - test_case.expected_source_names_after_replace - ) - assert replaced_records[0].data_version == "2" - assert replaced_records[0].data_version_hash == "hash-3" - assert replaced_records[0].observed_at == second_observed_at - stored_observed_at: datetime = connection.execute( - f'SELECT observed_at FROM "{test_case.schema}"."{SOURCE_FRESHNESS_OBSERVATION_TABLE}"' - ).fetchone()[0] - assert stored_observed_at == second_observed_at.replace(tzinfo=None) - planning_result: DirectSourceFreshnessPlanningResult = _direct_source_freshness_result( - replaced_records - ) - assert planning_result.observed_records[0].observed_at == second_observed_at - direct_record: DirectSourceFreshnessRecord = _direct_record_from_virtual_record( - replaced_records[0] - ) - assert direct_record.observed_at == second_observed_at - - backend.delete_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - backend.get_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - == () - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendErrorTestCase( - description="blocks mismatched source freshness virtual environment", - schema="sqlbuild_state", - expected_error_type=Exception, - expected_message_fragment="must match replacement virtual_environment_name", - ) - ], - ids=lambda case: case.description, -) -def test_given_mismatched_duckdb_source_freshness_record_when_replacing_then_blocks_cleanly( - test_case: DuckDbStateBackendErrorTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, schema=test_case.schema, sqlbuild_version="0.0.test" - ) - record: SourceFreshnessRecord = SourceFreshnessRecord( - virtual_environment_name="other", - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="1", - data_version_hash="hash-1", - observed_at=datetime(2026, 1, 1, 12, 0, 0), - ) - with pytest.raises(test_case.expected_error_type) as exc_info: - backend.replace_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name="dev", - records=(record,), - ) - - assert test_case.expected_message_fragment in str(exc_info.value) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendErrorTestCase( - description="blocks duplicate source freshness records", - schema="sqlbuild_state", - expected_error_type=Exception, - expected_message_fragment="Duplicate source freshness record", - ) - ], - ids=lambda case: case.description, -) -def test_given_duplicate_duckdb_source_freshness_records_when_replacing_then_blocks_cleanly( - test_case: DuckDbStateBackendErrorTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, schema=test_case.schema, sqlbuild_version="0.0.test" - ) - records: tuple[SourceFreshnessRecord, ...] = ( - SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="1", - data_version_hash="hash-1", - observed_at=datetime(2026, 1, 1, 12, 0, 0), - ), - SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="sql", - value_kind="string", - data_version="batch-1", - data_version_hash="hash-2", - observed_at=datetime(2026, 1, 1, 12, 0, 0), - ), - ) - with pytest.raises(test_case.expected_error_type) as exc_info: - backend.replace_virtual_environment_source_freshness( - connection=connection, - schema=test_case.schema, - virtual_environment_name="dev", - records=records, - ) - - assert test_case.expected_message_fragment in str(exc_info.value) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendPythonNodeIdentityTestCase( - description="persists python node versions and separate VDE refs", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - first_virtual_environment_name="dev_alice", - second_virtual_environment_name="dev_bob", - node_type="task", - node_name="prepare_orders", - first_version_hash="version-one", - second_version_hash="version-two", - expected_ref_versions=("version-one", "version-two"), - orphan_version_hash="version-orphan", - expected_pruned_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_upserting_python_node_identity_then_round_trips_refs( - test_case: DuckDbStateBackendPythonNodeIdentityTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - first_record: PythonNodeVersionRecord = PythonNodeVersionRecord( - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.first_version_hash, - definition_hash="definition-one", - identity_metadata_hash="metadata-one", - definition_json_b64="e30=", - identity_metadata_json_b64="e30=", - status=ModelVersionStatus.READY, - ) - second_record: PythonNodeVersionRecord = PythonNodeVersionRecord( - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.second_version_hash, - definition_hash="definition-two", - identity_metadata_hash="metadata-two", - definition_json_b64="e30=", - identity_metadata_json_b64="e30=", - status=ModelVersionStatus.READY, - ) - orphan_record: PythonNodeVersionRecord = PythonNodeVersionRecord( - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.orphan_version_hash, - definition_hash="definition-orphan", - identity_metadata_hash="metadata-orphan", - definition_json_b64="e30=", - identity_metadata_json_b64="e30=", - status=ModelVersionStatus.READY, - ) - - backend.upsert_python_node_version( - connection=connection, schema=test_case.schema, record=first_record - ) - backend.upsert_python_node_version( - connection=connection, schema=test_case.schema, record=second_record - ) - backend.upsert_python_node_version( - connection=connection, schema=test_case.schema, record=orphan_record - ) - backend.upsert_virtual_environment_python_node_ref( - connection=connection, - schema=test_case.schema, - ref=VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=test_case.first_virtual_environment_name, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.first_version_hash, - ), - ) - backend.upsert_virtual_environment_python_node_ref( - connection=connection, - schema=test_case.schema, - ref=VirtualEnvironmentPythonNodeRefRecord( - virtual_environment_name=test_case.second_virtual_environment_name, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.second_version_hash, - ), - ) - - first_refs: tuple[VirtualEnvironmentPythonNodeRefRecord, ...] = ( - backend.get_virtual_environment_python_node_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.first_virtual_environment_name, - ) - ) - second_refs: tuple[VirtualEnvironmentPythonNodeRefRecord, ...] = ( - backend.get_virtual_environment_python_node_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.second_virtual_environment_name, - ) - ) - - assert ( - backend.get_python_node_version( - connection=connection, - schema=test_case.schema, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.first_version_hash, - ) - == first_record - ) - assert tuple(ref.version_hash for ref in (*first_refs, *second_refs)) == ( - test_case.expected_ref_versions - ) - assert ( - backend.count_unreferenced_python_node_versions( - connection=connection, schema=test_case.schema - ) - == test_case.expected_pruned_count - ) - assert ( - backend.prune_unreferenced_python_node_versions( - connection=connection, schema=test_case.schema - ) - == test_case.expected_pruned_count - ) - assert ( - backend.get_python_node_version( - connection=connection, - schema=test_case.schema, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.orphan_version_hash, - ) - is None - ) - assert ( - backend.get_python_node_version( - connection=connection, - schema=test_case.schema, - node_type=test_case.node_type, - node_name=test_case.node_name, - version_hash=test_case.second_version_hash, - ) - == second_record - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendCoreRecordsTestCase( - description="persists function versions refs and checkpoint refs", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_model_name="is_large_order", - expected_version_hash="function123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="unused", - expected_replaced_relation_name="unused", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_upserting_function_records_then_round_trips_state( - test_case: DuckDbStateBackendCoreRecordsTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - assert ( - backend.get_function_version( - connection=connection, - schema=test_case.schema, - function_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - is None - ) - function_record: FunctionVersionRecord = FunctionVersionRecord( - function_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - language="sql", - returns="BOOLEAN", - arguments_json_b64="W3sibmFtZSI6ImFtb3VudCIsInR5cGUiOiJJTlRFR0VSIn1d", - return_columns_json_b64="W10=", - packages_json_b64="W10=", - runtime_version=None, - entry_point=None, - body_sql_b64="YW1vdW50ID4gOQ==", - definition_text_b64="YW1vdW50ID4gOQ==", - status=ModelVersionStatus.READY, - ) - backend.upsert_function_version( - connection=connection, - schema=test_case.schema, - record=function_record, - ) - assert ( - backend.get_function_version( - connection=connection, - schema=test_case.schema, - function_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == function_record - ) - backend.replace_virtual_environment_node_ref_groups( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - refs_by_node_type={ - "udf": ( - VirtualEnvironmentNodeRefRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - node_type="udf", - node_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ), - ), - "table_fn": (), - }, - ) - function_refs: tuple[VirtualEnvironmentFunctionRefRecord, ...] = ( - backend.get_virtual_environment_function_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - ) - assert len(function_refs) == test_case.expected_ref_count - assert function_refs[0].function_name == test_case.expected_model_name - checkpoint: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id="chk_function", - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - backend.create_virtual_environment_checkpoint( - connection=connection, - schema=test_case.schema, - checkpoint=checkpoint, - refs=(), - function_refs=( - VirtualEnvironmentCheckpointFunctionRefRecord( - checkpoint_id=checkpoint.checkpoint_id, - function_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ), - ), - ) - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = ( - backend.get_virtual_environment_checkpoint_function_refs( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - ) - assert len(checkpoint_function_refs) == test_case.expected_ref_count - backend.delete_virtual_environment_checkpoint( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - assert ( - backend.get_virtual_environment_checkpoint_function_refs( - connection=connection, - schema=test_case.schema, - checkpoint_id=checkpoint.checkpoint_id, - ) - == () - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendLockTestCase( - description="acquires blocks releases and replaces expired locks", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - lock_key="virtual_env:dev", - first_owner="run-1", - second_owner="run-2", - expected_active_lock_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_managing_locks_then_enforces_active_owner( - test_case: DuckDbStateBackendLockTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - future_expiry: datetime = datetime.now() + timedelta(hours=1) - expired_at: datetime = datetime.now() - timedelta(hours=1) - - assert backend.acquire_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=future_expiry, - ) - assert not backend.acquire_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=future_expiry, - ) - renewed_expiry: datetime = future_expiry + timedelta(hours=1) - assert backend.renew_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=renewed_expiry, - ) - assert not backend.renew_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=renewed_expiry, - ) - active_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema=test_case.schema, - ) - assert len(active_locks) == test_case.expected_active_lock_count - assert active_locks[0].owner_id == test_case.first_owner - assert not backend.release_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - ) - assert backend.release_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - ) - assert backend.list_active_locks(connection=connection, schema=test_case.schema) == () - - assert backend.acquire_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=expired_at, - ) - assert backend.acquire_lock( - connection=connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=future_expiry, - ) - replacement_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=connection, - schema=test_case.schema, - ) - assert len(replacement_locks) == test_case.expected_active_lock_count - assert replacement_locks[0].owner_id == test_case.second_owner - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendIndexValidationTestCase( - description="reports missing node results latest index", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - dropped_index_name="idx_sqb_node_results_latest", - expected_issue_kind=StateSchemaValidationIssueKind.MISSING_INDEX.value, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_required_index_is_missing_then_validation_reports_it( - test_case: DuckDbStateBackendIndexValidationTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - - connection.execute(f"DROP INDEX {test_case.schema}.{test_case.dropped_index_name}") - - validation_result: StateSchemaValidationResult = backend.inspect_schema( - connection=connection, - schema=test_case.schema, - ) - - assert test_case.expected_issue_kind in tuple( - issue.kind.value for issue in validation_result.issues - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendColumnValidationTestCase( - description="reports missing node results payload column", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - dropped_table_name="node_results", - dropped_column_name="payload_json_b64", - expected_issue_kind=StateSchemaValidationIssueKind.MISSING_COLUMN.value, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_node_results_column_is_missing_then_validation_reports_it( - test_case: DuckDbStateBackendColumnValidationTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - - index_name: str - for index_name in STATE_TABLE_INDEXES[test_case.dropped_table_name]: - connection.execute(f"DROP INDEX {test_case.schema}.{index_name}") - connection.execute( - f"ALTER TABLE {test_case.schema}.{test_case.dropped_table_name} " - f"DROP COLUMN {test_case.dropped_column_name}" - ) - - validation_result: StateSchemaValidationResult = backend.inspect_schema( - connection=connection, - schema=test_case.schema, - ) - - assert test_case.expected_issue_kind in tuple( - issue.kind.value for issue in validation_result.issues - ) - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendTransactionRollbackTestCase( - description="rolls back VDE ref replacement when duplicate rows violate unique index", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - virtual_environment_name="dev", - model_name="fact_orders", - original_version_hash="abc123", - duplicate_version_hash="def456", - expected_ref_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_ref_replace_fails_then_transaction_rolls_back( - test_case: DuckDbStateBackendTransactionRollbackTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - backend.replace_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.original_version_hash, - ), - ), - ) - - with pytest.raises(StateBackendConfigError): - backend.replace_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.original_version_hash, - ), - VirtualEnvironmentModelRefRecord( - virtual_environment_name=test_case.virtual_environment_name, - model_name=test_case.model_name, - version_hash=test_case.duplicate_version_hash, - ), - ), - ) - - refs: tuple[VirtualEnvironmentModelRefRecord, ...] = ( - backend.get_virtual_environment_model_refs( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.virtual_environment_name, - ) - ) - assert len(refs) == test_case.expected_ref_count - assert refs[0].version_hash == test_case.original_version_hash - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendCoreRecordsTestCase( - description="preserves created_at across current-state replacements", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_model_name="fact_orders", - expected_version_hash="abc123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="fact_orders__v_abc123", - expected_replaced_relation_name="fact_orders__v_abc123_replaced", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_upserting_same_identity_then_created_at_is_preserved( - test_case: DuckDbStateBackendCoreRecordsTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - model_record: ModelVersionRecord = ModelVersionRecord( - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - definition_identity_hash="definition-identity-hash", - identity_metadata_hash="identity-metadata-hash", - status=ModelVersionStatus.READY, - ) - backend.upsert_model_version( - connection=connection, schema=test_case.schema, record=model_record - ) - original_created_at: datetime = fetch_all( - connection, - f"SELECT created_at FROM {test_case.schema}.model_versions " - f"WHERE model_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - - backend.upsert_model_version( - connection=connection, schema=test_case.schema, record=model_record - ) - - replaced_created_at: datetime = fetch_all( - connection, - f"SELECT created_at FROM {test_case.schema}.model_versions " - f"WHERE model_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - assert ( - backend.get_model_version( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == model_record - ) - assert replaced_created_at == original_created_at - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendCoreRecordsTestCase( - description="preserves created_at for physical relations and virtual environments", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_model_name="fact_orders", - expected_version_hash="abc123", - expected_virtual_environment_name="dev", - expected_ref_count=1, - expected_ref_count_after_replace=0, - expected_relation_name="fact_orders__v_abc123", - expected_replaced_relation_name="fact_orders__v_abc123_replaced", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_replacing_rows_then_preserves_created_at( - test_case: DuckDbStateBackendCoreRecordsTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - relation_record: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=test_case.expected_relation_name, - relation_type="table", - ) - backend.upsert_physical_relation( - connection=connection, - schema=test_case.schema, - record=relation_record, - ) - physical_created_at: datetime = fetch_all( - connection, - f"SELECT created_at FROM {test_case.schema}.physical_relations " - "WHERE artifact_type = 'model' " - f"AND artifact_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - backend.upsert_physical_relation( - connection=connection, - schema=test_case.schema, - record=relation_record, - ) - replaced_physical_created_at: datetime = fetch_all( - connection, - f"SELECT created_at FROM {test_case.schema}.physical_relations " - "WHERE artifact_type = 'model' " - f"AND artifact_name = '{test_case.expected_model_name}' " - f"AND version_hash = '{test_case.expected_version_hash}'", - )[0][0] - - virtual_environment_record: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=test_case.expected_virtual_environment_name, - status=VirtualEnvironmentStatus.FINALIZED, - baseline_virtual_environment_name=None, - ) - backend.upsert_virtual_environment( - connection=connection, - schema=test_case.schema, - record=virtual_environment_record, - ) - virtual_created_at: datetime = fetch_all( - connection, - f"SELECT created_at FROM {test_case.schema}.virtual_environments " - f"WHERE virtual_environment_name = '{test_case.expected_virtual_environment_name}'", - )[0][0] - backend.upsert_virtual_environment( - connection=connection, - schema=test_case.schema, - record=virtual_environment_record, - ) - replaced_virtual_created_at: datetime = fetch_all( - connection, - f"SELECT created_at FROM {test_case.schema}.virtual_environments " - f"WHERE virtual_environment_name = '{test_case.expected_virtual_environment_name}'", - )[0][0] - - assert ( - backend.get_physical_relation( - connection=connection, - schema=test_case.schema, - model_name=test_case.expected_model_name, - version_hash=test_case.expected_version_hash, - ) - == relation_record - ) - assert ( - backend.get_virtual_environment( - connection=connection, - schema=test_case.schema, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - == virtual_environment_record - ) - assert replaced_physical_created_at == physical_created_at - assert replaced_virtual_created_at == virtual_created_at - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendOperationEventTestCase( - description="records operation and reconcile events", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_operation_id="op-1", - expected_virtual_environment_name="dev", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_recording_operation_events_then_they_round_trip( - test_case: DuckDbStateBackendOperationEventTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - backend.upsert_state_operation( - connection=connection, - schema=test_case.schema, - record=StateOperationRecord( - operation_id=test_case.expected_operation_id, - operation_type=StateOperationType.PROMOTE, - status=StateOperationStatus.RUNNING, - virtual_environment_name=test_case.expected_virtual_environment_name, - ), - ) - backend.create_state_operation_event( - connection=connection, - schema=test_case.schema, - record=StateOperationEventRecord( - event_id="event-1", - operation_id=test_case.expected_operation_id, - action="start", - status=StateOperationStatus.RUNNING, - message="started", - ), - ) - backend.create_reconcile_event( - connection=connection, - schema=test_case.schema, - record=ReconcileEventRecord( - event_id="event-2", - action=ReconcileAction.REPORT, - status=StateOperationStatus.SUCCEEDED, - message="clean", - ), - ) - - assert backend.get_state_operation( - connection=connection, - schema=test_case.schema, - operation_id=test_case.expected_operation_id, - ) == StateOperationRecord( - operation_id=test_case.expected_operation_id, - operation_type=StateOperationType.PROMOTE, - status=StateOperationStatus.RUNNING, - virtual_environment_name=test_case.expected_virtual_environment_name, - ) - assert fetch_all( - connection, - f"SELECT action, status, message FROM {test_case.schema}.state_operation_events " - f"WHERE operation_id = '{test_case.expected_operation_id}'", - ) == [("start", "running", "started")] - assert fetch_all( - connection, - f"SELECT action, status, message FROM {test_case.schema}.reconcile_events " - "WHERE event_id = 'event-2'", - ) == [("report", "succeeded", "clean")] - finally: - backend.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendConcurrentLockTestCase( - description="allows only one winner across two DuckDB connections", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - lock_key="virtual_env:dev", - first_owner="run-1", - second_owner="run-2", - expected_success_count=1, - expected_active_lock_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_two_connections_acquire_same_lock_then_only_one_succeeds( - test_case: DuckDbStateBackendConcurrentLockTestCase, - tmp_path: Path, -) -> None: - db_path: Path = tmp_path / "state.duckdb" - backend, first_connection = open_duckdb_state_backend(db_path=db_path) - second_backend, second_connection = open_duckdb_state_backend(db_path=db_path) - try: - backend.initialize( - connection=first_connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - future_expiry: datetime = datetime.now() + timedelta(hours=1) - - with ThreadPoolExecutor(max_workers=2) as executor: - futures: list[Future[bool]] = [ - executor.submit( - backend.acquire_lock, - connection=first_connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.first_owner, - expires_at=future_expiry, - ), - executor.submit( - second_backend.acquire_lock, - connection=second_connection, - schema=test_case.schema, - lock_key=test_case.lock_key, - owner_id=test_case.second_owner, - expires_at=future_expiry, - ), - ] - results: list[bool] = [future.result() for future in futures] - - assert sum(results) == test_case.expected_success_count - active_locks: tuple[StateLockRecord, ...] = backend.list_active_locks( - connection=first_connection, - schema=test_case.schema, - ) - assert len(active_locks) == test_case.expected_active_lock_count - assert active_locks[0].owner_id in {test_case.first_owner, test_case.second_owner} - finally: - backend.close(first_connection) - second_backend.close(second_connection) - - -@pytest.mark.parametrize( - "test_case", - [ - StateReadContractTestCase( - description="round trips function versions relations environments and locks", - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - expected_environments=( - ("prod", VirtualEnvironmentStatus.FINALIZED), - ("dev", VirtualEnvironmentStatus.ACTIVE), - ), - expected_active_lock_keys=("lock_active",), - expected_expired_lock_keys=("lock_expired",), - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_reading_state_then_matches_read_contract( - test_case: StateReadContractTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - observation: StateReadContractObservation = exercise_state_read_contract( - backend=backend, connection=connection, schema=test_case.schema - ) - finally: - backend.close(connection) - - assert observation.function_before_upsert is None - assert observation.function_after_upsert == STATE_READ_CONTRACT_FUNCTION - assert observation.function_after_second_upsert == replace( - STATE_READ_CONTRACT_FUNCTION, status=ModelVersionStatus.FAILED - ) - assert observation.relation_v1 == STATE_READ_CONTRACT_RELATION_V1 - assert observation.relation_missing is None - assert observation.relations == ( - STATE_READ_CONTRACT_RELATION_V2, - STATE_READ_CONTRACT_RELATION_V1, - ) - assert observation.environments == test_case.expected_environments - assert observation.active_lock_keys == test_case.expected_active_lock_keys - assert observation.expired_lock_keys == test_case.expected_expired_lock_keys - - -@pytest.mark.parametrize( - "test_case", - [ - DuckDbStateBackendRefContractTestCase( - description="replaces node function and grouped refs per environment and type", - expected_observation=EXPECTED_STATE_REF_CONTRACT_OBSERVATION, - schema="sqlbuild_state", - sqlbuild_version="0.0.test", - ) - ], - ids=lambda case: case.description, -) -def test_given_duckdb_state_backend_when_replacing_refs_then_matches_ref_contract( - test_case: DuckDbStateBackendRefContractTestCase, - tmp_path: Path, -) -> None: - backend, connection = open_duckdb_state_backend(db_path=tmp_path / "state.duckdb") - try: - backend.initialize( - connection=connection, - schema=test_case.schema, - sqlbuild_version=test_case.sqlbuild_version, - ) - observation: StateRefContractObservation = exercise_state_ref_contract( - backend=backend, connection=connection, schema=test_case.schema - ) - finally: - backend.close(connection) - - assert observation == test_case.expected_observation diff --git a/tests/unit/scripts/fensu_policy/rules/test_source_hygiene.py b/tests/unit/scripts/fensu_policy/rules/test_source_hygiene.py index d054ada6df..eb2ac25c9a 100644 --- a/tests/unit/scripts/fensu_policy/rules/test_source_hygiene.py +++ b/tests/unit/scripts/fensu_policy/rules/test_source_hygiene.py @@ -62,16 +62,6 @@ ), expected_fault_count=0, ), - CustomRuleTestCase( - description="source connection terminology passes in virtual source logic", - path="src/sqlbuild/virtual/planner/main/plan.py", - source=( - "def resolve(connection: object) -> object:\n" - " source_connection = connection\n" - " return source_connection\n" - ), - expected_fault_count=0, - ), CustomRuleTestCase( description="source relation terminology passes outside reuse code", path="src/sqlbuild/example/main/build.py", diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py index 9406da96d3..1cb9d68a4e 100644 --- a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/_test_types.py @@ -2,6 +2,14 @@ from dataclasses import dataclass from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter +from sqlbuild.adapter.contract.types import StatementSizeLimit + + +@dataclass(frozen=True) +class StatementSizeLimitTestCase: + description: str + adapter_factory: Callable[[], BaseAdapter] + expected_limit: StatementSizeLimit | None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py new file mode 100644 index 0000000000..a7d666cb56 --- /dev/null +++ b/tests/unit/src/sqlbuild/adapter/contract/classes/base_adapter/test_statement_size.py @@ -0,0 +1,111 @@ +"""Warehouse SQL text limits and their measurement units.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.adapters.bigquery.classes.bigquery_adapter import BigQueryAdapter +from sqlbuild.adapters.databricks.classes.databricks_adapter import DatabricksAdapter +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.adapters.motherduck.classes.motherduck_adapter import MotherDuckAdapter +from sqlbuild.adapters.postgres.classes.postgres_adapter import PostgresAdapter +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.adapters.sqlserver.classes.sqlserver_adapter import SqlServerAdapter +from sqlbuild.compiler.compile.exceptions import CompileInputError +from sqlbuild.executor.testing.main._sql_length import validate_unit_test_sql_length +from tests.unit.src.sqlbuild.adapter.contract.classes.base_adapter._test_types import ( + StatementSizeLimitTestCase, +) +from tests.unit.src.sqlbuild.adapter.contract.classes.base_adapter.helpers import ( + RecordingBaseAdapter, +) + + +@pytest.mark.parametrize( + "test_case", + ( + StatementSizeLimitTestCase("bigquery", BigQueryAdapter, (1_048_576, "bytes")), + StatementSizeLimitTestCase("databricks", DatabricksAdapter, (16_777_216, "bytes")), + StatementSizeLimitTestCase("base", RecordingBaseAdapter, None), + StatementSizeLimitTestCase("duckdb", DuckDbAdapter, None), + StatementSizeLimitTestCase("motherduck", MotherDuckAdapter, None), + StatementSizeLimitTestCase("postgres", PostgresAdapter, None), + StatementSizeLimitTestCase("snowflake", SnowflakeAdapter, None), + StatementSizeLimitTestCase("sqlserver", SqlServerAdapter, None), + ), + ids=lambda case: case.description, +) +def test_given_adapter_when_reading_statement_limit_then_returns_documented_value_and_unit( + test_case: StatementSizeLimitTestCase, +) -> None: + assert test_case.adapter_factory().max_statement_size() == test_case.expected_limit + + +@pytest.mark.parametrize( + "test_case", + ( + StatementSizeLimitTestCase("bigquery", BigQueryAdapter, (1_048_576, "bytes")), + StatementSizeLimitTestCase("databricks", DatabricksAdapter, (16_777_216, "bytes")), + ), + ids=lambda case: case.description, +) +def test_given_multibyte_sql_at_byte_limit_when_validating_then_accepts_exact_boundary( + test_case: StatementSizeLimitTestCase, +) -> None: + assert test_case.expected_limit is not None + limit, _ = test_case.expected_limit + sql: str = "é" * (limit // 2) + validate_unit_test_sql_length( + sql=sql, adapter=test_case.adapter_factory(), test_name="orders", model_name="orders" + ) + + +@pytest.mark.parametrize( + "test_case", + ( + StatementSizeLimitTestCase("bigquery", BigQueryAdapter, (1_048_576, "bytes")), + StatementSizeLimitTestCase("databricks", DatabricksAdapter, (16_777_216, "bytes")), + ), + ids=lambda case: case.description, +) +def test_given_multibyte_sql_over_byte_limit_when_validating_then_reports_actual_bytes( + test_case: StatementSizeLimitTestCase, +) -> None: + assert test_case.expected_limit is not None + limit, _ = test_case.expected_limit + sql: str = "é" * (limit // 2) + "x" + assert len(sql) < limit + with pytest.raises( + CompileInputError, match=f"is {limit + 1} bytes, which exceeds the maximum of {limit} bytes" + ): + validate_unit_test_sql_length( + sql=sql, adapter=test_case.adapter_factory(), test_name="orders", model_name="orders" + ) + + +@pytest.mark.parametrize( + "test_case", + ( + StatementSizeLimitTestCase("base", RecordingBaseAdapter, None), + StatementSizeLimitTestCase("duckdb", DuckDbAdapter, None), + StatementSizeLimitTestCase("motherduck", MotherDuckAdapter, None), + StatementSizeLimitTestCase("postgres", PostgresAdapter, None), + StatementSizeLimitTestCase("snowflake", SnowflakeAdapter, None), + StatementSizeLimitTestCase("sqlserver", SqlServerAdapter, None), + ), + ids=lambda case: case.description, +) +def test_given_sql_over_old_limit_when_no_static_limit_then_does_not_reject( + test_case: StatementSizeLimitTestCase, +) -> None: + assert test_case.adapter_factory().max_statement_size() == test_case.expected_limit + validate_unit_test_sql_length( + sql="é" * 1_100_000, + adapter=test_case.adapter_factory(), + test_name="orders", + model_name="orders", + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/_test_types.py deleted file mode 100644 index c221534595..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/_test_types.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class VirtualCallbackCloseTestCase: - description: str - error_type: type[BaseException] - expected_scheduler_omitted: int - expected_query_omitted: int - expected_final_query_id: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/helpers.py deleted file mode 100644 index 171d8b4370..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/helpers.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from io import StringIO -from typing import cast -from unittest.mock import Mock - -import pytest - -from sqlbuild.cli.commands._helpers.build_virtual import virtual_execution -from sqlbuild.cli.commands.classes.build_progress_callbacks import BuildProgressCallbacks -from sqlbuild.cli.commands.classes.virtual_build_plan_hook import VirtualBuildPlanHook -from sqlbuild.cli.commands.models import VirtualBuildCliRequest, VirtualBuildExecution -from sqlbuild.compiler.planner.models import PlanOutput -from sqlbuild.cost.models import StatementExecutionTelemetry -from sqlbuild.executor.build.models import SchedulerState - - -class _TrackingBuildProgressCallbacks(BuildProgressCallbacks): - def __init__(self) -> None: - super().__init__(plan=PlanOutput(), use_color=False, verbose=True) - self.close_calls: int = 0 - - def close(self) -> None: - self.close_calls += 1 - super().close() - - -def build_high_volume_virtual_hook() -> tuple[ - VirtualBuildPlanHook, _TrackingBuildProgressCallbacks -]: - callbacks: _TrackingBuildProgressCallbacks = _TrackingBuildProgressCallbacks() - hook: VirtualBuildPlanHook = object.__new__(VirtualBuildPlanHook) - hook.callbacks = callbacks - hook._closed = False - return hook, callbacks - - -def emit_high_volume_virtual_diagnostics(hook: VirtualBuildPlanHook) -> None: - callbacks: BuildProgressCallbacks = cast(BuildProgressCallbacks, hook.callbacks) - for index in range(52): - callbacks.on_scheduler_state(SchedulerState(running=1, ready=0, waiting=index, limit=4)) - for index in range(27): - callbacks.on_statement_complete( - StatementExecutionTelemetry( - query_id=f"virtual-query-{index}", - status="success", - elapsed_seconds=0.1, - resource_type="model", - resource_name="orders", - phase="microbatch", - ) - ) - - -def patch_virtual_execution( - *, - monkeypatch: pytest.MonkeyPatch, - hook: VirtualBuildPlanHook, - pipeline: Mock, -) -> None: - monkeypatch.setattr(virtual_execution, "VirtualBuildPlanHook", Mock(return_value=hook)) - monkeypatch.setattr(virtual_execution, "ConnectionProgressReporter", Mock()) - monkeypatch.setattr(virtual_execution, "PlanningProgressReporter", Mock()) - monkeypatch.setattr(virtual_execution, "resolve_target_name", Mock(return_value=None)) - monkeypatch.setattr(virtual_execution, "run_virtual_build_pipeline", pipeline) - - -def execute_patched_virtual_build() -> VirtualBuildExecution: - discovered_inputs: Mock = Mock() - discovered_inputs.project_config.snapshots = None - return virtual_execution.execute_virtual_build( - project_dir=Mock(), - discovered_inputs=discovered_inputs, - adapter=Mock(), - adapter_name="test", - connection_config={}, - request=VirtualBuildCliRequest(verbose=True), - progress_stream=StringIO(), - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/test_virtual_execution_diagnostics.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/test_virtual_execution_diagnostics.py deleted file mode 100644 index e5798f857c..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/build_virtual/test_virtual_execution_diagnostics.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Virtual execution callback finalization tests.""" - -from __future__ import annotations - -from unittest.mock import Mock - -import pytest - -from sqlbuild.cli.commands.classes.virtual_build_plan_hook import VirtualBuildPlanHook -from sqlbuild.cli.commands.models import VirtualBuildExecution -from tests.unit.src.sqlbuild.cli.commands._helpers.build_virtual._test_types import ( - VirtualCallbackCloseTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.build_virtual.helpers import ( - build_high_volume_virtual_hook, - emit_high_volume_virtual_diagnostics, - execute_patched_virtual_build, - patch_virtual_execution, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCallbackCloseTestCase( - description="successful virtual build closes callbacks and emits bounded summaries", - error_type=RuntimeError, - expected_scheduler_omitted=2, - expected_query_omitted=2, - expected_final_query_id="virtual-query-26", - ) - ], - ids=lambda case: case.description, -) -def test_given_high_volume_success_when_virtual_build_finishes_then_callbacks_close_once( - test_case: VirtualCallbackCloseTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - hook, callbacks = build_high_volume_virtual_hook() - result: Mock = Mock() - - def succeed(**_kwargs: object) -> Mock: - emit_high_volume_virtual_diagnostics(hook) - return result - - patch_virtual_execution(monkeypatch=monkeypatch, hook=hook, pipeline=Mock(side_effect=succeed)) - - execution: VirtualBuildExecution = execute_patched_virtual_build() - hook.close() - - output: str = capsys.readouterr().out - assert execution.result is result - assert callbacks.close_calls == 1 - assert f"Scheduler diagnostics {test_case.expected_scheduler_omitted}" in output - assert f"Query diagnostics {test_case.expected_query_omitted}" in output - assert test_case.expected_final_query_id in output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCallbackCloseTestCase( - description="failed virtual build closes callbacks and preserves failure", - error_type=RuntimeError, - expected_scheduler_omitted=2, - expected_query_omitted=2, - expected_final_query_id="virtual-query-26", - ), - VirtualCallbackCloseTestCase( - description="interrupted virtual build closes callbacks and preserves interruption", - error_type=KeyboardInterrupt, - expected_scheduler_omitted=2, - expected_query_omitted=2, - expected_final_query_id="virtual-query-26", - ), - ], - ids=lambda case: case.description, -) -def test_given_high_volume_error_when_virtual_build_unwinds_then_callbacks_close_once( - test_case: VirtualCallbackCloseTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - hook: VirtualBuildPlanHook - hook, callbacks = build_high_volume_virtual_hook() - - def fail(**_kwargs: object) -> None: - emit_high_volume_virtual_diagnostics(hook) - raise test_case.error_type("original virtual outcome") - - patch_virtual_execution(monkeypatch=monkeypatch, hook=hook, pipeline=Mock(side_effect=fail)) - - with pytest.raises(test_case.error_type, match="original virtual outcome"): - execute_patched_virtual_build() - hook.close() - - output: str = capsys.readouterr().out - assert callbacks.close_calls == 1 - assert f"Scheduler diagnostics {test_case.expected_scheduler_omitted}" in output - assert f"Query diagnostics {test_case.expected_query_omitted}" in output - assert test_case.expected_final_query_id in output - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCallbackCloseTestCase( - description="callback close failure is attempted once and does not escape", - error_type=RuntimeError, - expected_scheduler_omitted=0, - expected_query_omitted=0, - expected_final_query_id="", - ) - ], - ids=lambda case: case.description, -) -def test_given_callback_close_failure_when_closing_hook_then_failure_is_non_degrading( - test_case: VirtualCallbackCloseTestCase, -) -> None: - close: Mock = Mock(side_effect=test_case.error_type("close failed")) - callbacks: Mock = Mock(close=close) - hook: VirtualBuildPlanHook = object.__new__(VirtualBuildPlanHook) - hook.callbacks = callbacks - hook._closed = False - - hook.close() - hook.close() - - assert close.call_count == test_case.expected_scheduler_omitted + 1 diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py index ea95a18383..6457e8f576 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/_test_types.py @@ -1,17 +1,6 @@ from dataclasses import dataclass from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult -from sqlbuild.virtual.executor.models import VirtualCloneResult - - -@dataclass(frozen=True) -class RenderVirtualCloneOutputTestCase: - description: str - result: VirtualCloneResult - verbose: bool - expected_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] = () - unexpected_fragments: tuple[str, ...] = () @dataclass(frozen=True) @@ -66,13 +55,3 @@ class CloneFingerprintProgressReporterTestCase: total: int pending_identities: tuple[str, ...] expected_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class VirtualCloneLifecycleCase: - description: str - missing_count: int - skipped_count: int - expected_exit_code: int - expected_progress_fragment: str - unexpected_progress_fragment: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/helpers.py deleted file mode 100644 index 4528772147..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/helpers.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -from sqlbuild.virtual.executor.models import VirtualCloneItemResult, VirtualCloneResult -from sqlbuild.virtual.state.types import PhysicalArtifactType - - -def build_virtual_clone_result(*, missing_count: int, skipped_count: int = 0) -> VirtualCloneResult: - missing_items: tuple[VirtualCloneItemResult, ...] = tuple( - VirtualCloneItemResult( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=f"missing_{index:02d}", - version_hash="hash", - action="missing", - ) - for index in range(missing_count) - ) - skipped_items: tuple[VirtualCloneItemResult, ...] = tuple( - VirtualCloneItemResult( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=f"skipped_{index:02d}", - version_hash="hash", - action="skipped_locked", - ) - for index in range(skipped_count) - ) - return VirtualCloneResult( - mode="workspace expected hashes", - origin_environment="prod", - destination_environment="dev", - item_results=missing_items + skipped_items, - ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_virtual.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_virtual.py deleted file mode 100644 index 3a0827eaaf..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_virtual.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from io import StringIO -from pathlib import Path -from unittest.mock import Mock - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.cli.commands._helpers.clone import virtual -from sqlbuild.cli.commands.models import CloneCommandRequest, CloneInvocation -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - VirtualCloneLifecycleCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.clone.helpers import build_virtual_clone_result - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualCloneLifecycleCase( - "missing result reports errors", - 1, - 0, - 1, - "Clone finished with errors.", - "Cloned virtual environment.", - ), - VirtualCloneLifecycleCase( - "skipped locked result reports success", - 0, - 1, - 0, - "Cloned virtual environment.", - "Clone finished with errors.", - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_clone_result_when_executed_then_exit_and_lifecycle_match( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - test_case: VirtualCloneLifecycleCase, -) -> None: - monkeypatch.setattr( - virtual, - "run_virtual_clone", - lambda **_: build_virtual_clone_result( - missing_count=test_case.missing_count, - skipped_count=test_case.skipped_count, - ), - ) - monkeypatch.setattr(virtual, "resolve_target_connection_config", lambda **_: {}) - monkeypatch.setattr(virtual, "resolve_external_sql_reference_resolver", lambda **_: None) - monkeypatch.setattr(virtual, "render_virtual_clone_output", lambda **_: None) - monkeypatch.setattr(virtual, "write_execution_json_output", lambda **_: None) - stream: StringIO = StringIO() - - exit_code: int = virtual.execute_virtual_clone( - request=CloneCommandRequest( - project_dir=tmp_path, - no_color=True, - no_sql_validation=False, - origin_target_name="prod", - destination_target_name="dev", - hard_copy=False, - skip_locked=test_case.skipped_count > 0, - ), - invocation=CloneInvocation( - effective_project_dir=tmp_path, - discovered_inputs=Mock(), - adapter_name="duckdb", - adapter=DuckDbAdapter(), - destination_target_name="dev", - use_color=False, - progress_stream=stream, - ), - ) - - assert exit_code == test_case.expected_exit_code - assert test_case.expected_progress_fragment in stream.getvalue() - assert test_case.unexpected_progress_fragment not in stream.getvalue() diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_virtual_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_virtual_output.py deleted file mode 100644 index 6a4fe29004..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/clone/test_virtual_output.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.clone.virtual_output import render_virtual_clone_output -from tests.unit.src.sqlbuild.cli.commands._helpers.clone._test_types import ( - RenderVirtualCloneOutputTestCase, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.clone.helpers import ( - build_virtual_clone_result, -) - - -@pytest.mark.parametrize( - "test_case", - ( - RenderVirtualCloneOutputTestCase( - description="caps missing and skipped model sets by default", - result=build_virtual_clone_result(missing_count=22, skipped_count=21), - verbose=False, - expected_fragments=( - "Virtual clone prod -> dev", - "origin state not used", - "destination refs unchanged", - "missing_19", - "skipped_19", - "... 2 more; use --verbose to show all", - "... 1 more; use --verbose to show all", - ), - expected_color_fragments=( - "\033[34m\033[1mVirtual clone\033[0m", - "prod -> dev", - "selected artifacts \033[34m43\033[0m", - "missing in origin \033[33m\033[1m22\033[0m", - "\033[2mmissing\033[0m: missing_00", - "\033[2m ... 2 more; use --verbose to show all\033[0m", - ), - unexpected_fragments=("missing_20", "skipped_20"), - ), - RenderVirtualCloneOutputTestCase( - description="verbose output shows full missing and skipped model sets", - result=build_virtual_clone_result(missing_count=22, skipped_count=21), - verbose=True, - expected_fragments=("missing_21", "skipped_20"), - unexpected_fragments=("use --verbose",), - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_clone_result_when_rendering_then_output_respects_caps( - test_case: RenderVirtualCloneOutputTestCase, - capsys: pytest.CaptureFixture[str], -) -> None: - render_virtual_clone_output(result=test_case.result, use_color=False, verbose=test_case.verbose) - - output: str = capsys.readouterr().out - render_virtual_clone_output(result=test_case.result, use_color=True, verbose=test_case.verbose) - color_output: str = capsys.readouterr().out - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in output, output - for fragment in test_case.unexpected_fragments: - assert fragment not in output, output - for fragment in test_case.expected_color_fragments: - assert fragment in color_output, color_output diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py index 91a2c73400..896b7cdcd0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/_test_types.py @@ -17,22 +17,6 @@ class RenderDiffOutputTestCase: expected_fragments: tuple[str, ...] -@dataclass(frozen=True) -class RenderVirtualDiffHeaderTestCase: - description: str - selected_names: tuple[str, ...] - skipped_names: tuple[str, ...] - from_stale: tuple[str, ...] - to_stale: tuple[str, ...] - from_working: bool - to_working: bool - allow_partial_diff: bool - verbose: bool - expected_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] = () - unexpected_fragments: tuple[str, ...] = () - - @dataclass(frozen=True) class RenderDiffEvidenceTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_virtual_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_virtual_output.py deleted file mode 100644 index abdeaf9bca..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/diff/test_virtual_output.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.diff.virtual_output import format_virtual_diff_header -from sqlbuild.cli.commands.models import VirtualDiffRunOutcome -from sqlbuild.executor.diff.models import DiffExecutionResult -from tests.unit.src.sqlbuild.cli.commands._helpers.diff._test_types import ( - RenderVirtualDiffHeaderTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderVirtualDiffHeaderTestCase( - description="renders compact no color layout with unchanged ref count", - selected_names=("dim_customers", "fact_orders", "stg_orders"), - skipped_names=("dim_customers",), - from_stale=("fact_orders", "stg_orders"), - to_stale=(), - from_working=True, - to_working=False, - allow_partial_diff=True, - verbose=False, - expected_fragments=( - "Virtual diff dev -> pr", - "selected models 3", - "compared models 2", - "unchanged refs skipped 1", - "working VDEs yes (partial allowed)", - ), - expected_color_fragments=( - "\033[34m\033[1mVirtual diff\033[0m", - "dev -> pr", - "selected models \033[34m3\033[0m", - "unchanged refs skipped \033[2m1\033[0m", - "working VDEs \033[33m\033[1myes\033[0m\033[2m (partial allowed)\033[0m", - ), - unexpected_fragments=("not current with workspace",), - ), - RenderVirtualDiffHeaderTestCase( - description="verbose renders workspace staleness details", - selected_names=("dim_customers", "fact_orders", "stg_orders"), - skipped_names=(), - from_stale=("fact_orders",), - to_stale=("stg_orders",), - from_working=False, - to_working=False, - allow_partial_diff=False, - verbose=True, - expected_fragments=( - "compared models 3", - "working VDEs no", - "dev not current with workspace: fact_orders", - "pr not current with workspace: stg_orders", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_virtual_diff_metadata_when_formatting_header_then_expected_details_render( - test_case: RenderVirtualDiffHeaderTestCase, -) -> None: - outcome: VirtualDiffRunOutcome = VirtualDiffRunOutcome( - result=DiffExecutionResult(), - selected_names=test_case.selected_names, - skipped_names=test_case.skipped_names, - from_stale=test_case.from_stale, - to_stale=test_case.to_stale, - from_working=test_case.from_working, - to_working=test_case.to_working, - ) - result: str = format_virtual_diff_header( - from_virtual_environment="dev", - to_virtual_environment="pr", - outcome=outcome, - allow_partial_diff=test_case.allow_partial_diff, - verbose=test_case.verbose, - use_color=False, - ) - color_result: str = format_virtual_diff_header( - from_virtual_environment="dev", - to_virtual_environment="pr", - outcome=outcome, - allow_partial_diff=test_case.allow_partial_diff, - verbose=test_case.verbose, - use_color=True, - ) - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result - for fragment in test_case.unexpected_fragments: - assert fragment not in result - for fragment in test_case.expected_color_fragments: - assert fragment in color_result diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index c761d923a7..1843eca6ed 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -84,14 +84,6 @@ def test_given_raw_query_diff_arguments_when_parsing_then_inputs_remain_distinct expected_argv=("clone", "--from", "prod", "--to", "dev"), ), VerboseCommandTestCase(description="diff", expected_argv=("diff", "prod:dev")), - VerboseCommandTestCase( - description="promote", - expected_argv=("promote", "--from", "preview", "--to", "dev"), - ), - VerboseCommandTestCase( - description="rollback", - expected_argv=("rollback", "--virtual-env", "dev"), - ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py index cc9423a686..6b08277a95 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/_test_types.py @@ -6,7 +6,5 @@ class JanitorCompletionTestCase: description: str archived: int deleted: int - deleted_state_backups: int - deleted_checkpoints: int pruned_direct_state: int expected_output: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py index c364340aa6..cb73832d38 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/janitor_output/test_outputs.py @@ -25,40 +25,23 @@ description="single archive and object use singular nouns", archived=1, deleted=1, - deleted_state_backups=0, - deleted_checkpoints=0, pruned_direct_state=0, expected_output="Archived 1 relation. Deleted 1 object.\n", ), JanitorCompletionTestCase( - description="single state item and pruned table use singular nouns", + description="single pruned state table uses a singular noun", archived=0, deleted=0, - deleted_state_backups=1, - deleted_checkpoints=0, pruned_direct_state=1, - expected_output=( - "Deleted 0 objects, deleted 1 state item, and pruned 1 direct state table.\n" - ), - ), - JanitorCompletionTestCase( - description="single checkpoint uses a singular noun", - archived=2, - deleted=2, - deleted_state_backups=0, - deleted_checkpoints=1, - pruned_direct_state=0, - expected_output="Archived 2 relations. Deleted 2 objects and 1 checkpoint.\n", + expected_output="Deleted 0 objects and pruned 1 direct state table.\n", ), JanitorCompletionTestCase( description="plural counts keep plural nouns", - archived=0, + archived=2, deleted=3, - deleted_state_backups=2, - deleted_checkpoints=0, pruned_direct_state=2, expected_output=( - "Deleted 3 objects, deleted 2 state items, and pruned 2 direct state tables.\n" + "Archived 2 relations. Deleted 3 objects and pruned 2 direct state tables.\n" ), ), ), @@ -70,8 +53,6 @@ def test_given_janitor_counts_when_writing_completion_then_nouns_match_counts( result: JanitorExecutionResult = JanitorExecutionResult( archived=placeholder_candidates(test_case.archived), deleted=placeholder_candidates(test_case.deleted), - deleted_state_backups=placeholder_candidates(test_case.deleted_state_backups), - deleted_checkpoints=placeholder_candidates(test_case.deleted_checkpoints), pruned_direct_state=placeholder_candidates(test_case.pruned_direct_state), ) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/__init__.py deleted file mode 100644 index 0270ec1c97..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Unit tests for promote output helpers.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/_test_types.py deleted file mode 100644 index 33d05b126c..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/_test_types.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class FormatPromoteOutputTestCase: - description: str - status: str - promoted_models: tuple[str, ...] - remaining_stale: tuple[str, ...] - verbose: bool - expected_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] = () - unexpected_fragments: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/test_output.py deleted file mode 100644 index b8beb72af2..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/promote/test_output.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.promote.output import format_promote_output -from tests.unit.src.sqlbuild.cli.commands._helpers.promote._test_types import ( - FormatPromoteOutputTestCase, -) - -PROMOTED_MODELS: tuple[str, ...] = tuple(f"model_{index:02d}" for index in range(25)) - - -@pytest.mark.parametrize( - "test_case", - [ - FormatPromoteOutputTestCase( - description="renders finalized promotion with capped model set", - status="finalized", - promoted_models=PROMOTED_MODELS, - remaining_stale=(), - verbose=False, - expected_fragments=( - "\u2713 Virtual promotion complete pr -> dev", - "target status finalized", - "promoted models 25", - "promoted model set: model_00", - "... 5 more; use --verbose to show all", - "remaining stale models 0", - ), - expected_color_fragments=( - "\033[32m\u2713\033[0m Virtual promotion complete", - "pr -> dev", - "\033[2mtarget status\033[0m \033[32m\033[1mfinalized\033[0m", - "\033[2mpromoted models\033[0m \033[34m25\033[0m", - "\033[2mpromoted model set\033[0m: model_00", - "\033[2m ... 5 more; use --verbose to show all\033[0m", - ), - unexpected_fragments=("model_24",), - ), - FormatPromoteOutputTestCase( - description="verbose renders uncapped model set", - status="working", - promoted_models=PROMOTED_MODELS, - remaining_stale=("orders_rollup",), - verbose=True, - expected_fragments=( - "target status working", - "promoted model set: model_00", - "model_24", - "remaining stale models 1", - "remaining stale set: orders_rollup", - ), - unexpected_fragments=("use --verbose",), - ), - ], - ids=lambda case: case.description, -) -def test_given_promotion_summary_when_formatting_then_output_is_capped_and_status_is_clear( - test_case: FormatPromoteOutputTestCase, -) -> None: - result: str = format_promote_output( - from_virtual_environment="pr", - to_virtual_environment="dev", - status=test_case.status, - promoted_models=test_case.promoted_models, - remaining_stale=test_case.remaining_stale, - verbose=test_case.verbose, - use_color=False, - ) - color_result: str = format_promote_output( - from_virtual_environment="pr", - to_virtual_environment="dev", - status=test_case.status, - promoted_models=test_case.promoted_models, - remaining_stale=test_case.remaining_stale, - verbose=test_case.verbose, - use_color=True, - ) - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result - for fragment in test_case.unexpected_fragments: - assert fragment not in result - for fragment in test_case.expected_color_fragments: - assert fragment in color_result diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/_test_types.py deleted file mode 100644 index 5d6d013303..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/_test_types.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class ReconcileOutputTestCase: - description: str - message: str - expected_text: str - expected_color_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/test_output.py deleted file mode 100644 index d8dde80881..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/reconcile/test_output.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.reconcile.output import format_reconcile_output -from tests.unit.src.sqlbuild.cli.commands._helpers.reconcile._test_types import ( - ReconcileOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - ReconcileOutputTestCase( - description="formats clean reconcile report", - message="Reconcile report for dev: no issues.", - expected_text=("\nVirtual reconcile\n\nReconcile report for dev: no issues.\n"), - expected_color_fragments=( - "\033[34m\033[1mVirtual reconcile\033[0m", - "\033[34mReconcile report for dev: no issues.\033[0m", - ), - ), - ReconcileOutputTestCase( - description="formats repair result rows", - message=( - "Repair\n" - "model fact_orders\n" - "VDE dev\n" - "action recreate logical view from state\n" - "result repaired" - ), - expected_text=( - "\n" - "Virtual reconcile\n" - "\n" - "Repair\n" - "model fact_orders\n" - "VDE dev\n" - "action recreate logical view from state\n" - "result repaired\n" - ), - expected_color_fragments=( - "\033[32mRepair\033[0m", - " \033[2mmodel \033[0m fact_orders", - " \033[2mVDE \033[0m dev", - " \033[2mresult \033[0m \033[32mrepaired\033[0m", - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_reconcile_message_when_formatting_then_returns_expected_output( - test_case: ReconcileOutputTestCase, -) -> None: - text: str = format_reconcile_output(message=test_case.message, use_color=False) - color_text: str = format_reconcile_output(message=test_case.message, use_color=True) - - assert text == test_case.expected_text - expected_fragment: str - for expected_fragment in test_case.expected_color_fragments: - assert expected_fragment in color_text diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/_test_types.py deleted file mode 100644 index 4581abcf6b..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/_test_types.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - - -@dataclass(frozen=True) -class FormatRollbackOutputTestCase: - description: str - status: str - rolled_back_models: tuple[str, ...] - verbose: bool - expected_fragments: tuple[str, ...] - expected_color_fragments: tuple[str, ...] = () - unexpected_fragments: tuple[str, ...] = () diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/test_output.py deleted file mode 100644 index 2d0cf5d595..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/rollback/test_output.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.rollback.output import format_rollback_output -from tests.unit.src.sqlbuild.cli.commands._helpers.rollback._test_types import ( - FormatRollbackOutputTestCase, -) - -ROLLED_BACK_MODELS: tuple[str, ...] = tuple(f"model_{index:02d}" for index in range(25)) - - -@pytest.mark.parametrize( - "test_case", - [ - FormatRollbackOutputTestCase( - description="renders finalized rollback with capped model set", - status="finalized", - rolled_back_models=ROLLED_BACK_MODELS, - verbose=False, - expected_fragments=( - "\u2713 Virtual rollback complete", - "virtual environment dev", - "checkpoint cp_123", - "status finalized", - "rolled back models 25", - "rolled back model set: model_00", - "... 5 more; use --verbose to show all", - ), - expected_color_fragments=( - "\033[32m\u2713\033[0m Virtual rollback complete", - "\033[2mvirtual environment\033[0m dev", - "\033[2mcheckpoint\033[0m cp_123", - "\033[2mstatus\033[0m \033[32m\033[1mfinalized\033[0m", - "\033[2mrolled back models\033[0m \033[34m25\033[0m", - "\033[2mrolled back model set\033[0m: model_00", - ), - unexpected_fragments=("model_24",), - ), - FormatRollbackOutputTestCase( - description="verbose rollback shows uncapped model set", - status="working", - rolled_back_models=ROLLED_BACK_MODELS, - verbose=True, - expected_fragments=("status working", "model_24"), - unexpected_fragments=("use --verbose",), - ), - ], - ids=lambda case: case.description, -) -def test_given_rollback_summary_when_formatting_then_output_is_capped_and_status_is_clear( - test_case: FormatRollbackOutputTestCase, -) -> None: - result: str = format_rollback_output( - virtual_environment="dev", - checkpoint_id="cp_123", - status=test_case.status, - rolled_back_models=test_case.rolled_back_models, - verbose=test_case.verbose, - use_color=False, - ) - color_result: str = format_rollback_output( - virtual_environment="dev", - checkpoint_id="cp_123", - status=test_case.status, - rolled_back_models=test_case.rolled_back_models, - verbose=test_case.verbose, - use_color=True, - ) - - fragment: str - for fragment in test_case.expected_fragments: - assert fragment in result - for fragment in test_case.unexpected_fragments: - assert fragment not in result - for fragment in test_case.expected_color_fragments: - assert fragment in color_result diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py index ca6d7cbef4..f847700a7d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/_test_types.py @@ -1,17 +1,6 @@ from dataclasses import dataclass -@dataclass(frozen=True) -class ModeGuardTestCase: - description: str - virtual_environments: bool - command_name: str - expected_error_fragment: str | None - defer_to: str | None = None - virtual_env: str | None = None - include_stale_upstreams: bool = False - - @dataclass(frozen=True) class ResolveProjectConnectionConfigTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_mode_policy.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_mode_policy.py deleted file mode 100644 index 46163c18b4..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_mode_policy.py +++ /dev/null @@ -1,140 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.cli.commands._helpers.runtime.mode_policy import ( - enforce_no_defer_to_in_virtual_mode, - enforce_virtual_only_flags_in_virtual_mode, -) -from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SettingsConfig -from tests.unit.src.sqlbuild.cli.commands._helpers.runtime._test_types import ModeGuardTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - ModeGuardTestCase( - description="blocks defer-to in virtual mode plan", - virtual_environments=True, - command_name="plan", - defer_to="prod", - expected_error_fragment=( - "plan does not support --defer-to when virtual_environments = true" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_mode_defer_to_when_enforcing_flag_support_then_raises_cli_user_error( - test_case: ModeGuardTestCase, -) -> None: - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=test_case.virtual_environments), - ), - local_config=LocalConfig(), - ) - - with pytest.raises(CliUserError) as exc_info: - enforce_no_defer_to_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name=test_case.command_name, - defer_to=test_case.defer_to, - ) - - assert test_case.expected_error_fragment is not None - assert test_case.expected_error_fragment in exc_info.value.message - - -@pytest.mark.parametrize( - "test_case", - [ - ModeGuardTestCase( - description="blocks --virtual-env on direct-mode plan", - virtual_environments=False, - command_name="plan", - virtual_env="pr_123", - expected_error_fragment=( - "plan does not support --virtual-env unless virtual_environments = true" - ), - ), - ModeGuardTestCase( - description="blocks --include-stale-upstreams on direct-mode build", - virtual_environments=False, - command_name="build", - include_stale_upstreams=True, - expected_error_fragment=( - "build does not support --include-stale-upstreams unless " - "virtual_environments = true" - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_direct_mode_virtual_only_flags_when_enforcing_then_raises_cli_user_error( - test_case: ModeGuardTestCase, -) -> None: - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=test_case.virtual_environments), - ), - local_config=LocalConfig(), - ) - - with pytest.raises(CliUserError) as exc_info: - enforce_virtual_only_flags_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name=test_case.command_name, - virtual_env=test_case.virtual_env, - include_stale_upstreams=test_case.include_stale_upstreams, - ) - - assert test_case.expected_error_fragment is not None - assert test_case.expected_error_fragment in exc_info.value.message - - -@pytest.mark.parametrize( - "test_case", - [ - ModeGuardTestCase( - description="allows both flags in virtual mode", - virtual_environments=True, - command_name="plan", - virtual_env="pr_123", - include_stale_upstreams=True, - expected_error_fragment=None, - ), - ModeGuardTestCase( - description="allows direct-mode plan without virtual-only flags", - virtual_environments=False, - command_name="plan", - expected_error_fragment=None, - ), - ], - ids=lambda case: case.description, -) -def test_given_supported_virtual_flag_usage_when_enforcing_then_allows_execution( - test_case: ModeGuardTestCase, -) -> None: - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=test_case.virtual_environments), - ), - local_config=LocalConfig(), - ) - - enforce_virtual_only_flags_in_virtual_mode( - discovered_inputs=discovered_inputs, - command_name=test_case.command_name, - virtual_env=test_case.virtual_env, - include_stale_upstreams=test_case.include_stale_upstreams, - ) - assert test_case.expected_error_fragment is None diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py index 651d4d3387..0108a2e288 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/_test_types.py @@ -1,6 +1,32 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class NamespaceCase: + description: str + cli: str | None = None + environment: tuple[tuple[str, str], ...] = () + local: str | None = None + project: str | None = None + expected_value: str | None = None + expected_source: str = "unset" + + +@dataclass(frozen=True) +class InvalidNamespaceCase: + description: str + value: object + expected_error: str = "1–128" + + +@dataclass(frozen=True) +class InvalidNamespaceConfigCase: + description: str + filename: str + value_toml: str + expected_error: str = "scenario.run_namespace must be 1–128" + + @dataclass(frozen=True) class SelectScenariosTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py index c73a2be3c8..0726159e65 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/helpers.py @@ -12,7 +12,8 @@ CompiledSqlScenario, ) from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.discovery.models import DiscoveredSqlScenarioFile +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlScenarioFile +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig class ScenarioProgressStream(StringIO): @@ -30,8 +31,10 @@ def configure_scenario_runner( stream: ScenarioProgressStream, compile_project: Any, ) -> None: - discovered_inputs: Mock = Mock() - discovered_inputs.project_config.name = "project" + discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig(name="project", adapter="duckdb"), + local_config=LocalConfig(), + ) monkeypatch.setattr(runner.sys, "stdout", stream) monkeypatch.setattr(runner, "supports_color", lambda: False) monkeypatch.setattr(runner, "discover_project_inputs", lambda **_: discovered_inputs) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py new file mode 100644 index 0000000000..d88a23fb5c --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/scenario_execution/test_namespace.py @@ -0,0 +1,191 @@ +"""Namespace precedence and rejection before execution.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.commands._helpers.scenario_execution.namespace import resolve_scenario_namespace +from sqlbuild.cli.commands.exceptions import CliUserError +from sqlbuild.compiler.discovery.exceptions import ProjectConfigError +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.spec.contracts.exceptions import SpecConfigError +from sqlbuild.spec.contracts.main.parse_scenario_run_namespace import parse_scenario_run_namespace +from sqlbuild.spec.contracts.main.resolve_effective_scenario_config import ( + resolve_effective_scenario_config, +) +from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, ScenarioConfig +from tests.unit.src.sqlbuild.cli.commands._helpers.scenario_execution._test_types import ( + InvalidNamespaceCase, + InvalidNamespaceConfigCase, + NamespaceCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + NamespaceCase( + "CLI wins", + cli="job-1", + environment=(("SQLBUILD_SCENARIO_NAMESPACE", "job-2"),), + local="local", + project="project", + expected_value="job-1", + expected_source="cli", + ), + NamespaceCase( + "environment wins", + environment=(("SQLBUILD_SCENARIO_NAMESPACE", "job-2"),), + local="local", + project="project", + expected_value="job-2", + expected_source="env", + ), + NamespaceCase( + "local wins", + local="local", + project="project", + expected_value="local", + expected_source="local config", + ), + NamespaceCase( + "project fallback", + project="project", + expected_value="project", + expected_source="project config", + ), + NamespaceCase("unset preserves default"), + NamespaceCase( + "CLI overrides invalid environment", + cli="job-1", + environment=(("SQLBUILD_SCENARIO_NAMESPACE", ""),), + expected_value="job-1", + expected_source="cli", + ), + ), + ids=lambda case: case.description, +) +def test_given_namespace_sources_when_resolving_then_highest_precedence_wins( + test_case: NamespaceCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("SQLBUILD_SCENARIO_NAMESPACE", raising=False) + for key, value in test_case.environment: + monkeypatch.setenv(key, value) + inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig( + name="orders", + adapter="duckdb", + scenario=ScenarioConfig(run_namespace=test_case.project), + ), + local_config=LocalConfig(scenario=ScenarioConfig(run_namespace=test_case.local)), + ) + resolved, namespace = resolve_scenario_namespace(inputs=inputs, cli_value=test_case.cli) + assert namespace.value == test_case.expected_value + assert namespace.source == test_case.expected_source + assert ( + resolve_effective_scenario_config( + project_config=resolved.project_config, local_config=resolved.local_config + ).run_namespace + == test_case.expected_value + ) + + +@pytest.mark.parametrize( + "test_case", + ( + InvalidNamespaceCase("empty", ""), + InvalidNamespaceCase("whitespace", " "), + InvalidNamespaceCase("slash", "job/1"), + InvalidNamespaceCase("colon", "job:1"), + InvalidNamespaceCase("newline", "job\n"), + InvalidNamespaceCase("non ASCII", "é"), + InvalidNamespaceCase("too long", "x" * 129), + InvalidNamespaceCase("numeric", 42), + InvalidNamespaceCase("explicit null", None), + ), + ids=lambda case: case.description, +) +def test_given_invalid_namespace_when_parsing_then_rejects(test_case: InvalidNamespaceCase) -> None: + with pytest.raises(SpecConfigError, match=test_case.expected_error): + parse_scenario_run_namespace(test_case.value) + + +@pytest.mark.parametrize( + "test_case", + ( + NamespaceCase("all allowed characters", cli="AZaz09-_.", expected_value="AZaz09-_."), + NamespaceCase("maximum length", cli="x" * 128, expected_value="x" * 128), + ), + ids=lambda case: case.description, +) +def test_given_valid_namespace_when_parsing_then_preserves_value(test_case: NamespaceCase) -> None: + assert parse_scenario_run_namespace(test_case.cli) == test_case.expected_value + + +@pytest.mark.parametrize( + "test_case", + (InvalidNamespaceCase("empty environment does not fall back", ""),), + ids=lambda case: case.description, +) +def test_given_empty_environment_when_resolving_then_does_not_fall_back( + test_case: InvalidNamespaceCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("SQLBUILD_SCENARIO_NAMESPACE", str(test_case.value)) + inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( + project_config=ProjectConfig( + name="orders", adapter="duckdb", scenario=ScenarioConfig(run_namespace="project") + ), + local_config=LocalConfig(), + ) + with pytest.raises(CliUserError, match=test_case.expected_error): + resolve_scenario_namespace(inputs=inputs, cli_value=None) + + +@pytest.mark.parametrize( + "test_case", + (NamespaceCase("both config files", project="project", local="local", expected_value="local"),), + ids=lambda case: case.description, +) +def test_given_project_and_local_config_when_discovering_then_loads_namespace( + test_case: NamespaceCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n[scenario]\nrun_namespace = "{test_case.project}"\n' + ) + (tmp_path / "sqlbuild_local.toml").write_text( + f'[scenario]\nrun_namespace = "{test_case.local}"\n' + ) + inputs: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + assert ( + resolve_effective_scenario_config( + project_config=inputs.project_config, local_config=inputs.local_config + ).run_namespace + == test_case.expected_value + ) + + +@pytest.mark.parametrize( + "test_case", + ( + InvalidNamespaceConfigCase("project empty", "sqlbuild_project.toml", '""'), + InvalidNamespaceConfigCase("local invalid characters", "sqlbuild_local.toml", '"job/1"'), + InvalidNamespaceConfigCase("project numeric", "sqlbuild_project.toml", "42"), + InvalidNamespaceConfigCase("local boolean", "sqlbuild_local.toml", "true"), + ), + ids=lambda case: case.description, +) +def test_given_invalid_config_namespace_when_discovering_then_schema_validation_rejects( + test_case: InvalidNamespaceConfigCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text('name = "orders"\nadapter = "duckdb"\n') + with (tmp_path / test_case.filename).open("a") as config: + config.write(f"\n[scenario]\nrun_namespace = {test_case.value_toml}\n") + with pytest.raises(ProjectConfigError, match=test_case.expected_error): + discover_project_inputs(project_dir=tmp_path) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/state/__init__.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/state/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/state/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/state/_test_types.py deleted file mode 100644 index ee6d943e32..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/state/_test_types.py +++ /dev/null @@ -1,13 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class CheckpointOutputTestCase: - description: str - expected_rendered: str - - -@dataclass(frozen=True) -class CheckpointColorOutputTestCase: - description: str - expected_fragments: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/state/test_checkpoints.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/state/test_checkpoints.py deleted file mode 100644 index a807c8df59..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/state/test_checkpoints.py +++ /dev/null @@ -1,218 +0,0 @@ -from __future__ import annotations - -from datetime import datetime - -import pytest - -from sqlbuild.cli.commands._helpers.state.checkpoints import ( - _format_checkpoint_diff, - _format_checkpoint_list, - _format_checkpoint_show, -) -from sqlbuild.presentation.classes.cli_style import CliStyle -from sqlbuild.virtual.state.models import ( - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) -from tests.unit.src.sqlbuild.cli.commands._helpers.state._test_types import ( - CheckpointColorOutputTestCase, - CheckpointOutputTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - CheckpointOutputTestCase( - description="preserves no-color checkpoint list output", - expected_rendered="\nVirtual environment checkpoints dev\n\n" - " cp_1 2026-05-29 12:00:00\n" - " cp_2 unknown\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_checkpoints_when_formatting_list_without_color_then_preserves_output( - test_case: CheckpointOutputTestCase, -) -> None: - rendered: str = _format_checkpoint_list( - virtual_environment_name="dev", - checkpoints=( - VirtualEnvironmentCheckpointRecord( - checkpoint_id="cp_1", - virtual_environment_name="dev", - created_at=datetime(2026, 5, 29, 12, 0, 0), - ), - VirtualEnvironmentCheckpointRecord( - checkpoint_id="cp_2", - virtual_environment_name="dev", - created_at=None, - ), - ), - style=CliStyle(use_color=False), - ) - - assert rendered == test_case.expected_rendered - - -@pytest.mark.parametrize( - "test_case", - [ - CheckpointOutputTestCase( - description="preserves no-color checkpoint show output", - expected_rendered="\nVirtual environment checkpoint\n\n" - " checkpoint cp_1\n\n" - "Refs\n" - " model_a hash_a\n\n" - "Seed refs\n" - " seed_a seed_hash_a\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_checkpoint_model_refs_when_formatting_show_without_color_then_preserves_output( - test_case: CheckpointOutputTestCase, -) -> None: - rendered: str = _format_checkpoint_show( - checkpoint_id="cp_1", - refs=( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id="cp_1", model_name="model_a", version_hash="hash_a" - ), - ), - seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id="cp_1", seed_name="seed_a", version_hash="seed_hash_a" - ), - ), - style=CliStyle(use_color=False), - ) - - assert rendered == test_case.expected_rendered - - -@pytest.mark.parametrize( - "test_case", - [ - CheckpointOutputTestCase( - description="preserves no-color checkpoint diff output", - expected_rendered="\nVirtual environment checkpoint diff dev\n\n" - " checkpoint cp_1\n" - " changed refs 1\n" - " current only 1\n" - " checkpoint only 1\n" - " changed seeds 1\n" - " current seeds 1\n" - " checkpoint seeds 1\n\n" - "Changed refs\n" - " shared current_hash -> checkpoint_hash\n\n" - "Current only\n" - " current_only current_only_hash -> \n\n" - "Checkpoint only\n" - " checkpoint_only -> checkpoint_only_hash\n\n" - "Changed seed refs\n" - " seed_a current_seed_hash -> checkpoint_seed_hash\n\n" - "Current only seed refs\n" - " seed_current_only current_only_seed_hash -> \n\n" - "Checkpoint only seed refs\n" - " seed_checkpoint_only -> checkpoint_only_seed_hash\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_checkpoint_model_refs_when_formatting_diff_without_color_then_preserves_output( - test_case: CheckpointOutputTestCase, -) -> None: - rendered: str = _format_checkpoint_diff( - virtual_environment_name="dev", - checkpoint_id="cp_1", - current_refs=( - VirtualEnvironmentModelRefRecord( - virtual_environment_name="dev", model_name="shared", version_hash="current_hash" - ), - VirtualEnvironmentModelRefRecord( - virtual_environment_name="dev", - model_name="current_only", - version_hash="current_only_hash", - ), - ), - current_seed_refs=( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name="dev", - seed_name="seed_a", - version_hash="current_seed_hash", - ), - VirtualEnvironmentSeedRefRecord( - virtual_environment_name="dev", - seed_name="seed_current_only", - version_hash="current_only_seed_hash", - ), - ), - checkpoint_model_refs=( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id="cp_1", model_name="shared", version_hash="checkpoint_hash" - ), - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id="cp_1", - model_name="checkpoint_only", - version_hash="checkpoint_only_hash", - ), - ), - checkpoint_seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id="cp_1", seed_name="seed_a", version_hash="checkpoint_seed_hash" - ), - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id="cp_1", - seed_name="seed_checkpoint_only", - version_hash="checkpoint_only_seed_hash", - ), - ), - style=CliStyle(use_color=False), - ) - - assert rendered == test_case.expected_rendered - - -@pytest.mark.parametrize( - "test_case", - [ - CheckpointColorOutputTestCase( - description="uses semantic checkpoint colors", - expected_fragments=( - "\033[34m\033[1mVirtual environment checkpoint\033[0m", - "\033[34mcp_1\033[0m", - "\033[32mRefs\033[0m", - "\033[32mSeed refs\033[0m", - "model_a", - "seed_a", - "\033[2mhash_a\033[0m", - "\033[2mseed_hash_a\033[0m", - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_checkpoint_model_refs_when_formatting_with_color_then_uses_semantic_colors( - test_case: CheckpointColorOutputTestCase, -) -> None: - rendered: str = _format_checkpoint_show( - checkpoint_id="cp_1", - refs=( - VirtualEnvironmentCheckpointModelRefRecord( - checkpoint_id="cp_1", model_name="model_a", version_hash="hash_a" - ), - ), - seed_refs=( - VirtualEnvironmentCheckpointSeedRefRecord( - checkpoint_id="cp_1", seed_name="seed_a", version_hash="seed_hash_a" - ), - ), - style=CliStyle(use_color=True), - ) - - for fragment in test_case.expected_fragments: - assert fragment in rendered diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py index 0367bed941..68593ec23b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/_test_types.py @@ -63,9 +63,7 @@ class MainTestCase: expected_no_sql_validation: bool = False expected_no_cache: bool = False expected_full_refresh: bool = False - expected_changes_only: bool = False expected_selection_diagnostics: bool = False - expected_virtual_env: str | None = None expected_load_sources: bool | None = None expected_reload: bool = False expected_allow_snapshot_full_refresh: bool = False diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py index 745101d0b3..edbb12303b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/helpers.py @@ -75,14 +75,10 @@ def build_handlers(**overrides: Any) -> CliEntrypointHandlers: run_load=overrides.get("run_load", noop_handler), run_clone=overrides.get("run_clone", noop_handler), run_diff=overrides.get("run_diff", noop_handler), - run_reconcile=overrides.get("run_reconcile", noop_handler), - run_promote=overrides.get("run_promote", noop_handler), - run_rollback=overrides.get("run_rollback", noop_handler), run_query=overrides.get("run_query", noop_handler), run_debug=overrides.get("run_debug", noop_handler), run_lineage=overrides.get("run_lineage", noop_handler), run_janitor=overrides.get("run_janitor", noop_handler), - run_state=overrides.get("run_state", noop_handler), run_init=overrides.get("run_init", noop_handler), run_playground=overrides.get("run_playground", noop_handler), run_skills_update=overrides.get("run_skills_update", noop_handler), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 50d4ebc625..2c0e47f625 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -24,8 +24,6 @@ LoadCommandRequest, PlanCommandRequest, PlaygroundCommandRequest, - PromoteCommandRequest, - RollbackCommandRequest, RulesCommandRequest, ScenarioCaptureCommandRequest, ScenarioTestCommandRequest, @@ -248,7 +246,7 @@ def test_given_compile_command_arguments_when_running_with_dependencies_then_it_ def test_given_dbt_plan_arguments_when_running_with_dependencies_then_it_dispatches_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, tuple[str, ...], bool]] = [] + received_args: list[tuple[object, ...]] = [] def run_dbt_plan( project_dir: Path | None, @@ -508,7 +506,7 @@ def run_dbt_plan(project_dir: Path | None, args: tuple[str, ...], no_color: bool def test_given_dbt_execution_arguments_when_running_with_dependencies_then_it_dispatches_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, tuple[str, ...], bool]] = [] + received_args: list[tuple[object, ...]] = [] def run_dbt_execution( project_dir: Path | None, @@ -648,9 +646,6 @@ def test_given_dbt_without_subcommand_when_running_with_dependencies_then_it_ret "dev", "--select", "orders", - "--virtual-env", - "preview", - "--skip-locked", "--verbose", ], expected_exit_code=8, @@ -681,8 +676,6 @@ def run_clone(request: CloneCommandRequest) -> int: origin_target_name="prod", destination_target_name="dev", hard_copy=False, - virtual_env="preview", - skip_locked=True, select=("orders",), exclude=(), verbose=True, @@ -702,7 +695,6 @@ def run_clone(request: CloneCommandRequest) -> int: "--full", "--select", "orders", - "--allow-partial-diff", ], expected_exit_code=6, ) @@ -740,147 +732,10 @@ def run_diff(request: DiffCommandRequest) -> int: exclude=(), verbose=False, cli_vars={}, - allow_partial_diff=True, ) ] -@pytest.mark.parametrize( - "test_case", - [ - MainTestCase( - description="dispatches promote command through injected handler", - argv=[ - "promote", - "--from", - "pr", - "--to", - "dev", - "--select", - "fact_orders", - "--include-stale-upstreams", - "--allow-partial-promotion", - "--verbose", - ], - expected_exit_code=7, - ) - ], - ids=lambda case: case.description, -) -def test_given_promote_command_arguments_when_running_with_dependencies_then_it_dispatches_handler( - test_case: MainTestCase, -) -> None: - received_args: list[ - tuple[ - Path | None, - bool, - bool, - str, - str, - tuple[str, ...], - tuple[str, ...], - bool, - bool, - bool, - ] - ] = [] - - def run_promote(request: PromoteCommandRequest) -> int: - received_args.append( - ( - request.project_dir, - request.no_color, - request.no_sql_validation, - request.from_virtual_environment, - request.to_virtual_environment, - request.select, - request.exclude, - request.allow_partial_promotion, - request.include_stale_upstreams, - request.verbose, - ) - ) - return test_case.expected_exit_code - - exit_code: int = _main_with_dependencies( - argv=test_case.argv, - handlers=build_handlers(run_promote=run_promote), - ) - - assert exit_code == test_case.expected_exit_code - assert received_args == [ - (None, False, False, "pr", "dev", ("fact_orders",), (), True, True, True) - ] - - -@pytest.mark.parametrize( - "test_case", - [ - MainTestCase( - description="dispatches rollback command through injected handler", - argv=[ - "rollback", - "--virtual-env", - "dev", - "--checkpoint-id", - "chk_1", - "--select", - "fact_orders", - "--allow-partial-rollback", - "--include-stale-upstreams", - "--verbose", - ], - expected_exit_code=7, - ) - ], - ids=lambda case: case.description, -) -def test_given_rollback_command_arguments_when_running_with_dependencies_then_it_dispatches_handler( - test_case: MainTestCase, -) -> None: - received_args: list[ - tuple[ - Path | None, - bool, - bool, - str | None, - bool, - str | None, - tuple[str, ...], - tuple[str, ...], - bool, - bool, - ] - ] = [] - - def run_rollback(request: RollbackCommandRequest) -> int: - received_args.append( - ( - request.project_dir, - request.no_color, - request.no_sql_validation, - request.virtual_environment, - request.verbose, - request.checkpoint_id, - request.select, - request.exclude, - request.allow_partial_rollback, - request.include_stale_upstreams, - ) - ) - return test_case.expected_exit_code - - exit_code: int = _main_with_dependencies( - argv=test_case.argv, - handlers=build_handlers(run_rollback=run_rollback), - ) - - assert exit_code == test_case.expected_exit_code - assert received_args == [ - (None, False, False, "dev", True, "chk_1", ("fact_orders",), (), True, True) - ] - - @pytest.mark.parametrize( "test_case", [ @@ -910,27 +765,7 @@ def run_rollback(request: RollbackCommandRequest) -> int: def test_given_scenario_test_arguments_when_running_with_dependencies_then_dispatches_selectors( test_case: MainTestCase, ) -> None: - received_args: list[ - tuple[ - Path | None, - bool, - bool, - tuple[str, ...], - tuple[str, ...], - bool, - bool, - bool, - bool, - bool, - bool, - int | None, - int | None, - int | None, - int | None, - bool, - Path | None, - ] - ] = [] + received_args: list[tuple[object, ...]] = [] def run_scenario(request: ScenarioTestCommandRequest) -> int: received_args.append( @@ -1081,8 +916,6 @@ def test_given_execution_command_json_flag_when_running_then_dispatches_json_out '{"tenant":"acme"}', "--fail-on-error", "--state", - "--virtual-env", - "dev", "--fail-on-stale", ], expected_exit_code=8, @@ -1092,7 +925,6 @@ def test_given_execution_command_json_flag_when_running_then_dispatches_json_out expected_fail_on_error=True, expected_state=True, expected_fail_on_stale=True, - expected_virtual_env="dev", ) ], ids=lambda case: case.description, @@ -1100,18 +932,7 @@ def test_given_execution_command_json_flag_when_running_then_dispatches_json_out def test_given_freshness_arguments_when_running_then_dispatches_expected_arguments( test_case: MainTestCase, ) -> None: - received_args: list[ - tuple[ - bool, - tuple[str, ...], - tuple[str, ...], - dict[str, object] | None, - bool, - bool, - bool, - str | None, - ] - ] = [] + received_args: list[tuple[object, ...]] = [] def run_freshness(request: FreshnessCommandRequest) -> int: received_args.append( @@ -1123,7 +944,6 @@ def run_freshness(request: FreshnessCommandRequest) -> int: request.fail_on_error, request.compare_state, request.fail_on_stale, - request.virtual_environment_name, ) ) return test_case.expected_exit_code @@ -1143,7 +963,6 @@ def run_freshness(request: FreshnessCommandRequest) -> int: test_case.expected_fail_on_error, test_case.expected_state, test_case.expected_fail_on_stale, - test_case.expected_virtual_env, ) ] @@ -1169,7 +988,7 @@ def run_freshness(request: FreshnessCommandRequest) -> int: def test_given_freshness_json_arguments_when_running_then_dispatches_expected_arguments( test_case: MainTestCase, ) -> None: - received_args: list[tuple[bool, Path | None]] = [] + received_args: list[tuple[object, ...]] = [] def run_freshness(request: FreshnessCommandRequest) -> int: received_args.append((request.json_output, request.json_output_path)) @@ -1325,21 +1144,7 @@ def test_given_scenario_command_when_no_sql_validation_flag_passed_then_parser_r def test_given_scenario_capture_arguments_when_running_with_dependencies_then_dispatches_selectors( test_case: MainTestCase, ) -> None: - received_args: list[ - tuple[ - Path | None, - bool, - bool, - tuple[str, ...], - tuple[str, ...], - bool, - bool, - int | None, - int | None, - int | None, - int | None, - ] - ] = [] + received_args: list[tuple[object, ...]] = [] def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: received_args.append( @@ -1403,7 +1208,7 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: def test_given_query_command_arguments_when_running_with_dependencies_then_it_dispatches_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, str | None, Path | None, str, int | None]] = [] + received_args: list[tuple[object, ...]] = [] def run_query( project_dir: Path | None, @@ -1440,7 +1245,7 @@ def run_query( def test_given_query_file_argument_when_running_with_dependencies_then_dispatches_path( test_case: MainTestCase, ) -> None: - received_args: list[tuple[str | None, Path | None, int | None]] = [] + received_args: list[tuple[object, ...]] = [] def run_query( project_dir: Path | None, @@ -1478,7 +1283,7 @@ def run_query( def test_given_debug_command_arguments_when_running_with_dependencies_then_it_dispatches_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, bool, bool, bool]] = [] + received_args: list[tuple[object, ...]] = [] def run_debug( project_dir: Path | None, @@ -1675,125 +1480,6 @@ def run_janitor(request: JanitorCommandRequest) -> int: ] -@pytest.mark.parametrize( - "test_case", - [ - MainTestCase( - description="dispatches state rollback command through injected handler", - argv=[ - "--project-dir", - "/tmp/demo", - "--no-color", - "state", - "rollback", - "--backup-id", - "b1", - ], - expected_exit_code=11, - expected_project_dir=Path("/tmp/demo"), - expected_no_color=True, - expected_state_command="rollback", - expected_state_backup_id="b1", - ), - MainTestCase( - description="dispatches state reset approval through injected handler", - argv=["state", "reset", "--auto-approve"], - expected_exit_code=12, - expected_state_command="reset", - expected_auto_approve=True, - ), - MainTestCase( - description="dispatches state checkpoints list through injected handler", - argv=["state", "checkpoints", "list", "--virtual-env", "dev"], - expected_exit_code=13, - expected_state_command="checkpoints", - expected_state_checkpoint_command="list", - expected_virtual_env="dev", - ), - MainTestCase( - description="dispatches state checkpoints show through injected handler", - argv=["state", "checkpoints", "show", "chk_1"], - expected_exit_code=14, - expected_state_command="checkpoints", - expected_state_checkpoint_command="show", - expected_state_checkpoint_id="chk_1", - ), - MainTestCase( - description="dispatches state checkpoints diff through injected handler", - argv=["state", "checkpoints", "diff", "chk_2", "--virtual-env", "dev"], - expected_exit_code=15, - expected_state_command="checkpoints", - expected_state_checkpoint_command="diff", - expected_state_checkpoint_id="chk_2", - expected_virtual_env="dev", - ), - ], - ids=lambda case: case.description, -) -def test_given_state_command_arguments_when_running_with_dependencies_then_it_dispatches_handler( - test_case: MainTestCase, -) -> None: - received_args: list[ - tuple[ - Path | None, - str, - str | None, - bool, - bool, - str | None, - str | None, - str | None, - bool, - ] - ] = [] - - def run_state( - project_dir: Path | None, - state_command: str, - backup_id: str | None, - auto_approve: bool, - no_color: bool, - checkpoint_command: str | None, - checkpoint_id: str | None, - virtual_environment: str | None, - allow_copy: bool, - ) -> int: - received_args.append( - ( - project_dir, - state_command, - backup_id, - auto_approve, - no_color, - checkpoint_command, - checkpoint_id, - virtual_environment, - allow_copy, - ) - ) - return test_case.expected_exit_code - - exit_code: int = _main_with_dependencies( - argv=test_case.argv, - handlers=build_handlers(run_state=run_state), - ) - - assert exit_code == test_case.expected_exit_code - assert received_args == [ - ( - test_case.expected_project_dir, - str(test_case.expected_state_command), - test_case.expected_state_backup_id, - test_case.expected_auto_approve, - test_case.expected_no_color, - test_case.expected_state_checkpoint_command, - test_case.expected_state_checkpoint_id, - test_case.expected_virtual_env, - False, - ) - ] - - @pytest.mark.parametrize( "test_case", [ @@ -1810,7 +1496,7 @@ def run_state( def test_given_playground_command_when_running_then_it_dispatches_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, str, str]] = [] + received_args: list[tuple[object, ...]] = [] def run_playground(request: PlaygroundCommandRequest) -> int: received_args.append((request.project_dir, request.target_path, request.template)) @@ -1855,7 +1541,7 @@ def run_playground(request: PlaygroundCommandRequest) -> int: def test_given_skills_command_when_running_then_it_dispatches_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, bool, tuple[str, ...], bool]] = [] + received_args: list[tuple[object, ...]] = [] def run_skills_update( project_dir: Path | None, @@ -2000,24 +1686,7 @@ def run_compile(_request: CompileCommandRequest) -> int: def test_given_compile_no_sql_validation_when_running_then_dispatches_expected_flag( test_case: MainTestCase, ) -> None: - received_args: list[ - tuple[ - Path | None, - bool, - bool, - str | None, - bool, - bool, - str | None, - bool, - CompileLineageMode, - dict[str, object] | None, - bool, - bool, - bool, - bool, - ] - ] = [] + received_args: list[tuple[object, ...]] = [] def run_compile(request: CompileCommandRequest) -> int: received_args.append( @@ -2088,7 +1757,7 @@ def run_compile(request: CompileCommandRequest) -> int: def test_given_dag_command_arguments_when_running_then_dispatches_expected_handler( test_case: MainTestCase, ) -> None: - received_args: list[tuple[Path | None, bool, bool, dict[str, object]]] = [] + received_args: list[tuple[object, ...]] = [] def run_dag( project_dir: Path | None, @@ -2173,12 +1842,6 @@ def run_dag( expected_run_tests=False, expected_run_audits=False, ), - MainTestCase( - description="passes changes-only to build handler", - argv=["build", "--changes-only"], - expected_exit_code=5, - expected_changes_only=True, - ), MainTestCase( description="passes selection diagnostics to build handler", argv=["build", "--selection-diagnostics"], @@ -2197,25 +1860,7 @@ def run_dag( def test_given_build_full_refresh_when_running_then_dispatches_expected_flag( test_case: MainTestCase, ) -> None: - received_args: list[ - tuple[ - bool, - bool, - bool, - str | None, - bool | None, - bool, - bool, - bool, - bool, - bool, - bool, - bool, - bool, - bool, - bool, - ] - ] = [] + received_args: list[tuple[object, ...]] = [] def run_build(request: BuildCommandRequest) -> int: received_args.append( @@ -2223,7 +1868,6 @@ def run_build(request: BuildCommandRequest) -> int: request.no_color, request.fail_fast, request.full_refresh, - request.virtual_env, request.load_sources, request.reload_sources, request.allow_snapshot_full_refresh, @@ -2232,7 +1876,6 @@ def run_build(request: BuildCommandRequest) -> int: request.run_audits, request.debug, request.verbose, - request.changes_only, request.no_cache, request.selection_diagnostics, ) @@ -2250,7 +1893,6 @@ def run_build(request: BuildCommandRequest) -> int: test_case.expected_no_color, False, test_case.expected_full_refresh, - test_case.expected_virtual_env, test_case.expected_load_sources, test_case.expected_reload, test_case.expected_allow_snapshot_full_refresh, @@ -2259,7 +1901,6 @@ def run_build(request: BuildCommandRequest) -> int: test_case.expected_run_audits, test_case.expected_debug, test_case.expected_verbose, - test_case.expected_changes_only, test_case.expected_no_cache, test_case.expected_selection_diagnostics, ) @@ -2277,12 +1918,6 @@ def run_build(request: BuildCommandRequest) -> int: expected_select=("orders",), expected_exclude=("customers",), ), - MainTestCase( - description="passes changes-only to plan handler", - argv=["plan", "--changes-only"], - expected_exit_code=4, - expected_changes_only=True, - ), MainTestCase( description="passes selection diagnostics to plan handler", argv=["plan", "--selection-diagnostics"], @@ -2301,27 +1936,7 @@ def run_build(request: BuildCommandRequest) -> int: def test_given_plan_flags_when_running_then_dispatches_expected_arguments( test_case: MainTestCase, ) -> None: - received_args: list[ - tuple[ - Path | None, - bool, - str | None, - str | None, - object, - bool, - bool, - str | None, - bool | None, - bool, - tuple[str, ...], - tuple[str, ...], - bool, - dict[str, object] | None, - bool, - bool, - bool, - ] - ] = [] + received_args: list[tuple[object, ...]] = [] def run_plan(request: PlanCommandRequest) -> int: received_args.append( @@ -2333,14 +1948,12 @@ def run_plan(request: PlanCommandRequest) -> int: request.cursor_overrides, request.json_output, request.full_refresh, - request.virtual_env, request.load_sources, request.no_color, request.select, request.exclude, request.verbose, request.cli_vars, - request.changes_only, request.no_cache, request.selection_diagnostics, ) @@ -2358,13 +1971,11 @@ def run_plan(request: PlanCommandRequest) -> int: False, False, None, - None, test_case.expected_no_color, test_case.expected_select, test_case.expected_exclude, False, {}, - test_case.expected_changes_only, test_case.expected_no_cache, test_case.expected_selection_diagnostics, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py index b6176e4a12..9276cb2d44 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/execution/_test_types.py @@ -27,12 +27,3 @@ class BuildPartialTimingOutputTestCase: execution_seconds: float | None expected_fragments: tuple[str, ...] expected_absent_fragments: tuple[str, ...] - - -@dataclass(frozen=True) -class VirtualBuildProjectionFailureTestCase: - description: str - build_status: str - expected_exit_code: int - expected_document_status: str - expected_completion_message: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_virtual_output_degradation.py b/tests/unit/src/sqlbuild/cli/commands/main/execution/test_virtual_output_degradation.py deleted file mode 100644 index 20ff793fdb..0000000000 --- a/tests/unit/src/sqlbuild/cli/commands/main/execution/test_virtual_output_degradation.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Virtual-build post-execution output degradation tests.""" - -from __future__ import annotations - -import json -from contextlib import nullcontext -from datetime import UTC, datetime -from io import StringIO -from pathlib import Path -from types import SimpleNamespace -from typing import cast -from unittest.mock import Mock - -import pytest - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.main.execution import _virtual_build -from sqlbuild.cli.commands.models import VirtualBuildCliRequest, VirtualBuildExecution -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.executor.build.models import BuildExecutionResult -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.runtime.observability.exceptions import ObservabilityValidationError -from sqlbuild.virtual.executor.models import VirtualBuildPipelineResult -from tests.unit.src.sqlbuild.cli.commands.main.execution._test_types import ( - VirtualBuildProjectionFailureTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualBuildProjectionFailureTestCase( - description="successful virtual execution projection failure", - build_status="success", - expected_exit_code=0, - expected_document_status="success", - expected_completion_message="warehouse execution completed successfully", - ), - VirtualBuildProjectionFailureTestCase( - description="failed virtual execution projection failure", - build_status="failed", - expected_exit_code=1, - expected_document_status="failed", - expected_completion_message="execution result remains authoritative", - ), - ], - ids=lambda case: case.description, -) -def test_given_completed_virtual_execution_when_projection_fails_then_exit_status_is_preserved( - test_case: VirtualBuildProjectionFailureTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - output_path: Path = tmp_path / "execution.json" - result: VirtualBuildPipelineResult = cast( - VirtualBuildPipelineResult, - SimpleNamespace( - display_plan_output=Mock(), - python_node_results=(), - execution_result=BuildExecutionResult(status=BuildStatus(test_case.build_status)), - execution_plan=Mock(), - project=SimpleNamespace( - run_id="run-1", - effective_target_name="dev", - effective_target_database="warehouse", - ), - ), - ) - request: VirtualBuildCliRequest = cast( - VirtualBuildCliRequest, - SimpleNamespace( - exclude=(), - reload_sources=False, - providers=(), - use_color=False, - execution_command="build", - json_output=False, - json_output_path=output_path, - ), - ) - monkeypatch.setattr(_virtual_build, "write_python_node_results", Mock()) - monkeypatch.setattr(_virtual_build.CostContext, "scope", Mock(return_value=nullcontext())) - monkeypatch.setattr(_virtual_build, "run_post_virtual_build_checks", Mock(return_value=())) - monkeypatch.setattr(_virtual_build, "format_build_footer", Mock(return_value="completed")) - monkeypatch.setattr(_virtual_build, "write_runtime_target", Mock()) - monkeypatch.setattr(_virtual_build, "write_python_check_runtime_target", Mock()) - monkeypatch.setattr(_virtual_build, "plan_has_executable_work", Mock(return_value=False)) - monkeypatch.setattr(_virtual_build, "finalize_build_cost", Mock(return_value=None)) - monkeypatch.setattr( - "sqlbuild.cli.commands._helpers.build_execution.outputs.format_build_execution_json", - Mock(side_effect=ObservabilityValidationError("injected projection failure")), - ) - monkeypatch.setattr(_virtual_build, "render_build_cost", Mock()) - monkeypatch.setattr(_virtual_build, "record_and_write_virtual_build_phase_timings", Mock()) - - exit_code: int = _virtual_build._complete_virtual_build( - project_dir=tmp_path, - discovered_inputs=cast( - DiscoveredProjectInputs, - SimpleNamespace(project_config=SimpleNamespace(cost=Mock())), - ), - adapter=cast(BaseAdapter, Mock()), - adapter_name="duckdb", - connection_config={}, - request=request, - execution=cast(VirtualBuildExecution, SimpleNamespace(elapsed=1.0)), - result=result, - stream=StringIO(), - build_started_at=datetime.now(UTC), - ) - - payload: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8")) - assert exit_code == test_case.expected_exit_code - assert payload["status"] == test_case.expected_document_status - assert payload["projection_degraded"] is True - assert test_case.expected_completion_message in capsys.readouterr().err - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py index 058912dca2..8cd6da5f9f 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/helpers.py @@ -7,16 +7,11 @@ JanitorArchiveCandidate, JanitorArchivedRelation, JanitorBlockedSchema, - JanitorCheckpointCandidate, JanitorDeleteCandidate, - JanitorDetachedVirtualEnvironmentCandidate, - JanitorExpiredLockCandidate, - JanitorExpiredVirtualEnvironmentCandidate, JanitorPlan, JanitorRelationKey, JanitorSkippedRelation, JanitorSkippedSchema, - JanitorStateBackupCandidate, ) @@ -43,33 +38,6 @@ def build_janitor_plan() -> JanitorPlan: age_timestamp=None, ), ), - checkpoint_candidates=( - JanitorCheckpointCandidate( - checkpoint_id="cp_1", virtual_environment_name="dev", created_at=None - ), - ), - detached_virtual_environment_candidates=( - JanitorDetachedVirtualEnvironmentCandidate( - virtual_environment_name="branch_old", updated_at=None - ), - ), - expired_virtual_environment_candidates=( - JanitorExpiredVirtualEnvironmentCandidate( - virtual_environment_name="branch_expired", updated_at=None - ), - ), - state_backup_candidates=( - JanitorStateBackupCandidate( - backup_id="backup_1", schema_name="sqlbuild_state", created_at=None - ), - ), - expired_lock_candidates=( - JanitorExpiredLockCandidate( - lock_key="lock_1", - owner_id="worker_1", - expires_at=datetime(2026, 5, 29), - ), - ), skipped_relations=( JanitorSkippedRelation( key=JanitorRelationKey(database=None, schema="dev", name="source_table"), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py index 259b2a7962..23c6540e30 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor.py @@ -7,14 +7,11 @@ import pytest -from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.cli.commands._helpers.janitor_output.output import write_disabled, write_plan from sqlbuild.cli.commands._helpers.janitor_output.outputs import confirm_janitor_plan from sqlbuild.cli.commands.models import JanitorPlanningResult from sqlbuild.executor.janitor.models import ( - JanitorDeleteCandidate, JanitorPlan, - JanitorRelationKey, ) from tests.unit.src.sqlbuild.cli.commands.main.janitor._test_types import ( JanitorConfirmationInterruptTestCase, @@ -35,8 +32,8 @@ description="treats keyboard interrupt at confirmation as cancellation", expected_result=False, expected_output_fragments=( - "Janitor will delete 1 objects from dev.", - "Type `delete 1 objects from dev` to continue: ", + "Janitor will archive 1 and delete 1 objects from dev.", + "Type `archive 1 and delete 1 objects from dev` to continue: ", ), unexpected_output_fragments=("KeyboardInterrupt", "Traceback"), ) @@ -48,22 +45,7 @@ def test_given_janitor_confirmation_when_keyboard_interrupt_then_returns_cancell capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, ) -> None: - plan: JanitorPlan = JanitorPlan( - target_name="dev", - retention_days=30, - candidates=( - JanitorDeleteCandidate( - key=JanitorRelationKey(database=None, schema="dev", name="stale_model"), - relation=RelationInfo( - database=None, - schema="dev", - name="stale_model", - relation_type="table", - ), - age_timestamp=None, - ), - ), - ) + plan: JanitorPlan = build_direct_archive_plan() def interrupting_input() -> str: raise KeyboardInterrupt @@ -144,21 +126,20 @@ def test_given_disabled_janitor_when_writing_with_color_then_styles_title( expected_output_fragments=( "Janitor preview dev", " retention 30 days", - " eligible for deletion 1", + " relations to archive 0", "Janitor blocked\n Managed target schemas contain active configured sources.", "blocked active sources: raw.events", - "suppressed deletion: blocked.old_model", + "suppressed archive: blocked.old_model", "No janitor actions will be performed.", "Skipped schemas\n dev contains active source raw.orders", - "Eligible objects\n dev.stale_model", - "Eligible checkpoints\n cp_1 dev", - "Eligible detached VDEs\n branch_old detached virtual environment", - "Eligible expired VDEs\n branch_expired expired virtual environment", - "Eligible state backups\n backup_1 sqlbuild_state", - "Eligible expired locks\n lock_1 worker_1", "Skipped objects\n dev.source_table source relation", ), - unexpected_output_fragments=("\033[",), + unexpected_output_fragments=( + "\033[", + "checkpoints pruned", + "VDEs pruned", + "virtual state pruned", + ), ) ], ids=lambda case: case.description, @@ -188,10 +169,8 @@ def test_given_janitor_plan_when_writing_without_color_then_preserves_output_fra "\033[34m\033[1mJanitor preview\033[0m", "dev", "\033[34m30 days\033[0m", - "eligible for deletion \033[33m1\033[0m", + "relations to archive \033[34m0\033[0m", "Janitor blocked", - "\033[32mEligible objects\033[0m", - "dev.stale_model", "\033[2msource relation\033[0m", ), ) @@ -226,7 +205,7 @@ def test_given_janitor_plan_when_writing_with_color_then_uses_semantic_colors( ), ), JanitorDirectArchiveConfirmationTestCase( - description="direct archive plan rejects the virtual-mode phrase", + description="direct archive plan requires the archive count in confirmation", typed_response="delete 1 objects from dev", expected_result=False, expected_output_fragments=("Janitor will archive 1 and delete 1 objects from dev.",), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py index a97d044d23..de59ca0779 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/janitor/test_janitor_settings.py @@ -22,22 +22,6 @@ DIRECT_PROJECT_TOML: str = ( 'name = "orders_direct"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' ) -VIRTUAL_PROJECT_TOML: str = ( - 'name = "orders_virtual"\n' - 'adapter = "duckdb"\n' - 'default_target = "dev"\n\n' - "[settings]\n" - "virtual_environments = true\n\n" - "[connection]\n" - 'database = "warehouse.duckdb"\n\n' - "[targets.dev]\n" - 'schema = "dev"\n\n' - "[targets.dev.state]\n" - 'backend = "duckdb"\n' - 'schema = "sqlbuild_state"\n\n' - "[targets.dev.state.connection]\n" - 'database = "state.duckdb"\n' -) @pytest.mark.parametrize( @@ -50,13 +34,6 @@ expected_retention_days=14, expected_archive_retention_days=14, ), - JanitorSettingsResolutionTestCase( - description="virtual mode without explicit retention resolves 30", - project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\n", - cli_retention_days=None, - expected_retention_days=30, - expected_archive_retention_days=14, - ), JanitorSettingsResolutionTestCase( description="explicit retention applies to direct mode", project_toml=DIRECT_PROJECT_TOML @@ -65,20 +42,6 @@ expected_retention_days=9, expected_archive_retention_days=2, ), - JanitorSettingsResolutionTestCase( - description="explicit retention applies to virtual mode", - project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\nretention_days = 9\n", - cli_retention_days=None, - expected_retention_days=9, - expected_archive_retention_days=14, - ), - JanitorSettingsResolutionTestCase( - description="cli retention override wins in virtual mode", - project_toml=VIRTUAL_PROJECT_TOML + "\n[janitor]\nenabled = true\nretention_days = 9\n", - cli_retention_days=4, - expected_retention_days=4, - expected_archive_retention_days=14, - ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 31e7652458..9840aba72e 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -168,63 +168,6 @@ ), ), ), - CreatePlaygroundProjectTestCase( - description="creates virtual environments playground from loader template", - target_relative_path=Path("virtual_playground"), - template="virtual", - expected_files=( - Path("README.md"), - Path("sqlbuild_project.toml"), - Path("models/fact_waffle_orders.sql"), - Path("models/customer_revenue.sql"), - Path("seeds/lookups.yml"), - Path("seeds/waffle_price_tiers.csv"), - Path("sources/raw.yml"), - Path("python/loaders/waffle_loaders.py"), - Path("tests/unit/test_fact_waffle_orders.sql"), - Path("tests/scenarios/customer_revenue_minimal.sql"), - ), - unexpected_paths=(Path("target"), Path("sqlbuild_local.toml")), - expected_file_fragments=( - ( - Path("sqlbuild_project.toml"), - ( - "[connections.developer]", - "[settings]", - "virtual_environments = true", - "[targets.dev.state]", - 'backend = "duckdb"', - 'unsuffixed_virtual_env = "dev"', - 'database = "loader_waffle_shop_state.duckdb"', - ), - ), - ( - Path("models/fact_waffle_orders.sql"), - ( - '__seed("waffle_price_tiers")', - "waffle_category", - ), - ), - ( - Path("README.md"), - ( - "sqb state init", - "sqb build --virtual-env pr", - "sqb scenario test", - "sqb diff dev:pr --schema-only", - "sqb promote --from pr --to dev", - ), - ), - ( - Path("tests/unit/test_fact_waffle_orders.sql"), - ("__expected__fact_waffle_orders", "__seed__waffle_price_tiers"), - ), - ( - Path("tests/scenarios/customer_revenue_minimal.sql"), - ("SCENARIO", "__expected__customer_revenue"), - ), - ), - ), CreatePlaygroundProjectTestCase( description="creates Python nodes playground from generated template", target_relative_path=Path("python_nodes_playground"), @@ -388,23 +331,6 @@ def test_given_existing_target_when_creating_playground_then_it_raises_user_erro "sqb build", ), ), - RunPlaygroundTestCase( - description="prints virtual environments next steps", - target_path="demo_virtual_shop", - template="virtual", - expected_stdout_fragments=( - "SQLBuild playground created", - "Project: demo_virtual_shop", - "Example: virtual environments waffle shop", - "sqb state init", - "sqb build --virtual-env pr", - "sqb test", - "sqb audit", - "sqb scenario test", - "sqb diff dev:pr --schema-only", - "sqb promote --from pr --to dev", - ), - ), RunPlaygroundTestCase( description="prints Python nodes next steps", target_path="demo_python_nodes", diff --git a/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/_test_types.py b/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/_test_types.py index 1732b5e78b..9bbc6b4e22 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/_test_types.py @@ -11,11 +11,3 @@ class CloneExecutionJsonTestCase: expected_asset_statuses: tuple[str, ...] expected_asset_actions: tuple[str, ...] expected_summary: dict[str, int] - - -@dataclass(frozen=True) -class VirtualCloneExecutionJsonTestCase: - description: str - expected_status: str - expected_asset_statuses: tuple[str, ...] - expected_summary: dict[str, int] diff --git a/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/test_execution_protocol_v1.py b/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/test_execution_protocol_v1.py index 11087d4153..d925074e36 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/test_execution_protocol_v1.py +++ b/tests/unit/src/sqlbuild/cli/output/main/clone_execution_json/test_execution_protocol_v1.py @@ -6,16 +6,10 @@ from sqlbuild.cli.output.main._clone_execution_json import ( format_clone_execution_json, ) -from sqlbuild.cli.output.main._virtual_clone_execution_json import ( - format_virtual_clone_execution_json, -) from sqlbuild.executor.clone.models import CloneExecutionResult, CloneItemResult from sqlbuild.executor.clone.types import CloneAction, CloneStatus -from sqlbuild.virtual.executor.models import VirtualCloneItemResult, VirtualCloneResult -from sqlbuild.virtual.state.types import PhysicalArtifactType from tests.unit.src.sqlbuild.cli.output.main.clone_execution_json._test_types import ( CloneExecutionJsonTestCase, - VirtualCloneExecutionJsonTestCase, ) @@ -71,52 +65,3 @@ def test_given_clone_results_when_formatting_execution_json_then_preserves_asset assert tuple(asset["status"] for asset in assets) == test_case.expected_asset_statuses assert tuple(asset["action"] for asset in assets) == test_case.expected_asset_actions assert payload["summary"] == test_case.expected_summary - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualCloneExecutionJsonTestCase( - description="virtual clone reports reused and locked assets as completed", - expected_status="success", - expected_asset_statuses=("success", "skipped"), - expected_summary={ - "success_count": 1, - "failure_count": 0, - "skipped_count": 1, - "total_count": 2, - }, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_clone_results_when_formatting_json_then_preserves_asset_outcomes( - test_case: VirtualCloneExecutionJsonTestCase, -) -> None: - result: str = format_virtual_clone_execution_json( - result=VirtualCloneResult( - mode="target", - origin_environment="prod", - destination_environment="test", - item_results=( - VirtualCloneItemResult( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="abc", - action="reused", - ), - VirtualCloneItemResult( - artifact_type=PhysicalArtifactType.SEED, - artifact_name="countries", - version_hash="def", - action="skipped_locked", - ), - ), - ) - ) - - payload: dict[str, Any] = json.loads(result) - assets: list[dict[str, Any]] = payload["assets"] - assert payload["status"] == test_case.expected_status - assert tuple(asset["status"] for asset in assets) == test_case.expected_asset_statuses - assert payload["summary"] == test_case.expected_summary diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index ef0b9ac251..0cc878cffa 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -1149,98 +1149,6 @@ "\u2514\u2500\u2500 order_status_index delete_insert (integer)", ), ), - FormatPlanTestCase( - description="virtual metadata explains status and caps affected model list", - plan_output=build_plan_output( - metadata={ - "virtual_environment_name": "dev", - "virtual_environment_status": "working", - "virtual_stale_root_names": ("model_00", "model_01"), - "virtual_stale_model_names": tuple(f"model_{index:02d}" for index in range(55)), - }, - ), - expected_fragments=( - "Virtual environment dev (working, build required)", - "Models needing build (55)", - "directly affected (2) model_00, model_01", - "downstream affected (53) model_02", - "... (+33 more; use --verbose to show all)", - ), - expected_ordered_fragments=( - "Models needing build (55)", - "directly affected (2) model_00, model_01", - "downstream affected (53) model_02", - ), - ), - FormatPlanTestCase( - description="virtual metadata shows full affected sets in verbose output", - plan_output=build_plan_output( - metadata={ - "virtual_environment_name": "dev", - "virtual_environment_status": "working", - "virtual_stale_root_names": ("model_00",), - "virtual_stale_model_names": tuple(f"model_{index:02d}" for index in range(3)), - }, - ), - display_options=DisplayOptions(max_entries_per_section=None), - expected_fragments=( - "directly affected (1) model_00", - "downstream affected (2) model_01, model_02", - ), - unexpected_fragments=("use --verbose",), - ), - FormatPlanTestCase( - description="virtual metadata shows models outside partial selection", - plan_output=build_plan_output( - metadata={ - "virtual_environment_name": "dev", - "virtual_environment_status": "working", - "virtual_stale_root_names": ("stg_orders",), - "virtual_stale_model_names": ("fact_orders", "orders_rollup", "stg_orders"), - "virtual_remaining_stale_model_names": ("orders_rollup",), - }, - ), - expected_fragments=( - "Models needing build (3)", - "directly affected (1) stg_orders", - "downstream affected (2) fact_orders, orders_rollup", - "outside this plan (1) orders_rollup", - ), - expected_ordered_fragments=( - "directly affected (1) stg_orders", - "downstream affected (2) fact_orders, orders_rollup", - "outside this plan (1) orders_rollup", - ), - ), - FormatPlanTestCase( - description="virtual metadata explains source freshness outcomes", - plan_output=build_plan_output( - metadata={ - "virtual_environment_name": "dev", - "virtual_environment_status": "working", - "virtual_stale_root_names": ("fact_orders",), - "virtual_stale_model_names": ("fact_orders", "orders_rollup"), - "virtual_source_freshness_observed_source_names": ( - "raw_customers", - "raw_orders", - ), - "virtual_source_freshness_unchanged_source_names": ("raw_customers",), - "virtual_source_freshness_incomplete_source_names": ("raw_payments",), - "virtual_source_freshness_incomplete_model_names": ("fact_payments",), - }, - ), - expected_fragments=( - "Source freshness (2 of 3 checked)", - "new or changed (1) raw_orders", - "unchanged (1) raw_customers", - "not verifiable (1) raw_payments", - "affected models (1) fact_payments", - "Models needing build (2)", - "directly affected (1) fact_orders", - "downstream affected (1) orders_rollup", - ), - unexpected_fragments=("source freshness observed", "stale model set"), - ), FormatPlanTestCase( description="direct metadata caps remaining stale models after partial selection", plan_output=build_plan_output( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index a526cb8946..922ed461ff 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -190,7 +190,7 @@ def test_given_exact_compact_batch_when_entry_rows_are_absent_then_warm_compile_ write_repo_files(tmp_path, _CACHE_REPO_FILES) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v10" / "model-analysis.sqlite3" + cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: batch_count: int = connection.execute( "SELECT COUNT(*) FROM model_analysis_compact_batch" @@ -223,7 +223,7 @@ def test_given_one_changed_model_when_entry_rows_are_absent_then_compact_batch_r write_repo_files(tmp_path, _SELECTION_REPO_FILES) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v10" / "model-analysis.sqlite3" + cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: _ = connection.execute("DELETE FROM model_analysis") (tmp_path / "models" / "unrelated.sql").write_text( @@ -261,7 +261,7 @@ def test_given_corrupt_compact_batch_when_entry_rows_are_valid_then_warm_compile write_repo_files(tmp_path, _CACHE_REPO_FILES) monkeypatch.setattr(assembly_project, "_COMPACT_BATCH_CACHE_MIN_MODEL_COUNT", 1) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v10" / "model-analysis.sqlite3" + cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: _ = connection.execute( "UPDATE model_analysis_compact_batch SET payload = ?", @@ -277,7 +277,7 @@ def test_given_corrupt_compact_batch_when_entry_rows_are_valid_then_warm_compile @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="changed SQL cache miss", expected_count=1),), + (AnalysisCacheTestCase(description="changed SQL cache miss", expected_count=2),), ids=lambda case: case.description, ) def test_given_changed_expanded_sql_when_compiling_then_writes_a_new_analysis_object( @@ -301,7 +301,7 @@ def test_given_changed_expanded_sql_when_compiling_then_writes_a_new_analysis_ob @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="corrupt cache fallback", expected_count=1),), + (AnalysisCacheTestCase(description="corrupt cache fallback", expected_count=2),), ids=lambda case: case.description, ) def test_given_corrupt_analysis_when_compiling_then_reanalyzes_and_repairs_the_entry( @@ -312,7 +312,7 @@ def test_given_corrupt_analysis_when_compiling_then_reanalyzes_and_repairs_the_e ) -> None: write_repo_files(tmp_path, _CACHE_REPO_FILES) cold_project: CompiledProject = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v10" / "model-analysis.sqlite3" + cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: persisted_contents: str = connection.execute( "SELECT payload FROM model_analysis" @@ -335,14 +335,14 @@ def test_given_corrupt_analysis_when_compiling_then_reanalyzes_and_repairs_the_e _digest, _separator, serialized_payload = repaired_contents.partition("\n") repaired_payload: dict[str, object] = json.loads(serialized_payload) assert repaired_project.models == cold_project.models - assert repaired_payload["v"] == 10 + assert repaired_payload["v"] == 12 assert isinstance(repaired_payload["s"], str) assert analyzer.call_count == test_case.expected_count @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="non-text analysis cache fallback", expected_count=1),), + (AnalysisCacheTestCase(description="non-text analysis cache fallback", expected_count=2),), ids=lambda case: case.description, ) def test_given_non_text_analysis_cache_when_compiling_then_reanalyzes_safely( @@ -353,7 +353,7 @@ def test_given_non_text_analysis_cache_when_compiling_then_reanalyzes_safely( ) -> None: write_repo_files(tmp_path, _CACHE_REPO_FILES) _ = compile_project_with_cache(project_dir=tmp_path) - cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v10" / "model-analysis.sqlite3" + cache_path: Path = tmp_path / "target" / "cache" / "compiler" / "v12" / "model-analysis.sqlite3" with sqlite3.connect(cache_path) as connection: _ = connection.execute( "UPDATE model_analysis SET payload = ?", @@ -688,7 +688,7 @@ def test_given_analysis_inputs_when_building_keys_then_all_semantic_inputs_affec @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="propagated schema invalidation", expected_count=2),), + (AnalysisCacheTestCase(description="propagated schema invalidation", expected_count=4),), ids=lambda case: case.description, ) def test_given_schema_change_when_compiling_then_affected_consumers_miss_cache( @@ -764,7 +764,7 @@ def test_given_model_sql_change_with_stable_signature_when_compiling_then_downst @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="changed exported signature", expected_count=3),), + (AnalysisCacheTestCase(description="changed exported signature", expected_count=5),), ids=lambda case: case.description, ) def test_given_model_output_change_when_compiling_then_downstream_closure_misses_cache( @@ -794,7 +794,7 @@ def test_given_model_output_change_when_compiling_then_downstream_closure_misses (AnalysisCacheTestCase(description="parallel downstream invalidation", expected_count=2),), ids=lambda case: case.description, ) -def test_given_changed_output_signature_when_reanalyzing_downstream_then_uses_one_native_batch( +def test_given_changed_output_signature_when_reanalyzing_downstream_then_uses_dependency_ready_batches( test_case: AnalysisCacheTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -812,13 +812,13 @@ def test_given_changed_output_signature_when_reanalyzing_downstream_then_uses_on _ = compile_project_with_cache(project_dir=tmp_path) - assert batcher.call_count == 1 - assert len(batcher.call_args.kwargs["query_sqls"]) == test_case.expected_count + assert batcher.call_count == test_case.expected_count + assert all(len(call.kwargs["query_sqls"]) == 1 for call in batcher.call_args_list) @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="restored exported signature", expected_count=2),), + (AnalysisCacheTestCase(description="restored exported signature", expected_count=4),), ids=lambda case: case.description, ) def test_given_cached_model_signature_is_restored_when_compiling_then_downstream_reanalyzes( @@ -844,7 +844,7 @@ def test_given_cached_model_signature_is_restored_when_compiling_then_downstream @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="selected upstream change", expected_count=1),), + (AnalysisCacheTestCase(description="selected upstream change", expected_count=4),), ids=lambda case: case.description, ) def test_given_selected_upstream_change_when_compiling_full_project_then_stale_consumer_misses( @@ -945,7 +945,7 @@ def test_given_upstream_interface_change_when_compact_batch_exists_then_dependen @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="selected upstream analysis", expected_count=2),), + (AnalysisCacheTestCase(description="selected upstream analysis", expected_count=3),), ids=lambda case: case.description, ) def test_given_partial_selection_when_compiling_then_analyzes_only_upstream_closure( @@ -1010,7 +1010,7 @@ def test_given_unselected_invalid_reference_when_compiling_then_live_validation_ @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="explicit cache bypass", expected_count=2),), + (AnalysisCacheTestCase(description="explicit cache bypass", expected_count=4),), ids=lambda case: case.description, ) def test_given_compile_cache_bypass_when_compiling_twice_then_both_runs_analyze_cold( @@ -1046,7 +1046,7 @@ def test_given_compile_cache_bypass_when_compiling_twice_then_both_runs_analyze_ @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="command cache bypass", expected_count=2),), + (AnalysisCacheTestCase(description="command cache bypass", expected_count=4),), ids=lambda case: case.description, ) def test_given_command_cache_bypass_when_compiling_twice_then_both_runs_analyze_cold( @@ -1067,7 +1067,7 @@ def test_given_command_cache_bypass_when_compiling_twice_then_both_runs_analyze_ @pytest.mark.parametrize( "test_case", - (AnalysisCacheTestCase(description="target cache bypass", expected_count=2),), + (AnalysisCacheTestCase(description="target cache bypass", expected_count=4),), ids=lambda case: case.description, ) def test_given_target_cache_disabled_when_compiling_twice_then_both_runs_analyze_cold( diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py index 22e7f48a90..105003b8b3 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_sql_analysis_columns.py @@ -131,7 +131,7 @@ def analyze_project_queries_compact_json(request_json: str) -> str: } assert "orders" in results[0].cleaned_sql - assert results[0].analysis == {"hasStar": False} + assert results[0].analysis == {"hasStar": False, "starResolved": False} assert results[0].compact_rows == [0] assert results[1].analysis is None assert results[1].projected is True diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index d37e523799..8b80b23607 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -263,7 +263,6 @@ class LoadProjectConfigTestCase: expected_targets: dict[str, dict[str, object]] expected_janitor_enabled: bool expected_retention_days: int | None - expected_janitor_max_checkpoints: int expected_janitor_delete_tracked_only: bool expected_janitor_exclude_patterns: tuple[str, ...] expected_seed_database: str | None = None @@ -291,8 +290,6 @@ class LoadProjectConfigTestCase: expected_dbt_target_path: str | None = None expected_dbt_vars: dict[str, object] = field(default_factory=dict) expected_auto_load_sources: bool = True - expected_virtual_environments: bool = False - expected_changes_only: bool = False expected_enforce_placement: bool = True @@ -311,7 +308,6 @@ class LoadLocalConfigTestCase: expected_dbt_target: str | None = None expected_dbt_vars: dict[str, object] = field(default_factory=dict) expected_auto_load_sources: bool = True - expected_changes_only: bool = False expected_targets: dict[str, dict[str, object]] = field(default_factory=dict) expected_missing_attributes: tuple[str, ...] = () expected_scenario_local_type_overrides: dict[str, dict[str, str]] = field(default_factory=dict) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py index 8205c0b06c..dee0663c5d 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_yml_project.py @@ -660,7 +660,6 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise "loader_schema": None, "defer_sources_to": None, "defer_clone_from": None, - "changes_only": None, "compile_cache": None, "allow_as_clone_origin": False, "allow_as_clone_destination": False, @@ -668,7 +667,6 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise }, expected_janitor_enabled=False, expected_retention_days=None, - expected_janitor_max_checkpoints=20, expected_janitor_delete_tracked_only=True, expected_janitor_exclude_patterns=(), ), @@ -683,12 +681,10 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise path = "data.db" [settings] -virtual_environments = true sql_analysis = false query_change_tracking = true concurrency = 8 auto_load_sources = false -changes_only = true [scopes] enforce_placement = false @@ -720,7 +716,6 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise loader_schema = "raw_${user}" defer_sources_to = "prod" defer_clone_from = "prod" -changes_only = true compile_cache = false [targets.dev.connection] @@ -737,7 +732,6 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise enabled = true retention_days = 21 archive_retention_days = 3 -max_checkpoints = 3 delete_tracked_only = false exclude_patterns = ["partition_*"] direct_state_history_versions = 5 @@ -774,13 +768,11 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise """.strip(), expected_name="demo", expected_adapter="duckdb", - expected_virtual_environments=True, expected_default_target="dev", expected_connection={"path": "data.db"}, expected_sql_analysis=False, expected_max_concurrency=8, expected_auto_load_sources=False, - expected_changes_only=True, expected_enforce_placement=False, expected_materialized="table", expected_row_diff_exclude_columns=("loaded_at",), @@ -808,7 +800,6 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise "loader_schema": "raw_${user}", "defer_sources_to": "prod", "defer_clone_from": "prod", - "changes_only": True, "compile_cache": False, "allow_as_clone_origin": True, "allow_as_clone_destination": True, @@ -817,7 +808,6 @@ def test_given_non_day_retention_when_loading_project_then_config_error_is_raise expected_janitor_enabled=True, expected_retention_days=21, expected_janitor_archive_retention_days=3, - expected_janitor_max_checkpoints=3, expected_janitor_delete_tracked_only=False, expected_janitor_exclude_patterns=("partition_*",), expected_janitor_direct_state_history_versions=5, @@ -857,13 +847,11 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e assert config.name == test_case.expected_name assert config.adapter == test_case.expected_adapter - assert config.settings.virtual_environments is test_case.expected_virtual_environments assert config.default_target == test_case.expected_default_target assert config.connection == test_case.expected_connection assert config.settings.sql_analysis is test_case.expected_sql_analysis assert config.settings.concurrency == test_case.expected_max_concurrency assert config.settings.auto_load_sources is test_case.expected_auto_load_sources - assert config.settings.changes_only is test_case.expected_changes_only assert config.scopes.enforce_placement is test_case.expected_enforce_placement assert config.defaults.materialized == test_case.expected_materialized assert config.defaults.row_diff_exclude_columns == test_case.expected_row_diff_exclude_columns @@ -875,28 +863,11 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e assert config.defaults.function_schema == test_case.expected_function_schema assert config.path_defaults == test_case.expected_path_defaults assert config.vars == test_case.expected_vars - assert { - target_name: { - "connection": target_config.connection, - "vars": target_config.vars, - "database": target_config.database, - "schema": target_config.schema, - "loader_schema": target_config.loader_schema, - "defer_sources_to": target_config.defer_sources_to, - "defer_clone_from": target_config.defer_clone_from, - "changes_only": target_config.changes_only, - "compile_cache": target_config.compile_cache, - "allow_as_clone_origin": target_config.clone.allow_as_clone_origin, - "allow_as_clone_destination": target_config.clone.allow_as_clone_destination, - } - for target_name, target_config in config.targets.items() - } == test_case.expected_targets assert config.janitor.enabled is test_case.expected_janitor_enabled assert config.janitor.retention_days == test_case.expected_retention_days assert ( config.janitor.archive_retention_days == test_case.expected_janitor_archive_retention_days ) - assert config.janitor.max_checkpoints == test_case.expected_janitor_max_checkpoints assert config.janitor.delete_tracked_only is test_case.expected_janitor_delete_tracked_only assert config.janitor.exclude_patterns == test_case.expected_janitor_exclude_patterns assert ( @@ -956,7 +927,6 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e sql_validation = false concurrency = 4 auto_load_sources = false -changes_only = true [vars] user = "kevin" @@ -991,13 +961,11 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e "sql_analysis", "concurrency", "auto_load_sources", - "changes_only", } ), expected_vars={"user": "kevin"}, expected_dbt_target="pat", expected_dbt_vars={"shared": "local", "threads": 2}, - expected_changes_only=True, expected_scenario_local_type_overrides={ "snowflake": { "NUMBER(*,0)": "BIGINT", @@ -1070,7 +1038,6 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e loader_schema = "local_raw" defer_sources_to = "prod" defer_clone_from = "prod" -changes_only = false compile_cache = false [targets.dev.connection] @@ -1101,7 +1068,6 @@ def test_given_project_config_file_when_loading_project_config_then_it_returns_e "loader_schema": "local_raw", "defer_sources_to": "prod", "defer_clone_from": "prod", - "changes_only": False, "compile_cache": False, "allow_as_clone_origin": True, "allow_as_clone_destination": False, @@ -1131,7 +1097,6 @@ def test_given_local_config_state_when_loading_local_config_then_it_returns_expe assert config.settings.sql_validation is test_case.expected_sql_validation assert config.settings.concurrency == test_case.expected_max_concurrency assert config.settings.auto_load_sources is test_case.expected_auto_load_sources - assert config.settings.changes_only is test_case.expected_changes_only assert config.setting_overrides == test_case.expected_setting_overrides assert config.vars == test_case.expected_vars assert config.dbt.target == test_case.expected_dbt_target @@ -1143,22 +1108,6 @@ def test_given_local_config_state_when_loading_local_config_then_it_returns_expe "max_bytes_per_relation": config.scenario.snapshot_limits.max_bytes_per_relation, "max_total_bytes": config.scenario.snapshot_limits.max_total_bytes, } == test_case.expected_snapshot_limits - assert { - target_name: { - "connection": target_config.connection, - "vars": target_config.vars, - "database": target_config.database, - "schema": target_config.schema, - "loader_schema": target_config.loader_schema, - "defer_sources_to": target_config.defer_sources_to, - "defer_clone_from": target_config.defer_clone_from, - "changes_only": target_config.changes_only, - "compile_cache": target_config.compile_cache, - "allow_as_clone_origin": target_config.clone.allow_as_clone_origin, - "allow_as_clone_destination": target_config.clone.allow_as_clone_destination, - } - for target_name, target_config in config.targets.items() - } == test_case.expected_targets attribute_name: str for attribute_name in test_case.expected_missing_attributes: assert not hasattr(config, attribute_name) @@ -1277,17 +1226,6 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( """.strip(), expected_error_fragment="Expected 'enforce_placement' to be a boolean when provided", ), - LoadProjectConfigErrorTestCase( - description="raises when virtual environments setting is not a boolean", - project_file_contents=""" -name = "demo" -adapter = "duckdb" - -[settings] -virtual_environments = "yes" -""".strip(), - expected_error_fragment="Expected 'virtual_environments' to be a boolean when provided", - ), LoadProjectConfigErrorTestCase( description="raises when settings sql_analysis is not a boolean", project_file_contents=""" @@ -1545,11 +1483,6 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( project_file_contents='name = "demo"\nadapter = "duckdb"\n[targets.dev.clone]\nconnections = true', expected_error_fragment=r"targets.dev.clone contains unknown key\(s\): connections", ), - LoadProjectConfigErrorTestCase( - description="raises when state contains unknown key", - project_file_contents='name = "demo"\nadapter = "duckdb"\n[targets.dev.state]\nconnections = {}', - expected_error_fragment=r"targets.dev.state contains unknown key\(s\): connections", - ), LoadProjectConfigErrorTestCase( description="raises when environment clone allow_as_clone_origin is not a boolean", project_file_contents=""" @@ -1652,17 +1585,6 @@ def test_given_local_target_execution_limits_when_loading_then_limits_are_typed( """.strip(), expected_error_fragment="janitor.delete_tracked_only requires", ), - LoadProjectConfigErrorTestCase( - description="raises when janitor max checkpoints is less than one", - project_file_contents=""" -name = "demo" -adapter = "duckdb" - -[janitor] -max_checkpoints = 0 -""".strip(), - expected_error_fragment="janitor.max_checkpoints must be >= 1", - ), LoadProjectConfigErrorTestCase( description="raises when janitor direct state history versions is negative", project_file_contents=""" diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py index 1e8b04cec4..c41ac24208 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/_test_types.py @@ -216,16 +216,6 @@ class SelectionStalenessGraphWarningTestCase: unexpected_warning_fragments: tuple[str, ...] = () -@dataclass(frozen=True) -class VersionStalenessTestCase: - description: str - model_names: tuple[str, ...] - expected_version_hashes: dict[str, str] - built_version_hashes: dict[str, str | None] - forced_stale_model_names: tuple[str, ...] - expected_stale_model_names: tuple[str, ...] - - @dataclass(frozen=True) class GraphIdentityExpectedHashesTestCase: description: str @@ -255,7 +245,6 @@ class GraphIdentityWritePerfTestCase: class ModelClosureTestCase: description: str expected_downstream_model_names: frozenset[str] - expected_upstream_model_names: frozenset[str] @dataclass(frozen=True) @@ -545,6 +534,7 @@ class ScenarioHashPrefixTestCase: project_name: str scenario_name: str expected_hash_prefix: str + run_namespace: str | None = None @dataclass(frozen=True) @@ -553,6 +543,7 @@ class ScenarioHashCollisionTestCase: scenario_names: tuple[str, ...] prefix_length: int expected_error_fragment: str + run_namespace: str | None = None @dataclass(frozen=True) @@ -914,18 +905,6 @@ class ResolveEffectiveRunScopeTestCase: expected_effective_run_scope: AuditRunScope -@dataclass(frozen=True) -class RunDespiteUnchangedPlanningTestCase: - description: str - run_despite_unchanged: object - materialized: str - data_version: str | None - value_kind: str - expected_root_model_names: frozenset[str] - expected_stale_model_names: frozenset[str] - expected_error_fragment: str | None = None - - @dataclass(frozen=True) class PlannerStaleWarningScopeTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index 1ab2d85e0a..75d88ebc68 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -63,7 +63,6 @@ ChangeDetectionResult, GraphIdentityNode, GraphNodeKey, - PlannerScope, PlanOutput, ScenarioArtifactIdentity, ScenarioRelationMap, @@ -74,11 +73,6 @@ GraphResourceKind, ) from sqlbuild.compiler.references.types import SqlReferenceKind -from sqlbuild.compiler.source_freshness.models import ( - DirectSourceFreshnessPlanningResult, - SourceFreshnessIdentity, - SourceFreshnessRecord, -) from sqlbuild.spec.contracts.models import ( SchemaColumn, SchemaModelEntry, @@ -88,7 +82,6 @@ SourceEntry, ) from sqlbuild.spec.contracts.types import ( - SourceFreshnessStrategy, SourceWriteStrategy, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( @@ -220,62 +213,15 @@ def model_key(name: str) -> CompiledObjectKey: return CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=name) -def build_run_despite_unchanged_scope( - *, run_despite_unchanged: object, materialized: str -) -> PlannerScope: - """Build a source -> rolling table -> mart planner scope.""" - - source_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SOURCE, - name="raw_orders", - ) - rolling_key: CompiledObjectKey = model_key("rolling_orders") - mart_key: CompiledObjectKey = model_key("orders_mart") - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { - rolling_key: (source_key,), - mart_key: (rolling_key,), - source_key: (), - } - return PlannerScope( - upstream_deps=upstream_deps, - downstream_deps=build_downstream_deps(upstream_deps), - all_keys={ - "raw_orders": source_key, - "rolling_orders": rolling_key, - "orders_mart": mart_key, - }, - models_by_name={ - "rolling_orders": build_run_despite_unchanged_model( - key=rolling_key, - name="rolling_orders", - materialized=materialized, - run_despite_unchanged=run_despite_unchanged, - ), - "orders_mart": build_run_despite_unchanged_model( - key=mart_key, - name="orders_mart", - materialized="table", - run_despite_unchanged=None, - ), - }, - selected_keys=frozenset({rolling_key, mart_key}), - execution_order=(source_key, rolling_key, mart_key), - ) - - -def build_run_despite_unchanged_model( +def build_selection_model( *, key: CompiledObjectKey, name: str, materialized: str, - run_despite_unchanged: object | None, ) -> CompiledModel: - """Build a minimal model for run_despite_unchanged helper tests.""" + """Build a minimal model for selection helper tests.""" - values: dict[str, object] = {"materialized": materialized} | ( - {}, - {"run_despite_unchanged": run_despite_unchanged}, - )[run_despite_unchanged is not None] + values: dict[str, object] = {"materialized": materialized} return CompiledModel( key=key, deps=(), @@ -292,39 +238,6 @@ def build_run_despite_unchanged_model( ) -def build_run_despite_unchanged_source_freshness( - *, data_version: str | None, value_kind: str, observed_at: datetime -) -> DirectSourceFreshnessPlanningResult: - """Build source freshness state for run_despite_unchanged helper tests.""" - - record: SourceFreshnessRecord = SourceFreshnessRecord( - source_name="raw_orders", - target_database=None, - target_schema=None, - target_name=None, - run_id="run-1", - strategy=SourceFreshnessStrategy.SQL.value, - value_kind=value_kind, - data_version=data_version or "", - data_version_hash="hash", - observed_at=observed_at, - ) - populated_result: DirectSourceFreshnessPlanningResult = DirectSourceFreshnessPlanningResult( - observed_records=(record,), - unchanged_identities=frozenset( - { - SourceFreshnessIdentity( - source_name="raw_orders", - target_database=None, - target_schema=None, - target_name=None, - ) - } - ), - ) - return (DirectSourceFreshnessPlanningResult(), populated_result)[data_version is not None] - - def source_key(name: str) -> CompiledObjectKey: """Build a source object key.""" @@ -1498,8 +1411,6 @@ def build_scheduling_graph( ]: """Build upstream and downstream dep dicts from simple name-based edges.""" - from sqlbuild.compiler.planner._helpers.graph.core import build_downstream_deps - upstream: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = {} name: str deps: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py index 9f9316e95e..208b1a2173 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/identity/test_merge_recomputed_model_changes.py @@ -57,14 +57,6 @@ expected_backfill_action=BackfillAction.BOUNDED, expected_version_hash="resolved-hash", ), - MergeRecomputedModelChangesTestCase( - description="run_despite_unchanged keeps resolved change when no recomputed change exists", - resolved_change_kind=ChangeKind.RUN_DESPITE_UNCHANGED, - recomputed_change_kind=None, - expected_change_kind=ChangeKind.RUN_DESPITE_UNCHANGED, - expected_backfill_action=BackfillAction.BOUNDED, - expected_version_hash="resolved-hash", - ), MergeRecomputedModelChangesTestCase( description="no_change keeps resolved change when no recomputed change exists", resolved_change_kind=ChangeKind.NO_CHANGE, @@ -163,22 +155,6 @@ def test_given_no_recomputed_change_when_merging_then_keeps_resolved_change( expected_backfill_action=BackfillAction.FORWARD_ONLY, expected_version_hash="recomputed-hash", ), - MergeRecomputedModelChangesTestCase( - description="run_despite_unchanged survives recomputed no_change with resolved backfill", - resolved_change_kind=ChangeKind.RUN_DESPITE_UNCHANGED, - recomputed_change_kind=ChangeKind.NO_CHANGE, - expected_change_kind=ChangeKind.RUN_DESPITE_UNCHANGED, - expected_backfill_action=BackfillAction.BOUNDED, - expected_version_hash="recomputed-hash", - ), - MergeRecomputedModelChangesTestCase( - description="run_despite_unchanged adopts recomputed query_changed", - resolved_change_kind=ChangeKind.RUN_DESPITE_UNCHANGED, - recomputed_change_kind=ChangeKind.QUERY_CHANGED, - expected_change_kind=ChangeKind.QUERY_CHANGED, - expected_backfill_action=BackfillAction.FORWARD_ONLY, - expected_version_hash="recomputed-hash", - ), MergeRecomputedModelChangesTestCase( description="no_change adopts recomputed no_change", resolved_change_kind=ChangeKind.NO_CHANGE, diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py index 94e2889dcf..cb71d34ebc 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_model_closure.py @@ -6,7 +6,6 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.planner._helpers.graph.model_closure import ( build_downstream_model_name_closure, - build_upstream_model_name_closure, ) from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ModelClosureTestCase @@ -36,11 +35,10 @@ "test_case", [ ModelClosureTestCase( - description="collects model names in downstream and upstream graph closures", + description="collects model names in downstream graph closures", expected_downstream_model_names=frozenset( {"stg_orders", "fact_orders", "order_rollup"} ), - expected_upstream_model_names=frozenset({"stg_orders", "fact_orders", "order_rollup"}), ) ], ids=lambda case: case.description, @@ -55,22 +53,10 @@ def test_given_dependency_graph_when_building_model_closures_then_returns_reacha FACT_ORDERS: (ORDER_ROLLUP,), ORDER_ROLLUP: (), } - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = { - SOURCE_ORDERS: (), - FUNCTION_NORMALIZE: (SOURCE_ORDERS,), - STG_ORDERS: (FUNCTION_NORMALIZE,), - FACT_ORDERS: (STG_ORDERS,), - ORDER_ROLLUP: (FACT_ORDERS,), - } downstream_result: frozenset[str] = build_downstream_model_name_closure( start_keys=(SOURCE_ORDERS,), downstream_deps=downstream_deps, ) - upstream_result: frozenset[str] = build_upstream_model_name_closure( - start_keys=(ORDER_ROLLUP,), - upstream_deps=upstream_deps, - ) assert downstream_result == test_case.expected_downstream_model_names - assert upstream_result == test_case.expected_upstream_model_names diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_run_despite_unchanged.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_run_despite_unchanged.py deleted file mode 100644 index aaf4955a9d..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_run_despite_unchanged.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime - -import pytest - -from sqlbuild.compiler.planner._helpers.pruning.run_despite_unchanged import ( - build_run_despite_unchanged_planning_result, -) -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.models import PlannerScope, RunDespiteUnchangedPlanningResult -from sqlbuild.compiler.source_freshness.models import DirectSourceFreshnessPlanningResult -from sqlbuild.spec.contracts.types import SourceFreshnessValueKind -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import ( - RunDespiteUnchangedPlanningTestCase, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( - build_run_despite_unchanged_scope, - build_run_despite_unchanged_source_freshness, -) - -_NOW: datetime = datetime(2026, 6, 11, tzinfo=UTC) - - -@pytest.mark.parametrize( - "test_case", - [ - RunDespiteUnchangedPlanningTestCase( - description="always mode marks root and downstream stale", - run_despite_unchanged="always", - materialized="table", - data_version=None, - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset({"rolling_orders"}), - expected_stale_model_names=frozenset({"rolling_orders", "orders_mart"}), - ), - RunDespiteUnchangedPlanningTestCase( - description="active duration marks root and downstream stale", - run_despite_unchanged="30d", - materialized="table", - data_version="2026-06-01T00:00:00+00:00", - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset({"rolling_orders"}), - expected_stale_model_names=frozenset({"rolling_orders", "orders_mart"}), - ), - RunDespiteUnchangedPlanningTestCase( - description="active hour duration marks root and downstream stale", - run_despite_unchanged="12h", - materialized="table", - data_version="2026-06-10T18:00:00+00:00", - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset({"rolling_orders"}), - expected_stale_model_names=frozenset({"rolling_orders", "orders_mart"}), - ), - RunDespiteUnchangedPlanningTestCase( - description="active minute duration marks root and downstream stale", - run_despite_unchanged="90m", - materialized="table", - data_version="2026-06-10T23:00:00+00:00", - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset({"rolling_orders"}), - expected_stale_model_names=frozenset({"rolling_orders", "orders_mart"}), - ), - RunDespiteUnchangedPlanningTestCase( - description="expired duration leaves root and downstream current", - run_despite_unchanged="1d", - materialized="table", - data_version="2026-06-01T00:00:00+00:00", - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset(), - expected_stale_model_names=frozenset(), - ), - ], - ids=lambda case: case.description, -) -def test_given_valid_run_despite_unchanged_config_when_planning_then_returns_expected_roots( - test_case: RunDespiteUnchangedPlanningTestCase, -) -> None: - scope: PlannerScope = build_run_despite_unchanged_scope( - run_despite_unchanged=test_case.run_despite_unchanged, - materialized=test_case.materialized, - ) - source_freshness: DirectSourceFreshnessPlanningResult = ( - build_run_despite_unchanged_source_freshness( - data_version=test_case.data_version, - value_kind=test_case.value_kind, - observed_at=_NOW, - ) - ) - - result: RunDespiteUnchangedPlanningResult = build_run_despite_unchanged_planning_result( - scope=scope, - source_freshness=source_freshness, - already_stale_model_names=frozenset(), - now=_NOW, - ) - - assert result.root_model_names == test_case.expected_root_model_names - assert result.stale_model_names == test_case.expected_stale_model_names - - -@pytest.mark.parametrize( - "test_case", - [ - RunDespiteUnchangedPlanningTestCase( - description="invalid duration raises clear error", - run_despite_unchanged="30days", - materialized="table", - data_version="2026-06-01T00:00:00+00:00", - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset(), - expected_stale_model_names=frozenset(), - expected_error_fragment="must be 'always' or a positive duration", - ), - RunDespiteUnchangedPlanningTestCase( - description="unsupported second duration raises clear error", - run_despite_unchanged="30s", - materialized="table", - data_version="2026-06-01T00:00:00+00:00", - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset(), - expected_stale_model_names=frozenset(), - expected_error_fragment="must be 'always' or a positive duration", - ), - RunDespiteUnchangedPlanningTestCase( - description="non-table materialization raises clear error", - run_despite_unchanged="always", - materialized="view", - data_version=None, - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - expected_root_model_names=frozenset(), - expected_stale_model_names=frozenset(), - expected_error_fragment="only table materializations support it", - ), - RunDespiteUnchangedPlanningTestCase( - description="integer freshness raises clear error for duration", - run_despite_unchanged="30d", - materialized="table", - data_version="1", - value_kind=SourceFreshnessValueKind.INTEGER.value, - expected_root_model_names=frozenset(), - expected_stale_model_names=frozenset(), - expected_error_fragment="requires timestamp source freshness", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_run_despite_unchanged_config_when_planning_then_raises_error( - test_case: RunDespiteUnchangedPlanningTestCase, -) -> None: - scope: PlannerScope = build_run_despite_unchanged_scope( - run_despite_unchanged=test_case.run_despite_unchanged, - materialized=test_case.materialized, - ) - source_freshness: DirectSourceFreshnessPlanningResult = ( - build_run_despite_unchanged_source_freshness( - data_version=test_case.data_version, - value_kind=test_case.value_kind, - observed_at=_NOW, - ) - ) - - with pytest.raises(PlannerInputError) as exc_info: - build_run_despite_unchanged_planning_result( - scope=scope, - source_freshness=source_freshness, - already_stale_model_names=frozenset(), - now=_NOW, - ) - - assert test_case.expected_error_fragment is not None - assert test_case.expected_error_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py index 6febf0ca65..13545647f5 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_scenario_artifacts.py @@ -41,6 +41,13 @@ @pytest.mark.parametrize( "test_case", [ + ScenarioHashPrefixTestCase( + description="namespace is inserted between project and scenario", + project_name="waffle_shop", + scenario_name="revenue__customer_refund", + run_namespace="job-1", + expected_hash_prefix="60b4ca1f43a9", + ), ScenarioHashPrefixTestCase( description="uses stable project and scenario identity hash", project_name="waffle_shop", @@ -62,6 +69,7 @@ def test_given_project_and_scenario_when_hashing_then_returns_expected_prefix( result: str = compute_scenario_hash_prefix( project_name=test_case.project_name, scenario_name=test_case.scenario_name, + run_namespace=test_case.run_namespace, ) assert result == test_case.expected_hash_prefix @@ -75,7 +83,14 @@ def test_given_project_and_scenario_when_hashing_then_returns_expected_prefix( scenario_names=("scenario_2", "scenario_5"), prefix_length=1, expected_error_fragment="both map to hash prefix '9'", - ) + ), + ScenarioHashCollisionTestCase( + description="collision detection applies within a namespace", + scenario_names=tuple(f"scenario_{index}" for index in range(17)), + prefix_length=1, + run_namespace="job-1", + expected_error_fragment="Scenario artifact hash collision", + ), ], ids=lambda case: case.description, ) @@ -89,6 +104,7 @@ def test_given_colliding_scenario_hashes_when_building_index_then_raises_clear_e build_compiled_scenario_with_name(name) for name in test_case.scenario_names ), prefix_length=test_case.prefix_length, + run_namespace=test_case.run_namespace, ) assert exc_info.value.code == SCENARIO_PLAN_HASH_COLLISION diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py index fcf7f4338a..8fff444c99 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_selection_staleness.py @@ -43,7 +43,7 @@ StaleWarningMessageTestCase, ) from tests.unit.src.sqlbuild.compiler.planner._helpers.helpers import ( - build_run_despite_unchanged_model, + build_selection_model, ) MODEL_KEY: CompiledObjectKey = CompiledObjectKey( @@ -66,11 +66,10 @@ def test_given_complete_change_results_when_building_stale_warnings_then_models_ test_case: CompleteModelChangesWorkTestCase, monkeypatch: pytest.MonkeyPatch, ) -> None: - model: CompiledModel = build_run_despite_unchanged_model( + model: CompiledModel = build_selection_model( key=MODEL_KEY, name=MODEL_KEY.name, materialized="table", - run_despite_unchanged=None, ) scope: PlannerScope = PlannerScope( upstream_deps={MODEL_KEY: ()}, @@ -181,35 +180,31 @@ def test_given_partial_selection_when_resolving_scopes_then_inspects_only_upstre resource_type=CompiledResourceType.MODEL, name="unrelated", ) - root: CompiledModel = build_run_despite_unchanged_model( + root: CompiledModel = build_selection_model( key=root_key, name=root_key.name, materialized="table", - run_despite_unchanged=None, ) middle: CompiledModel = replace( - build_run_despite_unchanged_model( + build_selection_model( key=middle_key, name=middle_key.name, materialized="table", - run_despite_unchanged=None, ), deps=(root_key,), ) leaf: CompiledModel = replace( - build_run_despite_unchanged_model( + build_selection_model( key=leaf_key, name=leaf_key.name, materialized="table", - run_despite_unchanged=None, ), deps=(middle_key,), ) - unrelated: CompiledModel = build_run_despite_unchanged_model( + unrelated: CompiledModel = build_selection_model( key=unrelated_key, name=unrelated_key.name, materialized="table", - run_despite_unchanged=None, ) project: CompiledProject = CompiledProject( run_id="run", diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_version_staleness.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_version_staleness.py deleted file mode 100644 index 7d62743a4e..0000000000 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/test_version_staleness.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.planner._helpers.pruning.version_staleness import ( - build_stale_model_names_from_version_identities, -) -from tests.unit.src.sqlbuild.compiler.planner._helpers._test_types import VersionStalenessTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - VersionStalenessTestCase( - description="preserves model order while marking missing mismatched and forced stale", - model_names=("current", "missing", "mismatched", "forced"), - expected_version_hashes={ - "current": "hash_a", - "missing": "hash_b", - "mismatched": "hash_c", - "forced": "hash_d", - }, - built_version_hashes={ - "current": "hash_a", - "mismatched": "old_hash", - "forced": "hash_d", - }, - forced_stale_model_names=("forced",), - expected_stale_model_names=("missing", "mismatched", "forced"), - ) - ], - ids=lambda case: case.description, -) -def test_given_version_identity_maps_when_collecting_stale_models_then_returns_expected_names( - test_case: VersionStalenessTestCase, -) -> None: - result: tuple[str, ...] = build_stale_model_names_from_version_identities( - model_names=test_case.model_names, - expected_version_hashes=test_case.expected_version_hashes, - built_version_hashes=test_case.built_version_hashes, - forced_stale_model_names=test_case.forced_stale_model_names, - ) - - assert result == test_case.expected_stale_model_names diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py new file mode 100644 index 0000000000..9ed9362053 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/_test_types.py @@ -0,0 +1,10 @@ +"""Cases for bounded source metadata inspection.""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SemanticSourceBatchCase: + description: str + expected_batches: int + expected_sources: frozenset[str] diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py new file mode 100644 index 0000000000..b202e513fc --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/warehouse/test_semantic_sources.py @@ -0,0 +1,74 @@ +"""Multiple source readers share one metadata batch per database.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from unittest.mock import Mock + +import duckdb +import pytest + +from sqlbuild.adapter.contract.models import ColumnInfo +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.pipeline.main.project import compile_project +from sqlbuild.compiler.planner._helpers.warehouse.semantic_sources import ( + get_semantic_source_columns, +) +from tests.unit.src.sqlbuild.compiler.planner._helpers.warehouse._test_types import ( + SemanticSourceBatchCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticSourceBatchCase( + description="two unqualified sources use one database batch", + expected_batches=1, + expected_sources=frozenset({"orders", "customers"}), + ) + ], + ids=lambda case: case.description, +) +def test_given_multiple_missing_source_schemas_when_inspecting_then_metadata_is_batched( + test_case: SemanticSourceBatchCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files( + tmp_path, + { + "sqlbuild_project.toml": 'name = "source_batch"\nadapter = "duckdb"\n', + "sources/orders.yml": "sources:\n - name: orders\n table: orders\n - name: customers\n table: customers\n", + "models/combined.sql": 'MODEL ();\nSELECT o.id FROM __source("orders") o JOIN __source("customers") c ON o.id = c.id', + }, + ) + adapter: DuckDbAdapter = DuckDbAdapter() + project: CompiledProject = compile_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), adapter=adapter + ) + listings: Mock = Mock(wraps=adapter.list_relations) + columns: Mock = Mock(wraps=adapter.get_columns_for_relations) + monkeypatch.setattr(adapter, "list_relations", listings) + monkeypatch.setattr(adapter, "get_columns_for_relations", columns) + with duckdb.connect() as connection: + connection.execute("CREATE TABLE orders(id INTEGER); CREATE TABLE customers(id INTEGER)") + result: dict[str, tuple[ColumnInfo, ...]] = get_semantic_source_columns( + project=project, + adapter=adapter, + connection=connection, + source_read_map={source.name: source.source_entry for source in project.sources}, + columns={}, + selected_keys=frozenset(model.key for model in project.models), + ) + assert frozenset(result) == test_case.expected_sources + assert listings.call_count == test_case.expected_batches + assert columns.call_count == test_case.expected_batches + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/virtual/freshness/_test_types.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py similarity index 55% rename from tests/unit/src/sqlbuild/virtual/freshness/_test_types.py rename to tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py index c21b09b401..bfde40ce0b 100644 --- a/tests/unit/src/sqlbuild/virtual/freshness/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/_test_types.py @@ -1,5 +1,4 @@ from dataclasses import dataclass -from datetime import datetime from sqlbuild.spec.contracts.types import SourceFreshnessValueKind @@ -39,43 +38,9 @@ class UnsupportedTableFreshnessMetadataGuardTestCase: expected_metadata_requested: bool -@dataclass(frozen=True) -class SourceFreshnessStateTestCase: - description: str - source_name: str - strategy: str - value_kind: str - data_version: object - observed_at: datetime - expected_data_version: str - expected_hash_changes_with_observed_at: bool - - @dataclass(frozen=True) class SourceFreshnessStateErrorTestCase: description: str value_kind: str data_version: object expected_error_fragment: str - - -@dataclass(frozen=True) -class SourceFreshnessRuntimeTestCase: - description: str - expected_record_sources: tuple[str, ...] - expected_unknown_sources: tuple[str, ...] - expected_preserved_sources: tuple[str, ...] - expected_generated_sources: tuple[str, ...] - - -@dataclass(frozen=True) -class SourceFreshnessRuntimeLagToleranceTestCase: - description: str - current_data_version: str - expected_record_data_version: str - - -@dataclass(frozen=True) -class VirtualSourceFreshnessCompatibilityTestCase: - description: str - expected_same_object: bool diff --git a/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py new file mode 100644 index 0000000000..f69c76ceed --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/source_freshness/main/configured_observation/test_observation.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +import pytest + +from sqlbuild.adapter.contract.models import TableFreshnessMetadata +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError +from sqlbuild.compiler.source_freshness.main.normalization import ( + normalize_source_freshness_data_version, +) +from sqlbuild.compiler.source_freshness.main.observation import ( + observe_configured_source_freshness, +) +from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation +from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig +from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind +from tests.unit.src.sqlbuild.compiler.source_freshness.main.configured_observation._test_types import ( + SourceFreshnessObservationErrorTestCase, + SourceFreshnessObservationTestCase, + SourceFreshnessStateErrorTestCase, + UnsupportedTableFreshnessMetadataGuardTestCase, +) + + +class UnsupportedFreshnessMetadataDuckDbAdapter(DuckDbAdapter): + metadata_requested: bool + + def __init__(self) -> None: + super().__init__() + self.metadata_requested = False + + def supports_table_freshness_metadata(self) -> bool: + return False + + def get_table_freshness_metadata( + self, + connection: Any, + *, + database: str | None, + schema: str | None, + name: str, + ) -> TableFreshnessMetadata: + self.metadata_requested = True + return TableFreshnessMetadata( + data_version=datetime(2026, 1, 1, 12, 0, 0), + value_kind="timestamp", + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SourceFreshnessObservationTestCase( + description="observes max column data version", + setup_sql=( + "CREATE TABLE raw_orders (updated_at INTEGER)", + "INSERT INTO raw_orders VALUES (1), (3), (2)", + ), + source_name="raw_orders", + table="raw_orders", + strategy="column", + column="updated_at", + value_kind=SourceFreshnessValueKind.INTEGER, + expected_data_version=3, + ), + SourceFreshnessObservationTestCase( + description="observes sql data version", + setup_sql=(), + source_name="raw_orders", + table=None, + strategy="sql", + query="SELECT 'version-1' AS data_version", + value_kind="string", + expected_data_version="version-1", + ), + ], + ids=lambda case: case.description, +) +def test_given_source_freshness_config_when_observing_then_returns_data_version( + test_case: SourceFreshnessObservationTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + observed_at: datetime = datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC) + try: + statement: str + for statement in test_case.setup_sql: + adapter.execute(connection=connection, sql=statement) + observation: SourceFreshnessObservation = observe_configured_source_freshness( + adapter=adapter, + connection=connection, + source=SourceEntry( + name=test_case.source_name, + table=test_case.table, + freshness=SourceFreshnessConfig( + strategy=SourceFreshnessStrategy(test_case.strategy), + value_kind=SourceFreshnessValueKind(test_case.value_kind), + column=test_case.column, + query=test_case.query, + ), + ), + observed_at=observed_at, + ) + finally: + adapter.close(connection) + + assert observation.source_name == test_case.source_name + assert observation.data_version == test_case.expected_data_version + assert observation.value_kind == SourceFreshnessValueKind(test_case.value_kind) + assert observation.observed_at == observed_at + + +@pytest.mark.parametrize( + "test_case", + [ + SourceFreshnessObservationErrorTestCase( + description="raises when sql freshness returns multiple columns", + setup_sql=(), + source_name="raw_orders", + table=None, + strategy="sql", + query="SELECT 1 AS left_value, 2 AS right_value", + value_kind=SourceFreshnessValueKind.INTEGER, + expected_error_fragment="must return exactly one column", + ), + SourceFreshnessObservationErrorTestCase( + description="raises when sql freshness returns zero rows", + setup_sql=(), + source_name="raw_orders", + table=None, + strategy="sql", + query="SELECT 1 AS data_version WHERE FALSE", + value_kind=SourceFreshnessValueKind.INTEGER, + expected_error_fragment="must return exactly one row", + ), + SourceFreshnessObservationErrorTestCase( + description="raises when sql freshness returns null", + setup_sql=(), + source_name="raw_orders", + table=None, + strategy="sql", + query="SELECT NULL AS data_version", + value_kind=SourceFreshnessValueKind.INTEGER, + expected_error_fragment="data_version cannot be null", + ), + SourceFreshnessObservationErrorTestCase( + description="raises when adapter metadata is unsupported", + setup_sql=(), + source_name="raw_orders", + table="raw_orders", + strategy="adapter", + value_kind=None, + expected_error_fragment="does not support table freshness metadata", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_source_freshness_result_when_observing_then_raises_clear_error( + test_case: SourceFreshnessObservationErrorTestCase, +) -> None: + adapter: DuckDbAdapter = DuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + try: + statement: str + for statement in test_case.setup_sql: + adapter.execute(connection=connection, sql=statement) + source: SourceEntry = SourceEntry( + name=test_case.source_name, + table=test_case.table, + freshness=SourceFreshnessConfig( + strategy=SourceFreshnessStrategy(test_case.strategy), + value_kind=test_case.value_kind, + column=test_case.column, + query=test_case.query, + ), + ) + with pytest.raises( + SourceFreshnessObservationError, match=test_case.expected_error_fragment + ): + observe_configured_source_freshness( + adapter=adapter, + connection=connection, + source=source, + observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), + ) + finally: + adapter.close(connection) + + +@pytest.mark.parametrize( + "test_case", + [ + UnsupportedTableFreshnessMetadataGuardTestCase( + description="unsupported adapter guard prevents metadata lookup", + source_name="raw_orders", + table="raw_orders", + expected_error_fragment="does not support table freshness metadata", + expected_metadata_requested=False, + ) + ], + ids=lambda case: case.description, +) +def test_given_adapter_freshness_is_unsupported_when_observing_then_metadata_is_not_requested( + test_case: UnsupportedTableFreshnessMetadataGuardTestCase, +) -> None: + adapter: UnsupportedFreshnessMetadataDuckDbAdapter = UnsupportedFreshnessMetadataDuckDbAdapter() + connection: Any = adapter.connect({"database": ":memory:"}) + try: + source: SourceEntry = SourceEntry( + name=test_case.source_name, + table=test_case.table, + freshness=SourceFreshnessConfig( + strategy=SourceFreshnessStrategy.ADAPTER, + value_kind=None, + ), + ) + + with pytest.raises( + SourceFreshnessObservationError, match=test_case.expected_error_fragment + ): + observe_configured_source_freshness( + adapter=adapter, + connection=connection, + source=source, + observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), + ) + finally: + adapter.close(connection) + + assert adapter.metadata_requested is test_case.expected_metadata_requested + + +@pytest.mark.parametrize( + "test_case", + [ + SourceFreshnessStateErrorTestCase( + description="rejects non datetime timestamp value", + value_kind="timestamp", + data_version="2026-01-01T12:00:00Z", + expected_error_fragment="must be datetime values", + ), + SourceFreshnessStateErrorTestCase( + description="rejects bool integer value", + value_kind="integer", + data_version=True, + expected_error_fragment="must be integer values", + ), + SourceFreshnessStateErrorTestCase( + description="rejects non integer value", + value_kind="integer", + data_version="123", + expected_error_fragment="must be integer values", + ), + SourceFreshnessStateErrorTestCase( + description="rejects non string value", + value_kind="string", + data_version=123, + expected_error_fragment="must be string values", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_source_freshness_value_when_normalizing_then_raises_clear_error( + test_case: SourceFreshnessStateErrorTestCase, +) -> None: + with pytest.raises(SourceFreshnessObservationError, match=test_case.expected_error_fragment): + normalize_source_freshness_data_version( + value=test_case.data_version, + value_kind=SourceFreshnessValueKind(test_case.value_kind), + ) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py index 3568e0fe3b..b50c6ad60d 100644 --- a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/_test_types.py @@ -3,6 +3,12 @@ from dataclasses import dataclass +@dataclass(frozen=True) +class SemanticDiagnosticMappingCase: + description: str + expected_codes: tuple[str, ...] + + @dataclass(frozen=True) class SchemaValidationScopeTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py new file mode 100644 index 0000000000..038b660a85 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_normalize_analysis.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.sql_analysis.main._normalize_analysis import normalize_analysis_sql +from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( + PolyglotSqlNormalizationTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + ( + PolyglotSqlNormalizationTestCase( + description="package-qualified dbt reference becomes its model relation", + sql='SELECT order_id FROM __dbt_ref("analytics", "dbt_orders")', + dialect="duckdb", + expected_sql="SELECT order_id FROM dbt_orders", + ), + PolyglotSqlNormalizationTestCase( + description="bare dbt reference becomes its model relation", + sql='SELECT order_id FROM __dbt_ref("dbt_orders")', + dialect="duckdb", + expected_sql="SELECT order_id FROM dbt_orders", + ), + PolyglotSqlNormalizationTestCase( + description="project references keep their relation names", + sql='SELECT order_id FROM __ref("orders") JOIN __source("raw_orders") USING (order_id)', + dialect="duckdb", + expected_sql="SELECT order_id FROM orders JOIN raw_orders USING (order_id)", + ), + ), + ids=lambda case: case.description, +) +def test_given_reference_intrinsics_when_normalizing_for_analysis_then_relations_are_named( + test_case: PolyglotSqlNormalizationTestCase, +) -> None: + assert ( + normalize_analysis_sql(sql=test_case.sql, dialect=test_case.dialect) + == test_case.expected_sql + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py new file mode 100644 index 0000000000..528eb27bdc --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/sql_analysis/main/test_semantic_diagnostics.py @@ -0,0 +1,94 @@ +"""Native semantic diagnostics survive the SQLBuild boundary and dialect gate.""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +import sqlbuild._native as native +from sqlbuild.compiler.sql_analysis.main._schema_validation import get_schema_validations +from sqlbuild.compiler.sql_analysis.models import SqlBindingResult, SqlSchemaValidationRequest +from tests.unit.src.sqlbuild.compiler.sql_analysis.main._test_types import ( + SemanticDiagnosticMappingCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SemanticDiagnosticMappingCase( + description="semantic, prepared type, and implicit-conversion diagnostics", + expected_codes=( + "B101", + "B102", + "B004", + "B005", + "B210", + "B211", + "B212", + "B213", + "B214", + "B215", + "B216", + "B217", + "B218", + "B219", + "B230", + "B231", + "B232", + "B233", + "B234", + "B217", + "B218", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_native_semantic_errors_when_validating_then_codes_and_locations_survive( + monkeypatch: pytest.MonkeyPatch, + test_case: SemanticDiagnosticMappingCase, +) -> None: + codes: dict[str, str] = {"E202": "B101", "E203": "B102", "E222": "B004", "E223": "B005"} + codes.update( + {f"E{number}": f"B{number}" for number in (*range(210, 220), 230, 231, 232, 233, 234)} + ) + codes.update({"W210": "B217", "W213": "B218"}) + + def validate(payload: str) -> str: + requests: list[dict[str, Any]] = json.loads(payload) + assert all(request["options"]["semantic"] for request in requests) + assert all(request["options"]["check_types"] for request in requests) + errors: list[dict[str, object]] = [ + { + "code": code, + "severity": {True: "warning", False: "error"}[code.startswith("W")], + "message": "Invalid expression", + "line": 2, + "column": 8, + } + for code in codes + ] + return json.dumps([{"errors": errors}] * len(requests)) + + monkeypatch.setattr(native, "validate_sql_with_schemas_json", validate) + results: tuple[SqlBindingResult, ...] = get_schema_validations( + requests=tuple( + SqlSchemaValidationRequest(sql="SELECT\n invalid", dialect=dialect, schema={}) + for dialect in ("duckdb", "snowflake", "postgres", "bigquery") + ) + ) + for result in results: + assert ( + tuple(diagnostic.code for diagnostic in result.diagnostics) == test_case.expected_codes + ) + assert all( + (diagnostic.line, diagnostic.column) == (2, 8) for diagnostic in result.diagnostics + ) + assert all(diagnostic.severity == "error" for diagnostic in result.diagnostics) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/executor/custom/_test_types.py b/tests/unit/src/sqlbuild/executor/custom/_test_types.py index 2e324c5394..b24119802a 100644 --- a/tests/unit/src/sqlbuild/executor/custom/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/custom/_test_types.py @@ -1,13 +1,13 @@ from collections.abc import Callable from dataclasses import dataclass -from sqlbuild.executor.custom.models import MaterializationContext, PrepareVersionContext +from sqlbuild.executor.custom.models import MaterializationContext @dataclass(frozen=True) class CustomContextExecutionTestCase: description: str - context_builder: Callable[..., MaterializationContext | PrepareVersionContext] + context_builder: Callable[..., MaterializationContext] sql: str expected_result: object expected_operation_order: tuple[str, ...] @@ -17,7 +17,7 @@ class CustomContextExecutionTestCase: @dataclass(frozen=True) class CustomContextQualificationTestCase: description: str - context_builder: Callable[..., MaterializationContext | PrepareVersionContext] + context_builder: Callable[..., MaterializationContext] name: str database: str | None schema: str | None diff --git a/tests/unit/src/sqlbuild/executor/custom/helpers.py b/tests/unit/src/sqlbuild/executor/custom/helpers.py index bd96978a98..e71ec35395 100644 --- a/tests/unit/src/sqlbuild/executor/custom/helpers.py +++ b/tests/unit/src/sqlbuild/executor/custom/helpers.py @@ -6,7 +6,7 @@ from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.executor.auditing.models import AuditExecutionResult -from sqlbuild.executor.custom.models import MaterializationContext, PrepareVersionContext +from sqlbuild.executor.custom.models import MaterializationContext class RecordingCustomAdapter(BaseAdapter): @@ -78,28 +78,3 @@ def build_materialization_context( logger=logging.getLogger(__name__), statement_recorder=statement_recorder, ) - - -def build_prepare_version_context( - *, - adapter: BaseAdapter, - connection: object, - statement_recorder: StatementRecorder, -) -> PrepareVersionContext: - return PrepareVersionContext( - adapter=adapter, - connection=connection, - origin_relation="warehouse.analytics.orders_origin", - destination="warehouse.analytics.orders", - destination_database="warehouse", - destination_schema="analytics", - destination_name="orders", - config={}, - placeholders={}, - run_id="test-run", - environment="dev", - vars={}, - unique_key=(), - declared_columns=(), - statement_recorder=statement_recorder, - ) diff --git a/tests/unit/src/sqlbuild/executor/custom/test_context_operations.py b/tests/unit/src/sqlbuild/executor/custom/test_context_operations.py index 9176f408e0..e0448e6284 100644 --- a/tests/unit/src/sqlbuild/executor/custom/test_context_operations.py +++ b/tests/unit/src/sqlbuild/executor/custom/test_context_operations.py @@ -4,7 +4,7 @@ from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder from sqlbuild.adapter.contract.models import LifeCycleEvent -from sqlbuild.executor.custom.models import MaterializationContext, PrepareVersionContext +from sqlbuild.executor.custom.models import MaterializationContext from tests.unit.src.sqlbuild.executor.custom._test_types import ( CustomContextExecutionTestCase, CustomContextQualificationTestCase, @@ -13,7 +13,6 @@ OrderingStatementRecorder, RecordingCustomAdapter, build_materialization_context, - build_prepare_version_context, ) @@ -28,14 +27,6 @@ expected_operation_order=("record", "execute"), expected_recorded_sql="CREATE TABLE scratch_orders AS SELECT 1", ), - CustomContextExecutionTestCase( - description="prepare version context records before adapter execution", - context_builder=build_prepare_version_context, - sql="INSERT INTO scratch_orders SELECT 1", - expected_result="prepare-result", - expected_operation_order=("record", "execute"), - expected_recorded_sql="INSERT INTO scratch_orders SELECT 1", - ), ], ids=lambda case: case.description, ) @@ -49,7 +40,7 @@ def test_given_custom_context_when_executing_sql_then_records_before_adapter_cal ) connection: object = object() recorder: OrderingStatementRecorder = OrderingStatementRecorder(operation_order=operation_order) - context: MaterializationContext | PrepareVersionContext = test_case.context_builder( + context: MaterializationContext = test_case.context_builder( adapter=adapter, connection=connection, statement_recorder=recorder, @@ -76,14 +67,6 @@ def test_given_custom_context_when_executing_sql_then_records_before_adapter_cal schema=None, expected_qualified_name="warehouse.analytics.order_state", ), - CustomContextQualificationTestCase( - description="prepare version context uses destination qualification", - context_builder=build_prepare_version_context, - name="order_state", - database=None, - schema=None, - expected_qualified_name="warehouse.analytics.order_state", - ), CustomContextQualificationTestCase( description="materialization context uses explicit qualification", context_builder=build_materialization_context, @@ -92,14 +75,6 @@ def test_given_custom_context_when_executing_sql_then_records_before_adapter_cal schema="history", expected_qualified_name="archive.history.order_state", ), - CustomContextQualificationTestCase( - description="prepare version context uses explicit qualification", - context_builder=build_prepare_version_context, - name="order_state", - database="archive", - schema="history", - expected_qualified_name="archive.history.order_state", - ), CustomContextQualificationTestCase( description="materialization context preserves qualified input", context_builder=build_materialization_context, @@ -108,14 +83,6 @@ def test_given_custom_context_when_executing_sql_then_records_before_adapter_cal schema="history", expected_qualified_name="external.order_state", ), - CustomContextQualificationTestCase( - description="prepare version context preserves qualified input", - context_builder=build_prepare_version_context, - name="external.order_state", - database="archive", - schema="history", - expected_qualified_name="external.order_state", - ), ], ids=lambda case: case.description, ) @@ -126,7 +93,7 @@ def test_given_custom_context_when_qualifying_relation_then_preserves_context_be operation_order=[], execute_result=object(), ) - context: MaterializationContext | PrepareVersionContext = test_case.context_builder( + context: MaterializationContext = test_case.context_builder( adapter=adapter, connection=object(), statement_recorder=StatementRecorder(), diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index 68a1d40b84..5a4971c5c2 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -6,7 +6,6 @@ from sqlbuild.adapter.contract.models import RelationInfo from sqlbuild.executor.janitor.models import ( JanitorRelationKey, - JanitorVirtualStatePruneCandidate, ) @@ -25,14 +24,10 @@ class JanitorPlanTestCase: protected_relation_keys: frozenset[JanitorRelationKey] = frozenset() expected_candidate_names: tuple[str, ...] = field(default_factory=tuple) expected_direct_state_table_names: tuple[str, ...] = field(default_factory=tuple) - expected_virtual_state_table_names: tuple[str, ...] = field(default_factory=tuple) expected_skipped_relation_reasons: tuple[str, ...] = field(default_factory=tuple) expected_skipped_schema_sources: tuple[str, ...] = field(default_factory=tuple) expected_blocked_schema_sources: tuple[str, ...] = field(default_factory=tuple) expected_suppressed_candidate_names: tuple[str, ...] = field(default_factory=tuple) - virtual_state_prune_candidates: tuple[JanitorVirtualStatePruneCandidate, ...] = field( - default_factory=tuple - ) @dataclass(frozen=True) @@ -42,10 +37,6 @@ class JanitorExecuteTestCase: expected_dropped_targets: tuple[str, ...] direct_mode: bool = False expected_pruned_table_names: tuple[str, ...] = field(default_factory=tuple) - expected_pruned_virtual_table_names: tuple[str, ...] = field(default_factory=tuple) - virtual_state_prune_candidates: tuple[JanitorVirtualStatePruneCandidate, ...] = field( - default_factory=tuple - ) @dataclass(frozen=True) @@ -130,7 +121,6 @@ class JanitorRetentionResolutionTestCase: description: str override: int | None configured: int | None - virtual_environments: bool expected_retention_days: int diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py index 394a338b22..16c14497cc 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_execute.py @@ -9,86 +9,25 @@ from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import ( - JanitorDeleteCandidate, - JanitorDetachedVirtualEnvironmentCandidate, JanitorDirectModeSettings, JanitorExecutionResult, JanitorPlan, - JanitorRelationKey, ) from sqlbuild.executor.janitor_events.constants import JANITOR_EVENTS_TABLE_NAME from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorArchiveEventFailureTestCase, JanitorArchiveExecutionTestCase, - JanitorExecutionOrderTestCase, relation_info, ) from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( - FailingDropAdapter, FailingJanitorEventAdapter, FakeJanitorAdapter, build_project, - relation_info_for_test, ) OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) -@pytest.mark.parametrize( - "test_case", - [ - JanitorExecutionOrderTestCase( - description="state cleanup is skipped when physical deletion fails", - expected_error_fragment="drop failed", - expected_deleted_state_items=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_physical_drop_failure_when_executing_janitor_then_state_cleanup_is_skipped( - test_case: JanitorExecutionOrderTestCase, -) -> None: - deleted_state_items: list[str] = [] - adapter: FailingDropAdapter = FailingDropAdapter(message=test_case.expected_error_fragment) - - with pytest.raises(RuntimeError) as exc_info: - execute_janitor_plan( - plan=JanitorPlan( - target_name="dev", - retention_days=0, - candidates=( - JanitorDeleteCandidate( - key=JanitorRelationKey( - database=None, - schema="dev__sqb_physical", - name="orders__v_old", - ), - relation=relation_info_for_test( - schema="dev__sqb_physical", - name="orders__v_old", - ), - age_timestamp=None, - ), - ), - detached_virtual_environment_candidates=( - JanitorDetachedVirtualEnvironmentCandidate( - virtual_environment_name="dev", - updated_at=None, - ), - ), - ), - adapter=adapter, - connection=object(), - delete_detached_virtual_environment=lambda candidate: deleted_state_items.append( - candidate.virtual_environment_name - ), - ) - - assert test_case.expected_error_fragment in str(exc_info.value) - assert tuple(deleted_state_items) == test_case.expected_deleted_state_items - assert adapter.dropped_targets == ["dev__sqb_physical.orders__v_old"] - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index cc77f5b520..c50974b1d2 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -5,6 +5,10 @@ import pytest from sqlbuild.compiler.fingerprints.constants import FINGERPRINT_TABLE_NAME +from sqlbuild.compiler.planner._helpers.scenario.artifacts import ( + build_scenario_artifact_name, + compute_scenario_hash_prefix, +) from sqlbuild.compiler.source_freshness.constants import SOURCE_FRESHNESS_TABLE_NAME from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.main.plan import build_janitor_plan @@ -12,12 +16,8 @@ JanitorDirectModeSettings, JanitorExecutionResult, JanitorPlan, - JanitorRelationKey, JanitorRelationScope, - JanitorStateCandidates, - JanitorVirtualStatePruneCandidate, ) -from sqlbuild.virtual.state.constants import PYTHON_NODE_VERSION_TABLE from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorExecuteTestCase, JanitorPlanTestCase, @@ -30,6 +30,13 @@ OLD_TIME: datetime = datetime.now(UTC) - timedelta(days=30) NEW_TIME: datetime = datetime.now(UTC) - timedelta(days=1) +NAMESPACED_ARTIFACT: str = build_scenario_artifact_name( + hash_prefix=compute_scenario_hash_prefix( + project_name="orders", scenario_name="totals", run_namespace="job-1" + ), + kind="model", + logical_name="orders", +) @pytest.mark.parametrize( @@ -59,14 +66,6 @@ expected_blocked_schema_sources=("raw_orders",), expected_suppressed_candidate_names=("old_orders",), ), - JanitorPlanTestCase( - description="virtual schema with active source remains skipped", - relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), - source_schema="analytics", - direct_mode=False, - expected_candidate_names=(), - expected_skipped_schema_sources=("raw_orders",), - ), JanitorPlanTestCase( description="new relation is skipped by retention", relation_infos=(relation_info("old_orders", created_at=NEW_TIME),), @@ -135,6 +134,19 @@ delete_tracked_only=True, expected_candidate_names=("__sqb_a13f09c2e7b8__model__daily_revenue",), ), + JanitorPlanTestCase( + description="old namespaced scenario artifact is eligible without namespace knowledge", + relation_infos=(relation_info(NAMESPACED_ARTIFACT, created_at=OLD_TIME),), + delete_tracked_only=True, + expected_candidate_names=(NAMESPACED_ARTIFACT,), + ), + JanitorPlanTestCase( + description="recent namespaced scenario artifact is protected by retention", + relation_infos=(relation_info(NAMESPACED_ARTIFACT, created_at=NEW_TIME),), + delete_tracked_only=True, + expected_candidate_names=(), + expected_skipped_relation_reasons=("relation is newer than 7 days",), + ), JanitorPlanTestCase( description="dbt scenario artifact is eligible when tracked-only is enabled", relation_infos=( @@ -152,27 +164,6 @@ expected_candidate_names=(), expected_skipped_relation_reasons=("relation is not tracked by SQLBuild",), ), - JanitorPlanTestCase( - description="checkpoint protected relation is skipped", - relation_infos=( - relation_info( - "orders__v_old", schema="analytics__sqb_physical", created_at=OLD_TIME - ), - ), - protected_relation_keys=frozenset( - ( - JanitorRelationKey( - database=None, - schema="analytics__sqb_physical", - name="orders__v_old", - ), - ) - ), - expected_candidate_names=(), - expected_skipped_relation_reasons=( - "relation is referenced by a retained virtual checkpoint", - ), - ), JanitorPlanTestCase( description="existing direct state tables are eligible for history pruning", relation_infos=( @@ -200,18 +191,6 @@ "relation matches exclude pattern '_sqlbuild_fingerprints'", ), ), - JanitorPlanTestCase( - description="virtual state pruning candidates are preserved", - relation_infos=(), - expected_virtual_state_table_names=(PYTHON_NODE_VERSION_TABLE,), - virtual_state_prune_candidates=( - JanitorVirtualStatePruneCandidate( - schema="sqlbuild_state", - table_name=PYTHON_NODE_VERSION_TABLE, - reason="1 unreferenced Python identity version(s)", - ), - ), - ), ], ids=lambda case: case.description, ) @@ -234,9 +213,6 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp relation_scope=JanitorRelationScope( protected_relation_keys=test_case.protected_relation_keys, ), - state_candidates=JanitorStateCandidates( - virtual_state_prune_candidates=test_case.virtual_state_prune_candidates, - ), direct_settings=JanitorDirectModeSettings( enabled=test_case.direct_mode, state_history_versions=test_case.direct_state_history_versions, @@ -252,9 +228,6 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp assert tuple(candidate.table_name for candidate in plan.direct_state_prune_candidates) == ( test_case.expected_direct_state_table_names ) - assert tuple(candidate.table_name for candidate in plan.virtual_state_prune_candidates) == ( - test_case.expected_virtual_state_table_names - ) skipped_schema_sources: list[str] = [] for skipped_schema in plan.skipped_schemas: for source_name in skipped_schema.source_names: @@ -274,11 +247,6 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp @pytest.mark.parametrize( "test_case", [ - JanitorExecuteTestCase( - description="drops all eligible candidates", - relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), - expected_dropped_targets=("analytics.old_orders",), - ), JanitorExecuteTestCase( description="direct mode archives eligible candidates instead of dropping them", relation_infos=(relation_info("old_orders", created_at=OLD_TIME),), @@ -291,19 +259,6 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp expected_dropped_targets=(), expected_pruned_table_names=(FINGERPRINT_TABLE_NAME,), ), - JanitorExecuteTestCase( - description="prunes virtual state orphan tables", - relation_infos=(), - expected_dropped_targets=(), - expected_pruned_virtual_table_names=(PYTHON_NODE_VERSION_TABLE,), - virtual_state_prune_candidates=( - JanitorVirtualStatePruneCandidate( - schema="sqlbuild_state", - table_name=PYTHON_NODE_VERSION_TABLE, - reason="1 unreferenced Python identity version(s)", - ), - ), - ), ], ids=lambda case: case.description, ) @@ -317,20 +272,13 @@ def test_given_janitor_plan_when_executing_then_drops_expected_relations( connection=object(), retention_days=7, delete_tracked_only=False, - state_candidates=JanitorStateCandidates( - virtual_state_prune_candidates=test_case.virtual_state_prune_candidates, - ), direct_settings=JanitorDirectModeSettings(enabled=test_case.direct_mode), ) - pruned_virtual_table_names: list[str] = [] result: JanitorExecutionResult = execute_janitor_plan( plan=plan, adapter=adapter, connection=object(), - prune_virtual_state=lambda candidate: pruned_virtual_table_names.append( - candidate.table_name - ), ) assert tuple(adapter.dropped_targets) == test_case.expected_dropped_targets @@ -340,7 +288,3 @@ def test_given_janitor_plan_when_executing_then_drops_expected_relations( assert tuple(candidate.table_name for candidate in result.pruned_direct_state) == ( test_case.expected_pruned_table_names ) - assert tuple(candidate.table_name for candidate in result.pruned_virtual_state) == ( - test_case.expected_pruned_virtual_table_names - ) - assert tuple(pruned_virtual_table_names) == test_case.expected_pruned_virtual_table_names diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py index 0a5c041f8d..547b933001 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_archives.py @@ -109,14 +109,6 @@ "_sqb_archive__20200101t000000z__old_products", ), ), - JanitorArchivePlanTestCase( - description="virtual mode neither archives nor expires archive-named relations", - relation_infos=( - relation_info("old_orders", created_at=OLD_TIME), - relation_info("_sqb_archive__20200101t000000z__old_products", created_at=OLD_TIME), - ), - direct_mode=False, - ), ], ids=lambda case: case.description, ) @@ -256,12 +248,6 @@ def test_given_long_relation_name_when_planning_archive_then_fits_name_with_time ("_SQB_ARCHIVE__20200101T000000Z__OLD_INVOICES", UNQUOTED_ADDRESSING_REASON), ), ), - JanitorAddressingPlanTestCase( - description="virtual mode keeps deleting mixed-case relations as before", - relation_infos=(relation_info("LegacyCustomers", created_at=OLD_TIME),), - direct_mode=False, - expected_candidate_names=("LegacyCustomers",), - ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py index f12ffb05f1..885fbc3d2e 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_case_safety.py @@ -4,7 +4,6 @@ import pytest -from sqlbuild.executor.janitor.constants import CASE_COLLISION_REASON from sqlbuild.executor.janitor.main.execute import execute_janitor_plan from sqlbuild.executor.janitor.main.plan import build_janitor_plan from sqlbuild.executor.janitor.models import JanitorDirectModeSettings, JanitorPlan @@ -21,50 +20,6 @@ @pytest.mark.parametrize( "test_case", [ - JanitorCaseSafetyTestCase( - description="virtual mode never drops an excluded table through a quoted twin", - relation_names=("orders__v0", "Orders__v0"), - direct_mode=False, - tracked_names=("orders__v0",), - exclude_patterns=("orders__v0",), - expected_skipped_relations=( - ("analytics.orders__v0", "relation matches exclude pattern 'orders__v0'"), - ("analytics.Orders__v0", "relation matches exclude pattern 'orders__v0'"), - ), - expected_dropped_targets=(), - ), - JanitorCaseSafetyTestCase( - description="virtual mode skips relations whose folded names collide", - relation_names=("orders__v0", "Orders__v0"), - direct_mode=False, - tracked_names=("orders__v0",), - exclude_patterns=(), - expected_skipped_relations=( - ("analytics.orders__v0", CASE_COLLISION_REASON), - ("analytics.Orders__v0", CASE_COLLISION_REASON), - ), - expected_dropped_targets=(), - ), - JanitorCaseSafetyTestCase( - description="virtual mode still drops a non-colliding tracked relation", - relation_names=("orders__v0",), - direct_mode=False, - tracked_names=("orders__v0",), - exclude_patterns=(), - expected_skipped_relations=(), - expected_dropped_targets=("analytics.orders__v0",), - ), - JanitorCaseSafetyTestCase( - description="virtual exclude patterns match regardless of case", - relation_names=("orders__v0",), - direct_mode=False, - tracked_names=("orders__v0",), - exclude_patterns=("ORDERS__*",), - expected_skipped_relations=( - ("analytics.orders__v0", "relation matches exclude pattern 'ORDERS__*'"), - ), - expected_dropped_targets=(), - ), JanitorCaseSafetyTestCase( description="direct exclude patterns match regardless of case", relation_names=("old_orders",), diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py index b9c009d4ee..01024ea183 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_folded_identifiers.py @@ -13,7 +13,6 @@ JanitorDeleteCandidate, JanitorDirectModeSettings, JanitorPlan, - JanitorRelationKey, JanitorRelationScope, ) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( @@ -107,40 +106,6 @@ "ANALYTICS.DEV_ORDERS._sqb_archive__20200101t000000z__old_products", ), ), - JanitorFoldedIdentifierPlanTestCase( - description="virtual protection matches physical relations across identifier case", - relation_infos=( - relation_info( - "orders__v1", - database="ANALYTICS", - schema="DEV_ORDERS__SQB_PHYSICAL", - created_at=OLD_TIME, - ), - relation_info( - "orders__v0", - database="ANALYTICS", - schema="DEV_ORDERS__SQB_PHYSICAL", - created_at=OLD_TIME, - ), - ), - direct_mode=False, - protected_relation_keys=frozenset( - ( - JanitorRelationKey( - database="ANALYTICS", - schema="DEV_ORDERS__SQB_PHYSICAL", - name="ORDERS__V1", - ), - ) - ), - expected_candidate_display_names=("ANALYTICS.DEV_ORDERS__SQB_PHYSICAL.orders__v0",), - expected_skipped_relations=( - ( - "ANALYTICS.DEV_ORDERS__SQB_PHYSICAL.orders__v1", - "relation is referenced by a retained virtual checkpoint", - ), - ), - ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py index 1c42809a90..3c008eea98 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan_separate_age_metadata.py @@ -10,7 +10,6 @@ from sqlbuild.executor.janitor.models import ( JanitorDirectModeSettings, JanitorPlan, - JanitorRelationKey, JanitorRelationScope, ) from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( @@ -102,23 +101,6 @@ def test_given_adapter_reading_ages_separately_when_planning_janitor_then_uses_t protected_relation_keys=frozenset(), expected_age_requests=(("old_orders",),), ), - JanitorAgeReadScopeTestCase( - description="virtual mode reads ages only for unprotected stale relations", - relation_infos=( - relation_info("orders"), - relation_info("orders__v0"), - relation_info("orders__v1"), - relation_info("tmp_orders"), - ), - direct_mode=False, - delete_tracked_only=False, - tracked_relations=(), - exclude_patterns=("tmp_*",), - protected_relation_keys=frozenset( - (JanitorRelationKey(database=None, schema="analytics", name="orders__v1"),) - ), - expected_age_requests=(("orders__v0",),), - ), JanitorAgeReadScopeTestCase( description="no age read happens when nothing survives the filters", relation_infos=(relation_info("orders"), relation_info("stray_orders")), diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py index 69253bd50d..2afce5d559 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_resolve_retention_days.py @@ -17,42 +17,24 @@ description="unset retention in direct mode defaults to 14", override=None, configured=None, - virtual_environments=False, expected_retention_days=14, ), - JanitorRetentionResolutionTestCase( - description="unset retention in virtual mode defaults to 30", - override=None, - configured=None, - virtual_environments=True, - expected_retention_days=30, - ), JanitorRetentionResolutionTestCase( description="explicit retention applies in direct mode", override=None, configured=7, - virtual_environments=False, - expected_retention_days=7, - ), - JanitorRetentionResolutionTestCase( - description="explicit retention applies in virtual mode", - override=None, - configured=7, - virtual_environments=True, expected_retention_days=7, ), JanitorRetentionResolutionTestCase( description="cli override wins over explicit config", override=3, configured=7, - virtual_environments=True, expected_retention_days=3, ), JanitorRetentionResolutionTestCase( description="cli zero override wins over the mode default", override=0, configured=None, - virtual_environments=True, expected_retention_days=0, ), ], @@ -64,7 +46,6 @@ def test_given_retention_inputs_when_resolving_janitor_retention_then_returns_ex resolved: int = resolve_janitor_retention_days( override=test_case.override, configured=test_case.configured, - virtual_environments=test_case.virtual_environments, ) assert resolved == test_case.expected_retention_days diff --git a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py index 1c0fd91d57..29c8526925 100644 --- a/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py +++ b/tests/unit/src/sqlbuild/executor/testing/main/test_comparison_sql.py @@ -92,7 +92,7 @@ def test_given_adapter_when_building_comparison_sql_then_it_uses_expected_set_di BuildComparisonSqlTestCase( description="Snowflake comparison lifting preserves STARTSWITH", adapter_name="snowflake", - expected_fragments=("STARTSWITH(name, 'A')", "picked AS ("), + expected_fragments=("STARTSWITH(name, 'A')", "__sqb_cte_0 AS (", "AS picked"), expected_absent_fragments=("STARTS_WITH",), ) ], @@ -205,10 +205,11 @@ def test_given_assertion_step_when_building_comparison_sql_then_it_counts_failin "test_case", [ BuildComparisonSqlTestCase( - description="matching helper CTEs are lifted once from actual and expected SQL", + description="matching helper CTEs retain independent actual and expected scopes", adapter_name="duckdb", expected_fragments=( - "input_values AS", + "__sqb_cte_0 AS", + "INPUT_VALUES AS", "__actual__orders AS (", "__expected__orders AS (", ), @@ -216,7 +217,7 @@ def test_given_assertion_step_when_building_comparison_sql_then_it_counts_failin ], ids=lambda case: case.description, ) -def test_given_matching_helper_ctes_when_building_comparison_sql_then_lifts_once( +def test_given_matching_helper_ctes_when_building_comparison_sql_then_keeps_scopes_separate( test_case: BuildComparisonSqlTestCase, ) -> None: adapter: BaseAdapter = build_comparison_test_adapter(test_case.adapter_name) @@ -254,7 +255,7 @@ def test_given_unasserted_transitive_steps_when_building_comparison_then_emits_o for expected_fragment in test_case.expected_fragments: assert expected_fragment in comparison_sql assert "__actual__stg_orders AS" not in comparison_sql - assert comparison_sql.lower().count("shared as") == 1 + assert comparison_sql.lower().count("__sqb_cte_0 as (") == 1 @pytest.mark.parametrize( @@ -349,9 +350,9 @@ def test_given_preanalyzed_step_with_authored_cte_when_building_comparison_then_ ) assert ( - comparison_sql.index("__ref__raw_orders AS") < comparison_sql.index("picked AS") + comparison_sql.index("__ref__raw_orders AS") < comparison_sql.index("__sqb_cte_0 AS") ) is test_case.expected_result - assert comparison_sql.index("picked AS") < comparison_sql.index("__actual__orders AS") + assert comparison_sql.index("__sqb_cte_0 AS") < comparison_sql.index("__actual__orders AS") assert "__actual__orders AS (\nWITH picked" not in comparison_sql diff --git a/tests/unit/src/sqlbuild/integrations/dbt/test_mode.py b/tests/unit/src/sqlbuild/integrations/dbt/test_mode.py deleted file mode 100644 index e5e2837a55..0000000000 --- a/tests/unit/src/sqlbuild/integrations/dbt/test_mode.py +++ /dev/null @@ -1,76 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.integrations.dbt._helpers.cli.mode import enforce_dbt_interop_direct_mode -from sqlbuild.integrations.dbt.exceptions import DbtInteropConfigError -from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig, SettingsConfig -from tests.unit.src.sqlbuild.integrations.dbt._test_types import DbtModeGuardTestCase - - -@pytest.mark.parametrize( - "test_case", - [ - DbtModeGuardTestCase( - description="allows dbt interop in direct mode", - virtual_environments=False, - expected_error_fragment=None, - expected_code=None, - expected_help_fragment=None, - ) - ], - ids=lambda case: case.description, -) -def test_given_direct_project_mode_when_enforcing_dbt_interop_support_then_allows_execution( - test_case: DbtModeGuardTestCase, -) -> None: - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=test_case.virtual_environments), - ), - local_config=LocalConfig(), - ) - - enforce_dbt_interop_direct_mode(discovered_inputs=discovered_inputs) - - assert test_case.expected_code is None - - -@pytest.mark.parametrize( - "test_case", - [ - DbtModeGuardTestCase( - description="blocks dbt interop in virtual mode", - virtual_environments=True, - expected_error_fragment="sqb dbt is not supported when virtual_environments = true", - expected_code="C241", - expected_help_fragment="Disable virtual_environments", - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_project_mode_when_enforcing_dbt_interop_support_then_raises_error( - test_case: DbtModeGuardTestCase, -) -> None: - discovered_inputs: DiscoveredProjectInputs = DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=test_case.virtual_environments), - ), - local_config=LocalConfig(), - ) - - with pytest.raises(DbtInteropConfigError) as exc_info: - enforce_dbt_interop_direct_mode(discovered_inputs=discovered_inputs) - - error: DbtInteropConfigError = exc_info.value - assert test_case.expected_error_fragment is not None - assert test_case.expected_error_fragment in error.message - assert error.code == test_case.expected_code - assert error.help is not None - assert test_case.expected_help_fragment is not None - assert test_case.expected_help_fragment in error.help diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py index e2a0a18ff3..5e3827ed1d 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_native_sql.py @@ -10,7 +10,6 @@ from sqlbuild.compiler.compile.models import ExpansionSpan from sqlbuild.lint._helpers import native_sql -from sqlbuild.lint.exceptions import NativeLintError from sqlbuild.lint.models import LintBody, LintConfig, LintViolation from tests.unit.src.sqlbuild.lint._helpers._test_types import ( GeneratedRangeFallbackTestCase, @@ -88,12 +87,16 @@ def test_given_invalid_native_response_when_linting_then_boundary_fails_closed( passes=(), ) - with pytest.raises(NativeLintError, match=test_case.expected_message): - _ = native_sql.run_native_sql_lint( - bodies=(body,), - contents_by_path={target: "SELECT 1"}, - config=LintConfig(dialect="duckdb"), - ) + violations: dict[Path, tuple[LintViolation, ...]] = native_sql.run_native_sql_lint( + bodies=(body,), + contents_by_path={target: "SELECT 1"}, + config=LintConfig(dialect="duckdb"), + ) + (fault,) = violations[target] + assert fault.code == "L003" + assert fault.severity == "fault" + assert test_case.expected_message in fault.message + assert fault.fix is None @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/microbatches/test_event_codec.py b/tests/unit/src/sqlbuild/microbatches/test_event_codec.py index 555e360f5d..db947e387c 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_event_codec.py +++ b/tests/unit/src/sqlbuild/microbatches/test_event_codec.py @@ -59,7 +59,7 @@ def test_given_cursor_event_when_encoding_then_preserves_state_table_string_form values: tuple[object | None, ...] = MicrobatchEventCodec.values(event) - assert values[17:21] == test_case.expected_values + assert values[15:19] == test_case.expected_values @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/microbatches/test_sql.py b/tests/unit/src/sqlbuild/microbatches/test_sql.py index 646a4203f7..7acacadbca 100644 --- a/tests/unit/src/sqlbuild/microbatches/test_sql.py +++ b/tests/unit/src/sqlbuild/microbatches/test_sql.py @@ -46,7 +46,7 @@ description="stored aware offsets retain deterministic identity", partition_start="2026-01-01T08:00:00+08:00", partition_end="2026-01-01T09:00:00+08:00", - expected_event_id="d15ad5651a6a2461e611a3ac97c708fa74c3dab481e9feacbd976c0b34a85ed4", + expected_event_id="781f0c13965b66aba3b4db7cfe8cb58d50e983bc5ac41b659048072de849f572", ), ), ids=lambda case: case.description, @@ -190,7 +190,7 @@ def test_given_completion_when_building_insert_then_provenance_and_numeric_rows_ assert sql == ( "INSERT INTO analytics._sqlbuild_microbatches (event_id, record_type, scope_kind, " "scope_key, model_name, target_database, target_schema, target_name, " - "physical_generation_id, virtual_environment_name, virtual_model_version_hash, " + "physical_generation_id, " "origin_run_id, origin_run_started_at, execution_run_id, execution_run_started_at, " "run_type, completion_type, run_start, run_end, partition_start, partition_end, " "batch_size, cursor_column, cursor_type, cursor_grain, model_version_hash, " @@ -199,7 +199,7 @@ def test_given_completion_when_building_insert_then_provenance_and_numeric_rows_ "rows_affected, completed_at, coverage_source, observed_row_count, observed_at, " "synthetic_reason, unaccounted_policy, created_at) SELECT 'event-1', " "'partition_completion', 'direct_logical', 'duckdb:analytics.orders', 'orders', NULL, " - "'analytics', 'orders', '*', NULL, NULL, 'origin-run', CAST(NULL AS TIMESTAMP), " + "'analytics', 'orders', '*', 'origin-run', CAST(NULL AS TIMESTAMP), " "'execution-run', CAST(NULL AS TIMESTAMP), 'replay_on_change', 'recovery', '0', '1', " "'0', '1', '1', 'batch_id', 'integer', NULL, 'F2', 'fingerprint''definition', " "'known', NULL, NULL, NULL, NULL, CAST(0 AS BIGINT), CAST(NULL AS TIMESTAMP), NULL, " diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py index 621a08611a..ea16b93e94 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/_test_types.py @@ -12,31 +12,6 @@ from sqlbuild.sql_values.types import CollectionRendering -@dataclass(frozen=True) -class TargetConfigResolutionTestCase: - description: str - project_config: ProjectConfig - local_config: LocalConfig - target_name: str - expected_backend: str | None - expected_schema: str | None - expected_connection: dict[str, object] - expected_allow_reset: bool - expected_loader_schema: str | None = None - expected_defer_clone_from: str | None = None - expected_changes_only: bool | None = None - expected_connection_name: str | None = None - - -@dataclass(frozen=True) -class EffectiveChangesOnlyResolutionTestCase: - description: str - project_config: ProjectConfig - local_config: LocalConfig - cli_changes_only: bool - expected_changes_only: bool - - @dataclass(frozen=True) class EffectiveCollectionRenderingResolutionTestCase: description: str diff --git a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py index 3f6bccbbae..79bd7b2ac5 100644 --- a/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py +++ b/tests/unit/src/sqlbuild/spec/contracts/main/test_targets.py @@ -2,204 +2,21 @@ import pytest -from sqlbuild.spec.contracts.main.resolve_effective_changes_only import ( - resolve_effective_changes_only, -) from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.models import ( AuthoredTimeTravelRetention, ExecutionLimitsConfig, LocalConfig, - LocalStateConfig, LocalTargetConfig, ProjectConfig, - SettingsConfig, - StateConfig, TargetConfig, ) from tests.unit.src.sqlbuild.spec.contracts.main._test_types import ( - EffectiveChangesOnlyResolutionTestCase, ExecutionLimitsResolutionTestCase, - TargetConfigResolutionTestCase, TargetRetentionResolutionTestCase, ) -@pytest.mark.parametrize( - "test_case", - [ - EffectiveChangesOnlyResolutionTestCase( - description="uses default false when nothing is configured", - project_config=ProjectConfig(name="demo", adapter="duckdb"), - local_config=LocalConfig(), - cli_changes_only=False, - expected_changes_only=False, - ), - EffectiveChangesOnlyResolutionTestCase( - description="uses global changes-only setting when target does not override", - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - default_target="dev", - settings=SettingsConfig(changes_only=True), - targets={"dev": TargetConfig()}, - ), - local_config=LocalConfig(), - cli_changes_only=False, - expected_changes_only=True, - ), - EffectiveChangesOnlyResolutionTestCase( - description="allows target false to override global true", - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - default_target="prod", - settings=SettingsConfig(changes_only=True), - targets={"prod": TargetConfig(changes_only=False)}, - ), - local_config=LocalConfig(), - cli_changes_only=False, - expected_changes_only=False, - ), - EffectiveChangesOnlyResolutionTestCase( - description="allows CLI changes-only to override target false", - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - default_target="prod", - targets={"prod": TargetConfig(changes_only=False)}, - ), - local_config=LocalConfig(), - cli_changes_only=True, - expected_changes_only=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_changes_only_config_when_resolving_then_precedence_is_applied( - test_case: EffectiveChangesOnlyResolutionTestCase, -) -> None: - changes_only: bool = resolve_effective_changes_only( - project_config=test_case.project_config, - local_config=test_case.local_config, - selected_target=None, - cli_changes_only=test_case.cli_changes_only, - ) - - assert changes_only is test_case.expected_changes_only - - -@pytest.mark.parametrize( - "test_case", - [ - TargetConfigResolutionTestCase( - description="merges local state overrides over project state config", - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - targets={ - "dev": TargetConfig( - changes_only=True, - compile_cache=True, - loader_schema="raw_shared", - state=StateConfig( - backend="postgres", - schema="sqlbuild_state", - connection={"host": "shared-state", "port": 5432}, - allow_reset=False, - ), - defer_clone_from="prod", - ), - "prod": TargetConfig(), - }, - ), - local_config=LocalConfig( - targets={ - "dev": LocalTargetConfig( - state=LocalStateConfig( - backend="duckdb", - connection={"database": "local-state.duckdb"}, - allow_reset=True, - ), - changes_only=False, - compile_cache=False, - loader_schema="raw_local", - defer_clone_from="staging", - ) - } - ), - target_name="dev", - expected_backend="duckdb", - expected_schema="sqlbuild_state", - expected_connection={ - "host": "shared-state", - "port": 5432, - "database": "local-state.duckdb", - }, - expected_allow_reset=True, - expected_loader_schema="raw_local", - expected_defer_clone_from="staging", - expected_changes_only=False, - ), - TargetConfigResolutionTestCase( - description="local target connection name overrides project target reference", - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - targets={"dev": TargetConfig(connection_name="shared")}, - ), - local_config=LocalConfig( - targets={"dev": LocalTargetConfig(connection_name="developer", compile_cache=False)} - ), - target_name="dev", - expected_backend=None, - expected_schema=None, - expected_connection={}, - expected_allow_reset=False, - expected_connection_name="developer", - ), - TargetConfigResolutionTestCase( - description="local target inline mapping inherits project target reference", - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - targets={"dev": TargetConfig(connection_name="shared")}, - ), - local_config=LocalConfig( - targets={ - "dev": LocalTargetConfig(connection={"role": "local"}, compile_cache=False) - } - ), - target_name="dev", - expected_backend=None, - expected_schema=None, - expected_connection={}, - expected_allow_reset=False, - expected_connection_name="shared", - ), - ], - ids=lambda case: case.description, -) -def test_given_project_and_local_state_config_when_resolving_then_local_overrides_are_applied( - test_case: TargetConfigResolutionTestCase, -) -> None: - target_config: TargetConfig = resolve_target_config( - project_config=test_case.project_config, - local_config=test_case.local_config, - target_name=test_case.target_name, - ) - - assert target_config.state.backend == test_case.expected_backend - assert target_config.state.schema == test_case.expected_schema - assert target_config.state.connection == test_case.expected_connection - assert target_config.state.allow_reset is test_case.expected_allow_reset - assert target_config.loader_schema == test_case.expected_loader_schema - assert target_config.defer_clone_from == test_case.expected_defer_clone_from - assert target_config.changes_only is test_case.expected_changes_only - assert target_config.compile_cache is False - assert target_config.connection_name == test_case.expected_connection_name - - @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py deleted file mode 100644 index d501162b99..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/_test_types.py +++ /dev/null @@ -1,91 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class HydrationLifecycleTestCase: - description: str - resource_kind: str - resource_name: str - expected_action: str | None - expected_event_types: tuple[str, ...] - - -@dataclass(frozen=True) -class PhysicalTargetTestCase: - description: str - model_name: str - version_hash: str - expected_schema: str - expected_name: str - - -@dataclass(frozen=True) -class RelationTypeTestCase: - description: str - materialized: str | None - expected_relation_type: str - - -@dataclass(frozen=True) -class RewrittenTargetsTestCase: - description: str - selected_model_version_hashes: dict[str, str] - bound_relations: dict[str, str] - expected_selected_name: str - expected_bound_name: str - - -@dataclass(frozen=True) -class RewriteProjectTargetsTestCase: - description: str - selected_model_version_hashes: dict[str, str] - expected_rewritten_name: str - - -@dataclass(frozen=True) -class SeedingStrategyTestCase: - description: str - incremental_strategy: str - supports_durable_clone: bool - expected_strategy: str - expected_sql_fragment: str - - -@dataclass(frozen=True) -class SeedingIdempotencyTestCase: - description: str - target_exists: bool - expected_drop_count: int - expected_ancestry_count: int - expected_sql_count: int - - -@dataclass(frozen=True) -class SeedPhysicalRelationLookupTestCase: - description: str - seed_version_hashes: dict[str, str] - available_seed_names: tuple[str, ...] - expected_seed_names: tuple[str, ...] - - -@dataclass(frozen=True) -class MissingRollbackSeedRelationTestCase: - description: str - final_seed_hashes: dict[str, str] - expected_error_fragment: str - - -@dataclass(frozen=True) -class VirtualPhaseTimingTestCase: - description: str - clock_values: tuple[float, ...] - expected_compile_seconds: float - expected_planning_seconds: float - expected_total_seconds: float - - -@dataclass(frozen=True) -class VirtualStatementScopeTestCase: - description: str - expected_phases: tuple[str, ...] - expected_callback_count: int diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/helpers.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/helpers.py deleted file mode 100644 index 02b62fb537..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/helpers.py +++ /dev/null @@ -1,176 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import ( - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompileModelConfig, -) -from sqlbuild.compiler.compile.types import CompiledResourceType -from sqlbuild.compiler.planner.models import ( - BackfillResult, - CursorBounds, - ModelPlanEntry, - PlanOutput, -) -from sqlbuild.compiler.planner.types import ( - BackfillAction, - MaterializationType, - PlanAction, - PlanReason, -) -from sqlbuild.spec.contracts.models import SettingsConfig -from sqlbuild.virtual.state.models import PhysicalRelationRecord -from sqlbuild.virtual.state.types import PhysicalArtifactType - - -def build_relation_location(*, schema: str, name: str) -> CompiledRelationLocation: - return CompiledRelationLocation( - database=None, - schema=schema, - name=name, - qualified_name=f"{schema}.{name}", - ) - - -class SeedPhysicalRelationTestBackend: - def __init__(self, *, available_seed_names: tuple[str, ...]) -> None: - self.available_seed_names: frozenset[str] = frozenset(available_seed_names) - - def get_physical_relation_for_artifact( - self, - *, - connection: object, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - version_hash: str, - ) -> PhysicalRelationRecord | None: - del connection, schema, artifact_type - relation: PhysicalRelationRecord = build_seed_physical_relation( - seed_name=artifact_name, version_hash=version_hash - ) - return (None, relation)[artifact_name in self.available_seed_names] - - -def build_virtual_executor_test_project() -> CompiledProject: - stg_orders: CompiledModel = CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="stg_orders"), - deps=(), - name="stg_orders", - relative_path=Path("models/stg_orders.sql"), - query_sql="SELECT 1 AS id", - config=CompileModelConfig(values={"materialized": "table"}), - destination=CompiledRelationLocation( - database=None, - schema="dev", - name="stg_orders", - qualified_name="dev.stg_orders", - ), - ) - fact_orders: CompiledModel = CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="fact_orders"), - deps=(stg_orders.key,), - name="fact_orders", - relative_path=Path("models/fact_orders.sql"), - query_sql='SELECT id FROM __ref("stg_orders")', - config=CompileModelConfig(values={"materialized": "table"}), - destination=CompiledRelationLocation( - database=None, - schema="dev", - name="fact_orders", - qualified_name="dev.fact_orders", - ), - ) - return CompiledProject( - run_id="test_run", - effective_target_name="dev", - effective_connection={}, - effective_vars={}, - settings=SettingsConfig(), - models=(stg_orders, fact_orders), - ) - - -def build_bound_physical_relation(*, model_name: str, version_hash: str) -> PhysicalRelationRecord: - return PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=model_name, - version_hash=version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=f"{model_name}__v_{version_hash[:8]}", - relation_type="table", - ) - - -def build_seed_physical_relation(*, seed_name: str, version_hash: str) -> PhysicalRelationRecord: - return PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.SEED, - artifact_name=seed_name, - version_hash=version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=f"{seed_name}__v_{version_hash[:8]}", - relation_type="table", - ) - - -def build_optional_bound_physical_relations( - *, model_name: str, version_hash: str | None -) -> dict[str, PhysicalRelationRecord]: - relations: dict[str, PhysicalRelationRecord] = { - model_name: build_bound_physical_relation( - model_name=model_name, version_hash=version_hash or "" - ) - } - return ({}, relations)[version_hash is not None] - - -def build_adapter() -> DuckDbAdapter: - return DuckDbAdapter() - - -def build_seeded_incremental_plan_output( - *, - incremental_strategy: str, - resolved_sql: str = "SELECT id, ordered_at, amount_cents + 1 AS amount_cents FROM raw", - materialization_type: MaterializationType = MaterializationType.INCREMENTAL, - action: PlanAction = PlanAction.CREATE_TABLE, - cursor_bounds: CursorBounds | None = None, - include_cursor_bounds: bool = True, -) -> PlanOutput: - default_cursor_bounds: CursorBounds | None = ( - None, - CursorBounds(start="2026-01-02T00:00:00", end="2026-01-04T00:00:00"), - )[include_cursor_bounds] - effective_cursor_bounds: CursorBounds | None = (default_cursor_bounds, cursor_bounds)[ - cursor_bounds is not None - ] - entry: ModelPlanEntry = ModelPlanEntry( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="orders"), - name="orders", - relative_path=Path("models/orders.sql"), - materialization_type=materialization_type, - action=action, - reason=PlanReason.QUERY_CHANGED, - destination=CompiledRelationLocation( - database=None, - schema="dev__sqb_physical", - name="orders__v_newhash", - qualified_name='"dev__sqb_physical"."orders__v_newhash"', - ), - fingerprint_query_sql=resolved_sql, - resolved_sql=resolved_sql, - logical_ddl="", - incremental_strategy=incremental_strategy, - cursor_column="ordered_at", - cursor_type="timestamp", - cursor_bounds=effective_cursor_bounds, - backfill=BackfillResult(action=BackfillAction.BOUNDED, duration="7d"), - ) - return PlanOutput(model_entries=(entry,)) diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_clone_hydration_lifecycle.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_clone_hydration_lifecycle.py deleted file mode 100644 index 324293856e..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_clone_hydration_lifecycle.py +++ /dev/null @@ -1,114 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope -from sqlbuild.virtual.executor._helpers.clone import hydrate_and_register_relation -from tests.unit.src.sqlbuild.virtual.executor._helpers._test_types import ( - HydrationLifecycleTestCase, -) -from tests.unit.src.sqlbuild.virtual.executor._helpers.helpers import build_relation_location - - -@pytest.mark.parametrize( - "test_case", - ( - HydrationLifecycleTestCase( - "model registration failure", - "model", - "orders", - None, - ( - "resource_attempt_started", - "operation_started", - "operation_completed", - "resource_attempt_failed", - ), - ), - HydrationLifecycleTestCase( - "seed registration failure", - "seed", - "countries", - None, - ( - "resource_attempt_started", - "operation_started", - "operation_completed", - "resource_attempt_failed", - ), - ), - ), - ids=lambda case: case.description, -) -def test_given_fresh_hydration_when_registration_fails_then_resource_attempt_fails( - monkeypatch: pytest.MonkeyPatch, - test_case: HydrationLifecycleTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - monkeypatch.setattr(adapter, "relation_exists", lambda **_: False) - monkeypatch.setattr(adapter, "list_relations", lambda **_: ()) - monkeypatch.setattr(adapter, "ensure_schema", lambda **_: None) - monkeypatch.setattr(adapter, "durable_clone", lambda **_: None) - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - - def fail_registration() -> None: - raise RuntimeError("state registration failed") - - with dispatcher_scope(dispatcher): - with pytest.raises(RuntimeError, match="state registration failed"): - hydrate_and_register_relation( - adapter=adapter, - destination_connection=object(), - origin_location=build_relation_location( - schema="prod", name=test_case.resource_name - ), - destination_location=build_relation_location( - schema="dev", name=test_case.resource_name - ), - resource_kind=test_case.resource_kind, - resource_name=test_case.resource_name, - run_id="clone-run", - register=fail_registration, - ) - - assert tuple(event.event_type for event in events) == test_case.expected_event_types - assert tuple(event.run_id for event in events) == ("clone-run",) * len(events) - assert events[-1].payload["error_type"] == "RuntimeError" - - -@pytest.mark.parametrize( - "test_case", - (HydrationLifecycleTestCase("reused model", "model", "orders", "reused", ()),), - ids=lambda case: case.description, -) -def test_given_reused_hydration_when_registered_then_no_resource_attempt_is_fabricated( - monkeypatch: pytest.MonkeyPatch, - test_case: HydrationLifecycleTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - monkeypatch.setattr(adapter, "relation_exists", lambda **_: True) - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - registrations: list[str] = [] - - with dispatcher_scope(dispatcher): - action: str = hydrate_and_register_relation( - adapter=adapter, - destination_connection=object(), - origin_location=build_relation_location(schema="prod", name=test_case.resource_name), - destination_location=build_relation_location( - schema="dev", name=test_case.resource_name - ), - resource_kind=test_case.resource_kind, - resource_name=test_case.resource_name, - run_id="clone-run", - register=lambda: registrations.append(test_case.resource_name), - ) - - assert action == test_case.expected_action - assert registrations == [test_case.resource_name] - assert tuple(event.event_type for event in events) == test_case.expected_event_types diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_phase_timings.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_phase_timings.py deleted file mode 100644 index 4304e3afaa..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_phase_timings.py +++ /dev/null @@ -1,76 +0,0 @@ -"""Deterministic virtual compile and planning timing tests.""" - -from __future__ import annotations - -from unittest.mock import Mock - -import pytest - -from sqlbuild.diagnostics.classes.build_phase_timing_tracker import BuildPhaseTimingTracker -from sqlbuild.diagnostics.models import PartialBuildPhaseTimings -from sqlbuild.virtual.executor._helpers import build as virtual_build_module -from sqlbuild.virtual.executor.models import VirtualBuildHooks, VirtualBuildOptions -from tests.unit.src.sqlbuild.virtual.executor._helpers._test_types import ( - VirtualPhaseTimingTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPhaseTimingTestCase( - description="slow virtual planning connection is excluded from compilation", - clock_values=(0.0, 0.0, 2.0, 2.0, 12.0, 12.0), - expected_compile_seconds=2.0, - expected_planning_seconds=10.0, - expected_total_seconds=12.0, - ) - ], - ids=lambda case: case.description, -) -def test_given_slow_planning_connection_when_resolving_then_virtual_phases_are_disjoint( - test_case: VirtualPhaseTimingTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monotonic: Mock = Mock(side_effect=test_case.clock_values) - graph: Mock = Mock() - rewritten: Mock = Mock() - plan: Mock = Mock() - python_plan: Mock = Mock() - monkeypatch.setattr(virtual_build_module.time, "monotonic", monotonic) - monkeypatch.setattr(virtual_build_module, "build_project_graph", Mock(return_value=graph)) - monkeypatch.setattr( - virtual_build_module, "_resolve_virtual_environment_names", Mock(return_value=Mock()) - ) - monkeypatch.setattr( - virtual_build_module, "build_state_runtime", Mock(return_value=(Mock(), Mock())) - ) - monkeypatch.setattr( - virtual_build_module, "_read_virtual_build_state", Mock(return_value=Mock()) - ) - monkeypatch.setattr( - virtual_build_module, "_rewrite_virtual_project", Mock(return_value=rewritten) - ) - monkeypatch.setattr(virtual_build_module, "_plan_virtual_build", Mock(return_value=plan)) - monkeypatch.setattr( - virtual_build_module, "_prepare_virtual_python_execution", Mock(return_value=python_plan) - ) - tracker: BuildPhaseTimingTracker = BuildPhaseTimingTracker(monotonic=monotonic) - - with tracker.scope(): - resolution: virtual_build_module._VirtualBuildResolution = ( - virtual_build_module._resolve_virtual_build( - project_dir=Mock(), - discovered_inputs=Mock(), - adapter=Mock(), - connection_config={}, - options=VirtualBuildOptions(), - hooks=VirtualBuildHooks(), - ) - ) - timings: PartialBuildPhaseTimings = tracker.snapshot() - - assert resolution.compile_seconds == test_case.expected_compile_seconds - assert resolution.planning_seconds == test_case.expected_planning_seconds - assert timings.total_seconds == test_case.expected_total_seconds - assert resolution.compile_seconds + resolution.planning_seconds <= (timings.total_seconds or 0) diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_rewrite.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_rewrite.py deleted file mode 100644 index 30362c3d47..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_rewrite.py +++ /dev/null @@ -1,148 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledProject, CompiledRelationLocation -from sqlbuild.virtual.executor._helpers.rewrite import ( - build_physical_destination, - build_rewritten_model_locations, - relation_type_for_model, - rewrite_project_model_locations, -) -from tests.unit.src.sqlbuild.virtual.executor._helpers._test_types import ( - PhysicalTargetTestCase, - RelationTypeTestCase, - RewriteProjectTargetsTestCase, - RewrittenTargetsTestCase, -) -from tests.unit.src.sqlbuild.virtual.executor._helpers.helpers import ( - build_adapter, - build_bound_physical_relation, - build_virtual_executor_test_project, -) - - -@pytest.mark.parametrize( - "test_case", - [ - PhysicalTargetTestCase( - description="builds physical target name from model and version hash", - model_name="fact_orders", - version_hash="abcdef1234567890", - expected_schema="dev__sqb_physical", - expected_name="fact_orders__v_abcdef12", - ) - ], - ids=lambda case: case.description, -) -def test_given_model_target_when_building_physical_target_then_it_uses_physical_schema_and_suffix( - test_case: PhysicalTargetTestCase, -) -> None: - adapter: DuckDbAdapter = build_adapter() - project: CompiledProject = build_virtual_executor_test_project() - target: CompiledRelationLocation = build_physical_destination( - adapter=adapter, - target=project.models[1].destination, - model_name=test_case.model_name, - version_hash=test_case.version_hash, - ) - - assert target.schema == test_case.expected_schema - assert target.name == test_case.expected_name - - -@pytest.mark.parametrize( - "test_case", - [ - RewrittenTargetsTestCase( - description="selected and bound models resolve to physical targets", - selected_model_version_hashes={"fact_orders": "abcdef1234567890"}, - bound_relations={"stg_orders": "1111111122222222"}, - expected_selected_name="fact_orders__v_abcdef12", - expected_bound_name="stg_orders__v_11111111", - ) - ], - ids=lambda case: case.description, -) -def test_given_selected_and_bound_models_when_rewriting_targets_then_it_uses_physical( - test_case: RewrittenTargetsTestCase, -) -> None: - adapter: DuckDbAdapter = build_adapter() - project: CompiledProject = build_virtual_executor_test_project() - - rewritten_locations: dict[str, CompiledRelationLocation] = build_rewritten_model_locations( - project=project, - adapter=adapter, - selected_model_version_hashes=test_case.selected_model_version_hashes, - bound_physical_relations={ - model_name: build_bound_physical_relation( - model_name=model_name, - version_hash=version_hash, - ) - for model_name, version_hash in test_case.bound_relations.items() - }, - ) - - assert rewritten_locations["fact_orders"].schema == "dev__sqb_physical" - assert rewritten_locations["fact_orders"].name == test_case.expected_selected_name - assert rewritten_locations["stg_orders"].name == test_case.expected_bound_name - - -@pytest.mark.parametrize( - "test_case", - [ - RewriteProjectTargetsTestCase( - description="project rewrite changes only targeted models", - selected_model_version_hashes={"fact_orders": "abcdef1234567890"}, - expected_rewritten_name="fact_orders__v_abcdef12", - ) - ], - ids=lambda case: case.description, -) -def test_given_rewritten_locations_when_rewriting_project_then_only_model_locations_change( - test_case: RewriteProjectTargetsTestCase, -) -> None: - adapter: DuckDbAdapter = build_adapter() - project: CompiledProject = build_virtual_executor_test_project() - rewritten_locations: dict[str, CompiledRelationLocation] = build_rewritten_model_locations( - project=project, - adapter=adapter, - selected_model_version_hashes=test_case.selected_model_version_hashes, - bound_physical_relations={}, - ) - - rewritten_project: CompiledProject = rewrite_project_model_locations( - project=project, - rewritten_locations=rewritten_locations, - ) - - assert rewritten_project.models[0].destination == project.models[0].destination - assert rewritten_project.models[1].destination.name == test_case.expected_rewritten_name - - -@pytest.mark.parametrize( - "test_case", - [ - RelationTypeTestCase( - description="views persist as view relation type", - materialized="view", - expected_relation_type="view", - ), - RelationTypeTestCase( - description="tables persist as table relation type", - materialized="table", - expected_relation_type="table", - ), - RelationTypeTestCase( - description="incrementals persist as table relation type", - materialized="incremental", - expected_relation_type="table", - ), - ], - ids=lambda case: case.description, -) -def test_given_materialization_when_resolving_relation_type_then_it_matches_virtual_state( - test_case: RelationTypeTestCase, -) -> None: - assert relation_type_for_model(test_case.materialized) == test_case.expected_relation_type diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_seed_physical_relations.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_seed_physical_relations.py deleted file mode 100644 index 58e23123b8..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_seed_physical_relations.py +++ /dev/null @@ -1,89 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.virtual.executor._helpers.rollback import read_rollback_physical_relations -from sqlbuild.virtual.executor._helpers.seeding import read_seed_physical_relations -from sqlbuild.virtual.executor.models import RollbackResolution -from sqlbuild.virtual.state.models import PhysicalRelationRecord -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus -from tests.unit.src.sqlbuild.virtual.executor._helpers._test_types import ( - MissingRollbackSeedRelationTestCase, - SeedPhysicalRelationLookupTestCase, -) -from tests.unit.src.sqlbuild.virtual.executor._helpers.helpers import ( - SeedPhysicalRelationTestBackend, -) - - -@pytest.mark.parametrize( - "test_case", - [ - SeedPhysicalRelationLookupTestCase( - description="available seed relations are returned by seed name", - seed_version_hashes={"orders": "orders-hash", "customers": "customers-hash"}, - available_seed_names=("customers", "orders"), - expected_seed_names=("customers", "orders"), - ), - SeedPhysicalRelationLookupTestCase( - description="missing seed relations are omitted for caller policy", - seed_version_hashes={"orders": "orders-hash", "customers": "customers-hash"}, - available_seed_names=("orders",), - expected_seed_names=("orders",), - ), - ], - ids=lambda case: case.description, -) -def test_given_seed_version_hashes_when_reading_relations_then_available_records_are_returned( - test_case: SeedPhysicalRelationLookupTestCase, -) -> None: - backend: SeedPhysicalRelationTestBackend = SeedPhysicalRelationTestBackend( - available_seed_names=test_case.available_seed_names - ) - - result: dict[str, PhysicalRelationRecord] = read_seed_physical_relations( - backend=backend, - state_connection=object(), - schema="state", - seed_version_hashes=test_case.seed_version_hashes, - ) - - assert tuple(sorted(result)) == test_case.expected_seed_names - - -@pytest.mark.parametrize( - "test_case", - [ - MissingRollbackSeedRelationTestCase( - description="rollback rejects a checkpoint with a missing seed relation", - final_seed_hashes={"orders": "orders-hash"}, - expected_error_fragment=( - "checkpoint references missing physical relation for seed 'orders'" - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_checkpoint_seed_relation_when_reading_rollback_then_it_raises( - test_case: MissingRollbackSeedRelationTestCase, -) -> None: - backend: SeedPhysicalRelationTestBackend = SeedPhysicalRelationTestBackend( - available_seed_names=() - ) - resolution: RollbackResolution = RollbackResolution( - final_version_hashes={}, - final_seed_hashes=test_case.final_seed_hashes, - is_partial_scope=False, - status=VirtualEnvironmentStatus.FINALIZED, - rolled_back_model_names=(), - ) - - with pytest.raises(PlannerInputError, match=test_case.expected_error_fragment): - read_rollback_physical_relations( - backend=backend, - state_connection=object(), - schema="state", - checkpoint_id="checkpoint-1", - resolution=resolution, - ) diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_seeding.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_seeding.py deleted file mode 100644 index c22a1941f3..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_seeding.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -from typing import ClassVar - -import pytest - -from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.adapter.contract.classes.statement_recorder import StatementRecorder -from sqlbuild.adapter.contract.models import RelationInfo -from sqlbuild.compiler.planner.models import ModelPlanEntry -from sqlbuild.compiler.planner.types import PlanAction -from sqlbuild.virtual.executor._helpers.seeding import ( - _seed_physical_relation, - seed_virtual_physical_version, -) -from sqlbuild.virtual.state.models import PhysicalRelationAncestryRecord, PhysicalRelationRecord -from sqlbuild.virtual.state.types import PhysicalArtifactType -from tests.unit.src.sqlbuild.virtual.executor._helpers._test_types import ( - SeedingIdempotencyTestCase, - SeedingStrategyTestCase, -) -from tests.unit.src.sqlbuild.virtual.executor._helpers.helpers import ( - build_seeded_incremental_plan_output, -) - - -class FakeSeedAdapter(BaseAdapter): - adapter_name: ClassVar[str] = "seed-test" - - def __init__(self, *, supports_durable_clone: bool, target_exists: bool = False) -> None: - self._supports_durable_clone = supports_durable_clone - self._target_exists = target_exists - self.executed_sql: list[str] = [] - self.drop_count: int = 0 - - def connect(self, config: dict[str, object]) -> object: - del config - return object() - - def _execute(self, connection: object, sql: str) -> object: - del connection - self.executed_sql.append(sql) - return object() - - def close(self, connection: object) -> None: - del connection - - def supports_durable_clone(self) -> bool: - return self._supports_durable_clone - - def render_durable_clone( - self, *, origin: str, destination: str, origin_is_transient: bool = False - ) -> tuple[str, ...]: - del origin_is_transient - return (f"CREATE TABLE {destination} DEEP CLONE {origin}",) - - def relation_exists( - self, - connection: object, - *, - database: str | None, - schema: str | None, - name: str, - ) -> bool: - del connection, database, schema, name - return self._target_exists - - def list_relations( - self, - connection: object, - *, - database: str | None, - schemas: tuple[str, ...] | None, - names: tuple[str, ...] | None = None, - ) -> tuple[RelationInfo, ...]: - del connection, database, schemas, names - return () - - def ensure_schema( - self, - connection: object, - *, - database: str | None, - schema: str | None, - statement_recorder: StatementRecorder, - ) -> None: - del connection, database, schema, statement_recorder - - def drop( - self, - connection: object, - *, - destination: str, - if_exists: bool = True, - statement_recorder: StatementRecorder, - ) -> None: - del connection, if_exists, statement_recorder - self.drop_count += 1 - self.executed_sql.append(f"DROP {destination}") - - -class FakeStateBackend: - def __init__(self) -> None: - self.ancestry_records: list[PhysicalRelationAncestryRecord] = [] - - def upsert_physical_relation_ancestry( - self, - connection: object, - *, - schema: str, - record: PhysicalRelationAncestryRecord, - ) -> None: - del connection, schema - self.ancestry_records.append(record) - - -@pytest.mark.parametrize( - "test_case", - [ - SeedingStrategyTestCase( - description="uses durable clone when adapter supports it", - incremental_strategy="delete_insert", - supports_durable_clone=True, - expected_strategy="durable_clone", - expected_sql_fragment="DEEP CLONE source_relation", - ), - SeedingStrategyTestCase( - description="uses copy when adapter lacks durable clone", - incremental_strategy="delete_insert", - supports_durable_clone=False, - expected_strategy="copy", - expected_sql_fragment="SELECT * FROM source_relation", - ), - SeedingStrategyTestCase( - description="uses bounded append copy before durable clone", - incremental_strategy="append", - supports_durable_clone=True, - expected_strategy="bounded_append_copy", - expected_sql_fragment='WHERE "ordered_at" < TIMESTAMP', - ), - ], - ids=lambda case: case.description, -) -def test_given_incremental_seed_context_when_seeding_then_selects_expected_strategy( - test_case: SeedingStrategyTestCase, -) -> None: - adapter: FakeSeedAdapter = FakeSeedAdapter( - supports_durable_clone=test_case.supports_durable_clone - ) - entry: ModelPlanEntry = build_seeded_incremental_plan_output( - incremental_strategy=test_case.incremental_strategy - ).model_entries[0] - - strategy: str = _seed_physical_relation( - adapter=adapter, - connection=object(), - origin="source_relation", - destination="target_relation", - entry=entry, - origin_is_transient=False, - statement_recorder=StatementRecorder(), - ) - - assert strategy == test_case.expected_strategy - assert any(test_case.expected_sql_fragment in sql for sql in adapter.executed_sql) - - -@pytest.mark.parametrize( - "test_case", - [ - SeedingIdempotencyTestCase( - description="preserves existing immutable shared target", - target_exists=True, - expected_drop_count=0, - expected_ancestry_count=0, - expected_sql_count=0, - ), - SeedingIdempotencyTestCase( - description="seeds missing target without drop", - target_exists=False, - expected_drop_count=0, - expected_ancestry_count=1, - expected_sql_count=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_incremental_target_when_seeding_then_existing_target_is_preserved( - test_case: SeedingIdempotencyTestCase, -) -> None: - adapter: FakeSeedAdapter = FakeSeedAdapter( - supports_durable_clone=False, - target_exists=test_case.target_exists, - ) - backend: FakeStateBackend = FakeStateBackend() - entry: ModelPlanEntry = build_seeded_incremental_plan_output( - incremental_strategy="delete_insert", - action=PlanAction.INCREMENTAL_DELETE_INSERT, - ).model_entries[0] - parent_relation: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="oldhash", - database_name="", - schema_name="dev__sqb_physical", - relation_name="orders__v_oldhash", - relation_type="table", - ) - - seed_virtual_physical_version( - adapter=adapter, - connection=object(), - backend=backend, - state_connection=object(), - state_schema="sqlbuild_state", - entry=entry, - parent_relation=parent_relation, - version_hash="newhash", - ) - - assert adapter.drop_count == test_case.expected_drop_count - assert len(backend.ancestry_records) == test_case.expected_ancestry_count - assert len(adapter.executed_sql) == test_case.expected_sql_count diff --git a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_statement_diagnostics.py b/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_statement_diagnostics.py deleted file mode 100644 index 3eb2058ce0..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/_helpers/test_statement_diagnostics.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Virtual statement diagnostic callback scope tests.""" - -from __future__ import annotations - -from pathlib import Path -from typing import cast -from unittest.mock import Mock - -import pytest - -from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.cost.classes.cost_context import CostContext -from sqlbuild.cost.models import CostResourceContext -from sqlbuild.executor.build.models import BuildCallbacks, BuildExecutionResult -from sqlbuild.executor.build.types import BuildStatus -from sqlbuild.virtual.executor._helpers import build as virtual_build_module -from sqlbuild.virtual.executor.models import VirtualBuildExecutionHooks -from tests.unit.src.sqlbuild.virtual.executor._helpers._test_types import ( - VirtualStatementScopeTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualStatementScopeTestCase( - description="ingress and finalization inherit callback and reset after each scope", - expected_phases=("virtual_ingress", "virtual_finalize"), - expected_callback_count=2, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_statement_callback_when_executing_then_all_outer_scopes_propagate_and_reset( - test_case: VirtualStatementScopeTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - callback: Mock = Mock() - observed_phases: list[str] = [] - observed_callbacks: list[object] = [] - reset_contexts: list[CostResourceContext | None] = [] - - def capture_ingress(**_kwargs: object) -> None: - context: CostResourceContext | None = CostContext.current() - assert context is not None - observed_phases.append(context.phase) - observed_callbacks.append(context.on_statement_complete) - - def capture_finalize(**_kwargs: object) -> tuple[object, ...]: - context: CostResourceContext | None = CostContext.current() - assert context is not None - observed_phases.append(context.phase) - observed_callbacks.append(context.on_statement_complete) - return () - - def capture_execution(**_kwargs: object) -> BuildExecutionResult: - reset_contexts.append(CostContext.current()) - return BuildExecutionResult(status=BuildStatus.SUCCESS) - - monkeypatch.setattr(virtual_build_module, "_run_ingress_python_nodes", capture_ingress) - monkeypatch.setattr( - virtual_build_module, - "_execute_virtual_build_plan", - capture_execution, - ) - monkeypatch.setattr(virtual_build_module, "_persist_successful_virtual_build", Mock()) - monkeypatch.setattr(virtual_build_module, "_run_read_side_python_nodes", capture_finalize) - project: CompiledProject = CompiledProject( - run_id="scope-run", - effective_target_name="test", - effective_connection={}, - effective_vars={}, - ) - runtime: Mock = Mock(project_dir=Path(".")) - rewritten: Mock = Mock(project=project) - graph: Mock = Mock(project=project) - - virtual_build_module._execute_leased_virtual_build( - runtime=runtime, - graph=graph, - reads=Mock(), - rewritten=rewritten, - plan=Mock(), - python_plan=Mock(), - exec_hooks=VirtualBuildExecutionHooks(on_statement_complete=callback), - microbatch_lease_check=Mock(), - model_version_leases=(), - ) - - assert tuple(observed_phases) == test_case.expected_phases - assert observed_callbacks == [callback] * test_case.expected_callback_count - assert reset_contexts == [None] - assert CostContext.current() is None - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualStatementScopeTestCase( - description="build pipeline receives statement diagnostics", - expected_phases=("build",), - expected_callback_count=1, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_execution_callback_when_building_then_pipeline_receives_it( - test_case: VirtualStatementScopeTestCase, - monkeypatch: pytest.MonkeyPatch, -) -> None: - callback: Mock = Mock() - observed_phases: list[str] = [] - observed_callbacks: list[object] = [] - - def capture_build(**kwargs: object) -> BuildExecutionResult: - callbacks: BuildCallbacks = cast(BuildCallbacks, kwargs["callbacks"]) - observed_phases.append("build") - observed_callbacks.append(callbacks.on_statement_complete) - return BuildExecutionResult(status=BuildStatus.SUCCESS) - - monkeypatch.setattr(virtual_build_module, "load_custom_materializations", Mock(return_value={})) - monkeypatch.setattr( - virtual_build_module, "load_custom_prepare_version_functions", Mock(return_value={}) - ) - monkeypatch.setattr(virtual_build_module, "run_build_pipeline", capture_build) - project: CompiledProject = CompiledProject( - run_id="execution-scope-run", - effective_target_name="test", - effective_connection={}, - effective_vars={}, - ) - runtime: Mock = Mock(project_dir=Path("."), connection_config={}, adapter=Mock()) - runtime.options.concurrency = 1 - runtime.discovered_inputs.loader_functions = () - python_plan: Mock = Mock() - python_plan.lifecycle_plan.ingress_loader_names = frozenset() - - virtual_build_module._execute_virtual_build_plan( - runtime=runtime, - plan=Mock(), - project=project, - python_plan=python_plan, - reads=Mock(available_seed_physical_relations={}), - exec_hooks=VirtualBuildExecutionHooks(on_statement_complete=callback), - ingress_load_results=(), - microbatch_lease_check=Mock(), - ) - - assert tuple(observed_phases) == test_case.expected_phases - assert observed_callbacks == [callback] * test_case.expected_callback_count - assert CostContext.current() is None diff --git a/tests/unit/src/sqlbuild/virtual/executor/main/__init__.py b/tests/unit/src/sqlbuild/virtual/executor/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/virtual/executor/main/_test_types.py b/tests/unit/src/sqlbuild/virtual/executor/main/_test_types.py deleted file mode 100644 index bc688a4b8e..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/main/_test_types.py +++ /dev/null @@ -1,17 +0,0 @@ -from dataclasses import dataclass - - -@dataclass(frozen=True) -class VirtualCloneAggregateTestCase: - description: str - action: str - expected_terminal: str - expected_error_code: str | None - - -@dataclass(frozen=True) -class VirtualPromoteLifecycleTestCase: - description: str - expected_operation_id: str - expected_event_types: tuple[str, ...] - expected_state_result_calls: int diff --git a/tests/unit/src/sqlbuild/virtual/executor/main/test_clone.py b/tests/unit/src/sqlbuild/virtual/executor/main/test_clone.py deleted file mode 100644 index 938cebd48c..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/main/test_clone.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from unittest.mock import Mock - -import pytest - -from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope -from sqlbuild.virtual.executor.main import clone -from sqlbuild.virtual.executor.models import ( - CloneOptions, - VirtualCloneItemResult, - VirtualCloneResult, -) -from sqlbuild.virtual.state.types import PhysicalArtifactType -from tests.unit.src.sqlbuild.virtual.executor.main._test_types import ( - VirtualCloneAggregateTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualCloneAggregateTestCase( - "missing artifact fails aggregate", - "missing", - "operation_failed", - "clone_execution_failed", - ), - VirtualCloneAggregateTestCase( - "skipped lock completes aggregate", "skipped_locked", "operation_completed", None - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_clone_result_when_api_returns_then_aggregate_uses_destination_run( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, - test_case: VirtualCloneAggregateTestCase, -) -> None: - adapter: Mock = Mock() - destination_connection: object = object() - state_connection: object = object() - adapter.connect.return_value = destination_connection - backend: Mock = Mock() - backend.connect.return_value = state_connection - config: Mock = Mock(schema="state", connection={}) - pipeline: Mock = Mock() - pipeline.destination_project.run_id = "clone-run" - versions: Mock = Mock(mode="workspace", origin_state_used=False) - item: VirtualCloneItemResult = VirtualCloneItemResult( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash="hash", - action=test_case.action, - ) - monkeypatch.setattr(clone, "build_state_runtime", lambda **_: (config, backend)) - monkeypatch.setattr(clone, "compile_clone_pipeline", lambda **_: pipeline) - monkeypatch.setattr(clone, "build_clone_project_context", lambda _: Mock()) - monkeypatch.setattr(clone, "resolve_clone_versions", lambda **_: versions) - monkeypatch.setattr(clone, "build_clone_origin_lookup", lambda **_: Mock()) - monkeypatch.setattr(clone, "hydrate_clone_model_relations", lambda **_: (item,)) - monkeypatch.setattr(clone, "hydrate_clone_seed_relations", lambda **_: ()) - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - - with dispatcher_scope(dispatcher): - result: VirtualCloneResult = clone.run_virtual_clone( - project_dir=tmp_path, - discovered_inputs=Mock(), - adapter=adapter, - origin_target_name="prod", - destination_target_name="dev", - destination_connection_config={}, - options=CloneOptions(skip_locked=test_case.action == "skipped_locked"), - ) - - aggregate_events: tuple[LifecycleEvent, LifecycleEvent] = (events[4], events[-1]) - assert result.item_results == (item,) - assert tuple(event.event_type for event in aggregate_events) == ( - "operation_started", - test_case.expected_terminal, - ) - assert tuple(event.run_id for event in aggregate_events) == ("clone-run", "clone-run") - assert aggregate_events[-1].payload.get("error_code") == test_case.expected_error_code diff --git a/tests/unit/src/sqlbuild/virtual/executor/main/test_promote.py b/tests/unit/src/sqlbuild/virtual/executor/main/test_promote.py deleted file mode 100644 index 1def35a983..0000000000 --- a/tests/unit/src/sqlbuild/virtual/executor/main/test_promote.py +++ /dev/null @@ -1,137 +0,0 @@ -from pathlib import Path -from unittest.mock import Mock - -import pytest - -import sqlbuild.virtual.executor.main.promote as promote_module -from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope -from sqlbuild.virtual.executor.models import StateOperationHandle -from sqlbuild.virtual.state.types import StateOperationType -from tests.unit.src.sqlbuild.virtual.executor.main._test_types import ( - VirtualPromoteLifecycleTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualPromoteLifecycleTestCase( - description="state backend connection fails after canonical start", - expected_operation_id="promote-operation", - expected_event_types=("operation_started", "operation_failed"), - expected_state_result_calls=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_backend_connect_failure_when_promoting_then_handle_operation_fails( - test_case: VirtualPromoteLifecycleTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - backend: Mock = Mock() - backend.connect.side_effect = RuntimeError("state unavailable") - state_result: Mock = Mock() - monkeypatch.setattr( - promote_module, "resolve_virtual_project_context", Mock(return_value=Mock()) - ) - monkeypatch.setattr( - promote_module, - "build_state_runtime", - Mock(return_value=(Mock(connection=object(), schema="state"), backend)), - ) - monkeypatch.setattr( - promote_module, - "create_state_operation_handle", - Mock( - return_value=StateOperationHandle( - operation_id=test_case.expected_operation_id, - operation_type=StateOperationType.PROMOTE, - ) - ), - ) - monkeypatch.setattr(promote_module, "write_state_operation_result", state_result) - - with dispatcher_scope(dispatcher), pytest.raises(RuntimeError, match="state unavailable"): - promote_module.run_virtual_promote( - project_dir=Path("."), - discovered_inputs=Mock(), - adapter=Mock(adapter_name="private-adapter"), - connection_config={}, - from_virtual_environment_name="source", - to_virtual_environment_name="target", - options=Mock(), - hooks=Mock(on_progress=None), - ) - - assert tuple(event.event_type for event in events) == test_case.expected_event_types - assert events[0].operation_id == test_case.expected_operation_id - assert events[-1].operation_id == test_case.expected_operation_id - assert events[-1].payload["adapter"] == "custom" - assert state_result.call_count == test_case.expected_state_result_calls - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualPromoteLifecycleTestCase( - description="state startup write failure does not attempt failed state write", - expected_operation_id="promote-operation", - expected_event_types=("operation_started", "operation_failed"), - expected_state_result_calls=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_startup_state_write_failure_when_promoting_then_cleanup_canonical_failure_only( - test_case: VirtualPromoteLifecycleTestCase, monkeypatch: pytest.MonkeyPatch -) -> None: - events: list[LifecycleEvent] = [] - dispatcher: EventDispatcher = EventDispatcher() - dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) - backend: Mock = Mock() - state_result: Mock = Mock() - startup_write: Mock = Mock(side_effect=RuntimeError("state start failed")) - monkeypatch.setattr( - promote_module, "resolve_virtual_project_context", Mock(return_value=Mock()) - ) - monkeypatch.setattr( - promote_module, - "build_state_runtime", - Mock(return_value=(Mock(connection=object(), schema="state"), backend)), - ) - monkeypatch.setattr( - promote_module, - "create_state_operation_handle", - Mock( - return_value=StateOperationHandle( - operation_id=test_case.expected_operation_id, - operation_type=StateOperationType.PROMOTE, - ) - ), - ) - monkeypatch.setattr(promote_module, "write_state_operation_started", startup_write) - monkeypatch.setattr(promote_module, "write_state_operation_result", state_result) - - with dispatcher_scope(dispatcher), pytest.raises(RuntimeError, match="state start failed"): - promote_module.run_virtual_promote( - project_dir=Path("."), - discovered_inputs=Mock(), - adapter=Mock(adapter_name="duckdb"), - connection_config={}, - from_virtual_environment_name="source", - to_virtual_environment_name="target", - options=Mock(), - hooks=Mock(on_progress=None), - ) - - assert tuple(event.event_type for event in events) == test_case.expected_event_types - assert events[-1].operation_id == test_case.expected_operation_id - assert startup_write.call_count == 1 - assert state_result.call_count == test_case.expected_state_result_calls - backend.close.assert_called_once() - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/virtual/freshness/test_compatibility.py b/tests/unit/src/sqlbuild/virtual/freshness/test_compatibility.py deleted file mode 100644 index 9d3c680707..0000000000 --- a/tests/unit/src/sqlbuild/virtual/freshness/test_compatibility.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.source_freshness.models import ( - SourceFreshnessObservation as SharedSourceFreshnessObservation, -) -from sqlbuild.virtual.freshness.models import ( - SourceFreshnessObservation as VirtualSourceFreshnessObservation, -) -from tests.unit.src.sqlbuild.virtual.freshness._test_types import ( - VirtualSourceFreshnessCompatibilityTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualSourceFreshnessCompatibilityTestCase( - description="virtual observation model aliases shared model", - expected_same_object=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_observation_model_import_when_resolving_then_aliases_shared_model( - test_case: VirtualSourceFreshnessCompatibilityTestCase, -) -> None: - assert ( - VirtualSourceFreshnessObservation is SharedSourceFreshnessObservation - ) == test_case.expected_same_object diff --git a/tests/unit/src/sqlbuild/virtual/freshness/test_helpers.py b/tests/unit/src/sqlbuild/virtual/freshness/test_helpers.py deleted file mode 100644 index 8d3cf86231..0000000000 --- a/tests/unit/src/sqlbuild/virtual/freshness/test_helpers.py +++ /dev/null @@ -1,576 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any - -import pytest - -from sqlbuild.adapter.contract.models import TableFreshnessMetadata -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.python_nodes.types import SkipMode -from sqlbuild.compiler.source_freshness.exceptions import SourceFreshnessObservationError -from sqlbuild.compiler.source_freshness.main.normalization import ( - normalize_source_freshness_data_version, -) -from sqlbuild.compiler.source_freshness.main.observation import ( - observe_configured_source_freshness, -) -from sqlbuild.compiler.source_freshness.models import SourceFreshnessObservation -from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.scheduling.types import ExecutionStatus -from sqlbuild.spec.contracts.models import SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind -from sqlbuild.virtual.freshness._helpers.state import ( - source_freshness_record_from_observation, -) -from sqlbuild.virtual.freshness.main._runtime_observation import ( - observe_virtual_environment_source_freshness, -) -from sqlbuild.virtual.freshness.models import SourceFreshnessRuntimeResult -from sqlbuild.virtual.state.models import SourceFreshnessRecord -from tests.unit.src.sqlbuild.virtual.freshness._test_types import ( - SourceFreshnessObservationErrorTestCase, - SourceFreshnessObservationTestCase, - SourceFreshnessRuntimeLagToleranceTestCase, - SourceFreshnessRuntimeTestCase, - SourceFreshnessStateErrorTestCase, - SourceFreshnessStateTestCase, - UnsupportedTableFreshnessMetadataGuardTestCase, -) - - -class FreshnessMetadataDuckDbAdapter(DuckDbAdapter): - def supports_table_freshness_metadata(self) -> bool: - return True - - def get_table_freshness_metadata( - self, - connection: Any, - *, - database: str | None, - schema: str | None, - name: str, - ) -> TableFreshnessMetadata: - return TableFreshnessMetadata( - data_version=datetime(2026, 1, 1, 12, 0, 0), - value_kind="timestamp", - ) - - -class UnsupportedFreshnessMetadataDuckDbAdapter(DuckDbAdapter): - metadata_requested: bool - - def __init__(self) -> None: - super().__init__() - self.metadata_requested = False - - def supports_table_freshness_metadata(self) -> bool: - return False - - def get_table_freshness_metadata( - self, - connection: Any, - *, - database: str | None, - schema: str | None, - name: str, - ) -> TableFreshnessMetadata: - self.metadata_requested = True - return TableFreshnessMetadata( - data_version=datetime(2026, 1, 1, 12, 0, 0), - value_kind="timestamp", - ) - - -@pytest.mark.parametrize( - "test_case", - [ - SourceFreshnessObservationTestCase( - description="observes max column data version", - setup_sql=( - "CREATE TABLE raw_orders (updated_at INTEGER)", - "INSERT INTO raw_orders VALUES (1), (3), (2)", - ), - source_name="raw_orders", - table="raw_orders", - strategy="column", - column="updated_at", - value_kind=SourceFreshnessValueKind.INTEGER, - expected_data_version=3, - ), - SourceFreshnessObservationTestCase( - description="observes sql data version", - setup_sql=(), - source_name="raw_orders", - table=None, - strategy="sql", - query="SELECT 'version-1' AS data_version", - value_kind="string", - expected_data_version="version-1", - ), - ], - ids=lambda case: case.description, -) -def test_given_source_freshness_config_when_observing_then_returns_data_version( - test_case: SourceFreshnessObservationTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: Any = adapter.connect({"database": ":memory:"}) - observed_at: datetime = datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC) - try: - statement: str - for statement in test_case.setup_sql: - adapter.execute(connection=connection, sql=statement) - observation: SourceFreshnessObservation = observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=SourceEntry( - name=test_case.source_name, - table=test_case.table, - freshness=SourceFreshnessConfig( - strategy=SourceFreshnessStrategy(test_case.strategy), - value_kind=SourceFreshnessValueKind(test_case.value_kind), - column=test_case.column, - query=test_case.query, - ), - ), - observed_at=observed_at, - ) - finally: - adapter.close(connection) - - assert observation.source_name == test_case.source_name - assert observation.data_version == test_case.expected_data_version - assert observation.value_kind == SourceFreshnessValueKind(test_case.value_kind) - assert observation.observed_at == observed_at - - -@pytest.mark.parametrize( - "test_case", - [ - SourceFreshnessObservationErrorTestCase( - description="raises when sql freshness returns multiple columns", - setup_sql=(), - source_name="raw_orders", - table=None, - strategy="sql", - query="SELECT 1 AS left_value, 2 AS right_value", - value_kind=SourceFreshnessValueKind.INTEGER, - expected_error_fragment="must return exactly one column", - ), - SourceFreshnessObservationErrorTestCase( - description="raises when sql freshness returns zero rows", - setup_sql=(), - source_name="raw_orders", - table=None, - strategy="sql", - query="SELECT 1 AS data_version WHERE FALSE", - value_kind=SourceFreshnessValueKind.INTEGER, - expected_error_fragment="must return exactly one row", - ), - SourceFreshnessObservationErrorTestCase( - description="raises when sql freshness returns null", - setup_sql=(), - source_name="raw_orders", - table=None, - strategy="sql", - query="SELECT NULL AS data_version", - value_kind=SourceFreshnessValueKind.INTEGER, - expected_error_fragment="data_version cannot be null", - ), - SourceFreshnessObservationErrorTestCase( - description="raises when adapter metadata is unsupported", - setup_sql=(), - source_name="raw_orders", - table="raw_orders", - strategy="adapter", - value_kind=None, - expected_error_fragment="does not support table freshness metadata", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_source_freshness_result_when_observing_then_raises_clear_error( - test_case: SourceFreshnessObservationErrorTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: Any = adapter.connect({"database": ":memory:"}) - try: - statement: str - for statement in test_case.setup_sql: - adapter.execute(connection=connection, sql=statement) - source: SourceEntry = SourceEntry( - name=test_case.source_name, - table=test_case.table, - freshness=SourceFreshnessConfig( - strategy=SourceFreshnessStrategy(test_case.strategy), - value_kind=test_case.value_kind, - column=test_case.column, - query=test_case.query, - ), - ) - with pytest.raises( - SourceFreshnessObservationError, match=test_case.expected_error_fragment - ): - observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), - ) - finally: - adapter.close(connection) - - -@pytest.mark.parametrize( - "test_case", - [ - UnsupportedTableFreshnessMetadataGuardTestCase( - description="unsupported adapter guard prevents metadata lookup", - source_name="raw_orders", - table="raw_orders", - expected_error_fragment="does not support table freshness metadata", - expected_metadata_requested=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_adapter_freshness_is_unsupported_when_observing_then_metadata_is_not_requested( - test_case: UnsupportedTableFreshnessMetadataGuardTestCase, -) -> None: - adapter: UnsupportedFreshnessMetadataDuckDbAdapter = UnsupportedFreshnessMetadataDuckDbAdapter() - connection: Any = adapter.connect({"database": ":memory:"}) - try: - source: SourceEntry = SourceEntry( - name=test_case.source_name, - table=test_case.table, - freshness=SourceFreshnessConfig( - strategy=SourceFreshnessStrategy.ADAPTER, - value_kind=None, - ), - ) - - with pytest.raises( - SourceFreshnessObservationError, match=test_case.expected_error_fragment - ): - observe_configured_source_freshness( - adapter=adapter, - connection=connection, - source=source, - observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), - ) - finally: - adapter.close(connection) - - assert adapter.metadata_requested is test_case.expected_metadata_requested - - -@pytest.mark.parametrize( - "test_case", - [ - SourceFreshnessStateTestCase( - description="normalizes timestamp freshness value", - source_name="raw.orders", - strategy="column", - value_kind="timestamp", - data_version=datetime(2026, 1, 1, 12, 34, 56, tzinfo=UTC), - observed_at=datetime(2026, 1, 1, 13, 0, 0, tzinfo=UTC), - expected_data_version="2026-01-01T12:34:56+00:00", - expected_hash_changes_with_observed_at=False, - ), - SourceFreshnessStateTestCase( - description="normalizes integer freshness value", - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version=123, - observed_at=datetime(2026, 1, 1, 13, 0, 0, tzinfo=UTC), - expected_data_version="123", - expected_hash_changes_with_observed_at=False, - ), - SourceFreshnessStateTestCase( - description="preserves string freshness value", - source_name="raw.orders", - strategy="sql", - value_kind="string", - data_version="batch-001", - observed_at=datetime(2026, 1, 1, 13, 0, 0, tzinfo=UTC), - expected_data_version="batch-001", - expected_hash_changes_with_observed_at=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_source_freshness_observation_when_building_state_record_then_hashes_value( - test_case: SourceFreshnessStateTestCase, -) -> None: - observation: SourceFreshnessObservation = SourceFreshnessObservation( - source_name=test_case.source_name, - strategy=SourceFreshnessStrategy(test_case.strategy), - data_version=test_case.data_version, - value_kind=SourceFreshnessValueKind(test_case.value_kind), - observed_at=test_case.observed_at, - ) - - record: SourceFreshnessRecord = source_freshness_record_from_observation( - observation=observation, - virtual_environment_name="dev", - ) - record_with_later_observed_at: SourceFreshnessRecord = source_freshness_record_from_observation( - observation=SourceFreshnessObservation( - source_name=test_case.source_name, - strategy=SourceFreshnessStrategy(test_case.strategy), - data_version=test_case.data_version, - value_kind=SourceFreshnessValueKind(test_case.value_kind), - observed_at=datetime(2026, 1, 2, 13, 0, 0, tzinfo=UTC), - ), - virtual_environment_name="dev", - ) - - assert record.virtual_environment_name == "dev" - assert record.source_name == test_case.source_name - assert record.strategy == test_case.strategy - assert record.value_kind == test_case.value_kind - assert record.data_version == test_case.expected_data_version - assert record.data_version_hash == record_with_later_observed_at.data_version_hash - assert test_case.expected_hash_changes_with_observed_at is False - - -@pytest.mark.parametrize( - "test_case", - [ - SourceFreshnessStateErrorTestCase( - description="rejects non datetime timestamp value", - value_kind="timestamp", - data_version="2026-01-01T12:00:00Z", - expected_error_fragment="must be datetime values", - ), - SourceFreshnessStateErrorTestCase( - description="rejects bool integer value", - value_kind="integer", - data_version=True, - expected_error_fragment="must be integer values", - ), - SourceFreshnessStateErrorTestCase( - description="rejects non integer value", - value_kind="integer", - data_version="123", - expected_error_fragment="must be integer values", - ), - SourceFreshnessStateErrorTestCase( - description="rejects non string value", - value_kind="string", - data_version=123, - expected_error_fragment="must be string values", - ), - ], - ids=lambda case: case.description, -) -def test_given_invalid_source_freshness_value_when_normalizing_then_raises_clear_error( - test_case: SourceFreshnessStateErrorTestCase, -) -> None: - with pytest.raises(SourceFreshnessObservationError, match=test_case.expected_error_fragment): - normalize_source_freshness_data_version( - value=test_case.data_version, - value_kind=SourceFreshnessValueKind(test_case.value_kind), - ) - - -@pytest.mark.parametrize( - "test_case", - [ - SourceFreshnessRuntimeTestCase( - description="observes explicit config before adapter default and tracks unknowns", - expected_record_sources=("raw.customers", "raw.orders"), - expected_unknown_sources=("raw.unknown",), - expected_preserved_sources=(), - expected_generated_sources=(), - ) - ], - ids=lambda case: case.description, -) -def test_given_unmanaged_sources_when_observing_runtime_freshness_then_applies_precedence( - test_case: SourceFreshnessRuntimeTestCase, -) -> None: - adapter: FreshnessMetadataDuckDbAdapter = FreshnessMetadataDuckDbAdapter() - connection: Any = adapter.connect({"database": ":memory:"}) - try: - adapter.execute(connection=connection, sql="CREATE TABLE raw_orders (batch_id INTEGER)") - adapter.execute(connection=connection, sql="INSERT INTO raw_orders VALUES (7)") - - result: SourceFreshnessRuntimeResult = observe_virtual_environment_source_freshness( - adapter=adapter, - connection=connection, - sources=( - SourceEntry( - name="raw.orders", - table="raw_orders", - freshness=SourceFreshnessConfig( - strategy=SourceFreshnessStrategy.COLUMN, - value_kind=SourceFreshnessValueKind.INTEGER, - column="batch_id", - ), - ), - SourceEntry(name="raw.customers", table="raw_customers"), - SourceEntry(name="raw.unknown", expression="SELECT 1 AS id"), - ), - virtual_environment_name="dev", - observed_at=datetime(2026, 1, 1, 13, 0, 0, tzinfo=UTC), - ) - finally: - adapter.close(connection) - - records_by_source: dict[str, SourceFreshnessRecord] = { - record.source_name: record for record in result.records - } - assert tuple(records_by_source) == test_case.expected_record_sources - assert result.unknown_source_names == test_case.expected_unknown_sources - assert result.preserved_source_names == test_case.expected_preserved_sources - assert result.generated_source_names == test_case.expected_generated_sources - assert records_by_source["raw.orders"].strategy == "column" - assert records_by_source["raw.orders"].data_version == "7" - assert records_by_source["raw.customers"].strategy == "adapter" - assert records_by_source["raw.customers"].data_version == "2026-01-01T12:00:00+00:00" - - -@pytest.mark.parametrize( - "test_case", - [ - SourceFreshnessRuntimeTestCase( - description="preserves soft skipped source and generates load version", - expected_record_sources=("raw.inventory", "raw.orders"), - expected_unknown_sources=("raw.customers",), - expected_preserved_sources=("raw.orders",), - expected_generated_sources=("raw.inventory",), - ) - ], - ids=lambda case: case.description, -) -def test_given_managed_loader_results_when_observing_freshness_then_applies_loader_semantics( - test_case: SourceFreshnessRuntimeTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: Any = adapter.connect({"database": ":memory:"}) - previous_record: SourceFreshnessRecord = SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="column", - value_kind="integer", - data_version="10", - data_version_hash="previous-hash", - observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), - ) - try: - result: SourceFreshnessRuntimeResult = observe_virtual_environment_source_freshness( - adapter=adapter, - connection=connection, - sources=( - SourceEntry(name="raw.orders", table="raw_orders", managed=True), - SourceEntry(name="raw.inventory", table="raw_inventory", managed=True), - SourceEntry(name="raw.customers", table="raw_customers", managed=True), - ), - virtual_environment_name="dev", - observed_at=datetime(2026, 1, 2, 12, 0, 0, tzinfo=UTC), - run_id="run-123", - previous_records=(previous_record,), - load_results=( - LoadExecutionResult( - source_name="raw.orders", - loader_name="raw.orders", - status=ExecutionStatus.SKIPPED, - target="raw_orders", - skip_mode=SkipMode.SOFT, - ), - LoadExecutionResult( - source_name="raw.inventory", - loader_name="raw.inventory", - status=ExecutionStatus.SUCCESS, - target="raw_inventory", - ), - LoadExecutionResult( - source_name="raw.customers", - loader_name="raw.customers", - status=ExecutionStatus.SKIPPED, - target="raw_customers", - skip_mode=SkipMode.HARD, - ), - ), - ) - finally: - adapter.close(connection) - - records_by_source: dict[str, SourceFreshnessRecord] = { - record.source_name: record for record in result.records - } - assert tuple(records_by_source) == test_case.expected_record_sources - assert result.unknown_source_names == test_case.expected_unknown_sources - assert result.preserved_source_names == test_case.expected_preserved_sources - assert result.generated_source_names == test_case.expected_generated_sources - assert records_by_source["raw.orders"] == previous_record - assert records_by_source["raw.inventory"].strategy == "loader" - assert records_by_source["raw.inventory"].data_version == "run-123" - - -@pytest.mark.parametrize( - "test_case", - ( - SourceFreshnessRuntimeLagToleranceTestCase( - description="preserves previous within lag tolerance", - current_data_version="2026-01-01T12:05:00", - expected_record_data_version="2026-01-01T12:00:00", - ), - SourceFreshnessRuntimeLagToleranceTestCase( - description="preserves previous at lag tolerance boundary", - current_data_version="2026-01-01T12:10:00", - expected_record_data_version="2026-01-01T12:00:00", - ), - SourceFreshnessRuntimeLagToleranceTestCase( - description="advances beyond lag tolerance", - current_data_version="2026-01-01T12:11:00", - expected_record_data_version="2026-01-01T12:11:00+00:00", - ), - SourceFreshnessRuntimeLagToleranceTestCase( - description="advances on backwards timestamp movement", - current_data_version="2026-01-01T11:59:00", - expected_record_data_version="2026-01-01T11:59:00+00:00", - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_runtime_lag_tolerance_when_observing_then_preserves_baseline( - test_case: SourceFreshnessRuntimeLagToleranceTestCase, -) -> None: - adapter: DuckDbAdapter = DuckDbAdapter() - connection: Any = adapter.connect({"database": ":memory:"}) - previous_record: SourceFreshnessRecord = SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy=SourceFreshnessStrategy.SQL.value, - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - data_version="2026-01-01T12:00:00", - data_version_hash="previous-hash", - observed_at=datetime(2026, 1, 1, 12, 0, 0, tzinfo=UTC), - ) - try: - result: SourceFreshnessRuntimeResult = observe_virtual_environment_source_freshness( - adapter=adapter, - connection=connection, - sources=( - SourceEntry( - name="raw.orders", - freshness=SourceFreshnessConfig( - strategy=SourceFreshnessStrategy.SQL, - value_kind=SourceFreshnessValueKind.TIMESTAMP, - query=f"SELECT CAST('{test_case.current_data_version}' AS TIMESTAMP)", - lag_tolerance="10m", - ), - ), - ), - virtual_environment_name="dev", - observed_at=datetime(2026, 1, 1, 12, 30, 0, tzinfo=UTC), - previous_records=(previous_record,), - ) - finally: - adapter.close(connection) - - assert result.records[0].data_version == test_case.expected_record_data_version diff --git a/tests/unit/src/sqlbuild/virtual/planner/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/planner/_helpers/_test_types.py deleted file mode 100644 index 196d448f4b..0000000000 --- a/tests/unit/src/sqlbuild/virtual/planner/_helpers/_test_types.py +++ /dev/null @@ -1,124 +0,0 @@ -from dataclasses import dataclass - -from sqlbuild.compiler.planner.types import PlanReason, WorkSelectionPolicy - - -@dataclass(frozen=True) -class ExpectedVersionHashesTestCase: - description: str - upstream_query_sql: str - downstream_query_sql: str - expected_hashes_differ: bool - baseline_extra_config: dict[str, object] | None = None - changed_extra_config: dict[str, object] | None = None - - -@dataclass(frozen=True) -class VirtualSourceFreshnessLagToleranceTestCase: - description: str - current_data_version: str - expected_record_data_version: str - - -@dataclass(frozen=True) -class DefaultVirtualSelectionTestCase: - description: str - stale_model_names: tuple[str, ...] - expected_selection: tuple[str, ...] - - -@dataclass(frozen=True) -class VirtualModelSelectionTestCase: - description: str - select: tuple[str, ...] - default_selection: tuple[str, ...] - stale_model_names: tuple[str, ...] - include_stale_upstreams: bool - work_selection_policy: WorkSelectionPolicy - expected_selection: tuple[str, ...] - downstream_depends_on_dim_customers: bool = False - - -@dataclass(frozen=True) -class VirtualRunDespiteUnchangedSemanticsTestCase: - description: str - expected_stale_model_names: tuple[str, ...] - expected_default_selection: tuple[str, ...] - expected_root_reasons: dict[str, PlanReason] - - -@dataclass(frozen=True) -class StaleRequiredUpstreamClosureTestCase: - description: str - selected_model_names: tuple[str, ...] - stale_model_names: tuple[str, ...] - expected_stale_upstream_names: tuple[str, ...] - - -@dataclass(frozen=True) -class StaleModelNamesTestCase: - description: str - expected_version_hashes: dict[str, str] - bound_version_hashes: dict[str, str] - expected_stale_model_names: tuple[str, ...] - - -@dataclass(frozen=True) -class SeedRefPlanningTestCase: - description: str - seed_names: tuple[str, ...] - expected_seed_version_hashes: dict[str, str] - bound_seed_version_hashes: dict[str, str] - expected_stale_seed_names: tuple[str, ...] - expected_seed_plan_reasons: dict[str, PlanReason] - - -@dataclass(frozen=True) -class StaleRootReasonsTestCase: - description: str - stale_model_names: tuple[str, ...] - expected_local_hashes: dict[str, str] - bound_version_hashes: dict[str, str] - bound_local_hashes: dict[str, str] - expected_stale_root_reasons: dict[str, PlanReason] - current_query_sqls: dict[str, str] | None = None - bound_previous_query_sqls: dict[str, str] | None = None - expected_metadata_jsons: dict[str, str] | None = None - bound_metadata_jsons: dict[str, str] | None = None - - -@dataclass(frozen=True) -class StaleRootCausesTestCase: - description: str - stale_model_names: tuple[str, ...] - stale_root_reasons: dict[str, PlanReason] - expected_stale_root_causes: dict[str, str] - stale_root_source_causes: dict[str, str] | None = None - - -@dataclass(frozen=True) -class StaleRootCauseReasonsTestCase: - description: str - stale_root_reasons: dict[str, PlanReason] - stale_root_source_causes: dict[str, str] - expected_stale_root_cause_reasons: dict[str, PlanReason] - - -@dataclass(frozen=True) -class StaleRootSourceCausesTestCase: - description: str - stale_root_reasons: dict[str, PlanReason] - expected_metadata_jsons: dict[str, str] - bound_metadata_jsons: dict[str, str] - expected_stale_root_source_causes: dict[str, str] - - -@dataclass(frozen=True) -class PhysicalRelationDestinationTestCase: - description: str - database_name: str | None - schema_name: str - relation_name: str - fallback_logical_schema: str | None - fallback_logical_database: str | None - expected_qualified_name: str diff --git a/tests/unit/src/sqlbuild/virtual/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/virtual/planner/_helpers/helpers.py deleted file mode 100644 index 4ad255c59c..0000000000 --- a/tests/unit/src/sqlbuild/virtual/planner/_helpers/helpers.py +++ /dev/null @@ -1,262 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from types import MappingProxyType -from typing import cast - -from sqlbuild.compiler.compile.models import ( - CompiledFunction, - CompiledModel, - CompiledObjectKey, - CompiledProject, - CompiledRelationLocation, - CompiledSeed, - CompiledSource, - CompileModelConfig, - FunctionArgument, -) -from sqlbuild.compiler.compile.types import CompiledResourceType, FunctionLanguage -from sqlbuild.compiler.discovery.models import ( - DiscoveredSchemaFile, - DiscoveredSeedFile, - DiscoveredSourceFile, -) -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner._helpers.graph.core import ( - build_downstream_deps, - build_execution_upstream_deps, -) -from sqlbuild.spec.contracts.models import ( - SchemaColumn, - SchemaModelEntry, - SchemaSeedEntry, - SettingsConfig, - SourceEntry, -) - - -def build_virtual_planner_test_project( - *, - upstream_query_sql: str, - downstream_query_sql: str, - function_body_sql: str = "value + 1", - downstream_depends_on_dim_customers: bool = False, - upstream_model_name: str = "stg_orders", - upstream_schema: str = "staging", - upstream_materialized: str = "table", - upstream_extra_config: dict[str, object] | None = None, - upstream_source_name: str = "raw.orders", - upstream_schema_columns: tuple[SchemaColumn, ...] = (), - upstream_seed_file_path: Path | None = None, - upstream_seed_name: str = "order_statuses", -) -> ProjectGraph: - function: CompiledFunction = CompiledFunction( - key=CompiledObjectKey(resource_type=CompiledResourceType.UDF, name="normalize_order"), - deps=(), - name="normalize_order", - relative_path=Path("functions/sql/normalize_order.sql"), - arguments=(FunctionArgument(name="value", type="INTEGER"),), - returns="INTEGER", - body_sql=function_body_sql, - destination=CompiledRelationLocation( - database=None, - schema="staging", - name="normalize_order", - qualified_name="staging.normalize_order", - ), - fingerprint_destination=CompiledRelationLocation( - database=None, - schema="staging", - name="normalize_order", - qualified_name="staging.normalize_order", - ), - language=FunctionLanguage.SQL, - ) - upstream_config_values: dict[str, object] = { - "materialized": upstream_materialized, - "schema": upstream_schema, - } - upstream_config_values.update(upstream_extra_config or {}) - source_entry: SourceEntry = SourceEntry(name=upstream_source_name, table="raw_orders") - upstream_source: CompiledSource = CompiledSource( - key=CompiledObjectKey(resource_type=CompiledResourceType.SOURCE, name=upstream_source_name), - deps=(), - name=upstream_source_name, - source_entry=source_entry, - source_file=DiscoveredSourceFile( - file_path=Path("sources/raw.yml"), - relative_path=Path("sources/raw.yml"), - contents="", - source_entries=(source_entry,), - ), - ) - seed: CompiledSeed | None - seed_key: CompiledObjectKey | None - seed, seed_key = _VIRTUAL_SEED_BUILDERS[upstream_seed_file_path is not None]( - upstream_seed_file_path, upstream_seed_name - ) - upstream_model_deps: tuple[CompiledObjectKey, ...] = cast( - tuple[CompiledObjectKey, ...], - ((upstream_source.key,), (upstream_source.key, seed_key))[seed_key is not None], - ) - upstream_model: CompiledModel = CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name=upstream_model_name), - deps=upstream_model_deps, - name=upstream_model_name, - relative_path=Path(f"models/{upstream_model_name}.sql"), - query_sql=upstream_query_sql, - config=CompileModelConfig(values=upstream_config_values), - schema_entry=( - None, - SchemaModelEntry(name=upstream_model_name, columns=upstream_schema_columns), - )[bool(upstream_schema_columns)], - destination=CompiledRelationLocation( - database=None, - schema=upstream_schema, - name=upstream_model_name, - qualified_name=f"{upstream_schema}.{upstream_model_name}", - ), - ) - downstream_model: CompiledModel = CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="fact_orders"), - deps=(upstream_model.key, function.key), - name="fact_orders", - relative_path=Path("models/fact_orders.sql"), - query_sql=downstream_query_sql, - config=CompileModelConfig(values={"materialized": "table"}), - destination=CompiledRelationLocation( - database=None, - schema="marts", - name="fact_orders", - qualified_name="marts.fact_orders", - ), - ) - unrelated_model: CompiledModel = CompiledModel( - key=CompiledObjectKey(resource_type=CompiledResourceType.MODEL, name="dim_customers"), - deps=(), - name="dim_customers", - relative_path=Path("models/dim_customers.sql"), - query_sql="SELECT 1 AS customer_id", - config=CompileModelConfig(values={"materialized": "table"}), - destination=CompiledRelationLocation( - database=None, - schema="marts", - name="dim_customers", - qualified_name="marts.dim_customers", - ), - ) - downstream_deps: tuple[CompiledObjectKey, ...] = ( - (upstream_model.key, function.key), - (upstream_model.key, function.key, unrelated_model.key), - )[downstream_depends_on_dim_customers] - downstream_model = CompiledModel( - key=downstream_model.key, - deps=downstream_deps, - name=downstream_model.name, - relative_path=downstream_model.relative_path, - query_sql=downstream_model.query_sql, - config=downstream_model.config, - destination=downstream_model.destination, - ) - project: CompiledProject = CompiledProject( - run_id="test_run", - effective_target_name="dev", - effective_connection={}, - effective_vars={}, - settings=SettingsConfig(), - models=(upstream_model, downstream_model, unrelated_model), - sources=(upstream_source,), - seeds=cast(tuple[CompiledSeed, ...], ((), (seed,))[seed is not None]), - functions=(function,), - ) - upstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = ( - build_execution_upstream_deps(project) - ) - downstream_deps: dict[CompiledObjectKey, tuple[CompiledObjectKey, ...]] = build_downstream_deps( - upstream_deps - ) - all_keys: dict[str, CompiledObjectKey] = { - upstream_model.name: upstream_model.key, - upstream_source.name: upstream_source.key, - downstream_model.name: downstream_model.key, - unrelated_model.name: unrelated_model.key, - function.name: function.key, - } - _VIRTUAL_SEED_KEY_ADDERS[seed is not None](all_keys, seed) - return ProjectGraph( - project=project, - upstream_deps=upstream_deps, - downstream_deps=downstream_deps, - tag_index={}, - path_index={}, - all_keys=all_keys, - ) - - -def _build_virtual_seed( - seed_file_path: Path | None, seed_name: str -) -> tuple[CompiledSeed | None, CompiledObjectKey | None]: - resolved_seed_file_path: Path = cast(Path, seed_file_path) - seed_key: CompiledObjectKey = CompiledObjectKey( - resource_type=CompiledResourceType.SEED, - name=seed_name, - ) - seed_schema_entry: SchemaSeedEntry = SchemaSeedEntry( - name=seed_name, - columns=(SchemaColumn(name="status"),), - ) - seed_schema_file: DiscoveredSchemaFile = DiscoveredSchemaFile( - file_path=Path("seeds/schema.yml"), - relative_path=Path("seeds/schema.yml"), - contents="", - model_entries=(), - seed_entries=(seed_schema_entry,), - ) - seed: CompiledSeed = CompiledSeed( - key=seed_key, - deps=(), - name=seed_name, - seed_file=DiscoveredSeedFile( - file_path=resolved_seed_file_path, - relative_path=Path("seeds") / resolved_seed_file_path.name, - ), - schema_entry=seed_schema_entry, - schema_file=seed_schema_file, - destination=CompiledRelationLocation( - database=None, - schema="staging", - name=seed_name, - qualified_name=f"staging.{seed_name}", - ), - ) - return seed, seed_key - - -def _build_no_virtual_seed( - seed_file_path: Path | None, seed_name: str -) -> tuple[CompiledSeed | None, CompiledObjectKey | None]: - del seed_file_path, seed_name - return None, None - - -def _add_virtual_seed_key( - all_keys: dict[str, CompiledObjectKey], seed: CompiledSeed | None -) -> None: - resolved_seed: CompiledSeed = cast(CompiledSeed, seed) - all_keys[resolved_seed.name] = resolved_seed.key - - -def _skip_virtual_seed_key( - all_keys: dict[str, CompiledObjectKey], seed: CompiledSeed | None -) -> None: - del all_keys, seed - - -_VIRTUAL_SEED_BUILDERS: MappingProxyType[ - bool, - Callable[[Path | None, str], tuple[CompiledSeed | None, CompiledObjectKey | None]], -] = MappingProxyType({False: _build_no_virtual_seed, True: _build_virtual_seed}) -_VIRTUAL_SEED_KEY_ADDERS: MappingProxyType[ - bool, Callable[[dict[str, CompiledObjectKey], CompiledSeed | None], None] -] = MappingProxyType({False: _skip_virtual_seed_key, True: _add_virtual_seed_key}) diff --git a/tests/unit/src/sqlbuild/virtual/planner/_helpers/test_planning.py b/tests/unit/src/sqlbuild/virtual/planner/_helpers/test_planning.py deleted file mode 100644 index fc11714b72..0000000000 --- a/tests/unit/src/sqlbuild/virtual/planner/_helpers/test_planning.py +++ /dev/null @@ -1,1610 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from pathlib import Path - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.discovery.models import SqlHookEntry -from sqlbuild.compiler.pipeline.models import ProjectGraph -from sqlbuild.compiler.planner.exceptions import PlannerInputError -from sqlbuild.compiler.planner.types import PlanReason, WorkSelectionPolicy -from sqlbuild.spec.contracts.models import SchemaColumn, SourceEntry, SourceFreshnessConfig -from sqlbuild.spec.contracts.types import SourceFreshnessStrategy, SourceFreshnessValueKind -from sqlbuild.virtual.freshness._helpers.runtime import ( - build_current_virtual_source_freshness_records, -) -from sqlbuild.virtual.planner._helpers.planning import ( - build_default_virtual_selection, - build_expected_local_hashes, - build_expected_seed_version_hashes, - build_expected_version_hashes, - build_model_fingerprint_metadata_jsons, - build_seed_plan_reasons, - build_source_freshness_incomplete_model_names, - build_source_version_hashes, - build_stale_model_names, - build_stale_required_upstream_closure, - build_stale_root_cause_reasons, - build_stale_root_causes, - build_stale_root_reasons, - build_stale_root_source_causes, - build_stale_seed_names, - resolve_virtual_model_selection, -) -from sqlbuild.virtual.planner.main._semantics import build_virtual_plan_semantics -from sqlbuild.virtual.planner.models import VirtualPlanSemantics -from sqlbuild.virtual.state.models import ( - ModelVersionRecord, - SourceFreshnessRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import ModelVersionStatus -from tests.unit.src.sqlbuild.virtual.planner._helpers._test_types import ( - DefaultVirtualSelectionTestCase, - ExpectedVersionHashesTestCase, - SeedRefPlanningTestCase, - StaleModelNamesTestCase, - StaleRequiredUpstreamClosureTestCase, - StaleRootCauseReasonsTestCase, - StaleRootCausesTestCase, - StaleRootReasonsTestCase, - StaleRootSourceCausesTestCase, - VirtualModelSelectionTestCase, - VirtualRunDespiteUnchangedSemanticsTestCase, - VirtualSourceFreshnessLagToleranceTestCase, -) -from tests.unit.src.sqlbuild.virtual.planner._helpers.helpers import ( - build_virtual_planner_test_project, -) - - -@pytest.mark.parametrize( - "test_case", - ( - SeedRefPlanningTestCase( - description="missing bound seed ref is first run", - seed_names=("order_amounts",), - expected_seed_version_hashes={"order_amounts": "new-seed"}, - bound_seed_version_hashes={}, - expected_stale_seed_names=("order_amounts",), - expected_seed_plan_reasons={"order_amounts": PlanReason.FIRST_RUN}, - ), - SeedRefPlanningTestCase( - description="different bound seed ref is changed", - seed_names=("order_amounts",), - expected_seed_version_hashes={"order_amounts": "new-seed"}, - bound_seed_version_hashes={"order_amounts": "old-seed"}, - expected_stale_seed_names=("order_amounts",), - expected_seed_plan_reasons={"order_amounts": PlanReason.CONFIG_CHANGED}, - ), - SeedRefPlanningTestCase( - description="matching bound seed ref is current", - seed_names=("order_amounts",), - expected_seed_version_hashes={"order_amounts": "same-seed"}, - bound_seed_version_hashes={"order_amounts": "same-seed"}, - expected_stale_seed_names=(), - expected_seed_plan_reasons={"order_amounts": PlanReason.NO_CHANGE}, - ), - ), - ids=lambda case: case.description, -) -def test_given_seed_refs_when_building_virtual_seed_planning_then_returns_stale_names_and_reasons( - test_case: SeedRefPlanningTestCase, -) -> None: - stale_seed_names: tuple[str, ...] = build_stale_seed_names( - seed_names=test_case.seed_names, - expected_seed_version_hashes=test_case.expected_seed_version_hashes, - bound_seed_version_hashes=test_case.bound_seed_version_hashes, - ) - seed_plan_reasons: dict[str, PlanReason] = build_seed_plan_reasons( - seed_names=test_case.seed_names, - expected_seed_version_hashes=test_case.expected_seed_version_hashes, - bound_seed_version_hashes=test_case.bound_seed_version_hashes, - ) - - assert stale_seed_names == test_case.expected_stale_seed_names - assert seed_plan_reasons == test_case.expected_seed_plan_reasons - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="upstream query change changes downstream expected hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_upstream_change_when_building_expected_hashes_then_downstream_hash_changes( - test_case: ExpectedVersionHashesTestCase, -) -> None: - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 0 AS id", - downstream_query_sql=test_case.downstream_query_sql, - ) - changed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - changed_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_graph), - ) - - assert ( - baseline_hashes["fact_orders"] != changed_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="unchanged graph produces stable expected hashes", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT normalize_order(id) FROM stg_orders", - expected_hashes_differ=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_unchanged_graph_when_building_expected_hashes_then_hashes_match( - test_case: ExpectedVersionHashesTestCase, -) -> None: - first_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - second_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - - first_hashes: dict[str, str] = build_expected_version_hashes( - graph=first_graph, - expected_local_hashes=build_expected_local_hashes(graph=first_graph), - ) - second_hashes: dict[str, str] = build_expected_version_hashes( - graph=second_graph, - expected_local_hashes=build_expected_local_hashes(graph=second_graph), - ) - - assert ( - first_hashes["fact_orders"] != second_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="function body change updates dependent expected hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT normalize_order(id) FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_function_change_when_building_expected_hashes_then_downstream_hash_changes( - test_case: ExpectedVersionHashesTestCase, -) -> None: - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - function_body_sql="value + 1", - ) - changed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - function_body_sql="value + 2", - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - changed_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_graph), - ) - - assert ( - baseline_hashes["fact_orders"] != changed_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="source data version change updates downstream expected hash", - upstream_query_sql="SELECT * FROM __source('raw.orders')", - downstream_query_sql="SELECT id FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_source_data_version_change_when_building_hashes_then_downstream_hash_changes( - test_case: ExpectedVersionHashesTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=graph, - expected_local_hashes=build_expected_local_hashes(graph=graph), - source_version_hashes={"raw.orders": "source-version-1"}, - ) - changed_hashes: dict[str, str] = build_expected_version_hashes( - graph=graph, - expected_local_hashes=build_expected_local_hashes(graph=graph), - source_version_hashes={"raw.orders": "source-version-2"}, - ) - - assert ( - baseline_hashes["stg_orders"] != changed_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - assert ( - baseline_hashes["fact_orders"] != changed_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="seed identity change updates dependent expected hash", - upstream_query_sql='SELECT order_id FROM __source("raw.orders")', - downstream_query_sql="SELECT id FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_identity_change_when_building_hashes_then_downstream_hash_changes( - test_case: ExpectedVersionHashesTestCase, - tmp_path: Path, -) -> None: - baseline_seed_file: Path = tmp_path / "baseline_statuses.csv" - changed_seed_file: Path = tmp_path / "changed_statuses.csv" - baseline_seed_file.write_text("status\nplaced\n", encoding="utf-8") - changed_seed_file.write_text("status\nplaced\nshipped\n", encoding="utf-8") - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_seed_file_path=baseline_seed_file, - ) - changed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_seed_file_path=changed_seed_file, - ) - - baseline_seed_hashes: dict[str, str] = build_expected_seed_version_hashes(graph=baseline_graph) - changed_seed_hashes: dict[str, str] = build_expected_seed_version_hashes(graph=changed_graph) - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - seed_version_hashes=baseline_seed_hashes, - ) - changed_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_graph), - seed_version_hashes=changed_seed_hashes, - ) - - assert baseline_seed_hashes["order_statuses"] != changed_seed_hashes["order_statuses"] - assert ( - baseline_hashes["stg_orders"] != changed_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - assert ( - baseline_hashes["fact_orders"] != changed_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - assert baseline_hashes["dim_customers"] == changed_hashes["dim_customers"] - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="seed identity change marks dependent closure stale", - upstream_query_sql='SELECT order_id FROM __source("raw.orders")', - downstream_query_sql="SELECT id FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_identity_change_when_comparing_bound_hashes_then_dependent_models_are_stale( - test_case: ExpectedVersionHashesTestCase, - tmp_path: Path, -) -> None: - baseline_seed_file: Path = tmp_path / "baseline_statuses.csv" - changed_seed_file: Path = tmp_path / "changed_statuses.csv" - baseline_seed_file.write_text("status\nplaced\n", encoding="utf-8") - changed_seed_file.write_text("status\nplaced\nshipped\n", encoding="utf-8") - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_seed_file_path=baseline_seed_file, - ) - changed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_seed_file_path=changed_seed_file, - ) - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - changed_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_graph), - ) - - stale_model_names: tuple[str, ...] = build_stale_model_names( - model_names=tuple(model.name for model in changed_graph.project.models), - expected_version_hashes=changed_hashes, - bound_version_hashes=baseline_hashes, - ) - - assert bool(stale_model_names) is test_case.expected_hashes_differ - assert stale_model_names == ("stg_orders", "fact_orders") - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="old seed-composed bound model versions are stale", - upstream_query_sql='SELECT order_id FROM __source("raw.orders")', - downstream_query_sql="SELECT id FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_bound_model_versions_composed_with_old_seed_when_semantics_then_models_are_stale( - test_case: ExpectedVersionHashesTestCase, - tmp_path: Path, -) -> None: - baseline_seed_file: Path = tmp_path / "baseline_statuses.csv" - changed_seed_file: Path = tmp_path / "changed_statuses.csv" - baseline_seed_file.write_text("status\nplaced\n", encoding="utf-8") - changed_seed_file.write_text("status\nplaced\nshipped\n", encoding="utf-8") - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_seed_file_path=baseline_seed_file, - ) - changed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_seed_file_path=changed_seed_file, - ) - baseline_local_hashes: dict[str, str] = build_expected_local_hashes(graph=baseline_graph) - source_records: tuple[SourceFreshnessRecord, ...] = ( - SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="sql", - value_kind="integer", - data_version="1", - data_version_hash="source-hash", - observed_at=datetime(2026, 6, 11, tzinfo=UTC), - ), - ) - source_version_hashes: dict[str, str] = build_source_version_hashes(source_records) - baseline_version_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=baseline_local_hashes, - source_version_hashes=source_version_hashes, - ) - baseline_seed_hashes: dict[str, str] = build_expected_seed_version_hashes(graph=baseline_graph) - - semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=changed_graph, - bound_refs=tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name="dev", - model_name=model.name, - version_hash=baseline_version_hashes[model.name], - ) - for model in baseline_graph.project.models - ), - bound_model_versions={ - model.name: ModelVersionRecord( - model_name=model.name, - version_hash=baseline_version_hashes[model.name], - definition_identity_hash=baseline_local_hashes[model.name], - identity_metadata_hash="metadata-hash", - status=ModelVersionStatus.READY, - ) - for model in baseline_graph.project.models - }, - bound_seed_refs=tuple( - VirtualEnvironmentSeedRefRecord( - virtual_environment_name="dev", - seed_name=seed_name, - version_hash=version_hash, - ) - for seed_name, version_hash in baseline_seed_hashes.items() - ), - source_freshness_records=source_records, - ) - - assert bool(semantics.stale_model_names) is test_case.expected_hashes_differ - assert semantics.stale_seed_names == ("order_statuses",) - assert semantics.stale_model_names == ("fact_orders", "stg_orders") - assert semantics.default_selection == ("fact_orders", "stg_orders") - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="seed and function identities both affect dependent expected hash", - upstream_query_sql='SELECT order_id FROM __source("raw.orders")', - downstream_query_sql="SELECT normalize_order(id) FROM stg_orders", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_seed_and_function_changes_when_building_hashes_then_downstream_hash_changes( - test_case: ExpectedVersionHashesTestCase, - tmp_path: Path, -) -> None: - baseline_seed_file: Path = tmp_path / "baseline_statuses.csv" - changed_seed_file: Path = tmp_path / "changed_statuses.csv" - baseline_seed_file.write_text("status\nplaced\n", encoding="utf-8") - changed_seed_file.write_text("status\nplaced\nshipped\n", encoding="utf-8") - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - function_body_sql="value + 1", - upstream_seed_file_path=baseline_seed_file, - ) - changed_seed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - function_body_sql="value + 1", - upstream_seed_file_path=changed_seed_file, - ) - changed_function_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - function_body_sql="value + 2", - upstream_seed_file_path=baseline_seed_file, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - changed_seed_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_seed_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_seed_graph), - ) - changed_function_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_function_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_function_graph), - ) - - assert ( - baseline_hashes["fact_orders"] != changed_seed_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - assert ( - baseline_hashes["fact_orders"] != changed_function_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualSourceFreshnessLagToleranceTestCase( - description="virtual preserves previous within tolerance", - current_data_version="2026-01-15T12:05:00", - expected_record_data_version="2026-01-15T12:00:00", - ), - VirtualSourceFreshnessLagToleranceTestCase( - description="virtual preserves previous at tolerance boundary", - current_data_version="2026-01-15T12:10:00", - expected_record_data_version="2026-01-15T12:00:00", - ), - VirtualSourceFreshnessLagToleranceTestCase( - description="virtual uses current beyond tolerance", - current_data_version="2026-01-15T12:11:00", - expected_record_data_version="2026-01-15T12:11:00+00:00", - ), - VirtualSourceFreshnessLagToleranceTestCase( - description="virtual uses current for backwards timestamp movement", - current_data_version="2026-01-15T11:59:00", - expected_record_data_version="2026-01-15T11:59:00+00:00", - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_lag_tolerance_when_building_current_records_then_preserves_baseline( - test_case: VirtualSourceFreshnessLagToleranceTestCase, -) -> None: - previous_data_version: str = "2026-01-15T12:00:00" - previous_record: SourceFreshnessRecord = SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy=SourceFreshnessStrategy.SQL.value, - value_kind=SourceFreshnessValueKind.TIMESTAMP.value, - data_version=previous_data_version, - data_version_hash="previous-hash", - observed_at=datetime(2026, 1, 15, 12, 0, 0, tzinfo=UTC), - ) - adapter: DuckDbAdapter = DuckDbAdapter() - connection: object = adapter.connect({"database": ":memory:"}) - try: - records: tuple[SourceFreshnessRecord, ...] = build_current_virtual_source_freshness_records( - adapter=adapter, - connection=connection, - sources=( - SourceEntry( - name="raw.orders", - freshness=SourceFreshnessConfig( - strategy=SourceFreshnessStrategy.SQL, - value_kind=SourceFreshnessValueKind.TIMESTAMP, - query=f"SELECT CAST('{test_case.current_data_version}' AS TIMESTAMP)", - lag_tolerance="10m", - ), - ), - ), - virtual_environment_name="dev", - observed_at=datetime(2026, 1, 15, 12, 30, 0, tzinfo=UTC), - previous_records=(previous_record,), - ) - finally: - adapter.close(connection) - - assert records[0].data_version == test_case.expected_record_data_version - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="unchanged source data version keeps downstream expected hash stable", - upstream_query_sql="SELECT * FROM __source('raw.orders')", - downstream_query_sql="SELECT id FROM stg_orders", - expected_hashes_differ=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_same_source_data_version_when_building_expected_hashes_then_hashes_match( - test_case: ExpectedVersionHashesTestCase, -) -> None: - first_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - second_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - - first_hashes: dict[str, str] = build_expected_version_hashes( - graph=first_graph, - expected_local_hashes=build_expected_local_hashes(graph=first_graph), - source_version_hashes={"raw.orders": "source-version-1"}, - ) - second_hashes: dict[str, str] = build_expected_version_hashes( - graph=second_graph, - expected_local_hashes=build_expected_local_hashes(graph=second_graph), - source_version_hashes={"raw.orders": "source-version-1"}, - ) - - assert ( - first_hashes["fact_orders"] != second_hashes["fact_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - StaleModelNamesTestCase( - description="source data version change marks direct and transitive models stale", - expected_version_hashes={}, - bound_version_hashes={}, - expected_stale_model_names=("stg_orders", "fact_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_source_data_version_change_when_building_stale_models_then_marks_downstream_stale( - test_case: StaleModelNamesTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT * FROM __source('raw.orders')", - downstream_query_sql="SELECT id FROM stg_orders", - upstream_materialized="view", - ) - expected_local_hashes: dict[str, str] = build_expected_local_hashes(graph=graph) - bound_hashes: dict[str, str] = build_expected_version_hashes( - graph=graph, - expected_local_hashes=expected_local_hashes, - source_version_hashes={"raw.orders": "source-version-1"}, - ) - expected_hashes: dict[str, str] = build_expected_version_hashes( - graph=graph, - expected_local_hashes=expected_local_hashes, - source_version_hashes={"raw.orders": "source-version-2"}, - ) - - stale_model_names: tuple[str, ...] = build_stale_model_names( - model_names=tuple(model.name for model in graph.project.models), - expected_version_hashes=expected_hashes, - bound_version_hashes=bound_hashes, - ) - - assert stale_model_names == test_case.expected_stale_model_names - - -@pytest.mark.parametrize( - "test_case", - [ - StaleRequiredUpstreamClosureTestCase( - description="missing source freshness marks direct and downstream models incomplete", - selected_model_names=(), - stale_model_names=(), - expected_stale_upstream_names=("fact_orders", "stg_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_missing_source_freshness_when_finding_incomplete_models_then_returns_closure( - test_case: StaleRequiredUpstreamClosureTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT * FROM __source('raw.orders')", - downstream_query_sql="SELECT id FROM stg_orders", - ) - - incomplete_model_names: tuple[str, ...] = build_source_freshness_incomplete_model_names( - graph=graph, - source_version_hashes={}, - ) - - assert incomplete_model_names == test_case.expected_stale_upstream_names - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="target schema does not change model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_only_target_schema_differs_when_building_expected_hashes_then_hashes_match( - test_case: ExpectedVersionHashesTestCase, -) -> None: - prod_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_schema="prod", - ) - dev_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_schema="dev", - ) - - prod_hashes: dict[str, str] = build_expected_version_hashes( - graph=prod_graph, - expected_local_hashes=build_expected_local_hashes(graph=prod_graph), - ) - dev_hashes: dict[str, str] = build_expected_version_hashes( - graph=dev_graph, - expected_local_hashes=build_expected_local_hashes(graph=dev_graph), - ) - prod_metadata_jsons: dict[str, str] = build_model_fingerprint_metadata_jsons(graph=prod_graph) - - assert ( - prod_hashes["stg_orders"] != dev_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - assert '"schema"' not in prod_metadata_jsons["stg_orders"] - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="model identity changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_only_model_name_differs_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - first_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_model_name="stg_orders", - ) - second_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_model_name="stg_orders_copy", - ) - - first_hashes: dict[str, str] = build_expected_version_hashes( - graph=first_graph, - expected_local_hashes=build_expected_local_hashes(graph=first_graph), - ) - second_hashes: dict[str, str] = build_expected_version_hashes( - graph=second_graph, - expected_local_hashes=build_expected_local_hashes(graph=second_graph), - ) - - assert ( - first_hashes["stg_orders"] != second_hashes["stg_orders_copy"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="unknown validation config keys do not change model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_validation_config_keys_when_building_expected_hashes_then_hashes_match( - test_case: ExpectedVersionHashesTestCase, -) -> None: - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - extra_config_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={ - "audits": ["not_null(order_id)"], - "contract": "enforced", - "future_default_flag": "off", - "mode": "strict", - }, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - extra_config_hashes: dict[str, str] = build_expected_version_hashes( - graph=extra_config_graph, - expected_local_hashes=build_expected_local_hashes(graph=extra_config_graph), - ) - metadata_jsons: dict[str, str] = build_model_fingerprint_metadata_jsons( - graph=extra_config_graph - ) - - assert ( - baseline_hashes["stg_orders"] != extra_config_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - assert "audits" not in metadata_jsons["stg_orders"] - assert "contract" not in metadata_jsons["stg_orders"] - assert "future_default_flag" not in metadata_jsons["stg_orders"] - assert '"mode"' not in metadata_jsons["stg_orders"] - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="incremental mode changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_incremental_mode_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={"incremental_mode": "normal"}, - ) - microbatch_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={"incremental_mode": "microbatch"}, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - microbatch_hashes: dict[str, str] = build_expected_version_hashes( - graph=microbatch_graph, - expected_local_hashes=build_expected_local_hashes(graph=microbatch_graph), - ) - - assert ( - baseline_hashes["stg_orders"] != microbatch_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="version identity config key changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_version_identity_config_key_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - table_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_materialized="table", - ) - view_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_materialized="view", - ) - - table_hashes: dict[str, str] = build_expected_version_hashes( - graph=table_graph, - expected_local_hashes=build_expected_local_hashes(graph=table_graph), - ) - view_hashes: dict[str, str] = build_expected_version_hashes( - graph=view_graph, - expected_local_hashes=build_expected_local_hashes(graph=view_graph), - ) - - assert ( - table_hashes["stg_orders"] != view_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="merge exclusions change model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - baseline_extra_config={"merge_exclude_columns": ["ingested_at"]}, - changed_extra_config={"merge_exclude_columns": ["created_at"]}, - expected_hashes_differ=True, - ), - ExpectedVersionHashesTestCase( - description="full refresh guard changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - baseline_extra_config={"full_refresh": True}, - changed_extra_config={"full_refresh": False}, - expected_hashes_differ=True, - ), - ], - ids=lambda case: case.description, -) -def test_given_safety_policy_change_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config=test_case.baseline_extra_config, - ) - changed_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config=test_case.changed_extra_config, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - changed_hashes: dict[str, str] = build_expected_version_hashes( - graph=changed_graph, - expected_local_hashes=build_expected_local_hashes(graph=changed_graph), - ) - - assert ( - baseline_hashes["stg_orders"] != changed_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="enforced contract output shape changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_enforced_contract_shape_change_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - int_contract_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={"contract": "enforced"}, - upstream_schema_columns=(SchemaColumn(name="id", type="INTEGER", nullable=False),), - ) - string_contract_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={"contract": "enforced"}, - upstream_schema_columns=(SchemaColumn(name="id", type="TEXT", nullable=False),), - ) - - int_hashes: dict[str, str] = build_expected_version_hashes( - graph=int_contract_graph, - expected_local_hashes=build_expected_local_hashes(graph=int_contract_graph), - ) - string_hashes: dict[str, str] = build_expected_version_hashes( - graph=string_contract_graph, - expected_local_hashes=build_expected_local_hashes(graph=string_contract_graph), - ) - - assert ( - int_hashes["stg_orders"] != string_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="custom materialization config changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_custom_materialization_config_change_when_building_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - first_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_materialized="partition_tracked", - upstream_extra_config={"config": {"partition_column": "order_date"}}, - ) - second_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_materialized="partition_tracked", - upstream_extra_config={"config": {"partition_column": "created_date"}}, - ) - - first_hashes: dict[str, str] = build_expected_version_hashes( - graph=first_graph, - expected_local_hashes=build_expected_local_hashes(graph=first_graph), - ) - second_hashes: dict[str, str] = build_expected_version_hashes( - graph=second_graph, - expected_local_hashes=build_expected_local_hashes(graph=second_graph), - ) - - assert ( - first_hashes["stg_orders"] != second_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="custom materialization placeholder changes model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_custom_placeholder_change_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - first_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_materialized="partition_tracked", - upstream_extra_config={"placeholders": {"lower_bound": "2026-01-01"}}, - ) - second_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_materialized="partition_tracked", - upstream_extra_config={"placeholders": {"lower_bound": "2026-02-01"}}, - ) - - first_hashes: dict[str, str] = build_expected_version_hashes( - graph=first_graph, - expected_local_hashes=build_expected_local_hashes(graph=first_graph), - ) - second_hashes: dict[str, str] = build_expected_version_hashes( - graph=second_graph, - expected_local_hashes=build_expected_local_hashes(graph=second_graph), - ) - - assert ( - first_hashes["stg_orders"] != second_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="named hook definition and arguments change model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_hook_change_when_building_expected_hashes_then_hashes_differ( - test_case: ExpectedVersionHashesTestCase, -) -> None: - first_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={ - "pre_hooks": [ - SqlHookEntry( - statement="SELECT 'reader'", - name="record_access", - relative_path=Path("hooks/sql/record_access.sql"), - definition_sql="SELECT @role", - kwargs={"role": "'reader'"}, - ) - ], - "post_hooks": [SqlHookEntry(statement="SELECT 2")], - }, - ) - second_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={ - "pre_hooks": [ - SqlHookEntry( - statement="SELECT 'reader'", - name="record_access", - relative_path=Path("hooks/sql/record_access.sql"), - definition_sql="SELECT @'role'", - kwargs={"role": "reader"}, - ) - ], - "post_hooks": [SqlHookEntry(statement="SELECT 2")], - }, - ) - - first_hashes: dict[str, str] = build_expected_version_hashes( - graph=first_graph, - expected_local_hashes=build_expected_local_hashes(graph=first_graph), - ) - second_hashes: dict[str, str] = build_expected_version_hashes( - graph=second_graph, - expected_local_hashes=build_expected_local_hashes(graph=second_graph), - ) - - assert ( - first_hashes["stg_orders"] != second_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - - -@pytest.mark.parametrize( - "test_case", - [ - ExpectedVersionHashesTestCase( - description="validation and backfill metadata does not change model version hash", - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT 1 AS order_id", - expected_hashes_differ=False, - ) - ], - ids=lambda case: case.description, -) -def test_given_excluded_metadata_change_when_building_expected_hashes_then_hashes_match( - test_case: ExpectedVersionHashesTestCase, -) -> None: - baseline_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - ) - metadata_graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql=test_case.upstream_query_sql, - downstream_query_sql=test_case.downstream_query_sql, - upstream_extra_config={ - "audits": ["not_null(id)"], - "row_diff_exclude_columns": ["loaded_at"], - "row_diff_tolerances": {"amount": {"absolute": 1}}, - "replay_on_change": "bounded-7d", - "tags": ["nightly"], - }, - ) - - baseline_hashes: dict[str, str] = build_expected_version_hashes( - graph=baseline_graph, - expected_local_hashes=build_expected_local_hashes(graph=baseline_graph), - ) - metadata_hashes: dict[str, str] = build_expected_version_hashes( - graph=metadata_graph, - expected_local_hashes=build_expected_local_hashes(graph=metadata_graph), - ) - metadata_jsons: dict[str, str] = build_model_fingerprint_metadata_jsons(graph=metadata_graph) - - assert ( - baseline_hashes["stg_orders"] != metadata_hashes["stg_orders"] - ) is test_case.expected_hashes_differ - assert "replay_on_change" not in metadata_jsons["stg_orders"] - assert "replay_on_change" not in metadata_jsons["stg_orders"] - assert "row_diff" not in metadata_jsons["stg_orders"] - assert "tags" not in metadata_jsons["stg_orders"] - - -@pytest.mark.parametrize( - "test_case", - [ - DefaultVirtualSelectionTestCase( - description="selects stale models plus downstream closure only", - stale_model_names=("stg_orders",), - expected_selection=("fact_orders", "stg_orders"), - ) - ], - ids=lambda case: case.description, -) -def test_given_stale_models_when_building_default_selection_then_it_includes_downstream_only( - test_case: DefaultVirtualSelectionTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - ) - - selection: tuple[str, ...] = build_default_virtual_selection( - stale_model_names=test_case.stale_model_names, - graph=graph, - ) - - assert selection == test_case.expected_selection - - -@pytest.mark.parametrize( - "test_case", - [ - StaleRequiredUpstreamClosureTestCase( - description="finds stale required upstreams for selected downstream", - selected_model_names=("fact_orders",), - stale_model_names=("stg_orders", "fact_orders"), - expected_stale_upstream_names=("stg_orders",), - ) - ], - ids=lambda case: case.description, -) -def test_given_selected_downstream_when_building_closure_then_it_returns_stale_ancestors( - test_case: StaleRequiredUpstreamClosureTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - ) - - stale_upstream_names: tuple[str, ...] = build_stale_required_upstream_closure( - graph=graph, - selected_model_names=test_case.selected_model_names, - stale_model_names=test_case.stale_model_names, - ) - - assert stale_upstream_names == test_case.expected_stale_upstream_names - - -@pytest.mark.parametrize( - "test_case", - ( - VirtualModelSelectionTestCase( - description="include stale upstreams expands minimally", - select=("fact_orders",), - default_selection=("fact_orders", "stg_orders"), - stale_model_names=("fact_orders", "stg_orders"), - include_stale_upstreams=True, - work_selection_policy=WorkSelectionPolicy.ALL_SELECTED, - expected_selection=("fact_orders", "stg_orders"), - ), - VirtualModelSelectionTestCase( - description="changes only intersects selected models with default stale selection", - select=("fact_orders", "dim_customers"), - default_selection=("fact_orders", "stg_orders"), - stale_model_names=("fact_orders", "stg_orders"), - include_stale_upstreams=True, - work_selection_policy=WorkSelectionPolicy.STALE_ONLY, - expected_selection=("fact_orders", "stg_orders"), - ), - VirtualModelSelectionTestCase( - description="include stale upstreams excludes unchanged upstreams", - select=("fact_orders",), - default_selection=("fact_orders", "stg_orders"), - stale_model_names=("fact_orders", "stg_orders"), - include_stale_upstreams=True, - work_selection_policy=WorkSelectionPolicy.ALL_SELECTED, - expected_selection=("fact_orders", "stg_orders"), - downstream_depends_on_dim_customers=True, - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_selectors_when_resolving_selection_then_it_returns_coherent_scope( - test_case: VirtualModelSelectionTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - downstream_depends_on_dim_customers=test_case.downstream_depends_on_dim_customers, - ) - - selection: tuple[str, ...] = resolve_virtual_model_selection( - graph=graph, - select=test_case.select, - exclude=(), - default_selection=test_case.default_selection, - stale_model_names=test_case.stale_model_names, - include_stale_upstreams=test_case.include_stale_upstreams, - work_selection_policy=test_case.work_selection_policy, - ) - - assert selection == test_case.expected_selection - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualModelSelectionTestCase( - description="blocks selected downstream with stale required upstream", - select=("fact_orders",), - default_selection=("fact_orders", "stg_orders"), - stale_model_names=("fact_orders", "stg_orders"), - include_stale_upstreams=False, - work_selection_policy=WorkSelectionPolicy.ALL_SELECTED, - expected_selection=("stg_orders",), - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_selector_missing_stale_upstream_when_resolving_selection_then_it_raises( - test_case: VirtualModelSelectionTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - ) - - with pytest.raises(PlannerInputError) as exc_info: - resolve_virtual_model_selection( - graph=graph, - select=test_case.select, - exclude=(), - default_selection=test_case.default_selection, - stale_model_names=test_case.stale_model_names, - include_stale_upstreams=test_case.include_stale_upstreams, - work_selection_policy=test_case.work_selection_policy, - ) - - assert exc_info.value.code == "S010" - assert test_case.expected_selection[0] in exc_info.value.message - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualRunDespiteUnchangedSemanticsTestCase( - description="adds run_despite_unchanged root and downstream to stale selection", - expected_stale_model_names=("fact_orders", "stg_orders"), - expected_default_selection=("fact_orders", "stg_orders"), - expected_root_reasons={"stg_orders": PlanReason.RUN_DESPITE_UNCHANGED}, - ) - ], - ids=lambda case: case.description, -) -def test_given_current_vde_versions_and_runtime_stale_model_when_semantics_then_marks_stale( - test_case: VirtualRunDespiteUnchangedSemanticsTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - upstream_extra_config={"run_despite_unchanged": "30d"}, - ) - source_records: tuple[SourceFreshnessRecord, ...] = ( - SourceFreshnessRecord( - virtual_environment_name="dev", - source_name="raw.orders", - strategy="sql", - value_kind="timestamp", - data_version="2026-06-01T00:00:00+00:00", - data_version_hash="source-hash", - observed_at=datetime(2026, 6, 11, tzinfo=UTC), - ), - ) - expected_local_hashes: dict[str, str] = build_expected_local_hashes(graph=graph) - expected_version_hashes: dict[str, str] = build_expected_version_hashes( - graph=graph, - expected_local_hashes=expected_local_hashes, - source_version_hashes=build_source_version_hashes(source_records), - ) - model_names: tuple[str, ...] = tuple(model.name for model in graph.project.models) - - semantics: VirtualPlanSemantics = build_virtual_plan_semantics( - graph=graph, - bound_refs=tuple( - VirtualEnvironmentModelRefRecord( - virtual_environment_name="dev", - model_name=model_name, - version_hash=expected_version_hashes[model_name], - ) - for model_name in model_names - ), - bound_model_versions={ - model_name: ModelVersionRecord( - model_name=model_name, - version_hash=expected_version_hashes[model_name], - definition_identity_hash=expected_local_hashes[model_name], - identity_metadata_hash="metadata-hash", - status=ModelVersionStatus.READY, - ) - for model_name in model_names - }, - source_freshness_records=source_records, - ) - - assert semantics.stale_model_names == test_case.expected_stale_model_names - assert semantics.default_selection == test_case.expected_default_selection - assert semantics.stale_root_reasons == test_case.expected_root_reasons - - -@pytest.mark.parametrize( - "test_case", - [ - StaleModelNamesTestCase( - description="marks only mismatched bound hashes as stale", - expected_version_hashes={ - "stg_orders": "current-a", - "fact_orders": "current-b", - "dim_customers": "current-c", - }, - bound_version_hashes={ - "stg_orders": "old-a", - "fact_orders": "current-b", - "dim_customers": "current-c", - }, - expected_stale_model_names=("stg_orders",), - ) - ], - ids=lambda case: case.description, -) -def test_given_bound_hash_mismatch_when_building_stale_models_then_it_marks_only_mismatches( - test_case: StaleModelNamesTestCase, -) -> None: - stale_model_names: tuple[str, ...] = build_stale_model_names( - model_names=("stg_orders", "fact_orders", "dim_customers"), - expected_version_hashes=test_case.expected_version_hashes, - bound_version_hashes=test_case.bound_version_hashes, - ) - - assert stale_model_names == test_case.expected_stale_model_names - - -@pytest.mark.parametrize( - "test_case", - ( - StaleRootReasonsTestCase( - description="classifies first run and query changed stale roots", - stale_model_names=("stg_orders", "fact_orders", "dim_customers"), - expected_local_hashes={ - "stg_orders": "new-local-a", - "fact_orders": "same-local-b", - "dim_customers": "new-local-c", - }, - bound_version_hashes={ - "stg_orders": "old-version-a", - "fact_orders": "old-version-b", - }, - bound_local_hashes={ - "stg_orders": "old-local-a", - "fact_orders": "same-local-b", - }, - expected_stale_root_reasons={ - "stg_orders": PlanReason.QUERY_CHANGED, - "dim_customers": PlanReason.FIRST_RUN, - }, - ), - StaleRootReasonsTestCase( - description="classifies version identity config separately from function changes", - stale_model_names=("fact_orders", "orders_rollup"), - expected_local_hashes={ - "fact_orders": "new-local-a", - "orders_rollup": "new-local-b", - }, - bound_version_hashes={ - "fact_orders": "old-version-a", - "orders_rollup": "old-version-b", - }, - bound_local_hashes={ - "fact_orders": "old-local-a", - "orders_rollup": "old-local-b", - }, - current_query_sqls={ - "fact_orders": "SELECT id FROM stg_orders", - "orders_rollup": "SELECT COUNT(*) FROM fact_orders", - }, - bound_previous_query_sqls={ - "fact_orders": "SELECT id FROM stg_orders", - "orders_rollup": "SELECT COUNT(*) FROM fact_orders", - }, - expected_metadata_jsons={ - "fact_orders": '{"config":{},"local_function_hashes":{"is_large_order":"new"}}', - "orders_rollup": '{"config":{"materialized":"table"}}', - }, - bound_metadata_jsons={ - "fact_orders": '{"config":{},"local_function_hashes":{"is_large_order":"old"}}', - "orders_rollup": '{"config":{"materialized":"view"}}', - }, - expected_stale_root_reasons={ - "fact_orders": PlanReason.QUERY_CHANGED, - "orders_rollup": PlanReason.CONFIG_CHANGED, - }, - ), - ), - ids=lambda case: case.description, -) -def test_given_stale_models_when_building_stale_root_reasons_then_it_classifies_roots( - test_case: StaleRootReasonsTestCase, -) -> None: - stale_root_reasons: dict[str, PlanReason] = build_stale_root_reasons( - stale_model_names=test_case.stale_model_names, - expected_local_hashes=test_case.expected_local_hashes, - bound_version_hashes=test_case.bound_version_hashes, - bound_local_hashes=test_case.bound_local_hashes, - current_query_sqls=test_case.current_query_sqls, - bound_previous_query_sqls=test_case.bound_previous_query_sqls, - expected_metadata_jsons=test_case.expected_metadata_jsons, - bound_metadata_jsons=test_case.bound_metadata_jsons, - ) - - assert stale_root_reasons == test_case.expected_stale_root_reasons - - -@pytest.mark.parametrize( - "test_case", - ( - StaleRootCausesTestCase( - description="maps downstream stale models to their first stale root cause", - stale_model_names=("stg_orders", "fact_orders"), - stale_root_reasons={"stg_orders": PlanReason.QUERY_CHANGED}, - expected_stale_root_causes={"fact_orders": "stg_orders"}, - ), - StaleRootCausesTestCase( - description="maps downstream stale models to changed function source cause", - stale_model_names=("stg_orders", "fact_orders"), - stale_root_reasons={"stg_orders": PlanReason.QUERY_CHANGED}, - stale_root_source_causes={"stg_orders": "is_large_order"}, - expected_stale_root_causes={"fact_orders": "is_large_order"}, - ), - ), - ids=lambda case: case.description, -) -def test_given_stale_roots_when_building_root_causes_then_it_maps_downstream_models( - test_case: StaleRootCausesTestCase, -) -> None: - graph: ProjectGraph = build_virtual_planner_test_project( - upstream_query_sql="SELECT 1 AS id", - downstream_query_sql="SELECT id FROM stg_orders", - ) - - stale_root_causes: dict[str, str] = build_stale_root_causes( - stale_model_names=test_case.stale_model_names, - stale_root_reasons=test_case.stale_root_reasons, - graph=graph, - stale_root_source_causes=test_case.stale_root_source_causes, - ) - - assert stale_root_causes == test_case.expected_stale_root_causes - - -@pytest.mark.parametrize( - "test_case", - [ - StaleRootCauseReasonsTestCase( - description="maps function source causes to function changed display reasons", - stale_root_reasons={"fact_orders": PlanReason.QUERY_CHANGED}, - stale_root_source_causes={"fact_orders": "is_large_order"}, - expected_stale_root_cause_reasons={ - "is_large_order": PlanReason.FUNCTION_CHANGED, - }, - ) - ], - ids=lambda case: case.description, -) -def test_given_function_source_cause_when_building_cause_reasons_then_uses_function_reason( - test_case: StaleRootCauseReasonsTestCase, -) -> None: - result: dict[str, PlanReason] = build_stale_root_cause_reasons( - stale_root_reasons=test_case.stale_root_reasons, - stale_root_source_causes=test_case.stale_root_source_causes, - ) - - assert result == test_case.expected_stale_root_cause_reasons - - -@pytest.mark.parametrize( - "test_case", - [ - StaleRootSourceCausesTestCase( - description="maps function metadata diff to changed function source cause", - stale_root_reasons={"fact_orders": PlanReason.QUERY_CHANGED}, - expected_metadata_jsons={ - "fact_orders": '{"config":{},"local_function_hashes":{"is_large_order":"new"}}', - }, - bound_metadata_jsons={ - "fact_orders": '{"config":{},"local_function_hashes":{"is_large_order":"old"}}', - }, - expected_stale_root_source_causes={"fact_orders": "is_large_order"}, - ), - ], - ids=lambda case: case.description, -) -def test_given_function_metadata_diff_when_building_source_causes_then_maps_root_to_function( - test_case: StaleRootSourceCausesTestCase, -) -> None: - result: dict[str, str] = build_stale_root_source_causes( - stale_root_reasons=test_case.stale_root_reasons, - expected_metadata_jsons=test_case.expected_metadata_jsons, - bound_metadata_jsons=test_case.bound_metadata_jsons, - ) - - assert result == test_case.expected_stale_root_source_causes diff --git a/tests/unit/src/sqlbuild/virtual/planner/_helpers/test_targets.py b/tests/unit/src/sqlbuild/virtual/planner/_helpers/test_targets.py deleted file mode 100644 index dc630f5f50..0000000000 --- a/tests/unit/src/sqlbuild/virtual/planner/_helpers/test_targets.py +++ /dev/null @@ -1,123 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter -from sqlbuild.compiler.compile.models import CompiledRelationLocation -from sqlbuild.virtual.planner._helpers.targets import build_destination_from_physical_relation -from sqlbuild.virtual.planner.main._targets import build_virtual_destination_from_physical_relation -from sqlbuild.virtual.state.models import PhysicalRelationRecord -from sqlbuild.virtual.state.types import PhysicalArtifactType -from tests.unit.src.sqlbuild.virtual.planner._helpers._test_types import ( - PhysicalRelationDestinationTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - PhysicalRelationDestinationTestCase( - description="qualifies database schema and name from the stored record", - database_name="analytics", - schema_name="sqlbuild_physical", - relation_name="fact_orders__abc123", - fallback_logical_schema="dev", - fallback_logical_database=None, - expected_qualified_name="analytics.sqlbuild_physical.fact_orders__abc123", - ), - PhysicalRelationDestinationTestCase( - description="qualifies schema and name when the record has no database", - database_name=None, - schema_name="sqlbuild_physical", - relation_name="stg_orders__def456", - fallback_logical_schema="dev", - fallback_logical_database="analytics", - expected_qualified_name="sqlbuild_physical.stg_orders__def456", - ), - ], - ids=lambda case: case.description, -) -def test_given_physical_relation_when_building_destination_then_qualifies_location( - test_case: PhysicalRelationDestinationTestCase, -) -> None: - relation: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.relation_name.split("__")[0], - version_hash="hash", - database_name=test_case.database_name, - schema_name=test_case.schema_name, - relation_name=test_case.relation_name, - relation_type="table", - ) - fallback: CompiledRelationLocation = CompiledRelationLocation( - database=None, - schema="dev", - name=relation.artifact_name, - qualified_name="dev." + relation.artifact_name, - logical_schema=test_case.fallback_logical_schema, - logical_database=test_case.fallback_logical_database, - ) - - destination: CompiledRelationLocation = build_destination_from_physical_relation( - adapter=DuckDbAdapter(), - relation=relation, - fallback_target=fallback, - ) - - assert destination.qualified_name == test_case.expected_qualified_name - assert destination.database == test_case.database_name - assert destination.schema == test_case.schema_name - assert destination.name == test_case.relation_name - assert destination.logical_schema == test_case.fallback_logical_schema - assert destination.logical_database == test_case.fallback_logical_database - - -@pytest.mark.parametrize( - "test_case", - [ - PhysicalRelationDestinationTestCase( - description="public wrapper delegates to the single canonical implementation", - database_name="analytics", - schema_name="sqlbuild_physical", - relation_name="fact_orders__abc123", - fallback_logical_schema="dev", - fallback_logical_database=None, - expected_qualified_name="analytics.sqlbuild_physical.fact_orders__abc123", - ) - ], - ids=lambda case: case.description, -) -def test_given_physical_relation_when_using_public_wrapper_then_matches_canonical_result( - test_case: PhysicalRelationDestinationTestCase, -) -> None: - relation: PhysicalRelationRecord = PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name=test_case.relation_name.split("__")[0], - version_hash="hash", - database_name=test_case.database_name, - schema_name=test_case.schema_name, - relation_name=test_case.relation_name, - relation_type="table", - ) - fallback: CompiledRelationLocation = CompiledRelationLocation( - database=None, - schema="dev", - name=relation.artifact_name, - qualified_name="dev." + relation.artifact_name, - logical_schema=test_case.fallback_logical_schema, - logical_database=test_case.fallback_logical_database, - ) - - wrapped: CompiledRelationLocation = build_virtual_destination_from_physical_relation( - adapter=DuckDbAdapter(), - relation=relation, - fallback_target=fallback, - ) - canonical: CompiledRelationLocation = build_destination_from_physical_relation( - adapter=DuckDbAdapter(), - relation=relation, - fallback_target=fallback, - ) - - assert wrapped == canonical - assert wrapped.qualified_name == test_case.expected_qualified_name diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py deleted file mode 100644 index 68dcd3a0f0..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/_test_types.py +++ /dev/null @@ -1,63 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from datetime import datetime, timedelta - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.microbatches.models import MicrobatchEvent -from sqlbuild.virtual.state.models import MicrobatchReplayRoot -from sqlbuild.virtual.state.types import StateSchemaValidationIssueKind - - -@dataclass(frozen=True) -class StateValidationHelperTestCase: - description: str - existing_tables: set[str] - columns_by_table: dict[str, dict[str, str]] - existing_indexes_by_table: dict[str, set[str]] - expected_issue_kinds: tuple[StateSchemaValidationIssueKind, ...] - - -@dataclass(frozen=True) -class StateBackendConfigResolutionTestCase: - description: str - discovered_inputs: DiscoveredProjectInputs - expected_backend: str - expected_schema: str - expected_database_suffix: str - expected_allow_reset: bool - - -@dataclass(frozen=True) -class StateBackendConfigResolutionErrorTestCase: - description: str - discovered_inputs: DiscoveredProjectInputs - expected_error_type: type[Exception] - expected_message_fragment: str - - -@dataclass(frozen=True) -class StateLockServiceTestCase: - description: str - schema: str - owner_id: str - now: datetime - ttl: timedelta - expected_virtual_environment_lock_key: str - expected_model_version_lock_key: str - - -@dataclass(frozen=True) -class DetachedRetentionHelperTestCase: - description: str - retention_days: int - expected_cleanup_target_names: tuple[str, ...] - expected_cleanup_relation_names: tuple[str, ...] - expected_retained_relation_names: tuple[str, ...] - - -@dataclass(frozen=True) -class MicrobatchReplayRetentionTestCase: - description: str - events: tuple[MicrobatchEvent, ...] - expected_roots: tuple[MicrobatchReplayRoot, ...] diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py deleted file mode 100644 index 304f30ac97..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/helpers.py +++ /dev/null @@ -1,545 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any - -from sqlbuild.executor.node_results.models import ( - NodeResultEnvelope, - NodeResultQuery, - NodeResultRecord, -) -from sqlbuild.microbatches.constants import VIRTUAL_MICROBATCH_SCOPE_KIND -from sqlbuild.microbatches.models import MicrobatchEvent, MicrobatchScope -from sqlbuild.microbatches.types import ( - MicrobatchCompletionType, - MicrobatchFingerprintStatus, - MicrobatchRecordType, - MicrobatchRunType, -) -from sqlbuild.virtual.state._helpers.state_storage.validation import ( - validate_conditional_virtual_environment_publication, -) -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.models import ( - FunctionVersionRecord, - ModelVersionRecord, - PhysicalRelationAncestryRecord, - PhysicalRelationRecord, - PythonNodeVersionRecord, - ReconcileEventRecord, - SeedVersionRecord, - SourceFreshnessRecord, - StateBackupRecord, - StateLockLease, - StateLockRecord, - StateOperationEventRecord, - StateOperationRecord, - StateSchemaValidationResult, - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentFunctionRefRecord, - VirtualEnvironmentModelRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentPythonNodeRefRecord, - VirtualEnvironmentRecord, - VirtualEnvironmentRetentionRecord, - VirtualEnvironmentSeedRefRecord, -) -from sqlbuild.virtual.state.types import PhysicalArtifactType, StateColumnType - - -def state_type_matches_for_test(actual_type: str, expected_type: StateColumnType) -> bool: - return actual_type.lower() == expected_type.value or ( - expected_type == StateColumnType.TEXT and actual_type.lower() == "varchar" - ) - - -def state_columns_for_test( - expected_columns: dict[str, dict[str, StateColumnType]], -) -> dict[str, dict[str, str]]: - rendered_columns: dict[str, dict[str, str]] = {} - for table_name, columns in expected_columns.items(): - rendered_table_columns: dict[str, str] = {} - for column_name, column_type in columns.items(): - rendered_table_columns[column_name] = _state_column_type_for_test(column_type) - rendered_columns[table_name] = rendered_table_columns - return rendered_columns - - -def state_indexes_for_test( - expected_indexes: dict[str, dict[str, tuple[str, ...]]], -) -> dict[str, set[str]]: - return {table_name: set(indexes) for table_name, indexes in expected_indexes.items()} - - -def virtual_environment_ref_for_test( - virtual_environment_name: str, model_name: str, version_hash: str -) -> VirtualEnvironmentModelRefRecord: - return VirtualEnvironmentModelRefRecord( - virtual_environment_name=virtual_environment_name, - model_name=model_name, - version_hash=version_hash, - ) - - -def physical_relation_for_test(relation_name: str, version_hash: str) -> PhysicalRelationRecord: - return PhysicalRelationRecord( - artifact_type=PhysicalArtifactType.MODEL, - artifact_name="orders", - version_hash=version_hash, - database_name=None, - schema_name="dev__sqb_physical", - relation_name=relation_name, - relation_type="table", - ) - - -def virtual_replay_requirement_for_test( - *, version_hash: str, event_id: str, created_second: int -) -> MicrobatchEvent: - """Build one virtual replay requirement for retention projection tests.""" - - scope: MicrobatchScope = virtual_microbatch_scope_for_test(version_hash=version_hash) - return MicrobatchEvent( - event_id=event_id, - record_type=MicrobatchRecordType.REPLAY_REQUIREMENT, - scope=scope, - origin_run_id=event_id, - execution_run_id=event_id, - run_type=MicrobatchRunType.REPLAY_ON_CHANGE, - run_start="0", - run_end="2", - batch_size="1", - cursor_column="id", - cursor_type="integer", - model_version_hash=version_hash, - definition_hash=version_hash, - fingerprint_status=MicrobatchFingerprintStatus.KNOWN, - replay_requirement_id=event_id, - required_model_version_hash=version_hash, - previous_model_version_hash="previous", - replay_policy="full", - created_at=datetime(2026, 1, 1, 0, 0, created_second, tzinfo=UTC), - ) - - -def virtual_replay_completion_for_test( - *, - version_hash: str, - event_id: str, - requirement_id: str, - start: str, - end: str, - created_second: int, -) -> MicrobatchEvent: - """Build one virtual completion attributed to a replay requirement.""" - - return MicrobatchEvent( - event_id=event_id, - record_type=MicrobatchRecordType.PARTITION_COMPLETION, - scope=virtual_microbatch_scope_for_test(version_hash=version_hash), - origin_run_id=requirement_id, - execution_run_id=event_id, - run_type=MicrobatchRunType.REPLAY_ON_CHANGE, - run_start="0", - run_end="2", - batch_size="1", - cursor_column="id", - cursor_type="integer", - model_version_hash=version_hash, - definition_hash=version_hash, - fingerprint_status=MicrobatchFingerprintStatus.KNOWN, - completion_type=MicrobatchCompletionType.RECOVERY, - partition_start=start, - partition_end=end, - replay_requirement_id=requirement_id, - created_at=datetime(2026, 1, 1, 0, 0, created_second, tzinfo=UTC), - ) - - -def virtual_microbatch_scope_for_test(*, version_hash: str) -> MicrobatchScope: - """Build one virtual physical scope in a stable warehouse realm.""" - - return MicrobatchScope( - scope_kind=VIRTUAL_MICROBATCH_SCOPE_KIND, - scope_key=f"duckdb:state:warehouse-realm:orders:{version_hash}:physical.orders", - model_name="orders", - target_database=None, - target_schema="physical", - target_name="orders", - physical_generation_id=f"warehouse-realm:{version_hash}:physical.orders", - virtual_environment_name="dev", - virtual_model_version_hash=version_hash, - ) - - -def _state_column_type_for_test(column_type: StateColumnType) -> str: - return {StateColumnType.TEXT: "VARCHAR"}.get(column_type, column_type.value.upper()) - - -class FakeStateBackend(StateBackend): - def __init__(self, *, acquire_result: bool) -> None: - self.acquire_result: bool = acquire_result - self.acquire_calls: list[tuple[str, str, datetime]] = [] - self.release_calls: list[tuple[str, str]] = [] - - def connect(self, config: dict[str, object]) -> Any: - return object() - - def close(self, connection: Any) -> None: - return None - - def initialize(self, connection: Any, *, schema: str, sqlbuild_version: str) -> None: - return None - - def inspect_schema(self, connection: Any, *, schema: str) -> StateSchemaValidationResult: - return StateSchemaValidationResult() - - def create_backup(self, connection: Any, *, schema: str) -> str: - return "backup" - - def rollback(self, connection: Any, *, schema: str, backup_id: str | None = None) -> str: - return backup_id or "backup" - - def reset(self, connection: Any, *, schema: str) -> None: - return None - - def upsert_model_version( - self, connection: Any, *, schema: str, record: ModelVersionRecord - ) -> None: - return None - - def get_model_version( - self, connection: Any, *, schema: str, model_name: str, version_hash: str - ) -> ModelVersionRecord | None: - return None - - def upsert_function_version( - self, connection: Any, *, schema: str, record: FunctionVersionRecord - ) -> None: - return None - - def get_function_version( - self, connection: Any, *, schema: str, function_name: str, version_hash: str - ) -> FunctionVersionRecord | None: - return None - - def upsert_seed_version( - self, connection: Any, *, schema: str, record: SeedVersionRecord - ) -> None: - return None - - def get_seed_version( - self, connection: Any, *, schema: str, seed_name: str, version_hash: str - ) -> SeedVersionRecord | None: - return None - - def upsert_python_node_version( - self, connection: Any, *, schema: str, record: PythonNodeVersionRecord - ) -> None: - return None - - def get_python_node_version( - self, - connection: Any, - *, - schema: str, - node_type: str, - node_name: str, - version_hash: str, - ) -> PythonNodeVersionRecord | None: - return None - - def insert_node_result( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - record: NodeResultRecord, - ) -> None: - return None - - def read_node_results( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - query: NodeResultQuery, - ) -> tuple[NodeResultEnvelope, ...]: - return () - - def upsert_physical_relation( - self, connection: Any, *, schema: str, record: PhysicalRelationRecord - ) -> None: - return None - - def get_physical_relation_for_artifact( - self, - connection: Any, - *, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - version_hash: str, - ) -> PhysicalRelationRecord | None: - return None - - def list_physical_relations_for_artifact( - self, - connection: Any, - *, - schema: str, - artifact_type: PhysicalArtifactType, - artifact_name: str, - ) -> tuple[PhysicalRelationRecord, ...]: - return () - - def upsert_physical_relation_ancestry( - self, connection: Any, *, schema: str, record: PhysicalRelationAncestryRecord - ) -> None: - return None - - def upsert_virtual_environment( - self, connection: Any, *, schema: str, record: VirtualEnvironmentRecord - ) -> None: - return None - - def get_virtual_environment( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> VirtualEnvironmentRecord | None: - return None - - def list_virtual_environments( - self, connection: Any, *, schema: str - ) -> tuple[VirtualEnvironmentRetentionRecord, ...]: - return () - - def delete_virtual_environment( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> None: - return None - - def replace_virtual_environment_node_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - node_type: str, - refs: tuple[VirtualEnvironmentNodeRefRecord, ...], - ) -> None: - return None - - def get_virtual_environment_node_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - node_type: str, - ) -> tuple[VirtualEnvironmentNodeRefRecord, ...]: - return () - - def replace_virtual_environment_node_ref_groups( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - return None - - def upsert_virtual_environment_and_replace_node_ref_groups( - self, - connection: Any, - *, - schema: str, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - ) -> None: - return None - - def upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - self, - connection: Any, - *, - schema: str, - record: VirtualEnvironmentRecord, - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]], - leases: tuple[StateLockLease, ...], - checkpoint: VirtualEnvironmentCheckpointRecord | None = None, - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = (), - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = (), - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = (), - ) -> bool: - validate_conditional_virtual_environment_publication( - record=record, - refs_by_node_type=refs_by_node_type, - checkpoint=checkpoint, - checkpoint_refs=checkpoint_refs, - checkpoint_function_refs=checkpoint_function_refs, - checkpoint_seed_refs=checkpoint_seed_refs, - ) - return True - - def upsert_virtual_environment_node_ref( - self, - connection: Any, - *, - schema: str, - ref: VirtualEnvironmentNodeRefRecord, - ) -> None: - return None - - def replace_virtual_environment_model_refs( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - refs: tuple[VirtualEnvironmentModelRefRecord, ...], - ) -> None: - return None - - def get_virtual_environment_model_refs( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentModelRefRecord, ...]: - return () - - def replace_virtual_environment_source_freshness( - self, - connection: Any, - *, - schema: str, - virtual_environment_name: str, - records: tuple[SourceFreshnessRecord, ...], - ) -> None: - return None - - def get_virtual_environment_source_freshness( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> tuple[SourceFreshnessRecord, ...]: - return () - - def get_virtual_environment_function_refs( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentFunctionRefRecord, ...]: - return () - - def get_virtual_environment_seed_refs( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentSeedRefRecord, ...]: - return () - - def upsert_virtual_environment_python_node_ref( - self, - connection: Any, - *, - schema: str, - ref: VirtualEnvironmentPythonNodeRefRecord, - ) -> None: - return None - - def get_virtual_environment_python_node_refs( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentPythonNodeRefRecord, ...]: - return () - - def count_unreferenced_python_node_versions(self, connection: Any, *, schema: str) -> int: - return 0 - - def prune_unreferenced_python_node_versions(self, connection: Any, *, schema: str) -> int: - return 0 - - def create_virtual_environment_checkpoint( - self, - connection: Any, - *, - schema: str, - checkpoint: VirtualEnvironmentCheckpointRecord, - refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...], - function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = (), - seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = (), - ) -> None: - return None - - def list_virtual_environment_checkpoints( - self, connection: Any, *, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentCheckpointRecord, ...]: - return () - - def get_virtual_environment_checkpoint_model_refs( - self, connection: Any, *, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointModelRefRecord, ...]: - return () - - def get_virtual_environment_checkpoint_function_refs( - self, connection: Any, *, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...]: - return () - - def get_virtual_environment_checkpoint_seed_refs( - self, connection: Any, *, schema: str, checkpoint_id: str - ) -> tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...]: - return () - - def delete_virtual_environment_checkpoint( - self, connection: Any, *, schema: str, checkpoint_id: str - ) -> None: - return None - - def upsert_state_operation( - self, connection: Any, *, schema: str, record: StateOperationRecord - ) -> None: - return None - - def get_state_operation( - self, connection: Any, *, schema: str, operation_id: str - ) -> StateOperationRecord | None: - return None - - def create_state_operation_event( - self, connection: Any, *, schema: str, record: StateOperationEventRecord - ) -> None: - return None - - def create_reconcile_event( - self, connection: Any, *, schema: str, record: ReconcileEventRecord - ) -> None: - return None - - def acquire_lock( - self, - connection: Any, - *, - schema: str, - lock_key: str, - owner_id: str, - expires_at: datetime, - ) -> bool: - self.acquire_calls.append((lock_key, owner_id, expires_at)) - return self.acquire_result - - def release_lock(self, connection: Any, *, schema: str, lock_key: str, owner_id: str) -> bool: - self.release_calls.append((lock_key, owner_id)) - return True - - def list_active_locks(self, connection: Any, *, schema: str) -> tuple[StateLockRecord, ...]: - return () - - def list_expired_locks(self, connection: Any, *, schema: str) -> tuple[StateLockRecord, ...]: - return () - - def delete_lock(self, connection: Any, *, schema: str, lock_key: str) -> None: - return None - - def list_state_backups(self, connection: Any, *, schema: str) -> tuple[StateBackupRecord, ...]: - return () - - def delete_state_backup(self, connection: Any, *, schema: str, backup_id: str) -> None: - return None diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/_test_types.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/_test_types.py deleted file mode 100644 index cc221a361b..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/_test_types.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Test types for conditional virtual publication validation.""" - -from dataclasses import dataclass - -from sqlbuild.virtual.state.models import ( - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, -) - - -@dataclass(frozen=True) -class ConditionalPublicationValidationTestCase: - """One conditional publication payload and its expected validation result.""" - - description: str - record: VirtualEnvironmentRecord - refs_by_node_type: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] - checkpoint: VirtualEnvironmentCheckpointRecord | None - checkpoint_refs: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] - checkpoint_function_refs: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] - checkpoint_seed_refs: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] - expected_valid: bool = True - expected_error_fragment: str | None = None diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/helpers.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/helpers.py deleted file mode 100644 index 6210aa1ab7..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/helpers.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Helpers for conditional publication validation tests.""" - -from sqlbuild.virtual.state._helpers.state_storage.validation import ( - validate_conditional_virtual_environment_publication, -) -from tests.unit.src.sqlbuild.virtual.state._helpers.helpers import FakeStateBackend -from tests.unit.src.sqlbuild.virtual.state._helpers.state_storage._test_types import ( - ConditionalPublicationValidationTestCase, -) - - -def validate_case(test_case: ConditionalPublicationValidationTestCase) -> bool: - validate_conditional_virtual_environment_publication( - record=test_case.record, - refs_by_node_type=test_case.refs_by_node_type, - checkpoint=test_case.checkpoint, - checkpoint_refs=test_case.checkpoint_refs, - checkpoint_function_refs=test_case.checkpoint_function_refs, - checkpoint_seed_refs=test_case.checkpoint_seed_refs, - ) - return True - - -def publish_fake_case(test_case: ConditionalPublicationValidationTestCase) -> bool: - return FakeStateBackend( - acquire_result=True - ).upsert_virtual_environment_and_replace_node_ref_groups_if_locks_owned( - connection=object(), - schema="state", - record=test_case.record, - refs_by_node_type=test_case.refs_by_node_type, - leases=(), - checkpoint=test_case.checkpoint, - checkpoint_refs=test_case.checkpoint_refs, - checkpoint_function_refs=test_case.checkpoint_function_refs, - checkpoint_seed_refs=test_case.checkpoint_seed_refs, - ) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/test_validation.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/test_validation.py deleted file mode 100644 index 4b9e0d942e..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/state_storage/test_validation.py +++ /dev/null @@ -1,278 +0,0 @@ -"""Conditional virtual publication validation contract tests.""" - -from __future__ import annotations - -from dataclasses import replace - -import pytest - -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import ( - VirtualEnvironmentCheckpointFunctionRefRecord, - VirtualEnvironmentCheckpointModelRefRecord, - VirtualEnvironmentCheckpointRecord, - VirtualEnvironmentCheckpointSeedRefRecord, - VirtualEnvironmentNodeRefRecord, - VirtualEnvironmentRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus -from tests.unit.src.sqlbuild.virtual.state._helpers.state_storage._test_types import ( - ConditionalPublicationValidationTestCase, -) -from tests.unit.src.sqlbuild.virtual.state._helpers.state_storage.helpers import ( - publish_fake_case, - validate_case, -) - -_ENVIRONMENT: str = "dev" -_CHECKPOINT_ID: str = "checkpoint-1" -_FINALIZED_RECORD: VirtualEnvironmentRecord = VirtualEnvironmentRecord( - virtual_environment_name=_ENVIRONMENT, - status=VirtualEnvironmentStatus.FINALIZED, -) -_ACTIVE_RECORD: VirtualEnvironmentRecord = replace( - _FINALIZED_RECORD, - status=VirtualEnvironmentStatus.ACTIVE, -) -_REF_GROUPS: dict[str, tuple[VirtualEnvironmentNodeRefRecord, ...]] = { - "model": (VirtualEnvironmentNodeRefRecord(_ENVIRONMENT, "model", "orders", "model-v1"),), - "udf": (VirtualEnvironmentNodeRefRecord(_ENVIRONMENT, "udf", "normalize", "function-v1"),), - "seed": (VirtualEnvironmentNodeRefRecord(_ENVIRONMENT, "seed", "countries", "seed-v1"),), -} -_CHECKPOINT: VirtualEnvironmentCheckpointRecord = VirtualEnvironmentCheckpointRecord( - checkpoint_id=_CHECKPOINT_ID, - virtual_environment_name=_ENVIRONMENT, -) -_MODEL_REFS: tuple[VirtualEnvironmentCheckpointModelRefRecord, ...] = ( - VirtualEnvironmentCheckpointModelRefRecord(_CHECKPOINT_ID, "orders", "model-v1"), -) -_FUNCTION_REFS: tuple[VirtualEnvironmentCheckpointFunctionRefRecord, ...] = ( - VirtualEnvironmentCheckpointFunctionRefRecord(_CHECKPOINT_ID, "normalize", "function-v1"), -) -_SEED_REFS: tuple[VirtualEnvironmentCheckpointSeedRefRecord, ...] = ( - VirtualEnvironmentCheckpointSeedRefRecord(_CHECKPOINT_ID, "countries", "seed-v1"), -) - -_SUCCESS_CASES: tuple[ConditionalPublicationValidationTestCase, ...] = ( - ConditionalPublicationValidationTestCase( - description="finalized exact checkpoint payload succeeds", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - ), - ConditionalPublicationValidationTestCase( - description="active publication without checkpoint succeeds", - record=_ACTIVE_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=None, - checkpoint_refs=(), - checkpoint_function_refs=(), - checkpoint_seed_refs=(), - ), -) - -_ERROR_CASES: tuple[ConditionalPublicationValidationTestCase, ...] = ( - ConditionalPublicationValidationTestCase( - description="finalized publication requires checkpoint", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=None, - checkpoint_refs=(), - checkpoint_function_refs=(), - checkpoint_seed_refs=(), - expected_error_fragment="requires a checkpoint", - ), - ConditionalPublicationValidationTestCase( - description="active publication forbids complete checkpoint payload", - record=_ACTIVE_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="requires finalized virtual environment status", - ), - ConditionalPublicationValidationTestCase( - description="finalizing publication forbids complete checkpoint payload", - record=replace(_FINALIZED_RECORD, status=VirtualEnvironmentStatus.FINALIZING), - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="requires finalized virtual environment status", - ), - ConditionalPublicationValidationTestCase( - description="detached publication forbids complete checkpoint payload", - record=replace(_FINALIZED_RECORD, status=VirtualEnvironmentStatus.DETACHED), - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="requires finalized virtual environment status", - ), - ConditionalPublicationValidationTestCase( - description="failed publication forbids complete checkpoint payload", - record=replace(_FINALIZED_RECORD, status=VirtualEnvironmentStatus.FAILED), - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="requires finalized virtual environment status", - ), - ConditionalPublicationValidationTestCase( - description="checkpoint environment must match", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=replace(_CHECKPOINT, virtual_environment_name="other"), - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="must match the published environment", - ), - ConditionalPublicationValidationTestCase( - description="model checkpoint id must match", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=(replace(_MODEL_REFS[0], checkpoint_id="other"),), - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="checkpoint_id must match", - ), - ConditionalPublicationValidationTestCase( - description="function checkpoint id must match", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=(replace(_FUNCTION_REFS[0], checkpoint_id="other"),), - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="checkpoint_id must match", - ), - ConditionalPublicationValidationTestCase( - description="seed checkpoint id must match", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=(replace(_SEED_REFS[0], checkpoint_id="other"),), - expected_error_fragment="checkpoint_id must match", - ), - ConditionalPublicationValidationTestCase( - description="model version must correspond exactly", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=(replace(_MODEL_REFS[0], version_hash="other"),), - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="model refs must exactly match", - ), - ConditionalPublicationValidationTestCase( - description="function omission is rejected", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=(), - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="function refs must exactly match", - ), - ConditionalPublicationValidationTestCase( - description="seed extra ref is rejected", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=( - *_SEED_REFS, - VirtualEnvironmentCheckpointSeedRefRecord(_CHECKPOINT_ID, "extra", "seed-v2"), - ), - expected_error_fragment="seed refs must exactly match", - ), - ConditionalPublicationValidationTestCase( - description="checkpoint duplicate identity is rejected", - record=_FINALIZED_RECORD, - refs_by_node_type=_REF_GROUPS, - checkpoint=_CHECKPOINT, - checkpoint_refs=(*_MODEL_REFS, *_MODEL_REFS), - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="duplicate identities", - ), - ConditionalPublicationValidationTestCase( - description="published duplicate identity is rejected", - record=_FINALIZED_RECORD, - refs_by_node_type={**_REF_GROUPS, "model": (*_REF_GROUPS["model"], *_REF_GROUPS["model"])}, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="duplicate identities", - ), - ConditionalPublicationValidationTestCase( - description="published ref environment must match", - record=_FINALIZED_RECORD, - refs_by_node_type={ - **_REF_GROUPS, - "model": (replace(_REF_GROUPS["model"][0], virtual_environment_name="other"),), - }, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="ref virtual_environment_name must match", - ), - ConditionalPublicationValidationTestCase( - description="published ref node type must match group", - record=_FINALIZED_RECORD, - refs_by_node_type={ - **_REF_GROUPS, - "model": (replace(_REF_GROUPS["model"][0], node_type="seed"),), - }, - checkpoint=_CHECKPOINT, - checkpoint_refs=_MODEL_REFS, - checkpoint_function_refs=_FUNCTION_REFS, - checkpoint_seed_refs=_SEED_REFS, - expected_error_fragment="node_type must match", - ), -) - - -@pytest.mark.parametrize( - "test_case", - [ConditionalPublicationValidationTestCase(**case.__dict__) for case in _SUCCESS_CASES], - ids=lambda case: case.description, -) -def test_given_valid_conditional_publication_when_validating_then_succeeds( - test_case: ConditionalPublicationValidationTestCase, -) -> None: - assert validate_case(test_case) is test_case.expected_valid - assert publish_fake_case(test_case) is test_case.expected_valid - - -@pytest.mark.parametrize( - "test_case", - [ConditionalPublicationValidationTestCase(**case.__dict__) for case in _ERROR_CASES], - ids=lambda case: case.description, -) -def test_given_invalid_conditional_publication_when_validating_then_raises_structured_error( - test_case: ConditionalPublicationValidationTestCase, -) -> None: - with pytest.raises(StateBackendConfigError, match=test_case.expected_error_fragment or ""): - validate_case(test_case) - - with pytest.raises(StateBackendConfigError, match=test_case.expected_error_fragment or ""): - publish_fake_case(test_case) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_config.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_config.py deleted file mode 100644 index b6e3b4c818..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_config.py +++ /dev/null @@ -1,169 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.spec.contracts.exceptions import SpecConfigError -from sqlbuild.spec.contracts.models import ( - LocalConfig, - ProjectConfig, - SettingsConfig, - StateConfig, - TargetConfig, -) -from sqlbuild.virtual.state._helpers.state_runtime.config import resolve_state_backend_config -from sqlbuild.virtual.state.exceptions import StateBackendConfigError -from sqlbuild.virtual.state.models import StateBackendConfig -from tests.unit.src.sqlbuild.virtual.state._helpers._test_types import ( - StateBackendConfigResolutionErrorTestCase, - StateBackendConfigResolutionTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - StateBackendConfigResolutionTestCase( - description="resolves duckdb state config and project-relative database path", - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=True), - default_target="dev", - targets={ - "dev": TargetConfig( - state=StateConfig( - backend="duckdb", - schema="sqlbuild_state", - connection={"database": "state.duckdb"}, - allow_reset=True, - ) - ) - }, - ), - local_config=LocalConfig(), - ), - expected_backend="duckdb", - expected_schema="sqlbuild_state", - expected_database_suffix="state.duckdb", - expected_allow_reset=True, - ) - ], - ids=lambda case: case.description, -) -def test_given_state_config_when_resolving_backend_config_then_returns_effective_config( - test_case: StateBackendConfigResolutionTestCase, - tmp_path: Path, -) -> None: - config: StateBackendConfig = resolve_state_backend_config( - discovered_inputs=test_case.discovered_inputs, - project_dir=tmp_path, - ) - - assert config.backend.value == test_case.expected_backend - assert config.schema == test_case.expected_schema - assert config.connection["database"] == str(tmp_path / test_case.expected_database_suffix) - assert config.allow_reset is test_case.expected_allow_reset - - -@pytest.mark.parametrize( - "test_case", - ( - StateBackendConfigResolutionErrorTestCase( - description="blocks unsupported state backend", - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=True), - default_target="dev", - targets={ - "dev": TargetConfig( - state=StateConfig(backend="sqlite", schema="sqlbuild_state") - ) - }, - ), - local_config=LocalConfig(), - ), - expected_error_type=StateBackendConfigError, - expected_message_fragment="Unsupported state backend: sqlite", - ), - StateBackendConfigResolutionErrorTestCase( - description="blocks missing state backend", - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=True), - default_target="dev", - targets={"dev": TargetConfig(state=StateConfig(schema="sqlbuild_state"))}, - ), - local_config=LocalConfig(), - ), - expected_error_type=StateBackendConfigError, - expected_message_fragment="does not configure a state backend", - ), - StateBackendConfigResolutionErrorTestCase( - description="blocks missing state schema", - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=True), - default_target="dev", - targets={"dev": TargetConfig(state=StateConfig(backend="duckdb"))}, - ), - local_config=LocalConfig(), - ), - expected_error_type=StateBackendConfigError, - expected_message_fragment="state config must define schema", - ), - StateBackendConfigResolutionErrorTestCase( - description="blocks unknown active target", - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - settings=SettingsConfig(virtual_environments=True), - default_target="missing", - ), - local_config=LocalConfig(), - ), - expected_error_type=SpecConfigError, - expected_message_fragment="Unknown target 'missing'", - ), - StateBackendConfigResolutionErrorTestCase( - description="blocks state commands outside virtual mode", - discovered_inputs=DiscoveredProjectInputs( - project_config=ProjectConfig( - name="demo", - adapter="duckdb", - default_target="dev", - targets={ - "dev": TargetConfig( - state=StateConfig(backend="duckdb", schema="sqlbuild_state") - ) - }, - ), - local_config=LocalConfig(), - ), - expected_error_type=StateBackendConfigError, - expected_message_fragment="State commands require virtual_environments = true", - ), - ), - ids=lambda case: case.description, -) -def test_given_invalid_state_config_when_resolving_backend_config_then_raises_clear_error( - test_case: StateBackendConfigResolutionErrorTestCase, - tmp_path: Path, -) -> None: - with pytest.raises(test_case.expected_error_type) as exc_info: - resolve_state_backend_config( - discovered_inputs=test_case.discovered_inputs, - project_dir=tmp_path, - ) - - assert test_case.expected_message_fragment in str(exc_info.value) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_detached_retention.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_detached_retention.py deleted file mode 100644 index 6eb94dad56..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_detached_retention.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Tests for detached virtual environment retention helpers.""" - -from __future__ import annotations - -from datetime import UTC, datetime, timedelta - -import pytest - -from sqlbuild.virtual.state._helpers.state_lifecycle.detached_retention import ( - build_detached_environment_inspection, -) -from sqlbuild.virtual.state.models import ( - DetachedVirtualEnvironmentInspection, - VirtualEnvironmentRetentionRecord, -) -from sqlbuild.virtual.state.types import VirtualEnvironmentStatus -from tests.unit.src.sqlbuild.virtual.state._helpers._test_types import ( - DetachedRetentionHelperTestCase, -) -from tests.unit.src.sqlbuild.virtual.state._helpers.helpers import ( - physical_relation_for_test, - virtual_environment_ref_for_test, -) - -NOW: datetime = datetime(2026, 5, 27, 12, 0, 0, tzinfo=UTC) - - -@pytest.mark.parametrize( - "test_case", - ( - DetachedRetentionHelperTestCase( - description="retention zero cleans detached refs immediately and retains active refs", - retention_days=0, - expected_cleanup_target_names=("old_detached", "new_detached", "unknown_age"), - expected_cleanup_relation_names=("orders__v_new", "orders__v_old"), - expected_retained_relation_names=("orders__v_active",), - ), - DetachedRetentionHelperTestCase( - description="positive retention cleans only detached environments older than retention", - retention_days=7, - expected_cleanup_target_names=("old_detached",), - expected_cleanup_relation_names=("orders__v_old",), - expected_retained_relation_names=("orders__v_active", "orders__v_new"), - ), - ), - ids=lambda case: case.description, -) -def test_given_virtual_environments_when_inspecting_detached_retention_then_classifies_refs( - test_case: DetachedRetentionHelperTestCase, -) -> None: - inspection: DetachedVirtualEnvironmentInspection = build_detached_environment_inspection( - environments=( - VirtualEnvironmentRetentionRecord( - virtual_environment_name="old_detached", - status=VirtualEnvironmentStatus.DETACHED, - updated_at=NOW - timedelta(days=10), - ), - VirtualEnvironmentRetentionRecord( - virtual_environment_name="new_detached", - status=VirtualEnvironmentStatus.DETACHED, - updated_at=NOW - timedelta(days=2), - ), - VirtualEnvironmentRetentionRecord( - virtual_environment_name="unknown_age", - status=VirtualEnvironmentStatus.DETACHED, - updated_at=None, - ), - VirtualEnvironmentRetentionRecord( - virtual_environment_name="active", - status=VirtualEnvironmentStatus.ACTIVE, - updated_at=NOW - timedelta(days=20), - ), - ), - refs_by_environment={ - "old_detached": (virtual_environment_ref_for_test("old_detached", "orders", "old"),), - "new_detached": (virtual_environment_ref_for_test("new_detached", "orders", "new"),), - "unknown_age": (virtual_environment_ref_for_test("unknown_age", "missing", "missing"),), - "active": (virtual_environment_ref_for_test("active", "orders", "active"),), - }, - physical_relations_by_ref={ - ("orders", "old"): physical_relation_for_test("orders__v_old", "old"), - ("orders", "new"): physical_relation_for_test("orders__v_new", "new"), - ("orders", "active"): physical_relation_for_test("orders__v_active", "active"), - }, - retention_days=test_case.retention_days, - now=NOW, - ) - - assert ( - tuple( - environment.virtual_environment_name - for environment in inspection.cleanup_virtual_environments - ) - == test_case.expected_cleanup_target_names - ) - assert ( - tuple(relation.relation_name for relation in inspection.cleanup_physical_relations) - == test_case.expected_cleanup_relation_names - ) - assert ( - tuple(relation.relation_name for relation in inspection.retained_physical_relations) - == test_case.expected_retained_relation_names - ) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py deleted file mode 100644 index b4ecf1ed0f..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_locks.py +++ /dev/null @@ -1,118 +0,0 @@ -from __future__ import annotations - -from datetime import datetime, timedelta - -import pytest - -from sqlbuild.virtual.state._helpers.state_storage.locks import ( - acquire_model_version_lock, - acquire_virtual_environment_lock, - release_state_lock, -) -from sqlbuild.virtual.state.models import StateLockLease -from tests.unit.src.sqlbuild.virtual.state._helpers._test_types import ( - StateLockServiceTestCase, -) -from tests.unit.src.sqlbuild.virtual.state._helpers.helpers import FakeStateBackend - - -@pytest.mark.parametrize( - "test_case", - [ - StateLockServiceTestCase( - description="builds scoped lock keys and returns leases when acquired", - schema="sqlbuild_state", - owner_id="run-1", - now=datetime(2026, 5, 25, 12, 0, 0), - ttl=timedelta(minutes=15), - expected_virtual_environment_lock_key="virtual_env:dev", - expected_model_version_lock_key="model_version:fact_orders:abc123", - ) - ], - ids=lambda case: case.description, -) -def test_given_state_lock_service_when_acquiring_scoped_locks_then_uses_expected_lock_keys( - test_case: StateLockServiceTestCase, -) -> None: - backend: FakeStateBackend = FakeStateBackend(acquire_result=True) - connection: object = object() - - virtual_environment_lease: StateLockLease | None = acquire_virtual_environment_lock( - backend=backend, - connection=connection, - schema=test_case.schema, - virtual_environment_name="dev", - owner_id=test_case.owner_id, - ttl=test_case.ttl, - now=test_case.now, - ) - model_version_lease: StateLockLease | None = acquire_model_version_lock( - backend=backend, - connection=connection, - schema=test_case.schema, - model_name="fact_orders", - version_hash="abc123", - owner_id=test_case.owner_id, - ttl=test_case.ttl, - now=test_case.now, - ) - - assert virtual_environment_lease is not None - assert model_version_lease is not None - assert virtual_environment_lease.lock_key == test_case.expected_virtual_environment_lock_key - assert model_version_lease.lock_key == test_case.expected_model_version_lock_key - assert virtual_environment_lease.expires_at == test_case.now + test_case.ttl - assert tuple(call[0] for call in backend.acquire_calls) == ( - test_case.expected_virtual_environment_lock_key, - test_case.expected_model_version_lock_key, - ) - assert release_state_lock( - backend=backend, - connection=connection, - schema=test_case.schema, - lease=virtual_environment_lease, - ) - assert backend.release_calls == [ - (test_case.expected_virtual_environment_lock_key, test_case.owner_id) - ] - - -@pytest.mark.parametrize( - "test_case", - [ - StateLockServiceTestCase( - description="returns no lease when backend lock acquisition conflicts", - schema="sqlbuild_state", - owner_id="run-1", - now=datetime(2026, 5, 25, 12, 0, 0), - ttl=timedelta(minutes=15), - expected_virtual_environment_lock_key="virtual_env:dev", - expected_model_version_lock_key="model_version:fact_orders:abc123", - ) - ], - ids=lambda case: case.description, -) -def test_given_state_lock_service_when_backend_reports_conflict_then_returns_none( - test_case: StateLockServiceTestCase, -) -> None: - backend: FakeStateBackend = FakeStateBackend(acquire_result=False) - connection: object = object() - - lease: StateLockLease | None = acquire_virtual_environment_lock( - backend=backend, - connection=connection, - schema=test_case.schema, - virtual_environment_name="dev", - owner_id=test_case.owner_id, - ttl=test_case.ttl, - now=test_case.now, - ) - - assert lease is None - assert backend.acquire_calls == [ - ( - test_case.expected_virtual_environment_lock_key, - test_case.owner_id, - test_case.now + test_case.ttl, - ) - ] diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_microbatch_replay_retention.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_microbatch_replay_retention.py deleted file mode 100644 index a88fa46a7d..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_microbatch_replay_retention.py +++ /dev/null @@ -1,115 +0,0 @@ -"""Tests for active virtual microbatch replay retention roots.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.virtual.state._helpers.state_lifecycle.microbatch_replay_retention import ( - active_microbatch_replay_roots, -) -from sqlbuild.virtual.state.models import MicrobatchReplayRoot -from tests.unit.src.sqlbuild.virtual.state._helpers._test_types import ( - MicrobatchReplayRetentionTestCase, -) -from tests.unit.src.sqlbuild.virtual.state._helpers.helpers import ( - virtual_replay_completion_for_test, - virtual_replay_requirement_for_test, -) - - -@pytest.mark.parametrize( - "test_case", - [ - MicrobatchReplayRetentionTestCase( - description="partial latest replay protects its physical version", - events=( - virtual_replay_requirement_for_test( - version_hash="F2", event_id="requirement-F2", created_second=0 - ), - virtual_replay_completion_for_test( - version_hash="F2", - event_id="completion-F2-1", - requirement_id="requirement-F2", - start="0", - end="1", - created_second=1, - ), - ), - expected_roots=( - MicrobatchReplayRoot( - model_name="orders", - version_hash="F2", - previous_version_hash="previous", - ), - ), - ), - MicrobatchReplayRetentionTestCase( - description="verified latest replay releases its physical version", - events=( - virtual_replay_requirement_for_test( - version_hash="F2", event_id="requirement-F2", created_second=0 - ), - virtual_replay_completion_for_test( - version_hash="F2", - event_id="completion-F2", - requirement_id="requirement-F2", - start="0", - end="2", - created_second=1, - ), - ), - expected_roots=(), - ), - MicrobatchReplayRetentionTestCase( - description="new complete requirement supersedes incomplete old version", - events=( - virtual_replay_requirement_for_test( - version_hash="F2", event_id="requirement-F2", created_second=0 - ), - virtual_replay_requirement_for_test( - version_hash="F3", event_id="requirement-F3", created_second=1 - ), - virtual_replay_completion_for_test( - version_hash="F3", - event_id="completion-F3", - requirement_id="requirement-F3", - start="0", - end="2", - created_second=2, - ), - ), - expected_roots=(), - ), - MicrobatchReplayRetentionTestCase( - description="new all-failed requirement supersedes incomplete old version", - events=( - virtual_replay_requirement_for_test( - version_hash="F2", event_id="requirement-F2", created_second=0 - ), - virtual_replay_requirement_for_test( - version_hash="F3", event_id="requirement-F3", created_second=1 - ), - ), - expected_roots=( - MicrobatchReplayRoot( - model_name="orders", - version_hash="F3", - previous_version_hash="previous", - ), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_virtual_replay_history_when_projecting_retention_then_only_active_versions_are_roots( - test_case: MicrobatchReplayRetentionTestCase, -) -> None: - roots: tuple[MicrobatchReplayRoot, ...] = active_microbatch_replay_roots( - events=test_case.events - ) - - assert roots == test_case.expected_roots - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_validation.py b/tests/unit/src/sqlbuild/virtual/state/_helpers/test_validation.py deleted file mode 100644 index 2497028dc8..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/_helpers/test_validation.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.virtual.state._helpers.state_storage.validation import build_validation_result -from sqlbuild.virtual.state.constants import STATE_TABLE_COLUMNS, STATE_TABLE_INDEXES -from sqlbuild.virtual.state.models import StateSchemaValidationResult -from sqlbuild.virtual.state.types import StateSchemaValidationIssueKind -from tests.unit.src.sqlbuild.virtual.state._helpers._test_types import ( - StateValidationHelperTestCase, -) -from tests.unit.src.sqlbuild.virtual.state._helpers.helpers import ( - state_columns_for_test, - state_indexes_for_test, - state_type_matches_for_test, -) - -VALID_COLUMNS_BY_TABLE: dict[str, dict[str, str]] = state_columns_for_test(STATE_TABLE_COLUMNS) -VALID_INDEXES_BY_TABLE: dict[str, set[str]] = state_indexes_for_test(STATE_TABLE_INDEXES) - - -@pytest.mark.parametrize( - "test_case", - [ - StateValidationHelperTestCase( - description="accepts required tables and compatible column types", - existing_tables=set(STATE_TABLE_COLUMNS), - columns_by_table=VALID_COLUMNS_BY_TABLE, - existing_indexes_by_table=VALID_INDEXES_BY_TABLE, - expected_issue_kinds=(), - ), - StateValidationHelperTestCase( - description="reports missing table", - existing_tables=set(STATE_TABLE_COLUMNS) - {"state_migration_events"}, - columns_by_table={ - table_name: columns - for table_name, columns in VALID_COLUMNS_BY_TABLE.items() - if table_name != "state_migration_events" - }, - existing_indexes_by_table=VALID_INDEXES_BY_TABLE, - expected_issue_kinds=(StateSchemaValidationIssueKind.MISSING_TABLE,), - ), - StateValidationHelperTestCase( - description="reports missing column and wrong type", - existing_tables=set(STATE_TABLE_COLUMNS), - columns_by_table={ - **VALID_COLUMNS_BY_TABLE, - "state_versions": { - "schema_version": "VARCHAR", - "updated_at": "TIMESTAMP", - }, - }, - existing_indexes_by_table=VALID_INDEXES_BY_TABLE, - expected_issue_kinds=( - StateSchemaValidationIssueKind.WRONG_TYPE, - StateSchemaValidationIssueKind.MISSING_COLUMN, - ), - ), - StateValidationHelperTestCase( - description="reports missing index", - existing_tables=set(STATE_TABLE_COLUMNS), - columns_by_table=VALID_COLUMNS_BY_TABLE, - existing_indexes_by_table={ - table_name: indexes - for table_name, indexes in VALID_INDEXES_BY_TABLE.items() - if table_name != "locks" - }, - expected_issue_kinds=(StateSchemaValidationIssueKind.MISSING_INDEX,), - ), - ], - ids=lambda case: case.description, -) -def test_given_state_schema_metadata_when_validating_then_reports_expected_issues( - test_case: StateValidationHelperTestCase, -) -> None: - result: StateSchemaValidationResult = build_validation_result( - existing_tables=test_case.existing_tables, - columns_by_table=test_case.columns_by_table, - expected_columns=STATE_TABLE_COLUMNS, - type_matches=state_type_matches_for_test, - expected_indexes=STATE_TABLE_INDEXES, - existing_indexes_by_table=test_case.existing_indexes_by_table, - ) - - assert tuple(issue.kind for issue in result.issues) == test_case.expected_issue_kinds - assert result.valid is (not test_case.expected_issue_kinds) diff --git a/tests/unit/src/sqlbuild/virtual/state/main/_test_types.py b/tests/unit/src/sqlbuild/virtual/state/main/_test_types.py deleted file mode 100644 index 352e33e4cd..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/main/_test_types.py +++ /dev/null @@ -1,28 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.virtual.state.types import StateOperationStatus, StateOperationType - - -@dataclass(frozen=True) -class RecordStateOperationTestCase: - description: str - operation_id: str - operation_type: StateOperationType - virtual_environment_name: str - start_message: str - finish_message: str - expected_final_status: StateOperationStatus - expected_event_rows: tuple[tuple[str, StateOperationStatus, str], ...] - - -@dataclass(frozen=True) -class VirtualPythonNodeIdentityTestCase: - description: str - virtual_environment_name: str - expected_node_type: str - expected_node_name: str - expected_version_hash: str - expected_definition_json: str - expected_metadata_json: str diff --git a/tests/unit/src/sqlbuild/virtual/state/main/helpers.py b/tests/unit/src/sqlbuild/virtual/state/main/helpers.py deleted file mode 100644 index 0fde78bba4..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/main/helpers.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -from typing import Any, cast - -from sqlbuild.virtual.state.models import ( - PythonNodeVersionRecord, - StateOperationEventRecord, - StateOperationRecord, - VirtualEnvironmentPythonNodeRefRecord, -) - - -class RecordingStateBackend: - def __init__(self) -> None: - self.operation: StateOperationRecord | None = None - self.events: list[StateOperationEventRecord] = [] - - def get_state_operation( - self, - *, - connection: Any, - schema: str, - operation_id: str, - ) -> StateOperationRecord | None: - del connection, schema - operations_by_id: dict[str, StateOperationRecord] = { - getattr(self.operation, "operation_id", ""): cast(StateOperationRecord, self.operation) - } - return operations_by_id.get(operation_id) - - def upsert_state_operation( - self, - *, - connection: Any, - schema: str, - record: StateOperationRecord, - ) -> None: - del connection, schema - self.operation = record - - def create_state_operation_event( - self, - *, - connection: Any, - schema: str, - record: StateOperationEventRecord, - ) -> None: - del connection, schema - self.events.append(record) - - -class RecordingPythonIdentityStateBackend: - def __init__(self) -> None: - self.versions: dict[tuple[str, str, str], PythonNodeVersionRecord] = {} - self.refs: dict[tuple[str, str, str], VirtualEnvironmentPythonNodeRefRecord] = {} - - def upsert_python_node_version( - self, *, connection: Any, schema: str, record: PythonNodeVersionRecord - ) -> None: - del connection, schema - self.versions[(record.node_type, record.node_name, record.version_hash)] = record - - def get_python_node_version( - self, - *, - connection: Any, - schema: str, - node_type: str, - node_name: str, - version_hash: str, - ) -> PythonNodeVersionRecord | None: - del connection, schema - return self.versions.get((node_type, node_name, version_hash)) - - def upsert_virtual_environment_python_node_ref( - self, - *, - connection: Any, - schema: str, - ref: VirtualEnvironmentPythonNodeRefRecord, - ) -> None: - del connection, schema - self.refs[(ref.virtual_environment_name, ref.node_type, ref.node_name)] = ref - - def get_virtual_environment_python_node_refs( - self, *, connection: Any, schema: str, virtual_environment_name: str - ) -> tuple[VirtualEnvironmentPythonNodeRefRecord, ...]: - del connection, schema - refs_by_environment: dict[str, list[VirtualEnvironmentPythonNodeRefRecord]] = {} - key: tuple[str, str, str] - ref: VirtualEnvironmentPythonNodeRefRecord - for key, ref in sorted(self.refs.items()): - refs_by_environment.setdefault(key[0], []).append(ref) - return tuple(refs_by_environment.get(virtual_environment_name, ())) diff --git a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py deleted file mode 100644 index 163e6f83f1..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.fingerprints.models import Fingerprint -from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity -from sqlbuild.virtual.state.main.python_identities._python_node_identity_read import ( - read_virtual_python_identity_fingerprints, -) -from sqlbuild.virtual.state.main.python_identities._python_node_identity_write import ( - try_record_virtual_python_node_identity, -) -from tests.unit.src.sqlbuild.virtual.state.main._test_types import ( - VirtualPythonNodeIdentityTestCase, -) -from tests.unit.src.sqlbuild.virtual.state.main.helpers import ( - RecordingPythonIdentityStateBackend, -) - - -@pytest.mark.parametrize( - "test_case", - [ - VirtualPythonNodeIdentityTestCase( - description="writes and reads virtual python identity as fingerprint", - virtual_environment_name="dev_alice", - expected_node_type="task", - expected_node_name="prepare_orders", - expected_version_hash="version-hash", - expected_definition_json='{"node_name":"prepare_orders"}', - expected_metadata_json='{"dependencies":[]}', - ) - ], - ids=lambda case: case.description, -) -def test_given_virtual_python_identity_when_recording_then_reads_previous_fingerprint( - test_case: VirtualPythonNodeIdentityTestCase, -) -> None: - backend: RecordingPythonIdentityStateBackend = RecordingPythonIdentityStateBackend() - identity: PythonNodeIdentity = PythonNodeIdentity( - node_type=test_case.expected_node_type, - node_name=test_case.expected_node_name, - source_path="python/tasks/orders.py", - source_hash="source-hash", - definition_hash="definition-hash", - version_hash=test_case.expected_version_hash, - definition_json=test_case.expected_definition_json, - metadata_json=test_case.expected_metadata_json, - ) - - try_record_virtual_python_node_identity( - backend=backend, - state_connection=None, - schema="sqlbuild_state", - virtual_environment_name=test_case.virtual_environment_name, - identity=identity, - ) - fingerprints: dict[tuple[str, str], Fingerprint] = read_virtual_python_identity_fingerprints( - backend=backend, - state_connection=None, - schema="sqlbuild_state", - virtual_environment_name=test_case.virtual_environment_name, - ) - - fingerprint: Fingerprint = fingerprints[ - (test_case.expected_node_type, test_case.expected_node_name) - ] - assert fingerprint.node_type == test_case.expected_node_type - assert fingerprint.node_name == test_case.expected_node_name - assert fingerprint.version_hash == test_case.expected_version_hash - assert fingerprint.definition == test_case.expected_definition_json - assert fingerprint.metadata_json == test_case.expected_metadata_json diff --git a/tests/unit/src/sqlbuild/virtual/state/main/test_record_operation.py b/tests/unit/src/sqlbuild/virtual/state/main/test_record_operation.py deleted file mode 100644 index 2df94ee32a..0000000000 --- a/tests/unit/src/sqlbuild/virtual/state/main/test_record_operation.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -from typing import cast - -import pytest - -from sqlbuild.virtual.state.classes.state_backend import StateBackend -from sqlbuild.virtual.state.main.environments._record_operation import record_state_operation -from sqlbuild.virtual.state.models import StateOperationRecord -from sqlbuild.virtual.state.types import StateOperationStatus, StateOperationType -from tests.unit.src.sqlbuild.virtual.state.main._test_types import ( - RecordStateOperationTestCase, -) -from tests.unit.src.sqlbuild.virtual.state.main.helpers import RecordingStateBackend - - -@pytest.mark.parametrize( - "test_case", - [ - RecordStateOperationTestCase( - description="preserves operation type and VDE across follow-up events", - operation_id="detach:dev", - operation_type=StateOperationType.DETACH, - virtual_environment_name="dev", - start_message="starting detach", - finish_message="detached 1 models", - expected_final_status=StateOperationStatus.SUCCEEDED, - expected_event_rows=( - ("start", StateOperationStatus.RUNNING, "starting detach"), - ("finish", StateOperationStatus.SUCCEEDED, "detached 1 models"), - ), - ) - ], - ids=lambda case: case.description, -) -def test_given_existing_operation_when_recording_follow_up_event_then_transition_is_preserved( - test_case: RecordStateOperationTestCase, -) -> None: - backend: RecordingStateBackend = RecordingStateBackend() - - record_state_operation( - backend=cast(StateBackend, backend), - connection=None, - schema="sqlbuild_state", - operation_id=test_case.operation_id, - operation_type=test_case.operation_type, - status=StateOperationStatus.RUNNING, - action="start", - virtual_environment_name=test_case.virtual_environment_name, - message=test_case.start_message, - ) - record_state_operation( - backend=cast(StateBackend, backend), - connection=None, - schema="sqlbuild_state", - operation_id=test_case.operation_id, - operation_type=None, - status=StateOperationStatus.SUCCEEDED, - action="finish", - virtual_environment_name=None, - message=test_case.finish_message, - ) - - assert backend.operation == StateOperationRecord( - operation_id=test_case.operation_id, - operation_type=test_case.operation_type, - status=test_case.expected_final_status, - virtual_environment_name=test_case.virtual_environment_name, - ) - assert ( - tuple((event.action, event.status, event.message or "") for event in backend.events) - == test_case.expected_event_rows - ) diff --git a/uv.lock b/uv.lock index 9e25e68080..d96e4a64b6 100644 --- a/uv.lock +++ b/uv.lock @@ -2866,15 +2866,15 @@ wheels = [ [[package]] name = "polyglot-sql-chio" -version = "0.12.3" +version = "0.13.8" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/de/22/176f03abf1ed12718817e0b23958bc620792e7118272133fff5eaf65dd98/polyglot_sql_chio-0.12.3.tar.gz", hash = "sha256:2838af4dc1e318c56c2e39b35b0a87c68ef776114f09a9f183c54738a4b8efcb", size = 2004064, upload-time = "2026-09-23T05:37:32.777Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/21/dcacff0ca62efb8ed3b6e238e60f17c6d2de46347817d8326eeaccfc8f01/polyglot_sql_chio-0.13.8.tar.gz", hash = "sha256:dc0ac2278546da4008a8c73774723746ebc92caa35fa37f3c26d33dcfe8351cf", size = 2103046, upload-time = "2026-09-27T10:58:10.281Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/e9/14fc347556bef588dff15b12482891c712b3ce0a2ac5cb4e8ce32a268163/polyglot_sql_chio-0.12.3-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:17aa5d9b6e40965db1d42a1ce697522dc9db4ae90c88848c9f7da068f092807e", size = 12170576, upload-time = "2026-09-23T05:37:20.162Z" }, - { url = "https://files.pythonhosted.org/packages/4e/88/cf0f47bd5b6f855aae33fcc7aeafa09f82669e9facbd0d6e31fa85736012/polyglot_sql_chio-0.12.3-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:d3fb7bd68b857ca29bec17058c18e08a2296e62a87560d54b22ba7f83e14aee2", size = 11462028, upload-time = "2026-09-23T05:37:23.018Z" }, - { url = "https://files.pythonhosted.org/packages/64/d7/e47be834ecdd3c1cb2e70683f7cf0ce512edb85511e009912d160478c1ee/polyglot_sql_chio-0.12.3-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a4a8254389d2a07cbf14726181cf9a1a35d6e3e9127745b5fb6825791e14a2d5", size = 12313913, upload-time = "2026-09-23T05:37:25.499Z" }, - { url = "https://files.pythonhosted.org/packages/51/3c/2f33dad022391d5784ece619456807264344a71299619473a37c1902051b/polyglot_sql_chio-0.12.3-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75a69bc3d44cb63fd08c0b6f0d4d2841849dc7c3c65d7d846d3ec2a68d3960ed", size = 11334274, upload-time = "2026-09-23T05:37:28.409Z" }, - { url = "https://files.pythonhosted.org/packages/b0/ba/4d5afc99b280119dcfac07c3fdf54132a7c3f5c066a853836ba8edc8566c/polyglot_sql_chio-0.12.3-cp312-abi3-win_amd64.whl", hash = "sha256:3d4d3f9b84a0fb26a6705c0950d679d13a0844efffab0564ef3fc5e0eaa4988d", size = 11127160, upload-time = "2026-09-23T05:37:30.804Z" }, + { url = "https://files.pythonhosted.org/packages/25/42/b85b763f848761d32c71c74b84a255b6bb650c7aa371d262fefa668ab917/polyglot_sql_chio-0.13.8-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8fc651ea987e204ddc7e7373199fcda01b60b411f9c08c05701fa33c2401ded6", size = 12726373, upload-time = "2026-09-27T10:57:58.775Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f5/46e52ae235c5528dea8216ae1f34118e6eea74871e2dda9306a132ad3125/polyglot_sql_chio-0.13.8-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:7bfc7a5b0faf0837fe05b6fb58f77908559a98e4fe4df194ba7861c0b2b365e5", size = 11952980, upload-time = "2026-09-27T10:58:01.445Z" }, + { url = "https://files.pythonhosted.org/packages/b7/7d/fc30adbc583dc3f02834ed14392bd91a05fb00e827d3fccfe1f11fe14274/polyglot_sql_chio-0.13.8-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193a69cecc4d09c6c354e9e01e3c5eaae197e3058717bd7c83805744726064a7", size = 12864167, upload-time = "2026-09-27T10:58:03.921Z" }, + { url = "https://files.pythonhosted.org/packages/cf/44/008c6f32fd65f568a88aaff99de00619ae8a62ca8f819625a88a5d5dee46/polyglot_sql_chio-0.13.8-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a16fc856425a9905b1a55a07d7a44f2fbe7c09f0bc64efed85b41a8b490a57e", size = 11929491, upload-time = "2026-09-27T10:58:06.019Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d2/3f669e9fbc40236b15a750f531579a1af9b935963f949ae26fb8bec2a442/polyglot_sql_chio-0.13.8-cp312-abi3-win_amd64.whl", hash = "sha256:956c67ab78b470cb4cb320e290d7a1b3a987463e720667c7a71980efb34ea0d9", size = 11692992, upload-time = "2026-09-27T10:58:08.175Z" }, ] [[package]] @@ -4218,7 +4218,7 @@ requires-dist = [ { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.0.0" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, { name = "orjson", specifier = ">=3.10.0" }, - { name = "polyglot-sql-chio", specifier = "==0.12.3" }, + { name = "polyglot-sql-chio", specifier = "==0.13.8" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2.0" }, { name = "pydantic-settings", specifier = ">=2,<3" }, { name = "pymssql", marker = "extra == 'sqlserver'", specifier = ">=2.3.0" }, diff --git a/website/src/content/docs/docs/cli/scenario.mdx b/website/src/content/docs/docs/cli/scenario.mdx index 1956c499f1..46a717ec17 100644 --- a/website/src/content/docs/docs/cli/scenario.mdx +++ b/website/src/content/docs/docs/cli/scenario.mdx @@ -22,6 +22,7 @@ sqb scenario test [flags] | `--select`, `-s` | Select scenarios to run | | `--exclude` | Exclude scenarios from the selection | | `--retain` | Keep scenario-owned warehouse artifacts for inspection | +| `--scenario-namespace NAMESPACE` | Isolate warehouse artifacts for this run; overrides environment and config | | `--local` | Run locally against DuckDB using captured JSONL snapshots | | `--strict` | Treat missing/stale local snapshots as errors instead of skips | | `--sync-snapshots` | Capture missing/stale snapshots before local run (requires `--local`) | @@ -115,6 +116,7 @@ sqb scenario capture [flags] | `--select`, `-s` | Select scenarios to capture | | `--exclude` | Exclude scenarios from the selection | | `--retain` | Keep warehouse fixture artifacts after capture | +| `--scenario-namespace NAMESPACE` | Isolate warehouse artifacts for this capture; overrides environment and config | | `--force` | Bypass snapshot capture safety limits | | `--max-snapshot-rows` | Override per-relation row limit | | `--max-snapshot-total-rows` | Override total row limit | @@ -139,6 +141,11 @@ Snapshots are written to `tests/_scenario_snapshots//` with a `sc ## Runtime artifacts +Both subcommands report the effective namespace and its source. `scenario test --json` includes +`execution.scenario_namespace` (null when unset) and `execution.scenario_namespace_source` +(`cli`, `env`, `local config`, `project config`, or `unset`). Progress goes to stderr in JSON mode. +See [parallel CI runs](/docs/concepts/scenarios#parallel-ci-runs) for namespace configuration and cleanup. + Both remote and local scenario runs write runtime artifacts to `target/run/scenarios//`: ``` diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index 4f6bd23352..717f9c53f5 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -41,7 +41,7 @@ configuration. | Field | Description | |-------|-------------| -| `run_despite_unchanged` | Table-only change-aware policy. `always` rebuilds whenever selected. A duration such as `30d`, `12h`, or `90m` rebuilds while the newest timestamp-based upstream source observation is within that age; it is not a periodic schedule. | +| `run_despite_unchanged` | Removed with virtual environments. It is rejected in MODEL headers, project defaults, and path defaults. | Table promotion mode is a project setting rather than a `MODEL()` field. Staged promotion is the default. Immediate promotion is incompatible with model type enforcement and exact contracts; see [Materializations](/docs/concepts/models/materializations#table). diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 545e3478f2..85a4cb5534 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -444,9 +444,12 @@ See [janitor](/docs/cli/janitor) for the archive and delete lifecycle. ## Scenario -Configuration for scenario snapshot capture safety limits and local type overrides: +Configuration for scenario run isolation, snapshot capture safety limits, and local type overrides: ```toml +[scenario] +run_namespace = "orders-job-123" # Optional; omit to preserve existing scenario relation names + [scenario.snapshot_limits] max_rows_per_relation = 10000 max_total_rows = 50000 @@ -457,6 +460,7 @@ max_total_bytes = 52428800 | Field | Default | Description | |-------|---------|-------------| | `max_rows_per_relation` | none | Maximum rows per captured relation | +| `scenario.run_namespace` | unset | Optional 1–128-character namespace using ASCII letters, digits, `-`, `_`, or `.`. CLI `--scenario-namespace` overrides `SQLBUILD_SCENARIO_NAMESPACE`, then local config, then project config. | | `max_total_rows` | none | Maximum total rows across all relations in one scenario | | `max_bytes_per_relation` | none | Maximum JSONL bytes per relation file | | `max_total_bytes` | none | Maximum total JSONL bytes per scenario | diff --git a/website/src/content/docs/docs/concepts/scenarios.mdx b/website/src/content/docs/docs/concepts/scenarios.mdx index bd38f97f2d..94526bed1c 100644 --- a/website/src/content/docs/docs/concepts/scenarios.mdx +++ b/website/src/content/docs/docs/concepts/scenarios.mdx @@ -133,13 +133,54 @@ Scenario artifacts are physically isolated from production: ### Execution flow -1. Clean any existing scenario artifacts from a previous run +1. Clean any existing scenario artifacts from a previous run in the effective run namespace 2. Materialize source, ref, and seed fixtures as physical tables 3. Load required project seeds 4. Build required models in dependency order (incremental models run as full-refresh in scenarios) 5. Run expected-output comparisons (order-insensitive) 6. Run zero-row assertions -7. Clean up all scenario-owned artifacts (unless `--retain`) +7. Clean up this namespace's scenario-owned artifacts (unless `--retain`) + +### Parallel CI runs + +Give each concurrent CI job a unique scenario run namespace when jobs share a warehouse schema: + +```bash +export SQLBUILD_SCENARIO_NAMESPACE="ci-${CI_RUN_ID}-${CI_JOB_ID}" +sqb scenario test +# The same option applies to warehouse fixture capture: +sqb scenario capture --scenario-namespace "ci-${CI_RUN_ID}-${CI_JOB_ID}" +``` + +Use a unique job/run identifier, including an attempt or matrix identifier when necessary. +A pull-request identifier is suitable only when there is at most one active run for that identifier. +Concurrent runs with the **same namespace** are unsupported; namespaces do not add locking. +Keep separate working directories for concurrent jobs: namespaces isolate warehouse relation names, +not local snapshot files or `target/run/scenarios/` output paths. + +The precedence is `--scenario-namespace` > `SQLBUILD_SCENARIO_NAMESPACE` > +`sqlbuild_local.toml` > `sqlbuild_project.toml`. Both config files accept: + +```toml +[scenario] +run_namespace = "orders-job-123" +``` + +Values must contain 1–128 ASCII letters, digits, hyphens, underscores, or periods. +Empty or invalid values fail before warehouse work; values are case-sensitive and are not trimmed. +The effective namespace and its source appear in run output, including JSON output. + +With a namespace, the artifact prefix is the first 12 hexadecimal characters of +SHA-256 of `project_name:namespace:scenario_name`. Only the hash enters identifiers, so identifier +length limits are unchanged. Without a namespace, SQLBuild keeps the existing hash of +`project_name:scenario_name` and all existing physical names. Prefix collisions are checked among +the project's scenarios within the effective namespace. + +Preparation, normal teardown, and failure cleanup affect only the current namespace's relations. +`--retain` keeps that namespace's artifacts for inspection. The janitor still recognizes artifacts +from every namespace, including crashed CI jobs, and applies the existing age/retention policy +without needing their namespace values. Choose retention longer than active scenario jobs. +For SQLBuild versions without namespace support, use a separate warehouse schema per CI job. ### Inspecting with `--retain` diff --git a/website/src/content/docs/docs/concepts/semantic-compilation.mdx b/website/src/content/docs/docs/concepts/semantic-compilation.mdx new file mode 100644 index 0000000000..b5ed1f3068 --- /dev/null +++ b/website/src/content/docs/docs/concepts/semantic-compilation.mdx @@ -0,0 +1,181 @@ +--- +title: "Semantic Compilation" +description: "Offline reference, semantic and type checks that sqb compile runs before any warehouse work." +--- + +`sqb compile` stays offline. It validates SQL references against each input separately, including +models with no relation inputs. A join to an open source does not disable checks on a closed input. +An unqualified name that could belong to an open input cannot be rejected offline. + +## Authoritative column names + +- Enforced model and source contracts, seeds, and declared table-function outputs are closed shapes. +- Expression sources have the column names inferred from their SQL expression. +- An explicit model projection defines its output names even over an open input. Types propagate + through dependencies where they can be inferred; an unknown type does not make the names unknown. +- A star over closed inputs expands; a star over an open input remains open. +- Declaring some columns on a table source does **not** declare its complete shape. Use + `contract: enforced` when those declarations are complete. + +Adopt a physical source schema with: + +```bash +sqb contract generate --from --select source:raw_orders --write +``` + +`sqb plan` and `sqb build` recheck source readers against inspected physical columns before model +execution. Existing bulk metadata is reused. Selected table sources omitted from that metadata +are inspected in database-grouped batches: one relation-list call and one bulk-column call per +database, using the adapter's normal metadata batching. Missing managed sources and ambiguous +unqualified physical names stay open. + +## Checks and limits + +Binding checks apply to SQL clauses, CTEs, subqueries, and expanded macros. Semantic grouping, +aggregate and window-placement diagnostics are errors. Project UDF arity and known argument types +are checked separately; declared return types contribute to inference. SQL and Python UDFs and +table functions are registered with the native catalogue, along with types named in project +contracts and function signatures. Registration does not replace SQLBuild's signature checks. + +Known output shapes also validate `unique_key`, `cursor`, `cursor_type`, `cursor_inputs`, custom +`config.partition_column`, `row_diff_exclude_columns`, `row_diff_tolerances.by_column`, source +`cursor_column`, audit expressions/relationships/accepted values, and SQL-test mock/expected names. + +Expression type checks are enabled for **DuckDB (including MotherDuck), PostgreSQL, Snowflake, +and BigQuery**. Other adapters retain reference and semantic checks without expression type checks. + +Proven bind-time rejections and implicit conversions are both errors. For example, DuckDB rejects +`ordered_at > 5` for a TIMESTAMP column. Snowflake accepts `order_id = customer_code` for an +INTEGER and a VARCHAR column, but the conversion fails on non-numeric values, so SQLBuild reports +it as a blocking B217 error; write the conversion explicitly. Typed string literals such as +`ordered_at > '2026-01-01'` remain valid. Columns from `SELECT *` are type-checked through their +upstream shapes in every query. Unknown input types never cause type errors. Dialect coercions +differ: numeric predicates are accepted by DuckDB but rejected by PostgreSQL, Snowflake, and +BigQuery. +Snowflake's function catalogue is partial and deliberately does not reject unknown function names. +Function signatures and overload coverage remain partial; compilation is not a replacement for +executing SQL unit tests. + +Derived-table column aliases, UNPIVOT, and Snowflake GROUPING SETS are checked. DuckDB and Snowflake +reject excess relation aliases; fewer aliases than output columns remain valid. Missing UNPIVOT +input columns are errors. + +Project function registration prevents unknown-function errors but does not declare return types to +the validator: some expressions consuming a project function result remain unchecked. Raw external +relations and scope-ordering cases also remain +partially checked. A cast to a type SQLBuild does not know is a B218 error. Types named in project +contracts and function signatures are registered, so declare an extension-defined type there. + +One notice summarizes partial semantic checks. `sqb compile --json` includes per-model reasons in +`semantic_checks_partial`; progress and notices go to stderr. The notice does not increment the +diagnostic warning count. + +Escape hatches are `MODEL (sql_analysis false)`, the corresponding path default, and +`--no-sql-analysis`. Disabling SQL analysis also disables inference and lineage for that model. + +## Reading diagnostics + +Diagnostics use one layout: headline, resource and location, source snippet with a span underline, +notes, then code-specific help. SQL-test diagnostics include their authored fixture snippet and +use `sql test: name`, just as model diagnostics use `model: name`. JSON retains the location's +exclusive `end_line`/`end_column` boundary, notes, and help. + +An unknown column identifies its input and authored alias. Suggestions use edit distance with a +bounded typo cutoff and recognize short abbreviations such as `qty`. No suggestion is printed when +none is close. The available-column note puts the closest names first and shows at most ten names. + +```text +error[B002]: Unknown column 'qty' in stg_orders (as o) + model: fact_orders + --> models/marts/fact_orders.sql:16:5 + | + 16 | o.qty, + | ^^^ + = note: stg_orders has: quantity, status, waffle_type_id, customer_id, order_id, ordered_at + = note: 2 downstream uses of fact_orders.quantity were not checked because of this error + = help: did you mean 'quantity'? +``` + +Type errors report uppercase types and available operand evidence, for example +`o.ordered_at is TIMESTAMP, 5 is INTEGER`. Comparison help recommends a literal appropriate to the +type family and dialect. Native spans are mapped back through normalization and macro expansion; +simple binary expressions are underlined in full. Diagnostics without usable spans keep the +authored query-start fallback. + +Error recovery treats poisoned output names and their downstream lineage as provisional. A +downstream error traceable only to that root is suppressed and counted in a root-error note; +independent errors on other columns and other inputs remain errors. Recovery does not silence a +whole dependent query or make its unchecked uses valid. Affected models remain in +`semantic_checks_partial`, including on warm/cache-hit runs. + +Located binding and type errors also make the affected projection's type unknown. That uncertainty +propagates through column lineage. Failed dependants are checked again with only those input types +unknown; an independent error on another column or input remains visible. Root notes count unchecked +downstream output uses, including transitive uses. Contract type checks do not emit a secondary +unknown-type warning for those same poisoned outputs. If native evidence cannot locate the failing +projection, recovery retains the diagnostic rather than opening the entire input. + +Temporal cursor metadata accepts DATE, DATETIME, and TIMESTAMP families interchangeably, including +Snowflake timestamp variants. A timestamp cursor on a string, numeric, or Boolean column still +produces B301. + +Snowflake binding honours `QUOTED_IDENTIFIERS_IGNORE_CASE = true` from the effective target's +connection `session_parameters`, for both offline declared schemas and warehouse-backed rebinding. +False or absent settings preserve the dialect's existing case rules. For example: + +```toml +[targets.analytics] +schema = "preserve" + +[targets.analytics.connection] +session_parameters = { QUOTED_IDENTIFIERS_IGNORE_CASE = true } +``` + +The setting participates in the analysis-cache identity. Identifier folding is token-based and +does not alter string literals, comments, or authored SQL. Diagnostic offset mapping is lazy and +uses a bounded cache of token alignments, shared by all diagnostics on the same normalized query. + +Snowflake set-operation checks are directional: each branch is compared with the accumulated +result type, following the query's operator precedence (`INTERSECT` binds more tightly than +`UNION` and `EXCEPT`). Proven incompatibilities and implicit conversions whose success depends on +runtime values both produce blocking B215 errors. For example, combining a VARCHAR branch with a +numeric branch requires a value-dependent numeric conversion; cast the branch column explicitly. + +An unmodelled Snowflake function has an unknown result type rather than inheriting its first +argument's type. SQLBuild retains the output name and lineage, reports partial type coverage, and +uses explicit declared function signatures where available. This prevents a guessed return type +from creating downstream type errors. + +The single human partial-check notice groups model names by reason, gives the relevant repair or +rerun command, and truncates long lists. `sqb compile --json` contains the full selected-model +reason map. Unknown output types and unresolved stars are reported as explicitly as open sources; +the notice never reduces them to a count alone. + +## Diagnostic codes + +| SQLBuild | Native | Meaning | +|---|---|---| +| B002 | E201 | Unknown column | +| B003 | E221 | Ambiguous column | +| B004–B005 | E222–E223 | Reference/scope errors | +| B101 | E202 | Unknown function | +| B102 | E203 or project declaration | Invalid function arity | +| B210–B219 | E210–E219 | Proven native type/shape errors on enabled dialects | +| B211–B218 | W210–W216 | Implicit conversions; blocking, fixed by an explicit `CAST` or comparison | +| B216 | E216 | Set-operation, subquery, or row arity | +| B230–B232 | E230–E232 | Grouping, aggregate, and window semantics | +| B233 | E233 | Duplicate scope names, according to dialect rules | +| B234 | E234 | Invalid LIMIT/OFFSET bounds | +| B300 | SQLBuild | Unknown metadata/audit column | +| B301 | SQLBuild | Incompatible declared metadata, audit, or UDF argument type | +| B302 | SQLBuild | Unknown SQL-test fixture/expected column | + +Implicit conversions are enforced rather than reported as warnings. Each native W210–W216 +finding maps to the SQLBuild code of the incompatibility it risks: comparisons to B217, arithmetic +to B212, assignments to B214, casts and coercible literals to B218, set-operation branches to B215, +predicates to B211, and function arguments to B213. The remedy is always an explicit conversion, +for example `order_id = CAST(customer_code AS INTEGER)`, or an explicit comparison such as +`quantity <> 0`. There is no per-model suppression. + +B002–B005 predate these changes and overlap the executor's separately phased build codes. New +semantic codes use distinct ranges. Authored model diagnostics retain source paths and positions. diff --git a/website/src/sidebar.json b/website/src/sidebar.json index d3eb5201b1..433c45edab 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -22,6 +22,7 @@ "items": [ "docs/concepts/project-configuration", "docs/concepts/resource-identities", + "docs/concepts/semantic-compilation", { "label": "Adapters", "collapsed": true,