From c9d31bdf5603537121fac9b127e4b70e94ff66d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:43:25 +0100 Subject: [PATCH 01/39] fix: release the compute log lock before rejecting a write to a finished capture A write that reached a capture already marked complete built its rejection while still holding the storage lock. Building it imports Python, which can release the GIL; the main thread then pruning the logs waited for that lock while holding the GIL, so neither could continue and the command never exited. The lock is now released before the rejection is raised. --- .../bindings/_helpers/compute_logs/storage.rs | 2 + .../sqlbuild/runtime/compute_logs/__init__.py | 0 .../runtime/compute_logs/_test_types.py | 14 ++ .../test_storage_lock_without_python.py | 130 ++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py create mode 100644 tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py create mode 100644 tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs index 02bc5c5556..40b859097c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs @@ -90,6 +90,7 @@ fn exact_count(value: &Bound<'_, PyAny>, fallback: i64) -> i64 { } impl LocalFilesystemComputeLogStorage { + /// Callers wait for this lock holding the GIL, so never run Python while holding it. fn locked(&self) -> PyResult> { self.state .lock() @@ -116,6 +117,7 @@ impl LocalFilesystemComputeLogStorage { ) -> PyResult<()> { let mut state = self.locked()?; let Some(writers) = state.writers.get_mut(invocation_id) else { + drop(state); return Err(not_active(py)); }; writers[stream].write_all(data)?; diff --git a/tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py b/tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py b/tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py new file mode 100644 index 0000000000..d145089911 --- /dev/null +++ b/tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py @@ -0,0 +1,14 @@ +"""Test case types for compute log storage integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class StorageLockTestCase: + """A Python program making concurrent storage calls from two threads, and what it prints.""" + + description: str + script: str + expected_stdout: str diff --git a/tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py b/tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py new file mode 100644 index 0000000000..5e9370ac95 --- /dev/null +++ b/tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py @@ -0,0 +1,130 @@ +"""Compute log storage never runs Python while holding its lock, so it cannot deadlock on the GIL.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.integration.src.sqlbuild.runtime.compute_logs._test_types import ( + StorageLockTestCase, +) + +_DEADLOCK_TIMEOUT_SECONDS: int = 30 + +# A late write from a background thread reaches a capture that was already marked complete. +# Raising its rejection imports the storage exceptions, here forced to load afresh, and the +# import's file system calls release the GIL. The finder below holds that import open until the main thread is +# pruning, which takes the storage lock while holding the GIL. +_LATE_APPEND_DURING_PRUNE: str = dedent( + """ + import dataclasses + import sys + import threading + import time + from datetime import UTC, datetime + from pathlib import Path + + from sqlbuild.compute_logs import ( + COMPUTE_LOG_FORMAT_VERSION, + CaptureMetadata, + ComputeLogStream, + FinalCaptureMetadata, + LocalFilesystemComputeLogStorage, + ) + + EXCEPTIONS_MODULE = "sqlbuild.runtime.compute_logs.exceptions" + importing = threading.Event() + + + class SlowExceptionsImport: + def find_spec(self, name, path=None, target=None): + if name == EXCEPTIONS_MODULE and not importing.is_set(): + importing.set() + time.sleep(0.5) + return None + + + project_dir = Path(sys.argv[1]) + started_at = datetime.now(UTC) + metadata = CaptureMetadata( + format_version=COMPUTE_LOG_FORMAT_VERSION, + invocation_id="orders0000000000000000000000000a", + command="compile", + project_dir=str(project_dir), + started_at=started_at, + capture_date=started_at.date().isoformat(), + ) + storage = LocalFilesystemComputeLogStorage(project_dir=project_dir) + storage.start_capture(metadata) + storage.mark_complete( + FinalCaptureMetadata( + **dataclasses.asdict(metadata), + completed_at=datetime.now(UTC), + exit_code=1, + stdout_bytes=0, + stderr_bytes=0, + diagnostics_bytes=0, + ) + ) + rejected = [] + del sys.modules[EXCEPTIONS_MODULE] + sys.meta_path.insert(0, SlowExceptionsImport()) + + + def late_append(): + try: + storage.append( + invocation_id=metadata.invocation_id, + stream=ComputeLogStream.STDERR, + data=b"late customer export notice\\n", + ) + except Exception as error: + rejected.append(type(error).__name__) + + + writer = threading.Thread(target=late_append) + writer.start() + importing.wait() + storage.prune() + writer.join() + print(rejected) + """ +) + + +@pytest.mark.parametrize( + "test_case", + ( + StorageLockTestCase( + description="late append rejected while the main thread prunes", + script=_LATE_APPEND_DURING_PRUNE, + expected_stdout="['CaptureStateError']\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_rejected_append_releasing_gil_when_pruning_then_both_threads_finish( + test_case: StorageLockTestCase, tmp_path: Path +) -> None: + try: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", test_case.script, str(tmp_path)], + capture_output=True, + text=True, + timeout=_DEADLOCK_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"storage calls deadlocked for {_DEADLOCK_TIMEOUT_SECONDS}s") + + assert (completed.returncode, completed.stdout) == (0, test_case.expected_stdout), ( + completed.stderr + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From c10b78caccc50c2e4bf95205f57760584f33835f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:43:25 +0100 Subject: [PATCH 02/39] test: fail a hung E2E command with every thread's stack instead of stalling the job run_sqb now gives each command ten minutes. A command still running then has its process group aborted with Python's fault handler enabled, so the failing test reports every thread's stack, and the group is killed. Standard input ends at once when a test passes none. --- Makefile | 3 +- .../cli/commands/shared/_test_types.py | 24 +++++ .../sqlbuild/cli/commands/shared/helpers.py | 78 +++++++++++++-- .../shared/test_run_with_hang_dump.py | 95 +++++++++++++++++++ 4 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py diff --git a/Makefile b/Makefile index efba5d41b3..c1725a5a3d 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,8 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/snowflake \ tests/e2e/src/sqlbuild/cli/commands/main/sqlite \ tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \ - tests/e2e/src/sqlbuild/cli/commands/main/test + tests/e2e/src/sqlbuild/cli/commands/main/test \ + tests/e2e/src/sqlbuild/cli/commands/shared E2E_DUCKDB_INSPECTION_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/clone \ diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py new file mode 100644 index 0000000000..08d61d09e7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py @@ -0,0 +1,24 @@ +"""Test case types for the shared E2E command helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class HangDumpTestCase: + """A Python command that never finishes and what its hang report must show.""" + + description: str + script: str + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class FinishedCommandTestCase: + """A Python command that finishes and the result the helper must return for it.""" + + description: str + script: str + input_text: str | None + expected_result: tuple[int, str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py index 7b39be0dc4..d4cf1f3b99 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py @@ -6,6 +6,8 @@ import os import pty import re +import shlex +import signal import struct import subprocess import termios @@ -22,6 +24,10 @@ SOURCE_LOADER_STRATEGIES_DIR: Path = ( REPO_ROOT / "tests" / "e2e" / "fixtures" / "source_loader_strategies" ) +# Far above any healthy command, so only a hang trips it: one test fails instead of the CI job. +SQB_COMMAND_TIMEOUT_SECONDS: float = 600.0 +_HANG_DUMP_GRACE_SECONDS: float = 30.0 +_HANG_OUTPUT_TAIL_CHARACTERS: int = 20_000 def prepare_waffle_shop(tmp_path: Path) -> Path: @@ -208,18 +214,78 @@ def run_sqb( process_env: dict[str, str] = dict(os.environ) process_env.update(env or {}) + process_env.setdefault("PYTHONFAULTHANDLER", "1") - return subprocess.run( - ["uv", "run", "sqb", "--project-dir", str(project_dir), *command], + return run_with_hang_dump( + argv=("uv", "run", "sqb", "--project-dir", str(project_dir), *command), cwd=working_dir or REPO_ROOT, - capture_output=True, - text=True, - input=input_text, env=process_env, - check=False, + input_text=input_text, + timeout_seconds=SQB_COMMAND_TIMEOUT_SECONDS, + ) + + +def run_with_hang_dump( + *, + argv: tuple[str, ...], + cwd: Path, + env: Mapping[str, str], + input_text: str | None, + timeout_seconds: float, +) -> subprocess.CompletedProcess[str]: + """Run `argv` to completion, or fail with every Python thread's stack once it hangs. + + Standard input carries `input_text`, or ends at once without it. The command runs in its own + process group. Past `timeout_seconds` the whole group receives + SIGABRT, so Python processes started with `PYTHONFAULTHANDLER` print all thread stacks to + stderr before they exit, and the group is then killed. + """ + + process: subprocess.Popen[str] = subprocess.Popen( + argv, + cwd=cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=dict(env), + process_group=0, + ) + try: + stdout, stderr = process.communicate(input=input_text, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + stdout, stderr = _dump_and_kill_group(process=process) + raise AssertionError( + f"{shlex.join(argv)} did not finish within {timeout_seconds:.0f}s; " + "its process group was aborted to dump thread stacks.\n" + f"--- stdout (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n" + f"{stdout[-_HANG_OUTPUT_TAIL_CHARACTERS:]}\n" + f"--- stderr (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n" + f"{stderr[-_HANG_OUTPUT_TAIL_CHARACTERS:]}" + ) from None + return subprocess.CompletedProcess( + args=list(argv), returncode=process.returncode, stdout=stdout, stderr=stderr ) +def _dump_and_kill_group(*, process: subprocess.Popen[str]) -> tuple[str, str]: + _signal_group(process_group=process.pid, signal_number=signal.SIGABRT) + try: + stdout, stderr = process.communicate(timeout=_HANG_DUMP_GRACE_SECONDS) + except subprocess.TimeoutExpired: + _signal_group(process_group=process.pid, signal_number=signal.SIGKILL) + stdout, stderr = process.communicate() + _signal_group(process_group=process.pid, signal_number=signal.SIGKILL) + return stdout or "", stderr or "" + + +def _signal_group(*, process_group: int, signal_number: signal.Signals) -> None: + try: + os.killpg(process_group, signal_number) + except ProcessLookupError: + return + + def run_sqb_with_pty( *, command: tuple[str, ...], diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py b/tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py new file mode 100644 index 0000000000..0a2052dbcb --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py @@ -0,0 +1,95 @@ +"""A hung command fails its own test with every Python thread's stack instead of hanging the run.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.shared._test_types import ( + FinishedCommandTestCase, + HangDumpTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_with_hang_dump + + +@pytest.mark.parametrize( + "test_case", + [ + HangDumpTestCase( + description="main thread joins a worker blocked forever", + script=( + "import threading\n" + "def blocked_worker_wait():\n" + " threading.Event().wait()\n" + "def joining_main_thread():\n" + " worker = threading.Thread(target=blocked_worker_wait)\n" + " worker.start()\n" + " print('started', flush=True)\n" + " worker.join()\n" + "joining_main_thread()\n" + ), + expected_fragments=("started", "blocked_worker_wait", "joining_main_thread"), + ), + ], + ids=lambda case: case.description, +) +def test_given_hung_python_command_when_running_with_hang_dump_then_fails_with_thread_stacks( + test_case: HangDumpTestCase, tmp_path: Path +) -> None: + started: float = time.monotonic() + with pytest.raises(AssertionError) as raised: + run_with_hang_dump( + argv=(sys.executable, "-c", test_case.script), + cwd=tmp_path, + env={**os.environ, "PYTHONFAULTHANDLER": "1"}, + input_text=None, + timeout_seconds=2.0, + ) + + message: str = str(raised.value) + assert "did not finish within 2s" in message + assert [fragment in message for fragment in test_case.expected_fragments] == [True] * len( + test_case.expected_fragments + ), message + assert time.monotonic() - started < 30.0 + + +@pytest.mark.parametrize( + "test_case", + [ + FinishedCommandTestCase( + description="input reaches the command and its exit code and output return", + script="import sys; sys.stdout.write(sys.stdin.read().upper()); sys.exit(3)", + input_text="orders\n", + expected_result=(3, "ORDERS\n", ""), + ), + FinishedCommandTestCase( + description="without input the command reads end of input", + script="import sys; print(repr(sys.stdin.read()))", + input_text=None, + expected_result=(0, "''\n", ""), + ), + ], + ids=lambda case: case.description, +) +def test_given_finishing_command_when_running_with_hang_dump_then_returns_its_result( + test_case: FinishedCommandTestCase, tmp_path: Path +) -> None: + completed: subprocess.CompletedProcess[str] = run_with_hang_dump( + argv=(sys.executable, "-c", test_case.script), + cwd=tmp_path, + env=dict(os.environ), + input_text=test_case.input_text, + timeout_seconds=60.0, + ) + + assert (completed.returncode, completed.stdout, completed.stderr) == test_case.expected_result + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 3deb6a2121b6412e614026c8592a0b14443e9a67 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:10 +0100 Subject: [PATCH 03/39] refactor: share project scaffolding and rule listing text outside the Python bindings --- .../sqlbuild-core/src/init_command/errors.rs | 11 ++++ .../src/init_command/main/mod.rs | 1 + .../main/scaffold_blank_project.rs | 57 +++++++++++++++++ crates/sqlbuild-core/src/init_command/mod.rs | 1 + .../_helpers/init_command/scaffold.rs | 51 ++------------- .../bindings/_helpers/plan_command/rules.rs | 64 ++++++++----------- .../src/rule_catalogue/main/mod.rs | 1 + .../rule_catalogue/main/render_rule_list.rs | 44 +++++++++++++ .../src/rule_catalogue/models.rs | 11 ++++ fensu.toml | 2 +- 10 files changed, 157 insertions(+), 86 deletions(-) create mode 100644 crates/sqlbuild-core/src/init_command/errors.rs create mode 100644 crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs diff --git a/crates/sqlbuild-core/src/init_command/errors.rs b/crates/sqlbuild-core/src/init_command/errors.rs new file mode 100644 index 0000000000..752e7680d6 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/errors.rs @@ -0,0 +1,11 @@ +//! Why a blank project could not be created. + +use std::io; +use std::path::PathBuf; + +/// The I/O failure and the path it happened at, as Python's `OSError` names its `filename`. +#[derive(Debug)] +pub struct ScaffoldError { + pub error: io::Error, + pub path: PathBuf, +} diff --git a/crates/sqlbuild-core/src/init_command/main/mod.rs b/crates/sqlbuild-core/src/init_command/main/mod.rs index b8eae0db00..815db5c0cc 100644 --- a/crates/sqlbuild-core/src/init_command/main/mod.rs +++ b/crates/sqlbuild-core/src/init_command/main/mod.rs @@ -1,3 +1,4 @@ pub mod init_project_name; pub mod init_project_toml; pub mod render_init_summary; +pub mod scaffold_blank_project; diff --git a/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs b/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs new file mode 100644 index 0000000000..eb9c89ff78 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs @@ -0,0 +1,57 @@ +//! Create the blank project tree as Python's `pathlib` calls did. + +use std::fs::{self, OpenOptions}; +use std::io::ErrorKind; +use std::path::Path; + +use crate::constants::PROJECT_CONFIG_FILENAME; +use crate::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; +use crate::init_command::errors::ScaffoldError; + +fn failed(error: std::io::Error, path: &Path) -> ScaffoldError { + ScaffoldError { + error, + path: path.to_path_buf(), + } +} + +/// `Path.mkdir(parents=True, exist_ok=True)`. +fn make_directories(path: &Path) -> Result<(), ScaffoldError> { + match fs::create_dir(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => { + let Some(parent) = path.parent().filter(|parent| *parent != path) else { + return Err(failed(error, path)); + }; + make_directories(parent)?; + match fs::create_dir(path) { + Err(error) if !path.is_dir() => Err(failed(error, path)), + _ => Ok(()), + } + } + Err(_) if path.is_dir() => Ok(()), + Err(error) => Err(failed(error, path)), + } +} + +/// Create the resource directories, the project file, and `.gitkeep` in empty directories. +pub fn scaffold_blank_project(base_dir: &Path, project_toml: &str) -> Result<(), ScaffoldError> { + for directory in SCAFFOLD_DIRECTORIES { + make_directories(&base_dir.join(directory))?; + } + let project_file = base_dir.join(PROJECT_CONFIG_FILENAME); + fs::write(&project_file, project_toml).map_err(|error| failed(error, &project_file))?; + for directory in SCAFFOLD_DIRECTORIES { + let directory = base_dir.join(directory); + let mut entries = fs::read_dir(&directory).map_err(|error| failed(error, &directory))?; + if entries.next().is_none() { + let gitkeep = directory.join(GITKEEP_FILE); + OpenOptions::new() + .create(true) + .append(true) + .open(&gitkeep) + .map_err(|error| failed(error, &gitkeep))?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/init_command/mod.rs b/crates/sqlbuild-core/src/init_command/mod.rs index d9f45c6513..5aedb44cc0 100644 --- a/crates/sqlbuild-core/src/init_command/mod.rs +++ b/crates/sqlbuild-core/src/init_command/mod.rs @@ -1,6 +1,7 @@ //! `sqb init`: the blank project layout, its configuration, and the created summary. pub mod constants; +pub mod errors; pub mod main; #[cfg(test)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs index 8e969b0c1c..35d083baeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs @@ -1,13 +1,12 @@ //! Create the blank project tree, raising the `OSError` Python's `pathlib` would. -use std::fs::{self, OpenOptions}; -use std::io::{self, ErrorKind}; +use std::io; use std::path::Path; use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; -use sqlbuild_core::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; +use sqlbuild_core::init_command::errors::ScaffoldError; +use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_project as scaffold; /// `OSError(errno, strerror, filename)`, which Python narrows to its errno subclass. fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { @@ -35,52 +34,12 @@ fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { } } -/// `Path.mkdir(parents=True, exist_ok=True)`. -fn make_directories(path: &Path) -> Result<(), (io::Error, std::path::PathBuf)> { - match fs::create_dir(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == ErrorKind::NotFound => { - let Some(parent) = path.parent().filter(|parent| *parent != path) else { - return Err((error, path.to_path_buf())); - }; - make_directories(parent)?; - match fs::create_dir(path) { - Err(error) if !path.is_dir() => Err((error, path.to_path_buf())), - _ => Ok(()), - } - } - Err(error) if path.is_dir() => { - drop(error); - Ok(()) - } - Err(error) => Err((error, path.to_path_buf())), - } -} - /// Create the resource directories, the project file, and `.gitkeep` in empty directories. pub(crate) fn scaffold_blank_project( py: Python<'_>, base_dir: &Path, project_toml: &str, ) -> PyResult<()> { - for directory in SCAFFOLD_DIRECTORIES { - make_directories(&base_dir.join(directory)) - .map_err(|(error, path)| os_error(py, &error, &path))?; - } - let project_file = base_dir.join(PROJECT_FILE); - fs::write(&project_file, project_toml).map_err(|error| os_error(py, &error, &project_file))?; - for directory in SCAFFOLD_DIRECTORIES { - let directory = base_dir.join(directory); - let mut entries = - fs::read_dir(&directory).map_err(|error| os_error(py, &error, &directory))?; - if entries.next().is_none() { - let gitkeep = directory.join(GITKEEP_FILE); - OpenOptions::new() - .create(true) - .append(true) - .open(&gitkeep) - .map_err(|error| os_error(py, &error, &gitkeep))?; - } - } - Ok(()) + scaffold(base_dir, project_toml) + .map_err(|ScaffoldError { error, path }| os_error(py, &error, &path)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index a3be4ae5df..9fcc0ef2bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -3,8 +3,10 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::wrap_pyfunction; +use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; +use sqlbuild_rules::rule_catalogue::models::RuleListing; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -81,47 +83,31 @@ fn run_skills( Ok(if fresh { 0 } else { 1 }) } +fn rule_listing(rule: &Bound<'_, PyAny>) -> PyResult { + let subject: Bound<'_, PyAny> = native_field(rule, "subject")?; + Ok(RuleListing { + code: native_field(rule, "code")?.str()?.extract()?, + family: native_field(rule, "family")?.extract()?, + message: native_field(rule, "message")?.str()?.extract()?, + custom: native_field(rule, "custom")?.is_truthy()?, + subject: if subject.is_none() { + None + } else { + Some(native_field(&subject, "value")?.extract()?) + }, + }) +} + fn list_rules(catalogue: &Bound<'_, PyAny>, json_output: bool) -> PyResult { - let py: Python<'_> = catalogue.py(); - if !json_output { - let mut lines: Vec = Vec::new(); - for rule in catalogue.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - lines.push(format!( - "{} {}", - native_field(&rule, "code")?.str()?, - native_field(&rule, "message")?.str()? - )); - } - print_line(py, &lines.join("\n"), false)?; - return Ok(0); - } - let rules: Bound<'_, PyList> = PyList::empty(py); + let mut rules: Vec = Vec::new(); for rule in catalogue.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - let item: Bound<'_, PyDict> = PyDict::new(py); - for field in ["code", "family", "message", "custom"] { - item.set_item(field, native_field(&rule, field)?)?; - } - let subject: Bound<'_, PyAny> = native_field(&rule, "subject")?; - item.set_item( - "subject", - if subject.is_none() { - subject - } else { - native_field(&subject, "value")? - }, - )?; - rules.append(item)?; + rules.push(rule_listing(&rule?)?); } - let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item("rules", rules)?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("sort_keys", true)?; - let text: String = imported(py, "json", "dumps")? - .call((payload,), Some(&options))? - .extract()?; - print_line(py, &text, false)?; + print_line( + catalogue.py(), + &render_rule_list(&rules, json_output), + false, + )?; Ok(0) } diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs index 618095dd7e..0c4b11601a 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -1,5 +1,6 @@ pub mod format_rule_text; pub mod install_rendered_skills; +pub mod render_rule_list; pub mod render_skills_body; pub mod ruleset_fingerprint; pub mod skill_lock_name; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs new file mode 100644 index 0000000000..358d57d1b1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs @@ -0,0 +1,44 @@ +//! `sqb rules list` text: one `code message` line per rule, or the sorted JSON document. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::rule_catalogue::models::RuleListing; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn rule_object(rule: &RuleListing) -> JsonValue { + JsonValue::Object(vec![ + ("code".to_owned(), text(&rule.code)), + ("family".to_owned(), text(&rule.family)), + ("message".to_owned(), text(&rule.message)), + ("custom".to_owned(), JsonValue::Bool(rule.custom)), + ( + "subject".to_owned(), + rule.subject.as_deref().map_or(JsonValue::Null, text), + ), + ]) +} + +/// The listing without its trailing newline: `json.dumps({"rules": ...}, sort_keys=True)` or lines. +#[must_use] +pub fn render_rule_list(rules: &[RuleListing], json_output: bool) -> String { + if !json_output { + return rules + .iter() + .map(|rule| format!("{} {}", rule.code, rule.message)) + .collect::>() + .join("\n"); + } + let payload: JsonValue = JsonValue::Object(vec![( + "rules".to_owned(), + JsonValue::Array(rules.iter().map(rule_object).collect()), + )]); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + }; + dumps(&payload, &JsonDialect::Stdlib(options)).unwrap_or_default() +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/models.rs b/crates/sqlbuild-rules/src/rule_catalogue/models.rs index 2ebd66fc1b..a3f5fd695c 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/models.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/models.rs @@ -85,3 +85,14 @@ pub struct SkillTarget { pub path: std::path::PathBuf, pub state: String, } + +/// One catalogue rule as `sqb rules list` prints it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RuleListing { + pub code: String, + pub family: String, + pub message: String, + pub custom: bool, + /// The `RuleSubject` value, or `None` when the rule declares none. + pub subject: Option, +} diff --git a/fensu.toml b/fensu.toml index 154893a523..b09c54fa00 100644 --- a/fensu.toml +++ b/fensu.toml @@ -842,7 +842,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs=1", From cf463ff3524b7cd06b4148546ce0f1c052ce0028 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:11 +0100 Subject: [PATCH 04/39] feat: run help, usage errors, init, skills, clean and rules list/show in sqb without Python --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/sqlbuild-cli/Cargo.toml | 8 +- .../src/launcher/_helpers/dispatch.rs | 17 +- crates/sqlbuild-cli/src/lib.rs | 3 +- .../src/native_commands/_helpers/argv_host.rs | 119 +++++++ .../src/native_commands/_helpers/decision.rs | 92 +++++ .../src/native_commands/_helpers/execution.rs | 117 ++++++ .../src/native_commands/_helpers/init.rs | 101 ++++++ .../src/native_commands/_helpers/mod.rs | 10 + .../src/native_commands/_helpers/package.rs | 129 +++++++ .../src/native_commands/_helpers/paths.rs | 56 +++ .../native_commands/_helpers/project_facts.rs | 66 ++++ .../src/native_commands/_helpers/rules.rs | 169 +++++++++ .../src/native_commands/_helpers/skills.rs | 49 +++ .../src/native_commands/_helpers/terminal.rs | 93 +++++ .../src/native_commands/constants.rs | 75 ++++ .../src/native_commands/errors.rs | 16 + .../src/native_commands/main/mod.rs | 1 + .../main/run_native_invocation.rs | 52 +++ .../sqlbuild-cli/src/native_commands/mod.rs | 10 + .../src/native_commands/models.rs | 73 ++++ .../sqlbuild-cli/src/native_commands/tests.rs | 14 + .../src/native_commands/tests/helpers.rs | 28 ++ .../native_commands/tests/test_argv_host.rs | 154 ++++++++ .../native_commands/tests/test_decision.rs | 129 +++++++ .../src/native_commands/tests/test_paths.rs | 77 ++++ .../tests/test_project_facts.rs | 60 ++++ .../src/native_commands/tests/test_rules.rs | 108 ++++++ .../src/native_commands/tests/test_types.rs | 68 ++++ .../src/sqlbuild/cli/entry/_test_types.py | 31 ++ .../src/sqlbuild/cli/entry/helpers.py | 92 +++++ .../cli/entry/test_python_free_commands.py | 333 ++++++++++++++++++ 33 files changed, 2343 insertions(+), 11 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/init.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/package.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/constants.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/errors.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/models.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/helpers.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_types.rs create mode 100644 tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py diff --git a/Cargo.lock b/Cargo.lock index 3a3986c304..4907f98599 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,7 +1241,9 @@ dependencies = [ "serde", "serde_json", "sqlbuild-cache", + "sqlbuild-config", "sqlbuild-core", + "sqlbuild-rules", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index 0ffdd46f94..994caecd9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ ruff_text_size = "=0.0.4" rustc-hash = "2" # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" serde = { version = "1", features = ["derive"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 8daa9c84c3..cdfce87880 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sqlbuild-cli" version.workspace = true -description = "The native `sqb` executable: reused compiles served without Python, every other command handed to the Python CLI." +description = "The native `sqb` executable: reused compiles and Python-free commands served without Python, every other command handed to the Python CLI." edition.workspace = true license-file = "LICENSE" publish.workspace = true @@ -17,18 +17,18 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +ctrlc.workspace = true getrandom.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-cache.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true -[target.'cfg(windows)'.dependencies] -ctrlc.workspace = true - [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index c8c85f8e54..e6f689c242 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -1,4 +1,4 @@ -//! Decide how one invocation runs: natively, from a launch record, or in the Python CLI. +//! Decide how one invocation runs: from a launch record, natively, or in the Python CLI. use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -10,6 +10,7 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::run_native_invocation::run_native_invocation; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -43,10 +44,16 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - return HANDOFF_FAILURE_EXIT_CODE; }; let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); - if let Some((path, key)) = launch_record(&program, &script, &arguments) { - if let Some(code) = serve_reused_compile(&path, &key, started) { - return exit_code(code); - } + let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); + if let Some((path, key)) = &record + && let Some(code) = serve_reused_compile(path, key, started) + { + return exit_code(code); + } + if let Ok(code) = run_native_invocation(&arguments, &script) { + return code; + } + if let Some((path, _)) = record { let mut option = OsString::from(LAUNCH_RECORD_OPTION); option.push(path.as_os_str()); handed.push(option); diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 4bbb72fffb..fc36fb01fd 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -1,4 +1,4 @@ -//! The native `sqb` executable: replays reused compiles and hands other invocations to Python. +//! The native `sqb` executable: replays reused compiles, runs Python-free commands and hands the rest to Python. #![forbid(unsafe_code)] @@ -7,4 +7,5 @@ pub mod compute_logs; pub mod diff_report; pub mod dispatch; pub mod launcher; +pub mod native_commands; pub mod replay; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs new file mode 100644 index 0000000000..9e26816a90 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs @@ -0,0 +1,119 @@ +//! The parse's interpreter side without an interpreter: Python's `int`, `repr` and the version. + +use sqlbuild_core::text::main::decimal_digit_value::decimal_digit_value; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::argv::main::cli_namespace_defaults::cli_namespace_defaults; +use crate::argv::models::{NamespaceDefault, TypeFailure, Value}; +use crate::argv::types::ArgvHost; +use crate::native_commands::constants::{ + DIGIT_SEPARATOR, INT_ASCII_SPACES, INT_CONVERTER, NEGATIVE_SIGN, POSITIVE_INTEGER_CONVERTER, + POSITIVE_INTEGER_MESSAGE, POSITIVE_SIGN, PYTHON_INT_MAX_STR_DIGITS, ZERO_TEXT, +}; + +/// The ASCII text CPython parses: other decimal digits become ASCII, other spaces a space. +fn ascii_digits_and_spaces(text: &str) -> Option { + text.chars() + .map(|character| { + if character.is_ascii() { + Some(character) + } else if python_isspace(character) { + Some(' ') + } else { + decimal_digit_value(character).and_then(|digit| char::from_digit(digit, 10)) + } + }) + .collect() +} + +/// The canonical decimal text of Python's `int(text)`, or `None` where it raises `ValueError`. +pub(crate) fn python_int(text: &str) -> Option { + let ascii: String = ascii_digits_and_spaces(text)?; + let trimmed: &str = ascii.trim_matches(|character: char| INT_ASCII_SPACES.contains(character)); + let (negative, body): (bool, &str) = match trimmed.strip_prefix(NEGATIVE_SIGN) { + Some(body) => (true, body), + None => ( + false, + trimmed.strip_prefix(POSITIVE_SIGN).unwrap_or(trimmed), + ), + }; + let mut digits: String = String::with_capacity(body.len()); + let mut after_digit: bool = false; + for character in body.chars() { + if character.is_ascii_digit() { + digits.push(character); + after_digit = true; + } else if character == DIGIT_SEPARATOR && after_digit { + after_digit = false; + } else { + return None; + } + } + if !after_digit || digits.len() > PYTHON_INT_MAX_STR_DIGITS { + return None; + } + let significant: &str = digits.trim_start_matches('0'); + Some(match (significant.is_empty(), negative) { + (true, _) => ZERO_TEXT.to_owned(), + (false, true) => format!("{NEGATIVE_SIGN}{significant}"), + (false, false) => significant.to_owned(), + }) +} + +/// Converts what it can as Python would and records any converter only Python can apply. +#[derive(Debug, Default)] +pub(crate) struct NativeArgvHost { + /// Whether the parse met a converter, such as `--vars`, that needs the interpreter. + pub(crate) needs_python: bool, +} + +impl ArgvHost for NativeArgvHost { + fn convert(&mut self, converter: &str, text: &str) -> Result { + match converter { + INT_CONVERTER => python_int(text).map(Value::Int).ok_or(TypeFailure::Invalid), + POSITIVE_INTEGER_CONVERTER => { + let number: String = python_int(text).ok_or(TypeFailure::Invalid)?; + if number == ZERO_TEXT || number.starts_with(NEGATIVE_SIGN) { + return Err(TypeFailure::ArgumentType( + POSITIVE_INTEGER_MESSAGE.to_owned(), + )); + } + Ok(Value::Int(number)) + } + _ => { + self.needs_python = true; + Ok(Value::None) + } + } + } + + fn repr(&self, text: &str) -> String { + python_str_repr(text) + } + + fn program_version(&mut self) -> String { + env!("CARGO_PKG_VERSION").to_owned() + } +} + +/// The parse's starting namespace: every `CliNamespace` default. +pub(crate) fn namespace_defaults() -> Vec<(String, Value)> { + cli_namespace_defaults() + .iter() + .map(|(name, default)| { + let value: Value = match default { + NamespaceDefault::None => Value::None, + NamespaceDefault::Bool(flag) => Value::Bool(*flag), + NamespaceDefault::Int(number) => Value::Int(number.to_string()), + NamespaceDefault::Text(text) => Value::Str((*text).to_owned()), + NamespaceDefault::EmptyList => Value::List(Vec::new()), + NamespaceDefault::EmptyDict => Value::EmptyDict, + NamespaceDefault::CompileLineageFast | NamespaceDefault::ColumnLineageRich => { + Value::Host(0) + } + }; + ((*name).to_owned(), value) + }) + .collect() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs new file mode 100644 index 0000000000..30ede63f68 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -0,0 +1,92 @@ +//! Which parsed invocations run without Python, decided from the parsed values alone. + +use std::path::PathBuf; + +use crate::argv::models::{Namespace, Value}; +use crate::dispatch::main::invocation_policy::invocation_policy; +use crate::dispatch::models::{CommandFacts, InvocationPolicy}; +use crate::native_commands::constants::{ + CLEAN_COMMAND, INIT_COMMAND, RULES_COMMAND, RULES_LIST_ACTION, RULES_SHOW_ACTION, + SKILLS_COMMAND, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{ + NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, +}; + +fn text(namespace: &Namespace, name: &str) -> Option { + match namespace.get(name) { + Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), + _ => None, + } +} + +fn flag(namespace: &Namespace, name: &str) -> bool { + namespace.get(name) == Some(&Value::Bool(true)) +} + +/// The strings of a list attribute, or `None` when it holds anything else. +fn texts(namespace: &Namespace, name: &str) -> Option> { + match namespace.get(name) { + None | Some(Value::None) => Some(Vec::new()), + Some(Value::List(items)) => items + .iter() + .map(|item| match item { + Value::Str(text) => Some(text.clone()), + _ => None, + }) + .collect(), + Some(_) => None, + } +} + +fn rules_command(namespace: &Namespace) -> Result { + let action: RulesAction = match text(namespace, "rules_command").as_deref() { + None | Some(RULES_LIST_ACTION) => RulesAction::List, + Some(RULES_SHOW_ACTION) => RulesAction::Show, + Some(_) => return Err(PythonNeeded::Command), + }; + Ok(NativeCommand::Rules(RulesOptions { + action, + selector: text(namespace, "rules_rule_selector"), + json: flag(namespace, "json"), + })) +} + +fn native_command( + command: Option<&str>, + namespace: &Namespace, +) -> Result { + match command { + None | Some(CLEAN_COMMAND) => Ok(NativeCommand::Nothing), + Some(INIT_COMMAND) => Ok(NativeCommand::Init), + Some(SKILLS_COMMAND) => Ok(NativeCommand::Skills(SkillsOptions { + global_install: flag(namespace, "skills_global"), + targets: texts(namespace, "skills_target").ok_or(PythonNeeded::Arguments)?, + force: flag(namespace, "skills_force"), + })), + Some(RULES_COMMAND) => rules_command(namespace), + Some(_) => Err(PythonNeeded::Command), + } +} + +/// The command `namespace` runs, when it runs without Python; `--debug` always uses Python. +pub(crate) fn native_invocation(namespace: &Namespace) -> Result { + if flag(namespace, "debug") { + return Err(PythonNeeded::Command); + } + let command_name: Option = text(namespace, "command"); + let command: NativeCommand = native_command(command_name.as_deref(), namespace)?; + let policy: InvocationPolicy = invocation_policy(&CommandFacts { + command: command_name.clone(), + ..CommandFacts::default() + }); + Ok(NativeInvocation { + command, + command_name, + project_dir: text(namespace, "project_dir").map(PathBuf::from), + target: text(namespace, "target"), + creates_project: policy.creates_project, + maintains_skills: policy.maintains_skills, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs new file mode 100644 index 0000000000..278a397f81 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -0,0 +1,117 @@ +//! Running a Python-free command: its checks, then its output with the observed services. + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::argv::models::Stream; +use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::finish_capture::finish_capture; +use crate::compute_logs::main::open_log_root::open_log_root; +use crate::compute_logs::main::start_capture::start_capture; +use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::init::run_init; +use crate::native_commands::_helpers::package::installed_package; +use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; +use crate::native_commands::_helpers::rules::run_rules; +use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; +use crate::native_commands::_helpers::terminal::{exit_on_interrupt, write_text}; +use crate::native_commands::constants::UNKNOWN_COMMAND; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{ + CommandOutput, InstalledPackage, NativeCommand, NativeInvocation, ProjectLocation, +}; + +/// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. +fn open_capture( + invocation: &NativeInvocation, + project: &ProjectLocation, + started_at: SystemTime, +) -> io::Result { + let retention: i64 = i64::try_from(DEFAULT_RETENTION_COUNT).unwrap_or(i64::MAX); + let root: PathBuf = open_log_root(&project.given, None, Some(retention)) + .map_err(|error| io::Error::other(format!("{error:?}")))?; + start_capture(CaptureStart { + root, + command: invocation + .command_name + .clone() + .unwrap_or_else(|| UNKNOWN_COMMAND.to_owned()), + project_dir: project.resolved.display().to_string(), + target: invocation.target.clone(), + started_at, + }) +} + +/// Print an observed command's output after the skill notice, retaining both in its capture. +fn finish_observed( + invocation: &NativeInvocation, + (project, package): (&ProjectLocation, &InstalledPackage), + started_at: SystemTime, + output: CommandOutput, +) -> i32 { + let notice: String = if invocation.maintains_skills { + skill_freshness_notice(&project.given, package) + } else { + String::new() + }; + let capture: io::Result = open_capture(invocation, project, started_at); + let stderr: String = notice + &output.stderr; + write_text(Stream::Stderr, &stderr); + write_text(Stream::Stdout, &output.stdout); + if let Ok(capture) = capture { + let finished: io::Result<()> = finish_capture( + capture, + &CaptureEnd { + stdout: output.stdout.into_bytes(), + stderr: stderr.into_bytes(), + exit_code: i64::from(output.exit_code), + }, + ); + drop(finished); + } + output.exit_code +} + +/// What an observed command prints, unless it needs Python after all. +fn observed_output( + command: &NativeCommand, + project: &ProjectLocation, + package: &InstalledPackage, +) -> Result { + if !has_no_runtime_extensions(&project.given) { + return Err(PythonNeeded::Project); + } + match command { + NativeCommand::Nothing => Ok(CommandOutput::default()), + NativeCommand::Skills(options) => run_skills(&project.resolved, package, options), + NativeCommand::Rules(options) => run_rules(&project.resolved, options), + NativeCommand::Init => Err(PythonNeeded::Command), + } +} + +/// Run a parsed Python-free command; an error hands it to Python before any side effect. +pub(crate) fn run_command( + invocation: &NativeInvocation, + python_cli: &Path, + error_color: bool, +) -> Result { + exit_on_interrupt(); + let package: InstalledPackage = installed_package(python_cli)?; + let project: ProjectLocation = project_location(invocation.project_dir.as_deref())?; + if invocation.creates_project { + let output: CommandOutput = run_init(&project.resolved, &package, error_color)?; + write_text(Stream::Stderr, &output.stderr); + write_text(Stream::Stdout, &output.stdout); + return Ok(output.exit_code); + } + let started_at: SystemTime = SystemTime::now(); + let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; + Ok(finish_observed( + invocation, + (&project, &package), + started_at, + output, + )) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs new file mode 100644 index 0000000000..dfee4cd22b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs @@ -0,0 +1,101 @@ +//! `sqb init`: scaffold a blank project and install its skills. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::workspace::errors::WorkspaceError; +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::models::SkillInstallRequest; +use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::init_command::errors::ScaffoldError; +use sqlbuild_core::init_command::main::init_project_name::init_project_name; +use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; +use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; +use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_project; + +use crate::dispatch::main::failure_report::failure_report; +use crate::dispatch::models::CommandFailure; +use crate::native_commands::_helpers::skills::home_dir; +use crate::native_commands::_helpers::terminal::stdout_supports_color; +use crate::native_commands::constants::{INIT_FAILURE_PREFIX, USER_ERROR_CLASS}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, InstalledPackage}; + +/// A failure once the project file exists, which a Python rerun would report differently. +fn failed(stderr: String) -> Result { + Ok(CommandOutput { + stdout: String::new(), + stderr, + exit_code: 1, + }) +} + +/// The project's skills could not be installed: what the Python CLI prints for its error. +fn skills_failure(error: &WorkspaceError, use_color: bool) -> Result { + let WorkspaceError::User { + code, + message, + help, + } = error + else { + return failed(format!("{INIT_FAILURE_PREFIX}{error:?}\n")); + }; + let (_, log_message) = failure_report(CommandFailure::User); + let coded: String = format_coded_error( + (code, message, help.as_deref()), + true, + CliStyle { use_color }, + ); + failed(format!( + "{log_message}\n{USER_ERROR_CLASS}: {message}\n{coded}\n\n" + )) +} + +/// Create the project; a failure before the project file exists is Python's to report. +pub(crate) fn run_init( + base_dir: &Path, + package: &InstalledPackage, + error_color: bool, +) -> Result { + let project_file: PathBuf = base_dir.join(PROJECT_CONFIG_FILENAME); + if project_file.exists() { + return Err(PythonNeeded::Failure); + } + let directory_name: &str = match base_dir.file_name() { + Some(name) => name.to_str().ok_or(PythonNeeded::Arguments)?, + None => "", + }; + let home: PathBuf = home_dir()?; + let project_name: String = init_project_name(directory_name); + if let Err(ScaffoldError { error, path }) = + scaffold_blank_project(base_dir, &init_project_toml(&project_name)) + { + if !project_file.exists() { + return Err(PythonNeeded::Failure); + } + return failed(format!( + "{INIT_FAILURE_PREFIX}{error}: {}\n", + path.display() + )); + } + if let Err(error) = install_skills(&SkillInstallRequest { + project_dir: base_dir.to_path_buf(), + skills_root: package.skills_root.clone(), + home_dir: home, + global_install: false, + requested_targets: Vec::new(), + force: false, + }) { + return skills_failure(&error, error_color); + } + Ok(CommandOutput { + stdout: render_init_summary( + &project_name, + CliStyle { + use_color: stdout_supports_color(), + }, + ), + ..CommandOutput::default() + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs new file mode 100644 index 0000000000..c01acfd621 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod argv_host; +pub(crate) mod decision; +pub(crate) mod execution; +pub(crate) mod init; +pub(crate) mod package; +pub(crate) mod paths; +pub(crate) mod project_facts; +pub(crate) mod rules; +pub(crate) mod skills; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs new file mode 100644 index 0000000000..ce77e6aa57 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs @@ -0,0 +1,129 @@ +//! The installed `sqlbuild` package Python would import, found without starting Python. + +use std::path::{Path, PathBuf}; + +use crate::native_commands::constants::{ + PACKAGE_MARKER, PACKAGE_NAME, PATH_FILE_COMMENT, PATH_FILE_EXTENSION, + PATH_FILE_IMPORT_PREFIXES, PYTHON_PATH_VARIABLES, SKILLS_DIRECTORY, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::InstalledPackage; + +/// The site directories of the installation whose scripts directory holds `script`. +#[cfg(not(windows))] +fn site_directories(prefix: &Path) -> Vec { + use crate::native_commands::constants::{ + INTERPRETER_DIRECTORY_PREFIX, LIBRARY_DIRECTORY, SITE_DIRECTORIES, + }; + + let Ok(entries) = std::fs::read_dir(prefix.join(LIBRARY_DIRECTORY)) else { + return Vec::new(); + }; + let mut directories: Vec = Vec::new(); + for entry in entries.flatten() { + if !entry + .file_name() + .to_string_lossy() + .starts_with(INTERPRETER_DIRECTORY_PREFIX) + { + continue; + } + for site in SITE_DIRECTORIES { + let directory: PathBuf = entry.path().join(site); + if directory.is_dir() { + directories.push(directory); + } + } + } + directories.sort(); + directories +} + +#[cfg(windows)] +fn site_directories(prefix: &Path) -> Vec { + use crate::native_commands::constants::WINDOWS_SITE_DIRECTORY; + + let directory: PathBuf = WINDOWS_SITE_DIRECTORY + .iter() + .fold(prefix.to_path_buf(), |path, part| path.join(part)); + if directory.is_dir() { + vec![directory] + } else { + Vec::new() + } +} + +fn package_beneath(directory: &Path) -> Option { + let package: PathBuf = directory.join(PACKAGE_NAME); + package.join(PACKAGE_MARKER).is_file().then_some(package) +} + +/// The paths a site directory's `.pth` files add, in the order `site` reads them. +fn path_file_entries(site: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(site) else { + return Vec::new(); + }; + let mut files: Vec = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == PATH_FILE_EXTENSION) + }) + .collect(); + files.sort(); + let mut added: Vec = Vec::new(); + for file in files { + let Ok(text) = std::fs::read_to_string(&file) else { + continue; + }; + for line in text.lines() { + let line: &str = line.trim_end(); + if line.is_empty() + || line.starts_with(PATH_FILE_COMMENT) + || PATH_FILE_IMPORT_PREFIXES + .iter() + .any(|prefix| line.starts_with(prefix)) + { + continue; + } + added.push(site.join(line)); + } + } + added +} + +/// The one `sqlbuild` package the installation beside `script` holds, unless that is unclear. +pub(crate) fn installed_package(script: &Path) -> Result { + if PYTHON_PATH_VARIABLES + .iter() + .any(|name| std::env::var_os(name).is_some_and(|value| !value.is_empty())) + { + return Err(PythonNeeded::Installation); + } + let script: PathBuf = std::fs::canonicalize(script).map_err(|_| PythonNeeded::Installation)?; + let prefix: &Path = script + .parent() + .and_then(Path::parent) + .ok_or(PythonNeeded::Installation)?; + let mut found: Vec = Vec::new(); + for site in site_directories(prefix) { + let package: Option = package_beneath(&site).or_else(|| { + path_file_entries(&site) + .iter() + .find_map(|directory| package_beneath(directory)) + }); + if let Some(package) = package { + found.push(std::fs::canonicalize(&package).unwrap_or(package)); + } + } + found.dedup(); + let [package] = found.as_slice() else { + return Err(PythonNeeded::Installation); + }; + Ok(InstalledPackage { + skills_root: SKILLS_DIRECTORY + .iter() + .fold(package.clone(), |path, part| path.join(part)), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs new file mode 100644 index 0000000000..c50eea5af9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -0,0 +1,56 @@ +//! `pathlib.Path` construction and `Path.resolve()` over native paths. + +use std::path::{Component, Path, PathBuf}; + +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::ProjectLocation; + +/// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. +pub(crate) fn pathlib_path(text: &Path) -> PathBuf { + let path: PathBuf = text + .components() + .filter(|component| *component != Component::CurDir) + .collect(); + if path.as_os_str().is_empty() { + return PathBuf::from("."); + } + path +} + +/// `Path.resolve()`: absolute, with every existing prefix's links followed, as `realpath` does. +pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { + let absolute: PathBuf = if path.is_absolute() { + path.to_path_buf() + } else { + cwd.join(path) + }; + if let Ok(real) = std::fs::canonicalize(&absolute) { + return real; + } + let mut resolved: PathBuf = PathBuf::new(); + for component in absolute.components() { + match component { + Component::Prefix(_) | Component::RootDir => resolved.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + resolved.pop(); + } + Component::Normal(name) => { + let candidate: PathBuf = resolved.join(name); + resolved = std::fs::canonicalize(&candidate).unwrap_or(candidate); + } + } + } + resolved +} + +/// The project directory as given and as `Path.resolve()` reads it. +pub(crate) fn project_location(given: Option<&Path>) -> Result { + let cwd: PathBuf = std::env::current_dir().map_err(|_| PythonNeeded::Installation)?; + let given: PathBuf = match given { + Some(raw) => pathlib_path(raw), + None => cwd.clone(), + }; + let resolved: PathBuf = resolved_path(&given, &cwd); + Ok(ProjectLocation { given, resolved }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs new file mode 100644 index 0000000000..e94539aa29 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -0,0 +1,66 @@ +//! Project facts that decide, before any side effect, whether a command needs Python. + +use std::path::Path; + +use sqlbuild_config::project::main::load_project_config::load_project_config; +use sqlbuild_config::project::models::{Built, ConfigPaths}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + +use crate::native_commands::constants::{ + CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME, + LEGACY_PROJECT_CONFIG_FILENAME, LIFECYCLE_SINK_PATH, LOCAL_CONFIG_FILENAME, SINKS_DIRECTORY, +}; + +fn object_field<'a>(built: &'a Built, name: &str) -> Option<&'a Built> { + match built { + Built::Object(_, fields) => fields + .iter() + .find(|(field, _)| *field == name) + .map(|(_, value)| value), + _ => None, + } +} + +/// Whether the loaded configuration names a lifecycle sink, which only Python can bind. +fn names_lifecycle_sinks(config: &Built) -> bool { + let named: Option<&Built> = LIFECYCLE_SINK_PATH + .iter() + .try_fold(config, |value, name| object_field(value, name)); + match named { + None => false, + Some(Built::Dict(entries)) => !entries.is_empty(), + Some(_) => true, + } +} + +/// The four configuration files of `project_dir`. +fn config_paths(project_dir: &Path) -> ConfigPaths { + ConfigPaths { + project: project_dir.join(PROJECT_CONFIG_FILENAME), + legacy_project: project_dir.join(LEGACY_PROJECT_CONFIG_FILENAME), + local: project_dir.join(LOCAL_CONFIG_FILENAME), + legacy_local: project_dir.join(LEGACY_LOCAL_CONFIG_FILENAME), + } +} + +/// Whether `discover_runtime_extensions` would import nothing and accept the project file. +pub(crate) fn has_no_runtime_extensions(project_dir: &Path) -> bool { + if project_dir.join(EVENT_EXPORTERS_DIRECTORY).exists() + || project_dir.join(SINKS_DIRECTORY).is_dir() + { + return false; + } + let paths: ConfigPaths = config_paths(project_dir); + if !paths.project.exists() { + return true; + } + match load_project_config(&paths) { + Ok(config) => !names_lifecycle_sinks(&config), + Err(_) => false, + } +} + +/// Whether the project defines custom rules, which Python imports. +pub(crate) fn has_custom_rules(project_dir: &Path) -> bool { + project_dir.join(CUSTOM_RULES_DIRECTORY).is_dir() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs new file mode 100644 index 0000000000..c956dd94fe --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs @@ -0,0 +1,169 @@ +//! `sqb rules list|show` over the built-in catalogue, when no custom rule or option needs Python. + +use std::collections::BTreeMap; +use std::path::Path; + +use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config; +use sqlbuild_rules::models::{RuleMetadata, RulesConfig}; +use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; +use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; +use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::models::{ + CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleListing, RuleText, +}; +use sqlbuild_rules::rules::main::builtin_options::builtin_options; +use sqlbuild_rules::rules::main::catalogue::catalogue; +use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; + +use crate::native_commands::_helpers::project_facts::has_custom_rules; +use crate::native_commands::constants::{ + EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, PROJECT_SUBJECT, + RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, RulesAction, RulesOptions}; + +/// `RuleOption.boolean(...)` or `RuleOption.string_list(...)` as the catalogue validates it. +fn option_spec(option: &BuiltinOption) -> OptionSpec { + let (value_type, default, item_type) = match option.kind { + BuiltinOptionKind::Boolean => (OptionKind::Bool, OptionValue::Bool(false), None), + BuiltinOptionKind::StringList => ( + OptionKind::Tuple, + OptionValue::Tuple(Vec::new()), + Some(OptionKind::Str), + ), + }; + OptionSpec { + name: option.name.to_owned(), + description: option.description.to_owned(), + value_type, + default, + choices: Vec::new(), + minimum: None, + maximum: None, + minimum_items: None, + item_type, + } +} + +/// The option as `rules show` prints it: the `repr` of its default, which is also effective. +fn option_text(option: &BuiltinOption) -> OptionText { + let default_repr: &str = match option.kind { + BuiltinOptionKind::Boolean => FALSE_REPR, + BuiltinOptionKind::StringList => EMPTY_TUPLE_REPR, + }; + OptionText { + name: option.name.to_owned(), + description: option.description.to_owned(), + default_repr: default_repr.to_owned(), + effective_repr: default_repr.to_owned(), + } +} + +/// The rule code without its three-digit number, as the built-in `Rule` records it. +fn family(rule: &RuleMetadata) -> String { + rule.code[..rule.code.len().saturating_sub(FAMILY_SUFFIX_LENGTH)].to_owned() +} + +fn listing(rule: &RuleMetadata) -> RuleListing { + RuleListing { + code: rule.code.clone(), + family: family(rule), + message: rule.message.clone(), + custom: false, + subject: Some( + if rule.project_wide { + PROJECT_SUBJECT + } else { + MODEL_SUBJECT + } + .to_owned(), + ), + } +} + +fn rule_text(rule: &RuleMetadata) -> RuleText { + RuleText { + code: rule.code.clone(), + family: family(rule), + slug: rule.slug.clone(), + message: rule.message.clone(), + remediation: rule.remediation.clone(), + enabled_by_default: rule.enabled_by_default, + custom: false, + source: None, + options: builtin_options(&rule.code) + .iter() + .map(option_text) + .collect(), + good_example: rule + .guidance + .as_ref() + .map(|guidance| guidance.good_example.clone()), + } +} + +/// The built-in catalogue in the order `build_rule_catalogue` sorts it. +fn sorted_catalogue() -> Result, PythonNeeded> { + let rules: Vec = catalogue(); + let views: Vec = rules + .iter() + .map(|rule| CatalogueRule { + code: rule.code.clone(), + options: builtin_options(&rule.code) + .iter() + .map(option_spec) + .collect(), + }) + .collect(); + let order: Vec = + sorted_catalogue_order(&views, &BTreeMap::new()).map_err(|_| PythonNeeded::Failure)?; + Ok(order + .into_iter() + .filter_map(|index| rules.get(index).cloned()) + .collect()) +} + +fn printed(stdout: String, exit_code: i32) -> CommandOutput { + CommandOutput { + stdout: format!("{stdout}\n"), + stderr: String::new(), + exit_code, + } +} + +fn show(selector: Option<&str>, rules: &[RuleMetadata], config: &RulesConfig) -> CommandOutput { + let Some(selector) = selector else { + return printed(RULE_SELECTOR_REQUIRED.to_owned(), RULES_USAGE_EXIT_CODE); + }; + match rules.iter().find(|rule| rule.code == selector) { + Some(rule) => printed(format_rule_text(&rule_text(rule), Some(config)), 0), + None => printed( + format!("{UNKNOWN_RULE_PREFIX}{selector}"), + RULES_USAGE_EXIT_CODE, + ), + } +} + +/// What `rules list|show` prints for `project_dir`, unless it needs Python. +pub(crate) fn run_rules( + project_dir: &Path, + options: &RulesOptions, +) -> Result { + if has_custom_rules(project_dir) { + return Err(PythonNeeded::Project); + } + let config: RulesConfig = load_rules_config(project_dir).map_err(|_| PythonNeeded::Failure)?; + if forbidden_exceptions(&config).is_some() || !config.rule_options.is_empty() { + return Err(PythonNeeded::Project); + } + let rules: Vec = sorted_catalogue()?; + Ok(match options.action { + RulesAction::List => { + let listings: Vec = rules.iter().map(listing).collect(); + printed(render_rule_list(&listings, options.json), 0) + } + RulesAction::Show => show(options.selector.as_deref(), &rules, &config), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs new file mode 100644 index 0000000000..4272d40185 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs @@ -0,0 +1,49 @@ +//! `sqb skills` and the skill freshness check every observed command makes first. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::main::maintain_skills::maintain_skills; +use sqlbuild_config::workspace::models::SkillInstallRequest; + +use crate::native_commands::constants::{HOME_VARIABLE, SKILL_FILES_HEADER}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, InstalledPackage, SkillsOptions}; + +/// `Path.home()` from `HOME`, unless only Python's password lookup could tell. +pub(crate) fn home_dir() -> Result { + std::env::var_os(HOME_VARIABLE) + .filter(|home| !home.is_empty()) + .map(PathBuf::from) + .ok_or(PythonNeeded::Installation) +} + +/// The packaged skill files installed into `project_dir`; Python reports any failure. +pub(crate) fn run_skills( + project_dir: &Path, + package: &InstalledPackage, + options: &SkillsOptions, +) -> Result { + let written: Vec = install_skills(&SkillInstallRequest { + project_dir: project_dir.to_path_buf(), + skills_root: package.skills_root.clone(), + home_dir: home_dir()?, + global_install: options.global_install, + requested_targets: options.targets.clone(), + force: options.force, + }) + .map_err(|_| PythonNeeded::Failure)?; + let mut stdout: String = SKILL_FILES_HEADER.to_owned(); + for path in written { + stdout.push_str(&format!(" {}\n", path.display())); + } + Ok(CommandOutput { + stdout, + ..CommandOutput::default() + }) +} + +/// The stale-skill notice, refreshing generated skills; failures stay silent, as in Python. +pub(crate) fn skill_freshness_notice(project_dir: &Path, package: &InstalledPackage) -> String { + maintain_skills(project_dir, &package.skills_root).unwrap_or_default() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs new file mode 100644 index 0000000000..3a584e7abd --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -0,0 +1,93 @@ +//! The process's terminal: its width, colour support, stream writes and Ctrl+C. + +use std::io::{self, IsTerminal, Write}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::argv_host::python_int; +use crate::native_commands::constants::{COLUMNS_VARIABLE, FALLBACK_COLUMNS, NO_COLOR_VARIABLE}; +use crate::native_commands::errors::PythonNeeded; + +/// `int(os.environ["COLUMNS"])`, `0` when unset or not an integer; beyond 64 bits is Python's. +fn columns_variable() -> Result { + let Ok(text) = std::env::var(COLUMNS_VARIABLE) else { + return Ok(0); + }; + let Some(number) = python_int(&text) else { + return Ok(0); + }; + number.parse::().map_err(|_| PythonNeeded::Arguments) +} + +/// The width of the terminal on stdout, or the fallback when stdout is not one. +#[cfg(unix)] +fn stdout_columns() -> Result { + match rustix::termios::tcgetwinsize(io::stdout()) { + Ok(size) if size.ws_col > 0 => Ok(i64::from(size.ws_col)), + _ => Ok(FALLBACK_COLUMNS), + } +} + +/// A console's width needs the console API; only a redirected stdout has a known width. +#[cfg(not(unix))] +fn stdout_columns() -> Result { + if io::stdout().is_terminal() { + return Err(PythonNeeded::Arguments); + } + Ok(FALLBACK_COLUMNS) +} + +/// `shutil.get_terminal_size().columns`, unless only Python can tell. +pub(crate) fn terminal_columns() -> Result { + let columns: i64 = columns_variable()?; + if columns > 0 { + return Ok(columns); + } + stdout_columns() +} + +/// `stdout_supports_color`: stdout is a terminal and `NO_COLOR` is unset. +pub(crate) fn stdout_supports_color() -> bool { + std::env::var_os(NO_COLOR_VARIABLE).is_none() && io::stdout().is_terminal() +} + +/// `\n` as a Python text stream writes it on this platform. +fn console_bytes(text: &str) -> Vec { + if cfg!(windows) { + text.replace('\n', "\r\n").into_bytes() + } else { + text.as_bytes().to_vec() + } +} + +/// Write and flush one text; a closed or full stream drops it, as the parse's writes do. +pub(crate) fn write_text(stream: Stream, text: &str) { + if text.is_empty() { + return; + } + let bytes: Vec = console_bytes(text); + let written: io::Result<()> = match stream { + Stream::Stdout => { + let mut output = io::stdout().lock(); + output.write_all(&bytes).and_then(|()| output.flush()) + } + Stream::Stderr => { + let mut output = io::stderr().lock(); + output.write_all(&bytes).and_then(|()| output.flush()) + } + }; + drop(written); +} + +/// End the process with status 130 on Ctrl+C; handing off by `exec` restores the default. +#[cfg(unix)] +pub(crate) fn exit_on_interrupt() { + use crate::native_commands::constants::INTERRUPTED_EXIT_CODE; + + drop(ctrlc::set_handler(|| { + std::process::exit(INTERRUPTED_EXIT_CODE) + })); +} + +/// The Python child waited for alone handles Ctrl+C, so the native command keeps the default. +#[cfg(not(unix))] +pub(crate) fn exit_on_interrupt() {} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs new file mode 100644 index 0000000000..9c01a84a6b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -0,0 +1,75 @@ +//! Names and limits of the Python-free command path. + +/// `type=` converters the native parse applies itself; any other needs Python. +pub(crate) const INT_CONVERTER: &str = "int"; +pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; +pub(crate) const POSITIVE_INTEGER_MESSAGE: &str = "value must be a positive integer"; +pub(crate) const NO_COLOR_OPTION: &str = "--no-color"; +pub(crate) const NO_COLOR_VARIABLE: &str = "NO_COLOR"; +pub(crate) const COLUMNS_VARIABLE: &str = "COLUMNS"; +/// `shutil.get_terminal_size`'s fallback width. +pub(crate) const FALLBACK_COLUMNS: i64 = 80; +/// Variables that change where Python imports `sqlbuild` from. +pub(crate) const PYTHON_PATH_VARIABLES: [&str; 2] = ["PYTHONPATH", "PYTHONHOME"]; +pub(crate) const PACKAGE_NAME: &str = "sqlbuild"; +pub(crate) const PACKAGE_MARKER: &str = "__init__.py"; +pub(crate) const PATH_FILE_EXTENSION: &str = "pth"; +/// `.pth` lines that `site` executes instead of adding to the search path. +pub(crate) const PATH_FILE_IMPORT_PREFIXES: [&str; 2] = ["import ", "import\t"]; +pub(crate) const PATH_FILE_COMMENT: char = '#'; +#[cfg(not(windows))] +pub(crate) const LIBRARY_DIRECTORY: &str = "lib"; +#[cfg(not(windows))] +pub(crate) const INTERPRETER_DIRECTORY_PREFIX: &str = "python3"; +#[cfg(not(windows))] +pub(crate) const SITE_DIRECTORIES: [&str; 2] = ["site-packages", "dist-packages"]; +#[cfg(windows)] +pub(crate) const WINDOWS_SITE_DIRECTORY: [&str; 2] = ["Lib", "site-packages"]; +/// The packaged skill beneath the `sqlbuild` package directory. +pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"]; +pub(crate) const HOME_VARIABLE: &str = "HOME"; +/// The exit status of a command stopped by Ctrl+C. +#[cfg(unix)] +pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; +/// The command name compute logs record for a bare `sqb`. +pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; +/// Commands with no handler: observed, then successful. +pub(crate) const CLEAN_COMMAND: &str = "clean"; +pub(crate) const INIT_COMMAND: &str = "init"; +pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const RULES_COMMAND: &str = "rules"; +pub(crate) const RULES_LIST_ACTION: &str = "list"; +pub(crate) const RULES_SHOW_ACTION: &str = "show"; +/// Project directories holding Python the command would import. +pub(crate) const EVENT_EXPORTERS_DIRECTORY: &str = "event_exporters"; +pub(crate) const SINKS_DIRECTORY: &str = "sinks"; +pub(crate) const CUSTOM_RULES_DIRECTORY: &str = "rules"; +pub(crate) const SKILL_FILES_HEADER: &str = "Updated SQLBuild skill files:\n"; +pub(crate) const RULE_SELECTOR_REQUIRED: &str = "A rule code or family prefix is required"; +/// `run_rules_command`'s exit status for a missing or unknown rule. +pub(crate) const RULES_USAGE_EXIT_CODE: i32 = 2; +pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +/// The class Python's last-resort log handler names for a user error. +pub(crate) const USER_ERROR_CLASS: &str = "sqlbuild._native.CliUserError"; +pub(crate) const INIT_FAILURE_PREFIX: &str = "sqb init: "; +/// A built-in rule's family is its code without the trailing three-digit number. +pub(crate) const FAMILY_SUFFIX_LENGTH: usize = 3; +/// `RuleSubject` values. +pub(crate) const MODEL_SUBJECT: &str = "model"; +pub(crate) const PROJECT_SUBJECT: &str = "project"; +/// Python `repr` of the built-in option defaults: `False` and `()`. +pub(crate) const FALSE_REPR: &str = "False"; +pub(crate) const EMPTY_TUPLE_REPR: &str = "()"; +pub(crate) const UNKNOWN_RULE_PREFIX: &str = "Unknown rule: "; +/// The whitespace `int()` strips once other spaces are ASCII spaces. +pub(crate) const INT_ASCII_SPACES: &str = " \t\n\x0b\x0c\r"; +pub(crate) const NEGATIVE_SIGN: char = '-'; +pub(crate) const POSITIVE_SIGN: char = '+'; +pub(crate) const ZERO_TEXT: &str = "0"; +/// `sys.int_info.default_max_str_digits`: longer decimal text makes `int()` raise. +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4300; +pub(crate) const DIGIT_SEPARATOR: char = '_'; +/// Where a loaded project configuration names its lifecycle sinks. +pub(crate) const LIFECYCLE_SINK_PATH: [&str; 3] = ["sinks", "lifecycle", "named"]; diff --git a/crates/sqlbuild-cli/src/native_commands/errors.rs b/crates/sqlbuild-cli/src/native_commands/errors.rs new file mode 100644 index 0000000000..78e1542aa8 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/errors.rs @@ -0,0 +1,16 @@ +//! Why `sqb` hands an invocation to Python after all. + +/// What made the invocation need Python, found before any side effect. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PythonNeeded { + /// The arguments are not text, or a converter or terminal fact needs the interpreter. + Arguments, + /// The command, or an option such as `--debug`, runs in Python. + Command, + /// The installed package, home directory or working directory is not known natively. + Installation, + /// The project holds Python the command imports, or files Python reports on. + Project, + /// The command failed before any side effect, and Python reports the failure. + Failure, +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs new file mode 100644 index 0000000000..f45e2c475a --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -0,0 +1 @@ +pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs new file mode 100644 index 0000000000..222d953909 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -0,0 +1,52 @@ +//! Serve one invocation without Python when its parse and project facts allow it. + +use std::ffi::OsString; +use std::path::Path; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::native_invocation; +use crate::native_commands::_helpers::execution::run_command; +use crate::native_commands::_helpers::terminal::{ + stdout_supports_color, terminal_columns, write_text, +}; +use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeInvocation; + +/// The exit status of an invocation `sqb` served, or why the same arguments go to Python. +pub fn run_native_invocation( + arguments: &[OsString], + python_cli: &Path, +) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color: bool = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let settings: ArgvSettings = ArgvSettings { + columns: terminal_columns()?, + use_color, + }; + let mut host: NativeArgvHost = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(code) => { + for (stream, text) in &outcome.writes { + write_text(*stream, text); + } + return Ok(i32::try_from(code).unwrap_or(1)); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + let invocation: NativeInvocation = native_invocation(&namespace)?; + run_command(&invocation, python_cli, use_color) +} diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs new file mode 100644 index 0000000000..d99ca200f2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -0,0 +1,10 @@ +//! Python-free commands `sqb` runs itself: help, usage errors and the commands needing no Python. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs new file mode 100644 index 0000000000..f0ce868fa2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -0,0 +1,73 @@ +//! Python-free commands, the invocation facts they run with and what they print. + +use std::path::PathBuf; + +/// What `sqb skills` installs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SkillsOptions { + pub(crate) global_install: bool, + pub(crate) targets: Vec, + pub(crate) force: bool, +} + +/// Which `sqb rules` inspection runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RulesAction { + List, + Show, +} + +/// What `sqb rules list|show` prints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RulesOptions { + pub(crate) action: RulesAction, + pub(crate) selector: Option, + pub(crate) json: bool, +} + +/// A command that runs without Python. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NativeCommand { + /// `clean` and a bare `sqb`: observed, with no handler to run. + Nothing, + Init, + Skills(SkillsOptions), + Rules(RulesOptions), +} + +/// One parsed invocation `sqb` serves without Python. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NativeInvocation { + pub(crate) command: NativeCommand, + /// The parsed command name; `None` for a bare `sqb`. + pub(crate) command_name: Option, + /// `--project-dir` as given. + pub(crate) project_dir: Option, + pub(crate) target: Option, + pub(crate) creates_project: bool, + pub(crate) maintains_skills: bool, +} + +/// What a command printed and the status it ends with. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct CommandOutput { + pub(crate) stdout: String, + pub(crate) stderr: String, + pub(crate) exit_code: i32, +} + +/// Where the installed `sqlbuild` package keeps what the commands read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InstalledPackage { + /// The packaged `.agents/skills/sqlbuild` directory. + pub(crate) skills_root: PathBuf, +} + +/// The directories one command runs against. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProjectLocation { + /// `Path(--project-dir)`, or `Path.cwd()`, unresolved. + pub(crate) given: PathBuf, + /// `Path(--project-dir).resolve()`, or `Path.cwd()`. + pub(crate) resolved: PathBuf, +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs new file mode 100644 index 0000000000..9119c141c9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -0,0 +1,14 @@ +#[path = "tests/test_argv_host.rs"] +mod argv_host; +#[path = "tests/test_decision.rs"] +mod decision; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_paths.rs"] +mod paths; +#[path = "tests/test_project_facts.rs"] +mod project_facts; +#[path = "tests/test_rules.rs"] +mod rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs new file mode 100644 index 0000000000..e54ee1dd6d --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -0,0 +1,28 @@ +use std::path::Path; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; + +/// Parse `argv` onto the `CliNamespace` defaults with the native host. +pub(super) fn parse(argv: &[&str]) -> (ArgvOutcome, NativeArgvHost) { + let argv: Vec = argv.iter().map(|arg| (*arg).to_owned()).collect(); + let settings: ArgvSettings = ArgvSettings { + columns: 100, + use_color: false, + }; + let mut host: NativeArgvHost = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&argv, namespace_defaults(), settings, &mut host); + (outcome, host) +} + +/// Create `directories` and write `files` beneath `root`. +pub(super) fn write_tree(root: &Path, directories: &[&str], files: &[(&str, &str)]) { + for directory in directories { + std::fs::create_dir_all(root.join(directory)).expect("create directory"); + } + for (relative, contents) in files { + std::fs::write(root.join(relative), contents).expect("write file"); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs new file mode 100644 index 0000000000..be011f8c40 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs @@ -0,0 +1,154 @@ +use crate::argv::models::{TypeFailure, Value}; +use crate::argv::types::ArgvHost; +use crate::native_commands::_helpers::argv_host::NativeArgvHost; +use crate::native_commands::tests::test_types::ConversionTestCase; + +/// Integer expectations recorded from CPython 3.12 `int(text)`. +#[test] +fn given_converter_when_converting_natively_then_python_conversions_match() { + let test_cases = [ + ConversionTestCase { + description: "surrounding ASCII spaces", + converter: "int", + text: " 5 ", + expected_value: Ok(Value::Int("5".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "underscores between digits", + converter: "int", + text: "1_0", + expected_value: Ok(Value::Int("10".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a leading underscore", + converter: "int", + text: "_1", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a trailing underscore", + converter: "int", + text: "1_", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "doubled underscores", + converter: "int", + text: "1__0", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "an Arabic-Indic digit", + converter: "int", + text: "\u{663}", + expected_value: Ok(Value::Int("3".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a non-breaking space", + converter: "int", + text: "\u{a0}5", + expected_value: Ok(Value::Int("5".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "an ASCII separator is not stripped", + converter: "int", + text: "\u{1c}5", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a zero-width space is not a space", + converter: "int", + text: "\u{200b}7", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "negative zero", + converter: "int", + text: "-0", + expected_value: Ok(Value::Int("0".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a sign and leading zeros", + converter: "int", + text: "+007", + expected_value: Ok(Value::Int("7".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "beyond 64 bits", + converter: "int", + text: "-123456789012345678901234567890", + expected_value: Ok(Value::Int("-123456789012345678901234567890".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a decimal point", + converter: "int", + text: "1.5", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "empty text", + converter: "int", + text: "", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer rejects zero with its message", + converter: "_positive_integer", + text: "0", + expected_value: Err(TypeFailure::ArgumentType( + "value must be a positive integer".to_owned(), + )), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer rejects negatives", + converter: "_positive_integer", + text: "-3", + expected_value: Err(TypeFailure::ArgumentType( + "value must be a positive integer".to_owned(), + )), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer that is not an integer is invalid", + converter: "_positive_integer", + text: "many", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "--vars JSON needs Python", + converter: "parse_cli_vars", + text: "{}", + expected_value: Ok(Value::None), + expected_needs_python: true, + }, + ]; + + for test_case in test_cases { + let mut host: NativeArgvHost = NativeArgvHost::default(); + + let value = host.convert(test_case.converter, test_case.text); + + assert_eq!( + (value, host.needs_python), + (test_case.expected_value, test_case.expected_needs_python), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs new file mode 100644 index 0000000000..02dd4d48be --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs @@ -0,0 +1,129 @@ +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{NativeCommand, RulesAction, RulesOptions, SkillsOptions}; +use crate::native_commands::tests::helpers::parse; +use crate::native_commands::tests::test_types::{DecisionTestCase, ParseTestCase}; + +#[test] +fn given_parsed_command_when_deciding_then_only_python_free_commands_run_natively() { + let test_cases = [ + DecisionTestCase { + description: "a bare sqb has no handler", + argv: &[], + expected_command: Ok(NativeCommand::Nothing), + expected_creates_project: false, + }, + DecisionTestCase { + description: "clean has no handler", + argv: &["clean"], + expected_command: Ok(NativeCommand::Nothing), + expected_creates_project: false, + }, + DecisionTestCase { + description: "init creates a project", + argv: &["init", "--project-dir", "shop"], + expected_command: Ok(NativeCommand::Init), + expected_creates_project: true, + }, + DecisionTestCase { + description: "skills with targets", + argv: &[ + "skills", "--target", "claude", "--target", "agents", "--force", + ], + expected_command: Ok(NativeCommand::Skills(SkillsOptions { + global_install: false, + targets: vec!["claude".to_owned(), "agents".to_owned()], + force: true, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules list as JSON", + argv: &["rules", "--json", "list"], + expected_command: Ok(NativeCommand::Rules(RulesOptions { + action: RulesAction::List, + selector: None, + json: true, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules show", + argv: &["rules", "show", "SQBRMODEL101"], + expected_command: Ok(NativeCommand::Rules(RulesOptions { + action: RulesAction::Show, + selector: Some("SQBRMODEL101".to_owned()), + json: false, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules run evaluates the project in Python", + argv: &["rules", "run", "SQBRMODEL"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + DecisionTestCase { + description: "debug output is Python's", + argv: &["clean", "--debug"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + DecisionTestCase { + description: "compile needs Python", + argv: &["compile"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + ]; + + for test_case in test_cases { + let (outcome, _) = parse(test_case.argv); + let namespace: Namespace = outcome.parsed.expect(test_case.description); + + let decided = native_invocation(&namespace); + + assert_eq!( + decided + .as_ref() + .map(|invocation| invocation.command.clone()) + .map_err(|reason| *reason), + test_case.expected_command.clone(), + "{}", + test_case.description + ); + assert_eq!( + decided.is_ok_and(|invocation| invocation.creates_project), + test_case.expected_creates_project, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_command_line_when_parsing_natively_then_only_python_converters_need_python() { + let test_cases = [ + ParseTestCase { + description: "--vars JSON is parsed by Python", + argv: &["compile", "--vars", "{}"], + expected_needs_python: true, + }, + ParseTestCase { + description: "an integer option converts natively", + argv: &["build", "--concurrency", "4"], + expected_needs_python: false, + }, + ]; + + for test_case in test_cases { + let (_, host) = parse(test_case.argv); + + assert_eq!( + host.needs_python, test_case.expected_needs_python, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs new file mode 100644 index 0000000000..6183cb8ae0 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs @@ -0,0 +1,77 @@ +use std::path::{Path, PathBuf}; + +use crate::native_commands::_helpers::paths::{pathlib_path, resolved_path}; +use crate::native_commands::tests::test_types::{PathlibTestCase, ResolveTestCase}; + +#[test] +fn given_project_dir_text_when_building_a_path_then_pathlib_normalisation_applies() { + let test_cases = [ + PathlibTestCase { + description: "a leading dot and a trailing slash drop", + text: "./shop/", + expected_path: "shop", + }, + PathlibTestCase { + description: "inner dots and repeated slashes drop", + text: "shop/.//orders", + expected_path: "shop/orders", + }, + PathlibTestCase { + description: "a lone dot stays", + text: ".", + expected_path: ".", + }, + PathlibTestCase { + description: "parent references stay", + text: "../shop", + expected_path: "../shop", + }, + ]; + + for test_case in test_cases { + assert_eq!( + pathlib_path(Path::new(test_case.text)), + PathBuf::from(test_case.expected_path), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_path_when_resolving_then_existing_prefix_is_real_and_rest_is_kept() { + let test_cases = [ + ResolveTestCase { + description: "a missing directory is joined as written", + text: "new/shop", + expected_parts: &["new", "shop"], + }, + ResolveTestCase { + description: "a parent reference after a missing directory removes it", + text: "missing/../new", + expected_parts: &["new"], + }, + ResolveTestCase { + description: "the working directory itself", + text: ".", + expected_parts: &[], + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let real: PathBuf = std::fs::canonicalize(root.path()).expect("real root"); + + let resolved: PathBuf = resolved_path(Path::new(test_case.text), &real); + + assert_eq!( + resolved, + test_case + .expected_parts + .iter() + .fold(real.clone(), |path, part| path.join(part)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs new file mode 100644 index 0000000000..c7b317e626 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs @@ -0,0 +1,60 @@ +use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::ProjectFactsTestCase; + +const PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n"; +const NAMED_SINK_PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n\n[sinks.lifecycle.named.audit]\nmin_severity = \"error\"\n"; + +#[test] +fn given_project_files_when_checking_runtime_extensions_then_python_owned_ones_need_python() { + let test_cases = [ + ProjectFactsTestCase { + description: "no project file", + directories: &[], + files: &[], + expected_native: true, + }, + ProjectFactsTestCase { + description: "a plain project", + directories: &[], + files: &[("sqlbuild_project.toml", PROJECT)], + expected_native: true, + }, + ProjectFactsTestCase { + description: "a sinks directory is imported", + directories: &["sinks"], + files: &[("sqlbuild_project.toml", PROJECT)], + expected_native: false, + }, + ProjectFactsTestCase { + description: "the replaced event exporters directory is rejected in Python", + directories: &["event_exporters"], + files: &[], + expected_native: false, + }, + ProjectFactsTestCase { + description: "a named lifecycle sink must be bound in Python", + directories: &[], + files: &[("sqlbuild_project.toml", NAMED_SINK_PROJECT)], + expected_native: false, + }, + ProjectFactsTestCase { + description: "an invalid project file is reported by Python", + directories: &[], + files: &[("sqlbuild_project.toml", "name = \n")], + expected_native: false, + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + write_tree(root.path(), test_case.directories, test_case.files); + + assert_eq!( + has_no_runtime_extensions(root.path()), + test_case.expected_native, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs new file mode 100644 index 0000000000..4eedd7443e --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs @@ -0,0 +1,108 @@ +use crate::native_commands::_helpers::rules::run_rules; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{RulesAction, RulesOptions}; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::{RulesTestCase, output_start}; + +const PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n"; +const OPTIONS_PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n\n[rules.rule_options.SQBRCONTRACT102]\nallow_numeric_indicators = true\n"; + +#[test] +fn given_project_when_inspecting_rules_then_builtin_catalogue_prints_unless_python_owns_it() { + let test_cases = [ + RulesTestCase { + description: "list prints code and message lines", + directories: &[], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Ok(( + "SQBRCONTRACT101 models must declare an enforced output contract\n", + 0, + )), + }, + RulesTestCase { + description: "list --json prints the sorted document", + directories: &[], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: true, + expected_output: Ok(( + "{\"rules\": [{\"code\": \"SQBRCONTRACT101\", \"custom\": false, \"family\": \"SQBRCONTRACT\", ", + 0, + )), + }, + RulesTestCase { + description: "show prints the rule", + directories: &[], + project_file: PROJECT, + action: RulesAction::Show, + selector: Some("SQBRMODEL101"), + json: false, + expected_output: Ok(( + "SQBRMODEL101: dependency-import-ctes\nFamily: SQBRMODEL\n", + 0, + )), + }, + RulesTestCase { + description: "an unknown rule is a usage failure", + directories: &[], + project_file: PROJECT, + action: RulesAction::Show, + selector: Some("SQBRNOPE101"), + json: false, + expected_output: Ok(("Unknown rule: SQBRNOPE101\n", 2)), + }, + RulesTestCase { + description: "custom rules are imported by Python", + directories: &["rules"], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Err(PythonNeeded::Project), + }, + RulesTestCase { + description: "configured options are rendered by Python", + directories: &[], + project_file: OPTIONS_PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Err(PythonNeeded::Project), + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + write_tree( + root.path(), + test_case.directories, + &[("sqlbuild_project.toml", test_case.project_file)], + ); + + let output = run_rules( + root.path(), + &RulesOptions { + action: test_case.action, + selector: test_case.selector.map(str::to_owned), + json: test_case.json, + }, + ); + + let length: usize = test_case + .expected_output + .map_or(0, |(prefix, _)| prefix.chars().count()); + + assert_eq!( + output.map(|output| output_start(&output, length)), + test_case + .expected_output + .map(|(prefix, code)| (prefix.to_owned(), code)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs new file mode 100644 index 0000000000..528aeaf4e4 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -0,0 +1,68 @@ +use crate::argv::models::{TypeFailure, Value}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, NativeCommand, RulesAction}; + +/// One converter applied to one command-line string. +pub(super) struct ConversionTestCase { + pub(super) description: &'static str, + pub(super) converter: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, + pub(super) expected_needs_python: bool, +} + +/// One accepted command line, the native command it runs and its invocation services. +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_command: Result, + pub(super) expected_creates_project: bool, +} + +/// One command line and whether its parse needs Python. +pub(super) struct ParseTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_needs_python: bool, +} + +/// One `--project-dir` string and the `pathlib.Path` it names. +pub(super) struct PathlibTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_path: &'static str, +} + +/// A path beneath a real directory and what `Path.resolve()` makes of it. +pub(super) struct ResolveTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_parts: &'static [&'static str], +} + +/// Project directories and files, and whether an observed command can run without Python. +pub(super) struct ProjectFactsTestCase { + pub(super) description: &'static str, + pub(super) directories: &'static [&'static str], + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_native: bool, +} + +/// One `rules` inspection over a project and the start of what it prints. +pub(super) struct RulesTestCase { + pub(super) description: &'static str, + pub(super) directories: &'static [&'static str], + pub(super) project_file: &'static str, + pub(super) action: RulesAction, + pub(super) selector: Option<&'static str>, + pub(super) json: bool, + pub(super) expected_output: Result<(&'static str, i32), PythonNeeded>, +} + +/// The printed start and exit status of a command's output, for comparison. +pub(super) fn output_start(output: &CommandOutput, length: usize) -> (String, i32) { + ( + output.stdout.chars().take(length).collect(), + output.exit_code, + ) +} diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 2f6143377e..a39ae9398f 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -82,3 +82,34 @@ class LaunchedTeardownTestCase: description: str args: tuple[str, ...] expected_hits: int + + +@dataclass(frozen=True) +class PythonFreeTestCase: + """One command line `sqb` serves without Python, in a directory `prepare` sets up.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int + expected_captures: int + + +@dataclass(frozen=True) +class PythonStartTestCase: + """One command line `sqb` hands to Python because of the project `prepare` writes.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_python_starts: int + + +@dataclass(frozen=True) +class InterruptTestCase: + """One Python-free command line interrupted while it waits on the project `prepare` writes.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index e81c78ba3a..09fbd33876 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -2,6 +2,7 @@ import os import re import shutil +import signal import subprocess import sys from pathlib import Path @@ -199,3 +200,94 @@ def compiled_files(project_dir: Path) -> dict[str, bytes]: path.relative_to(compiled).as_posix(): path.read_bytes() for path in sorted(compiled.rglob("*.sql")) } + + +def counting_environment(directory: Path, marker: Path) -> Path: + """Install `sqb` in an environment whose `sqlbuild-python` logs each start; return `sqb`.""" + + program: Path = counting_install(directory / "bin", marker) + site_packages: Path = ( + directory / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" + ) / "site-packages" + site_packages.mkdir(parents=True) + _ = (site_packages / "sqlbuild_source.pth").write_text( + str(Path(sqlbuild.__file__).resolve().parent.parent) + "\n", encoding="utf-8" + ) + return program + + +def reset_directory(directory: Path) -> Path: + """An empty `directory`, removing whatever an earlier run left.""" + + shutil.rmtree(directory, ignore_errors=True) + directory.mkdir(parents=True) + return directory + + +def _capture_files(capture: Path) -> dict[str, object]: + metadata: dict[str, object] = capture_metadata(capture) + for volatile in ("invocation_id", "started_at", "completed_at", "capture_date"): + _ = metadata.pop(volatile) + streams: dict[str, bytes] = { + name: (capture / name).read_bytes() + for name in ("stdout.log", "stderr.log", "diagnostics.jsonl") + } + return {"complete": (capture / "complete").exists(), "metadata": metadata, **streams} + + +def compute_log_files(project_dir: Path) -> list[dict[str, object]]: + """Each capture's streams and metadata, without per-run identity and times.""" + + return [ + _capture_files(metadata.parent) + for metadata in sorted((project_dir / "logs").glob("*/*/metadata.json")) + ] + + +def prepare_nothing(directory: Path) -> None: + """Leave the directory empty.""" + + _ = directory + + +def prepare_project(directory: Path) -> None: + """Write the one-model project.""" + + _ = write_project(directory) + + +def prepare_custom_skill(directory: Path) -> None: + """Write a hand-written skill where `sqb init` installs the packaged one.""" + + custom: Path = directory / ".claude" / "skills" / "sqlbuild" / "SKILL.md" + custom.parent.mkdir(parents=True) + _ = custom.write_text("my own skill\n", encoding="utf-8") + + +def prepare_stale_skills(directory: Path) -> None: + """Write the project configured for Claude skills that are not installed yet.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + '\n[skills]\ntargets = ["claude"]\n', + encoding="utf-8", + ) + + +def prepare_custom_rules(directory: Path) -> None: + """Write the project with a custom rules directory.""" + + _ = write_project(directory) + (directory / "rules").mkdir() + + +def prepare_named_pipe_project(directory: Path) -> None: + """Make the project file a named pipe that blocks its reader.""" + + os.mkfifo(directory / "sqlbuild_project.toml") + + +def restore_default_interrupt() -> None: + """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py new file mode 100644 index 0000000000..6a787e7454 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -0,0 +1,333 @@ +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import ( + InterruptTestCase, + PythonFreeTestCase, + PythonStartTestCase, +) +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + PYTHON_CLI, + LauncherRun, + compute_log_files, + counting_environment, + prepare_custom_rules, + prepare_custom_skill, + prepare_named_pipe_project, + prepare_nothing, + prepare_project, + prepare_stale_skills, + python_starts, + reset_directory, + restore_default_interrupt, + run, +) + +INTERRUPT_DELAY_SECONDS: float = 1.0 +INTERRUPT_TIMEOUT_SECONDS: float = 30.0 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + PythonFreeTestCase( + description="top-level help", + args=("--help",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="short help after options", + args=("--no-color", "-h"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="command help", + args=("compile", "--help"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="nested command help", + args=("rules", "show", "--help"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="version among options", + args=("--no-color", "--version"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="unknown command", + args=("not-a-command",), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="invalid choice", + args=("skills", "--target", "bogus"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="invalid integer", + args=("build", "--concurrency", "many"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="post-parse rule", + args=("build", "--concurrency", "0"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="missing option value", + args=("--project-dir",), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="unrecognized argument", + args=("rules", "list", "--json"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="mutually exclusive options", + args=("lineage", "orders", "--json", "--format", "tree"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="bare sqb", + args=(), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="clean", + args=("clean",), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="clean outside a project", + args=("clean",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules list", + args=("rules", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules without an action", + args=("rules",), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="rules list json", + args=("rules", "--json", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show", + args=("rules", "show", "SQBRMODEL101"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show with thresholds", + args=("rules", "show", "SQBRTEST101"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show unknown", + args=("rules", "show", "SQBRNOPE101"), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules with stale skills", + args=("rules", "list"), + prepare=prepare_stale_skills, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="skills", + args=("skills",), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="skills for one target", + args=("skills", "--target", "claude"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="init", + args=("init",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="init a new directory", + args=("init", "--project-dir", "shop/"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="init beside a custom skill", + args=("init",), + prepare=prepare_custom_skill, + expected_exit_code=1, + expected_captures=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_free_invocation_when_running_sqb_then_python_never_starts_and_output_matches( + test_case: PythonFreeTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = tmp_path / "workspace" + test_case.prepare(reset_directory(workspace)) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=workspace) + python_logs: list[dict[str, object]] = compute_log_files(workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native == python + assert native[0] == test_case.expected_exit_code + assert python_starts(marker) == 0 + assert compute_log_files(workspace) == python_logs + assert len(python_logs) == test_case.expected_captures + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + PythonStartTestCase( + description="custom rules", + args=("rules", "list"), + prepare=prepare_custom_rules, + expected_python_starts=1, + ), + PythonStartTestCase( + description="dbt passthrough help", + args=("dbt", "plan", "--help"), + prepare=prepare_nothing, + expected_python_starts=1, + ), + PythonStartTestCase( + description="debug diagnostics", + args=("clean", "--debug"), + prepare=prepare_project, + expected_python_starts=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_invocation_needing_python_when_running_sqb_then_python_runs_it( + test_case: PythonStartTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = reset_directory(tmp_path / "workspace") + test_case.prepare(workspace) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native[:2] == python[:2] + assert python_starts(marker) == test_case.expected_python_starts + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals and named pipes") +@pytest.mark.parametrize( + "test_case", + [ + InterruptTestCase( + description="rules list waiting on the project file", + args=("rules", "list"), + prepare=prepare_named_pipe_project, + expected_exit_code=130, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_python( + test_case: InterruptTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + project_dir: Path = reset_directory(tmp_path / "project") + test_case.prepare(project_dir) + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(program), *test_case.args], + cwd=project_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=restore_default_interrupt, + ) + time.sleep(INTERRUPT_DELAY_SECONDS) + assert process.poll() is None, "sqb returned before reading the project file" + + process.send_signal(signal.SIGINT) + _, stderr = process.communicate(timeout=INTERRUPT_TIMEOUT_SECONDS) + + assert process.returncode == test_case.expected_exit_code, stderr + assert python_starts(marker) == 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 986b63d4c8a0a594a755b7a841f73da94311af8b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:28:38 +0100 Subject: [PATCH 05/39] docs: note that Recall does not record commands sqb answers without Python --- .../references/docs/concepts/observability/recall.md | 5 +++-- .../src/content/docs/docs/concepts/observability/recall.mdx | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 2b92ac0dcb..b481c1fbcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -41,8 +41,9 @@ of its history still succeeds, prints a warning followed by its own final status status `abandoned`. Every command is recorded except `sqb recall` itself, commands that create a project (such as -`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not -recorded either. +`sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not +recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, +`sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 69a1328e64..2511e7fb0f 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -26,8 +26,9 @@ of its history still succeeds, prints a warning followed by its own final status status `abandoned`. Every command is recorded except `sqb recall` itself, commands that create a project (such as -`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not -recorded either. +`sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not +recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, +`sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. From 2ca8a36757fa64e0a0019528a3b666ce3bda9f31 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:19:08 +0100 Subject: [PATCH 06/39] fix: resolve native project paths and the home directory as Python does on Windows --- .../src/native_commands/_helpers/paths.rs | 33 +++++++++++++++++-- .../src/native_commands/_helpers/skills.rs | 2 +- .../src/native_commands/constants.rs | 4 +++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index c50eea5af9..302e8a5a66 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -2,6 +2,8 @@ use std::path::{Component, Path, PathBuf}; +use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; + use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::ProjectLocation; @@ -24,8 +26,8 @@ pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { } else { cwd.join(path) }; - if let Ok(real) = std::fs::canonicalize(&absolute) { - return real; + if absolute.exists() { + return project_real_path(&absolute); } let mut resolved: PathBuf = PathBuf::new(); for component in absolute.components() { @@ -37,7 +39,11 @@ pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { } Component::Normal(name) => { let candidate: PathBuf = resolved.join(name); - resolved = std::fs::canonicalize(&candidate).unwrap_or(candidate); + resolved = if candidate.exists() { + project_real_path(&candidate) + } else { + candidate + }; } } } @@ -52,5 +58,26 @@ pub(crate) fn project_location(given: Option<&Path>) -> Result cwd.clone(), }; let resolved: PathBuf = resolved_path(&given, &cwd); + if !spelled_as_given(&resolved, &cwd.join(&given)) { + return Err(PythonNeeded::Installation); + } Ok(ProjectLocation { given, resolved }) } + +/// Links resolve as in Python on Unix; Windows drive substitutions and mappings stay Python's. +#[cfg(unix)] +fn spelled_as_given(_resolved: &Path, _absolute: &Path) -> bool { + true +} + +/// Whether the resolved path is the absolute path as given, apart from letter case. +#[cfg(not(unix))] +fn spelled_as_given(resolved: &Path, absolute: &Path) -> bool { + let absolute: PathBuf = absolute + .components() + .filter(|component| *component != Component::CurDir) + .collect(); + resolved + .to_string_lossy() + .eq_ignore_ascii_case(&absolute.to_string_lossy()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs index 4272d40185..ac7eee9a36 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs @@ -10,7 +10,7 @@ use crate::native_commands::constants::{HOME_VARIABLE, SKILL_FILES_HEADER}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, InstalledPackage, SkillsOptions}; -/// `Path.home()` from `HOME`, unless only Python's password lookup could tell. +/// `Path.home()` from `HOME` (`USERPROFILE` on Windows), unless only Python could tell. pub(crate) fn home_dir() -> Result { std::env::var_os(HOME_VARIABLE) .filter(|home| !home.is_empty()) diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 9c01a84a6b..fdef90a8cf 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -27,7 +27,11 @@ pub(crate) const SITE_DIRECTORIES: [&str; 2] = ["site-packages", "dist-packages" pub(crate) const WINDOWS_SITE_DIRECTORY: [&str; 2] = ["Lib", "site-packages"]; /// The packaged skill beneath the `sqlbuild` package directory. pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"]; +/// The variable `Path.home()` reads first; without it `sqb` defers to Python's other lookups. +#[cfg(not(windows))] pub(crate) const HOME_VARIABLE: &str = "HOME"; +#[cfg(windows)] +pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; /// The exit status of a command stopped by Ctrl+C. #[cfg(unix)] pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; From 8ccb01fc426492287a993b1d466440bf78775801 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:19:27 +0100 Subject: [PATCH 07/39] fix: report lost stdout, Ctrl+C, init failures and the version as the Python CLI does --- crates/sqlbuild-cli/Cargo.toml | 4 +- .../src/compute_logs/constants.rs | 5 + .../src/compute_logs/main/abandon_capture.rs | 72 ++++++++++++ .../sqlbuild-cli/src/compute_logs/main/mod.rs | 1 + .../src/launcher/_helpers/messages.rs | 11 +- crates/sqlbuild-cli/src/launcher/constants.rs | 2 + .../src/native_commands/_helpers/argv_host.rs | 3 +- .../src/native_commands/_helpers/execution.rs | 43 ++++--- .../src/native_commands/_helpers/init.rs | 43 ++++--- .../src/native_commands/_helpers/terminal.rs | 108 ++++++++++++++---- .../src/native_commands/constants.rs | 11 +- .../main/run_native_invocation.rs | 20 ++-- .../src/native_commands/models.rs | 11 ++ .../sqlbuild-cli/src/native_commands/tests.rs | 4 + .../src/native_commands/tests/test_init.rs | 56 +++++++++ .../native_commands/tests/test_terminal.rs | 75 ++++++++++++ .../src/native_commands/tests/test_types.rs | 15 +++ .../src/sqlbuild/cli/entry/_test_types.py | 11 ++ .../src/sqlbuild/cli/entry/helpers.py | 25 +++- .../cli/entry/test_python_free_commands.py | 63 +++++++++- 20 files changed, 515 insertions(+), 68 deletions(-) create mode 100644 crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_init.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index cdfce87880..791fafe765 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,7 +17,6 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true -ctrlc.workspace = true getrandom.workspace = true serde.workspace = true serde_json.workspace = true @@ -29,6 +28,9 @@ sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(windows)'.dependencies] +ctrlc.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs index ba6664c9b8..44dfb4738b 100644 --- a/crates/sqlbuild-cli/src/compute_logs/constants.rs +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -33,3 +33,8 @@ pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; pub const MONTHS_PER_YEAR: i64 = 12; pub const MAX_DAY_OF_MONTH: i64 = 31; pub const HOURS_PER_DAY: i64 = 24; +/// The debug record Python's capture logs when its streams could not be finalized. +pub const CAPTURE_FAILURE_LOGGER: &str = "sqlbuild.cli.compute_logs"; +pub const CAPTURE_FAILURE_MESSAGE: &str = "local compute log capture unavailable"; +pub const DIAGNOSTIC_PRODUCER: &str = "sqlbuild"; +pub const DIAGNOSTIC_SCHEMA_VERSION: i64 = 1; diff --git a/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs b/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs new file mode 100644 index 0000000000..1697a7896b --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs @@ -0,0 +1,72 @@ +//! Leave a capture incomplete after its final flush failed, as Python's capture cleanup does. + +use std::io::{self, Write}; +use std::time::SystemTime; + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, OrjsonOptions}; + +use crate::compute_logs::_helpers::timestamps::{epoch_microseconds, utc_timestamp}; +use crate::compute_logs::constants::{ + CAPTURE_FAILURE_LOGGER, CAPTURE_FAILURE_MESSAGE, DIAGNOSTIC_PRODUCER, DIAGNOSTIC_SCHEMA_VERSION, +}; +use crate::compute_logs::models::{CaptureEnd, OpenCapture}; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +/// The debug record `log_compute_capture_failure` writes to the capture's diagnostics. +fn capture_failure_record(invocation_id: &str, occurred_at: SystemTime) -> String { + let record: JsonValue = JsonValue::Object(vec![ + ("fields".to_owned(), JsonValue::Object(Vec::new())), + ("invocation_id".to_owned(), text(invocation_id)), + ("log_stream_id".to_owned(), JsonValue::Null), + ("logger".to_owned(), text(CAPTURE_FAILURE_LOGGER)), + ("message".to_owned(), text(CAPTURE_FAILURE_MESSAGE)), + ( + "occurred_at".to_owned(), + text(&utc_timestamp(epoch_microseconds(occurred_at))), + ), + ("operation_id".to_owned(), JsonValue::Null), + ("producer".to_owned(), text(DIAGNOSTIC_PRODUCER)), + ( + "producer_version".to_owned(), + text(env!("CARGO_PKG_VERSION")), + ), + ("resource_attempt_id".to_owned(), JsonValue::Null), + ("resource_id".to_owned(), JsonValue::Null), + ("run_id".to_owned(), JsonValue::Null), + ( + "schema_version".to_owned(), + JsonValue::Integer(JsonInteger::from(DIAGNOSTIC_SCHEMA_VERSION)), + ), + ("severity".to_owned(), text("debug")), + ("source".to_owned(), text("python_logging")), + ("statement_id".to_owned(), JsonValue::Null), + ]); + let options: OrjsonOptions = OrjsonOptions { + sort_keys: true, + ..OrjsonOptions::default() + }; + dumps(&record, &JsonDialect::Orjson(options)).unwrap_or_default() + "\n" +} + +/// Append the streams the command wrote and the capture failure, without completing it. +pub fn abandon_capture(capture: OpenCapture, end: &CaptureEnd) -> io::Result<()> { + let OpenCapture { + invocation_id, + mut streams, + .. + } = capture; + let [stdout, stderr, diagnostics] = streams.as_mut_slice() else { + return Err(io::Error::other("capture streams are not open")); + }; + stdout.write_all(&end.stdout)?; + stderr.write_all(&end.stderr)?; + diagnostics.write_all(capture_failure_record(&invocation_id, SystemTime::now()).as_bytes())?; + for stream in [stdout, stderr, diagnostics] { + stream.flush()?; + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs index 8e44c3eed2..60606a3a12 100644 --- a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs +++ b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs @@ -1,3 +1,4 @@ +pub mod abandon_capture; pub mod capture_inventory; pub mod capture_is_complete; pub mod check_completion; diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs index 8a91e6387d..6a9994f629 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs @@ -3,11 +3,16 @@ use std::io::{self, Write}; use std::path::Path; -use crate::launcher::constants::{PROGRAM_NAME, PYTHON_CLI_SCRIPT}; +use crate::launcher::constants::{PROGRAM_NAME, PROGRAM_VERSION, PYTHON_CLI_SCRIPT}; -/// argparse's `--version` line: `sqb ` on stdout. +/// argparse's `--version` line: `sqb `. +pub(crate) fn version_line() -> String { + format!("{PROGRAM_NAME} {PROGRAM_VERSION}\n") +} + +/// Print the `--version` line on stdout. pub(crate) fn print_version() { - let _ = writeln!(io::stdout(), "{PROGRAM_NAME} {}", env!("CARGO_PKG_VERSION")); + let _ = io::stdout().write_all(version_line().as_bytes()); } /// The Python CLI script is not installed beside this program. diff --git a/crates/sqlbuild-cli/src/launcher/constants.rs b/crates/sqlbuild-cli/src/launcher/constants.rs index 82cad78763..ce9a8716eb 100644 --- a/crates/sqlbuild-cli/src/launcher/constants.rs +++ b/crates/sqlbuild-cli/src/launcher/constants.rs @@ -11,6 +11,8 @@ pub const LAUNCH_RECORD_OPTION: &str = "--sqlbuild-launch-record="; /// The program name argparse reports, fixed by the Python parser. pub const PROGRAM_NAME: &str = "sqb"; pub const VERSION_OPTIONS: [&str; 2] = ["--version", "-V"]; +/// The version `--version` prints, alone or among other options; the wheels share it. +pub const PROGRAM_VERSION: &str = env!("CARGO_PKG_VERSION"); /// Global options naming the project directory, exactly as the Python parser spells them. pub const PROJECT_DIR_OPTIONS: [&str; 2] = ["--project-dir", "--sqb-project-dir"]; /// Prefixes argparse could expand to a project directory option. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs index 9e26816a90..4a7f8b62de 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs @@ -7,6 +7,7 @@ use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::argv::main::cli_namespace_defaults::cli_namespace_defaults; use crate::argv::models::{NamespaceDefault, TypeFailure, Value}; use crate::argv::types::ArgvHost; +use crate::launcher::constants::PROGRAM_VERSION; use crate::native_commands::constants::{ DIGIT_SEPARATOR, INT_ASCII_SPACES, INT_CONVERTER, NEGATIVE_SIGN, POSITIVE_INTEGER_CONVERTER, POSITIVE_INTEGER_MESSAGE, POSITIVE_SIGN, PYTHON_INT_MAX_STR_DIGITS, ZERO_TEXT, @@ -93,7 +94,7 @@ impl ArgvHost for NativeArgvHost { } fn program_version(&mut self) -> String { - env!("CARGO_PKG_VERSION").to_owned() + PROGRAM_VERSION.to_owned() } } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 278a397f81..7021116ad0 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -6,6 +6,7 @@ use std::time::SystemTime; use crate::argv::models::Stream; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; @@ -16,11 +17,11 @@ use crate::native_commands::_helpers::paths::project_location; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{exit_on_interrupt, write_text}; -use crate::native_commands::constants::UNKNOWN_COMMAND; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. @@ -44,6 +45,19 @@ fn open_capture( }) } +/// Close the capture the way Python's does once the command's output met its reader. +fn close_capture(capture: OpenCapture, mut end: CaptureEnd, lost: Option<&LostStdout>) { + match lost { + None => drop(finish_capture(capture, &end)), + Some(LostStdout::Raised(_)) => { + end.stdout.clear(); + end.exit_code = i64::from(UNCAUGHT_EXCEPTION_STATUS); + drop(finish_capture(capture, &end)); + } + Some(LostStdout::AtExit(_) | LostStdout::Dropped) => drop(abandon_capture(capture, &end)), + } +} + /// Print an observed command's output after the skill notice, retaining both in its capture. fn finish_observed( invocation: &NativeInvocation, @@ -58,20 +72,22 @@ fn finish_observed( }; let capture: io::Result = open_capture(invocation, project, started_at); let stderr: String = notice + &output.stderr; - write_text(Stream::Stderr, &stderr); - write_text(Stream::Stdout, &output.stdout); + let lost: Option = write_output(&[ + (Stream::Stderr, stderr.as_str()), + (Stream::Stdout, output.stdout.as_str()), + ]); if let Ok(capture) = capture { - let finished: io::Result<()> = finish_capture( + close_capture( capture, - &CaptureEnd { + CaptureEnd { stdout: output.stdout.into_bytes(), stderr: stderr.into_bytes(), exit_code: i64::from(output.exit_code), }, + lost.as_ref(), ); - drop(finished); } - output.exit_code + finish_lost_stdout(lost.as_ref(), output.exit_code) } /// What an observed command prints, unless it needs Python after all. @@ -97,14 +113,15 @@ pub(crate) fn run_command( python_cli: &Path, error_color: bool, ) -> Result { - exit_on_interrupt(); let package: InstalledPackage = installed_package(python_cli)?; let project: ProjectLocation = project_location(invocation.project_dir.as_deref())?; if invocation.creates_project { let output: CommandOutput = run_init(&project.resolved, &package, error_color)?; - write_text(Stream::Stderr, &output.stderr); - write_text(Stream::Stdout, &output.stdout); - return Ok(output.exit_code); + let lost: Option = write_output(&[ + (Stream::Stderr, output.stderr.as_str()), + (Stream::Stdout, output.stdout.as_str()), + ]); + return Ok(finish_lost_stdout(lost.as_ref(), output.exit_code)); } let started_at: SystemTime = SystemTime::now(); let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs index dfee4cd22b..ec54a0f30a 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs @@ -17,7 +17,7 @@ use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_pr use crate::dispatch::main::failure_report::failure_report; use crate::dispatch::models::CommandFailure; use crate::native_commands::_helpers::skills::home_dir; -use crate::native_commands::_helpers::terminal::stdout_supports_color; +use crate::native_commands::_helpers::terminal::{error_description, stdout_supports_color}; use crate::native_commands::constants::{INIT_FAILURE_PREFIX, USER_ERROR_CLASS}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, InstalledPackage}; @@ -31,15 +31,31 @@ fn failed(stderr: String) -> Result { }) } -/// The project's skills could not be installed: what the Python CLI prints for its error. -fn skills_failure(error: &WorkspaceError, use_color: bool) -> Result { - let WorkspaceError::User { - code, - message, - help, - } = error - else { - return failed(format!("{INIT_FAILURE_PREFIX}{error:?}\n")); +/// The project's skills could not be installed: what the Python CLI prints for its user error. +fn skills_failure( + base_dir: &Path, + error: &WorkspaceError, + use_color: bool, +) -> Result { + let (code, message, help) = match error { + WorkspaceError::User { + code, + message, + help, + } => (code, message, help), + WorkspaceError::Decode { path, .. } => { + return failed(format!( + "{INIT_FAILURE_PREFIX}cannot install SQLBuild skills: {} is not UTF-8 text\n", + path.display() + )); + } + WorkspaceError::Io(error) => { + return failed(format!( + "{INIT_FAILURE_PREFIX}cannot install SQLBuild skills into {}: {}\n", + base_dir.display(), + error_description(error) + )); + } }; let (_, log_message) = failure_report(CommandFailure::User); let coded: String = format_coded_error( @@ -75,8 +91,9 @@ pub(crate) fn run_init( return Err(PythonNeeded::Failure); } return failed(format!( - "{INIT_FAILURE_PREFIX}{error}: {}\n", - path.display() + "{INIT_FAILURE_PREFIX}cannot create {}: {}\n", + path.display(), + error_description(&error) )); } if let Err(error) = install_skills(&SkillInstallRequest { @@ -87,7 +104,7 @@ pub(crate) fn run_init( requested_targets: Vec::new(), force: false, }) { - return skills_failure(&error, error_color); + return skills_failure(base_dir, &error, error_color); } Ok(CommandOutput { stdout: render_init_summary( diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs index 3a584e7abd..44d55e4570 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -1,11 +1,15 @@ -//! The process's terminal: its width, colour support, stream writes and Ctrl+C. +//! The process's terminal: its width, colour support and stream writes. use std::io::{self, IsTerminal, Write}; use crate::argv::models::Stream; use crate::native_commands::_helpers::argv_host::python_int; -use crate::native_commands::constants::{COLUMNS_VARIABLE, FALLBACK_COLUMNS, NO_COLOR_VARIABLE}; +use crate::native_commands::constants::{ + BROKEN_PIPE_EXCEPTION, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, EXIT_FLUSH_FAILURE_STATUS, + FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PYTHON_STDOUT_BUFFER_SIZE, UNCAUGHT_EXCEPTION_STATUS, +}; use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::LostStdout; /// `int(os.environ["COLUMNS"])`, `0` when unset or not an integer; beyond 64 bits is Python's. fn columns_variable() -> Result { @@ -59,35 +63,93 @@ fn console_bytes(text: &str) -> Vec { } } -/// Write and flush one text; a closed or full stream drops it, as the parse's writes do. -pub(crate) fn write_text(stream: Stream, text: &str) { - if text.is_empty() { - return; - } - let bytes: Vec = console_bytes(text); - let written: io::Result<()> = match stream { +fn write_stream(stream: Stream, bytes: &[u8]) -> io::Result<()> { + match stream { Stream::Stdout => { let mut output = io::stdout().lock(); - output.write_all(&bytes).and_then(|()| output.flush()) + output.write_all(bytes).and_then(|()| output.flush()) } Stream::Stderr => { let mut output = io::stderr().lock(); - output.write_all(&bytes).and_then(|()| output.flush()) + output.write_all(bytes).and_then(|()| output.flush()) } - }; - drop(written); + } } -/// End the process with status 130 on Ctrl+C; handing off by `exec` restores the default. -#[cfg(unix)] -pub(crate) fn exit_on_interrupt() { - use crate::native_commands::constants::INTERRUPTED_EXIT_CODE; +/// The system's description of an I/O failure, without Rust's `(os error N)` suffix. +pub(crate) fn error_description(error: &io::Error) -> String { + let described: String = error.to_string(); + match error.raw_os_error() { + Some(errno) => described + .strip_suffix(&format!(" (os error {errno})")) + .unwrap_or(&described) + .to_owned(), + None => described, + } +} - drop(ctrlc::set_handler(|| { - std::process::exit(INTERRUPTED_EXIT_CODE) - })); +/// The last line of the traceback Python prints for a failed write. +fn exception_line(error: &io::Error) -> String { + if error.kind() == io::ErrorKind::BrokenPipe { + return BROKEN_PIPE_EXCEPTION.to_owned(); + } + match error.raw_os_error() { + Some(errno) => format!("OSError: [Errno {errno}] {}", error_description(error)), + None => format!("OSError: {error}"), + } } -/// The Python child waited for alone handles Ctrl+C, so the native command keeps the default. -#[cfg(not(unix))] -pub(crate) fn exit_on_interrupt() {} +/// CPython's 8 KiB stdout buffer keeps up to half until exit, drops more, and fails a full write. +pub(crate) fn lost_stdout(total: usize, error: &io::Error) -> LostStdout { + if total >= PYTHON_STDOUT_BUFFER_SIZE { + LostStdout::Raised(exception_line(error)) + } else if total <= PYTHON_STDOUT_BUFFER_SIZE / 2 { + LostStdout::AtExit(exception_line(error)) + } else { + LostStdout::Dropped + } +} + +/// Write each text in order and flush it; errors on stderr are ignored, as Python's are. +pub(crate) fn write_output(writes: &[(Stream, &str)]) -> Option { + let encoded: Vec<(Stream, Vec)> = writes + .iter() + .filter(|(_, text)| !text.is_empty()) + .map(|(stream, text)| (*stream, console_bytes(text))) + .collect(); + let total: usize = encoded + .iter() + .filter(|(stream, _)| *stream == Stream::Stdout) + .map(|(_, bytes)| bytes.len()) + .sum(); + let mut lost: Option = None; + for (stream, bytes) in &encoded { + let written: io::Result<()> = match (stream, &lost) { + (Stream::Stdout, Some(_)) => Ok(()), + _ => write_stream(*stream, bytes), + }; + if let (Stream::Stdout, Err(error)) = (stream, written) { + lost = Some(lost_stdout(total, &error)); + } + } + lost +} + +/// Report lost stdout as Python does and return the exit status that results. +pub(crate) fn finish_lost_stdout(lost: Option<&LostStdout>, exit_code: i32) -> i32 { + match lost { + None | Some(LostStdout::Dropped) => exit_code, + Some(LostStdout::AtExit(exception)) => { + let text: String = format!("{EXIT_FLUSH_FAILURE}\n{exception}\n"); + drop(write_stream(Stream::Stderr, &console_bytes(&text))); + EXIT_FLUSH_FAILURE_STATUS + } + Some(LostStdout::Raised(exception)) => { + drop(write_stream( + Stream::Stderr, + &console_bytes(&format!("{exception}\n")), + )); + UNCAUGHT_EXCEPTION_STATUS + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index fdef90a8cf..5b1dd8ff84 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -32,9 +32,6 @@ pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"] pub(crate) const HOME_VARIABLE: &str = "HOME"; #[cfg(windows)] pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; -/// The exit status of a command stopped by Ctrl+C. -#[cfg(unix)] -pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; /// The command name compute logs record for a bare `sqb`. pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; /// Commands with no handler: observed, then successful. @@ -77,3 +74,11 @@ pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4300; pub(crate) const DIGIT_SEPARATOR: char = '_'; /// Where a loaded project configuration names its lifecycle sinks. pub(crate) const LIFECYCLE_SINK_PATH: [&str; 3] = ["sinks", "lifecycle", "named"]; +/// CPython's default buffer size for a piped stdout. +pub(crate) const PYTHON_STDOUT_BUFFER_SIZE: usize = 8192; +pub(crate) const BROKEN_PIPE_EXCEPTION: &str = "BrokenPipeError: [Errno 32] Broken pipe"; +/// What Python prints when the final flush of a lost stdout fails, and the status it exits with. +pub(crate) const EXIT_FLUSH_FAILURE: &str = + "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; +pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; +pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 222d953909..1233e99081 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -4,16 +4,16 @@ use std::ffi::OsString; use std::path::Path; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace, Stream}; use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; use crate::native_commands::_helpers::terminal::{ - stdout_supports_color, terminal_columns, write_text, + finish_lost_stdout, stdout_supports_color, terminal_columns, write_output, }; use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::NativeInvocation; +use crate::native_commands::models::{LostStdout, NativeInvocation}; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( @@ -39,10 +39,16 @@ pub fn run_native_invocation( } let namespace: Namespace = match outcome.parsed { Err(code) => { - for (stream, text) in &outcome.writes { - write_text(*stream, text); - } - return Ok(i32::try_from(code).unwrap_or(1)); + let writes: Vec<(Stream, &str)> = outcome + .writes + .iter() + .map(|(stream, text)| (*stream, text.as_str())) + .collect(); + let lost: Option = write_output(&writes); + return Ok(finish_lost_stdout( + lost.as_ref(), + i32::try_from(code).unwrap_or(1), + )); } Ok(namespace) if outcome.writes.is_empty() => namespace, Ok(_) => return Err(PythonNeeded::Arguments), diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f0ce868fa2..9c6ece8ad5 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -71,3 +71,14 @@ pub(crate) struct ProjectLocation { /// `Path(--project-dir).resolve()`, or `Path.cwd()`. pub(crate) resolved: PathBuf, } + +/// What Python's buffered stdout does with output its reader no longer takes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum LostStdout { + /// Kept until exit, whose failed flush Python reports with this exception and status 120. + AtExit(String), + /// Dropped by a failed flush; the command's own status stands. + Dropped, + /// Raised by the write: an uncaught exception, with this last line and status 1. + Raised(String), +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs index 9119c141c9..2f86f371e1 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -4,11 +4,15 @@ mod argv_host; mod decision; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_init.rs"] +mod init; #[path = "tests/test_paths.rs"] mod paths; #[path = "tests/test_project_facts.rs"] mod project_facts; #[path = "tests/test_rules.rs"] mod rules; +#[path = "tests/test_terminal.rs"] +mod terminal; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs new file mode 100644 index 0000000000..29d90d5f7f --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs @@ -0,0 +1,56 @@ +use std::path::Path; + +use crate::native_commands::_helpers::init::run_init; +use crate::native_commands::models::InstalledPackage; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::InitFailureTestCase; + +#[cfg(unix)] +#[test] +fn given_unwritable_skill_directory_when_initialising_then_a_readable_error_names_the_path() { + let test_cases = [InitFailureTestCase { + description: "a file where the agents skill directory goes", + blocking_file: ".agents", + expected_stderr_start: "sqb init: cannot install SQLBuild skills into ", + expected_exit_code: 1, + }]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let skills = root.path().join("packaged"); + write_tree( + root.path(), + &["packaged", "shop"], + &[("packaged/SKILL.md", "# Skill\n")], + ); + let project: &Path = &root.path().join("shop"); + write_tree( + project, + &[], + &[(test_case.blocking_file, "not a directory\n")], + ); + + let output = run_init( + project, + &InstalledPackage { + skills_root: skills, + }, + false, + ) + .expect(test_case.description); + + assert!( + output.stderr.starts_with(test_case.expected_stderr_start) + && output.stderr.contains(&project.display().to_string()) + && output.stderr.ends_with("Not a directory\n"), + "{}: {}", + test_case.description, + output.stderr + ); + assert_eq!( + output.exit_code, test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs new file mode 100644 index 0000000000..19be6f26cd --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs @@ -0,0 +1,75 @@ +use std::io; + +use crate::argv::models::Stream; +use crate::launcher::constants::{PROGRAM_NAME, PROGRAM_VERSION}; +use crate::native_commands::_helpers::terminal::lost_stdout; +use crate::native_commands::models::LostStdout; +use crate::native_commands::tests::helpers::parse; +use crate::native_commands::tests::test_types::{LostStdoutTestCase, ParseTestCase}; + +/// Thresholds recorded from CPython 3.12 writing to a pipe without a reader. +#[test] +fn given_lost_stdout_bytes_when_classifying_then_python_buffer_outcome_matches() { + let test_cases = [ + LostStdoutTestCase { + description: "half the buffer stays until the final flush", + total: 4096, + expected_lost: LostStdout::AtExit("BrokenPipeError: [Errno 32] Broken pipe".to_owned()), + }, + LostStdoutTestCase { + description: "more than half is dropped by the failed flush", + total: 4097, + expected_lost: LostStdout::Dropped, + }, + LostStdoutTestCase { + description: "a full buffer fails the write", + total: 8192, + expected_lost: LostStdout::Raised("BrokenPipeError: [Errno 32] Broken pipe".to_owned()), + }, + ]; + + for test_case in test_cases { + let error: io::Error = io::Error::from(io::ErrorKind::BrokenPipe); + + assert_eq!( + lost_stdout(test_case.total, &error), + test_case.expected_lost, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_version_among_options_when_parsing_natively_then_the_lone_version_line_prints() { + let test_cases = [ + ParseTestCase { + description: "after --no-color", + argv: &["--no-color", "--version"], + expected_needs_python: false, + }, + ParseTestCase { + description: "the short option after --debug", + argv: &["--debug", "-V"], + expected_needs_python: false, + }, + ]; + + for test_case in test_cases { + let (outcome, host) = parse(test_case.argv); + + assert_eq!( + (outcome.writes, outcome.parsed, host.needs_python), + ( + vec![( + Stream::Stdout, + format!("{PROGRAM_NAME} {PROGRAM_VERSION}\n") + )], + Err(0), + test_case.expected_needs_python, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 528aeaf4e4..252ef24b95 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -66,3 +66,18 @@ pub(super) fn output_start(output: &CommandOutput, length: usize) -> (String, i3 output.exit_code, ) } + +/// Bytes a reader no longer takes and what Python's stdout buffer makes of them. +pub(super) struct LostStdoutTestCase { + pub(super) description: &'static str, + pub(super) total: usize, + pub(super) expected_lost: crate::native_commands::models::LostStdout, +} + +/// A directory `init` meets once the project file exists, and the start of its error. +pub(super) struct InitFailureTestCase { + pub(super) description: &'static str, + pub(super) blocking_file: &'static str, + pub(super) expected_stderr_start: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index a39ae9398f..5be5472a2a 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -113,3 +113,14 @@ class InterruptTestCase: args: tuple[str, ...] prepare: Callable[[Path], None] expected_exit_code: int + + +@dataclass(frozen=True) +class BrokenPipeTestCase: + """One Python-free command line whose stdout has no reader.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int + expected_stderr_end: bytes diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 09fbd33876..15825c8079 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -14,6 +14,7 @@ _ELAPSED_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d s\)") _PHASE_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d+s\)") +_DIAGNOSTIC_IDENTITY: re.Pattern[bytes] = re.compile(rb'"(invocation_id|occurred_at)":"[^"]*"') NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") PROJECT_FILES: dict[str, str] = { @@ -227,9 +228,9 @@ def reset_directory(directory: Path) -> Path: def _capture_files(capture: Path) -> dict[str, object]: metadata: dict[str, object] = capture_metadata(capture) for volatile in ("invocation_id", "started_at", "completed_at", "capture_date"): - _ = metadata.pop(volatile) + _ = metadata.pop(volatile, None) streams: dict[str, bytes] = { - name: (capture / name).read_bytes() + name: _DIAGNOSTIC_IDENTITY.sub(rb'"\1":""', (capture / name).read_bytes()) for name in ("stdout.log", "stderr.log", "diagnostics.jsonl") } return {"complete": (capture / "complete").exists(), "metadata": metadata, **streams} @@ -291,3 +292,23 @@ def restore_default_interrupt() -> None: """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" _ = signal.signal(signal.SIGINT, signal.SIG_DFL) + + +def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: + """Run with a stdout pipe whose reader has already closed; return exit code and stderr.""" + + reader, writer = os.pipe() + os.close(reader) + try: + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + [str(program), *args], + cwd=cwd, + stdout=writer, + stderr=subprocess.PIPE, + env={**os.environ, "NO_COLOR": "1"}, + check=False, + timeout=600, + ) + finally: + os.close(writer) + return completed.returncode, b"", completed.stderr diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py index 6a787e7454..fe5412e49b 100644 --- a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -7,6 +7,7 @@ import pytest from tests.integration.src.sqlbuild.cli.entry._test_types import ( + BrokenPipeTestCase, InterruptTestCase, PythonFreeTestCase, PythonStartTestCase, @@ -26,9 +27,11 @@ reset_directory, restore_default_interrupt, run, + run_without_reader, ) INTERRUPT_DELAY_SECONDS: float = 1.0 +TRACEBACK_LAST_LINE: bytes = b"BrokenPipeError: [Errno 32] Broken pipe\n" INTERRUPT_TIMEOUT_SECONDS: float = 30.0 @@ -299,12 +302,12 @@ def test_given_invocation_needing_python_when_running_sqb_then_python_runs_it( description="rules list waiting on the project file", args=("rules", "list"), prepare=prepare_named_pipe_project, - expected_exit_code=130, + expected_exit_code=-signal.SIGINT, ), ], ids=lambda case: case.description, ) -def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_python( +def test_given_python_free_command_when_interrupted_then_sqb_dies_by_sigint_without_python( test_case: InterruptTestCase, tmp_path: Path ) -> None: marker: Path = tmp_path / "python-starts.log" @@ -329,5 +332,61 @@ def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_p assert python_starts(marker) == 0 +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX pipes") +@pytest.mark.parametrize( + "test_case", + [ + BrokenPipeTestCase( + description="help kept until the final flush", + args=("--help",), + prepare=prepare_nothing, + expected_exit_code=120, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + BrokenPipeTestCase( + description="a short observed report kept until the final flush", + args=("rules", "show", "SQBRMODEL101"), + prepare=prepare_project, + expected_exit_code=120, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + BrokenPipeTestCase( + description="a report the failed flush drops", + args=("rules", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_stderr_end=b"", + ), + BrokenPipeTestCase( + description="a report whose write fails", + args=("rules", "--json", "list"), + prepare=prepare_project, + expected_exit_code=1, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + ], + ids=lambda case: case.description, +) +def test_given_stdout_without_reader_when_running_sqb_then_status_and_logs_match_python( + test_case: BrokenPipeTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = reset_directory(tmp_path / "workspace") + test_case.prepare(workspace) + python: LauncherRun = run_without_reader(PYTHON_CLI, args=test_case.args, cwd=workspace) + python_logs: list[dict[str, object]] = compute_log_files(workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run_without_reader(program, args=test_case.args, cwd=workspace) + + assert (native[0], python[0]) == (test_case.expected_exit_code,) * 2 + assert native[2].endswith(test_case.expected_stderr_end), native[2] + assert python[2].endswith(test_case.expected_stderr_end), python[2] + assert python_starts(marker) == 0 + assert compute_log_files(workspace) == python_logs + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 273df0c2fa933982625451815e4288caf7181a18 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 08/39] chore(deps): bump fensu to 0.27 --- pyproject.toml | 2 +- uv.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f861ac0fc3..c26e3ebbf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.23.0", + "fensu>=0.27.0", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/uv.lock b/uv.lock index 980908966c..10418364db 100644 --- a/uv.lock +++ b/uv.lock @@ -1345,31 +1345,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.23.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/f4/d037850c050343d2548b901a994a77783e7a02d352fe35aa2477644a185c/fensu-0.23.0.tar.gz", hash = "sha256:997c31c26e9b6ea88b2797898a5644a95ae543ba568fb5f3482efd2e17b54d61", size = 508009, upload-time = "2026-09-24T15:29:54.691Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/a3/d1eea20ab654f3c4b1b170cc8eeaf27d5092095cd74e6bc819bfff8d9312/fensu-0.27.0.tar.gz", hash = "sha256:023692a73c4ac2fdadbb167c59a58dc944c7a41b0d78a40fa260f6923a8b3e7b", size = 538895, upload-time = "2026-10-17T05:38:49.174Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/67/597b68a104ae906705e8fc27ec53f57eaad36d869a5323b71518aa77592d/fensu-0.23.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ee20effc98db0d4e682c3f3a55b72ffca81386228c9092cab82b4bd044d1ddc", size = 4026956, upload-time = "2026-09-24T15:29:34.609Z" }, - { url = "https://files.pythonhosted.org/packages/13/0e/1e4583e15a49bc066530ea88faa7257caa131a41bf084c89eb9b34a944a7/fensu-0.23.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:39f85e019e2a66a1cfa280efc02b47740fee8270a6e4befef4885f46f8e03753", size = 3892516, upload-time = "2026-09-24T15:29:36.742Z" }, - { url = "https://files.pythonhosted.org/packages/b4/d6/a10db3b1c7f5fb667bd6e634f494d527c29637b2fb9bc8f68d13f0300482/fensu-0.23.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3002c67ba494386e1c6e94f5f2abbb4b355a174dce0ecf8ef71756c203b7d26a", size = 4119441, upload-time = "2026-09-24T15:29:38.636Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a6/0ac54e58f3d0aec31b6ad6fa46bb8d3454540e7f6b066d6832880cf81fbb/fensu-0.23.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed8a57574977971b2f3c599b5beaac9bd905612aba89971b7b1697f790310e5", size = 4184904, upload-time = "2026-09-24T15:29:40.493Z" }, - { url = "https://files.pythonhosted.org/packages/99/57/7bb9c5c2b971c26bcbf1a1c10a54097925cbaff07f3c8bcb9aefad3d3865/fensu-0.23.0-cp312-abi3-win_amd64.whl", hash = "sha256:98d78a6582ea21fc456c18a31b7d01f3ff43b8ed687de9e51ef8497e54ee2514", size = 3864765, upload-time = "2026-09-24T15:29:42.311Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b7/7b4053d22df1307dbaebb4ed48280bb9c3663ee649005eb6ab5e2fec28b8/fensu-0.27.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d982c4fec0d2210b78fcf67098c32ca9f8b8592f338b2a327fcd1747b59bb310", size = 4154209, upload-time = "2026-10-17T05:38:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/b7058d187e9908ec18f245c1b168c24b746633a0b13666d69052157da3eb/fensu-0.27.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c3cca4a4391028d7b07bb9ead34340f6ba0662c36ea4ef31f69b44a46847289", size = 4004244, upload-time = "2026-10-17T05:38:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/f8/bc/b1eee94e617e954a2188c1802d3770b27624a97f775cbed310f54389c100/fensu-0.27.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5640ffe3cf8916b1533dfe51ca27d65bcea71c9fbd62ae5cfaa2079e8007bdaf", size = 4230195, upload-time = "2026-10-17T05:38:33.057Z" }, + { url = "https://files.pythonhosted.org/packages/7c/01/9f895e6a4427c8ee80ce5d779aceab5a2b994a156a4c1f3bf664108ab489/fensu-0.27.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d26d2713a69f2e05b07c6f778e1e2f86896e26b74247e60952018288381c04", size = 4307270, upload-time = "2026-10-17T05:38:34.827Z" }, + { url = "https://files.pythonhosted.org/packages/08/61/2a380a704a49eea91a3539935af7b2d4ad09aa1d87d787f87c1de847ae4b/fensu-0.27.0-cp312-abi3-win_amd64.whl", hash = "sha256:15a9aef82221f25812d9c4f74e782dd677c73d566980eaed60b762da13ee2b3d", size = 3977639, upload-time = "2026-10-17T05:38:36.612Z" }, ] [[package]] name = "fensu-cli" -version = "0.23.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4a/6f/48493231455bdaceeb637594765c15c46b59252b2526b5ee9478b0c0841c/fensu_cli-0.23.0.tar.gz", hash = "sha256:0a598d0013c662b4bc4e7785dc91f8d480665a06750c2114acbafe8de9ef3dba", size = 715137, upload-time = "2026-09-24T15:29:56.142Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/97/2540b6ea4bca959ac577502fe0c96b432c3b4e694daa1ca68df3a2a06b8a/fensu_cli-0.27.0.tar.gz", hash = "sha256:4276e7fcf7dddc4c36afc347536bf1fd7114fff246ad344544991a1edf7b4ee3", size = 760869, upload-time = "2026-10-17T05:38:50.61Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/ce/87ca84074f6ba2f278d2739505c6f6d1cde53b8b66085b373666873607a3/fensu_cli-0.23.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4ccbf9ed3182344ef9e6fefe1d6d62f0770b825d18059bce1954347dc1d3615e", size = 7745194, upload-time = "2026-09-24T15:29:44.933Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b1/21d23c39b2c8dd51e544b6546fada9bd317214a1e4a6f44b3351cc73d4be/fensu_cli-0.23.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f8d1851c1711359ad2688a3aaa1717af8e48a2c51700a54c647082c3fe51e15", size = 7271582, upload-time = "2026-09-24T15:29:46.978Z" }, - { url = "https://files.pythonhosted.org/packages/aa/56/c190cdedf6e2afa67e58bd6cd6188853983f160e14b73494a08906e84869/fensu_cli-0.23.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d21e87ba99a9c600783c7107e2d06cce15cdce379921a8c996884c16b4c8a407", size = 7362814, upload-time = "2026-09-24T15:29:49.078Z" }, - { url = "https://files.pythonhosted.org/packages/74/13/54147ab05067512127c99df415b73667f878693bf4bd50eb1e03fcb84f76/fensu_cli-0.23.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07ab456f0f8eb90a611e842e3ad1ddf44053b5eb5a5c69d4499f248bd21d01b9", size = 7795295, upload-time = "2026-09-24T15:29:50.828Z" }, - { url = "https://files.pythonhosted.org/packages/70/53/2c0a86d35cb37123dffd5644020cb5284169f83bd0571281e341896b8946/fensu_cli-0.23.0-py3-none-win_amd64.whl", hash = "sha256:8fa9948fd8ee64319a105f70085ddb790b4131bb5ff88636ae6d27ef51b14f57", size = 7490965, upload-time = "2026-09-24T15:29:53.097Z" }, + { url = "https://files.pythonhosted.org/packages/0b/69/9491487602985731af4a7363754970bb35baece4e27cf2a7a0a95ebe4ae7/fensu_cli-0.27.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e90c8d298904d02c639e87541ba51a31ea3912af0bc354cdfea9d95f4428fd41", size = 7935880, upload-time = "2026-10-17T05:38:38.606Z" }, + { url = "https://files.pythonhosted.org/packages/de/4e/7e84f824b3902708ce4c8399f3402fb278cba9c48ed37877d67a4bad5245/fensu_cli-0.27.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1a9f3a2fb8ec8930af9b5cd59f3f1c027cd80a1ebae1a22c7d07133b80d7c65a", size = 7457913, upload-time = "2026-10-17T05:38:40.624Z" }, + { url = "https://files.pythonhosted.org/packages/05/9f/fc20897baafdac0039faef2cc3a324b059fdfe7f4a1522bac386f1e5f8ae/fensu_cli-0.27.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf73108b3778d0d74e11857e743042f4241174ec8f16aeee7e5c8c4769e41e7b", size = 7537447, upload-time = "2026-10-17T05:38:42.834Z" }, + { url = "https://files.pythonhosted.org/packages/46/be/1fc7c87cbf320ff215dcb204df808c592fefec6a0892a1103564a1c69b76/fensu_cli-0.27.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:435dad914ec0d061d5fceb10d8b049f75cdb5c031a156efe2f8309647c744b8c", size = 8035160, upload-time = "2026-10-17T05:38:45.074Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5b/53050980866370c7a41dc7e3e61de86cf67e5848d8560e6a9d166040679a/fensu_cli-0.27.0-py3-none-win_amd64.whl", hash = "sha256:68e91a38dab7e081c4ba504e22a7709169d8d6c470137834adcad116b52fa280", size = 7652792, upload-time = "2026-10-17T05:38:47.26Z" }, ] [[package]] @@ -4350,7 +4350,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.23.0" }, + { name = "fensu", specifier = ">=0.27.0" }, { name = "ingestr", specifier = ">=1.0.0" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From 62334d9fc16db57475add1011420fda34b12acef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 09/39] chore: move function name data under the crate source --- .../semantic_validation/_helpers/function_names.rs | 2 +- .../_helpers/snowflake_functions.txt} | 0 fensu.toml | 13 +++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) rename crates/sqlbuild-analysis/{function_names/snowflake.txt => src/semantic_validation/_helpers/snowflake_functions.txt} (100%) diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs index fba1e4be4c..ca0337945e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs @@ -12,7 +12,7 @@ use std::mem::Discriminant; use std::sync::{LazyLock, OnceLock, PoisonError}; const UNKNOWN_FUNCTION: &str = "B101"; -const SNOWFLAKE_FUNCTIONS: &str = include_str!("../../../function_names/snowflake.txt"); +const SNOWFLAKE_FUNCTIONS: &str = include_str!("snowflake_functions.txt"); const DUCKDB_CATALOG: &str = "duckdb"; const PROBE_PREFIX: &str = "SELECT "; const TYPE_POSITION_KEYWORD: &str = "AS"; diff --git a/crates/sqlbuild-analysis/function_names/snowflake.txt b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/snowflake_functions.txt similarity index 100% rename from crates/sqlbuild-analysis/function_names/snowflake.txt rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/snowflake_functions.txt diff --git a/fensu.toml b/fensu.toml index 154893a523..9394b3890e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -448,6 +448,19 @@ rule = "FPRSL105" path = "crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs" reason = "This test-support entry exists only for the SQL lint tests in sqlbuild-rules, which the analyzer does not count as importers." +[targets.rust.rule_options.FPRSR311] +allowed_entries = [ + "Cargo.toml", + "build.rs", + "src/", + "tests/", + "benches/", + "examples/", + "README*", + "LICENSE*", + "pyproject.toml", +] + [targets.rust.rule_options.FPRSL301] forbidden_packages = [ "sqlbuild-structure-checker", From 912815b1df275ba351e4af782e5371b691bad374 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 10/39] refactor: name large tuple types --- .../_helpers/analysis_cache.rs | 31 +- .../_helpers/compact_batch.rs | 19 +- .../_helpers/dynamic_pivot.rs | 70 +- .../analysis_session/_helpers/session.rs | 2 +- .../src/assembly/analysis_session/models.rs | 2 +- .../analysis_session/tests/helpers.rs | 2 +- .../main/locate_binding_diagnostics.rs | 26 +- .../src/binding_locations/models.rs | 11 +- .../src/binding_locations/tests/helpers.rs | 20 +- .../src/binding_locations/tests/test_types.rs | 2 +- .../sqlbuild-analysis/src/lineage/models.rs | 52 +- .../src/lineage/tests/helpers.rs | 19 +- .../lineage/tests/test_rich_lineage_panics.rs | 2 +- .../_helpers/resolved_reads.rs | 6 +- .../planner_fixtures/main/required_columns.rs | 6 +- .../src/planner_fixtures/mod.rs | 1 - .../src/planner_fixtures/models.rs | 9 +- .../src/planner_fixtures/tests/test_reads.rs | 21 +- .../src/planner_fixtures/types.rs | 4 - .../src/query_analysis/_helpers/engine.rs | 41 +- .../_helpers/metadata_checks/audits.rs | 77 ++- .../_helpers/resource_sql/diagnostics.rs | 80 ++- .../_helpers/resource_sql/fixtures.rs | 33 +- .../_helpers/resource_sql/locations.rs | 83 +-- .../src/semantic_checks/models.rs | 24 +- .../tests/test_resource_sql.rs | 6 +- .../src/semantic_usage/main.rs | 26 +- .../semantic_validation/_helpers/catalog.rs | 26 +- .../_helpers/diagnostics.rs | 28 +- .../semantic_validation/_helpers/positions.rs | 36 +- .../semantic_validation/main/diagnostics.rs | 2 +- .../src/semantic_validation/models.rs | 23 +- .../tests/test_type_locations.rs | 25 +- .../src/semantic_validation/types.rs | 10 - .../src/functions/_helpers/namespace.rs | 22 +- .../main/resolve_function_namespace.rs | 20 +- .../main/render_dbt_execution_summary.rs | 5 +- .../src/lifecycle_events/constants.rs | 602 +++++++++--------- .../src/lifecycle_events/main/catalog.rs | 16 +- .../src/lifecycle_events/types.rs | 11 +- .../src/_helpers/locations.rs | 25 +- .../_helpers/analysis/binding_catalog.rs | 55 +- .../_helpers/analysis/binding_locations.rs | 62 +- .../src/bindings/_helpers/analysis/catalog.rs | 12 +- .../_helpers/analysis/explicit_references.rs | 42 +- .../bindings/_helpers/analysis/model_reads.rs | 32 +- .../_helpers/analysis/normalization.rs | 8 +- .../analysis_session/model_analyses.rs | 104 +-- .../_helpers/analysis_session/oracle.rs | 61 +- .../analysis_session/outcome_objects.rs | 106 +-- .../analysis_session/profile_inputs.rs | 15 +- .../analysis_session/query_columns.rs | 52 +- .../_helpers/analysis_session/session.rs | 283 +++++--- .../analysis_session/stage_functions.rs | 79 +-- .../src/bindings/_helpers/boundary/mod.rs | 4 + .../bindings/_helpers/boundary/positional.rs | 114 ++++ .../_helpers/boundary/positional_records.rs | 50 ++ .../_helpers/command_runtime/invocation.rs | 37 +- .../compile_attachment/cursor_roles.rs | 3 +- .../compile_attachment/model_inputs.rs | 13 +- .../bindings/_helpers/compile_delta/image.rs | 49 +- .../_helpers/compile_driver/analysis_cache.rs | 15 +- .../compile_driver/assembly_analysis.rs | 23 +- .../compile_driver/assembly_models.rs | 2 +- .../_helpers/compile_outputs/reuse.rs | 20 +- .../compile_outputs/runtime_target.rs | 27 +- .../_helpers/compiled_project/attachment.rs | 116 ++-- .../_helpers/compiled_project/audit_loop.rs | 119 ++-- .../_helpers/compiled_project/audit_views.rs | 9 +- .../compiled_model_analysis.rs | 47 +- .../compiled_project/compiled_model_view.rs | 34 +- .../compiled_project/function_loop.rs | 103 ++- .../compiled_project/function_views.rs | 50 +- .../compiled_project/model_input_view.rs | 22 +- .../_helpers/compiled_project/model_loop.rs | 71 ++- .../compiled_project/model_loop_calls.rs | 26 +- .../compiled_project/model_loop_external.rs | 8 +- .../compiled_project/model_loop_hooks.rs | 22 +- .../compiled_project/model_loop_host.rs | 35 +- .../compiled_project/model_loop_inputs.rs | 21 +- .../compiled_project/model_loop_macros.rs | 16 +- .../_helpers/compiled_project/project.rs | 89 ++- .../_helpers/compiled_project/project_view.rs | 10 +- .../_helpers/compiled_project/records.rs | 34 +- .../compiled_project/resource_diagnostics.rs | 74 ++- .../compiled_project/scenario_loop.rs | 23 +- .../_helpers/compiled_project/seed_views.rs | 56 +- .../_helpers/compiled_project/source_views.rs | 40 +- .../compiled_project/sql_test_loop.rs | 141 ++-- .../contracts/contract_diagnostics.rs | 30 +- .../_helpers/contracts/model_contracts.rs | 63 +- .../bindings/_helpers/diff_report/query.rs | 23 +- .../_helpers/discovery/header_values.rs | 9 +- .../_helpers/graph/execution_gates.rs | 17 +- .../bindings/_helpers/graph/project_graph.rs | 72 ++- .../_helpers/graph/python_selection.rs | 59 +- .../_helpers/hand_built/compiled_model.rs | 54 +- .../_helpers/lineage/column_lineage_graph.rs | 23 +- .../_helpers/lineage/lineage_cache.rs | 46 +- .../_helpers/lineage/project_lineage.rs | 2 +- .../bindings/_helpers/load/dlt_destination.rs | 25 +- .../src/bindings/_helpers/mod.rs | 4 +- .../_helpers/model_config/model_validator.rs | 56 +- .../_helpers/plan_output/plan_json.rs | 50 +- .../planner_entries/execution_plan.rs | 37 +- .../_helpers/planner_identity/graph.rs | 25 +- .../_helpers/planner_phases/models.rs | 36 ++ .../planner_phases/output_assembly.rs | 51 +- .../planner_strategy/fixture_reads.rs | 28 +- .../planner_strategy/fixture_types.rs | 10 +- .../planner_warehouse/fingerprint_cache.rs | 35 +- .../project_assembly/declared_relations.rs | 21 +- .../project_assembly/native_resources.rs | 90 ++- .../record_behaviours/lineage_facts.rs | 23 +- .../_helpers/record_behaviours/macros.rs | 8 +- .../_helpers/refactor/refactor_facts.rs | 70 +- .../bindings/_helpers/render/authored_sql.rs | 26 +- .../_helpers/render/declaration_expansion.rs | 28 +- .../_helpers/render/macro_call_rows.rs | 79 ++- .../_helpers/render/macro_expansion.rs | 56 +- .../_helpers/render/macro_expansion_result.rs | 25 +- .../bindings/_helpers/render/macro_modules.rs | 91 ++- .../_helpers/rules/custom_rule_evidence.rs | 20 +- .../bindings/_helpers/rules/fix_session.rs | 45 +- .../bindings/_helpers/rules/fix_verdict.rs | 43 +- .../src/bindings/_helpers/rules/lint.rs | 32 +- .../bindings/_helpers/rules/lint_inputs.rs | 6 +- .../bindings/_helpers/rules/lint_pipeline.rs | 48 +- .../src/bindings/_helpers/rules/lint_text.rs | 49 +- .../bindings/_helpers/rules/relation_keys.rs | 77 ++- .../src/bindings/_helpers/rules/rows.rs | 28 +- .../src/bindings/_helpers/rules/rule_cache.rs | 25 +- .../src/bindings/_helpers/rules/sql_rules.rs | 15 +- .../scenario_command/capture_command.rs | 16 +- .../_helpers/scenario_command/presentation.rs | 20 +- .../scenario_command/snapshot_sync.rs | 14 +- .../_helpers/scenario_command/test_command.rs | 16 +- .../_helpers/scopes/compile_usages.rs | 85 +-- .../src/bindings/_helpers/scopes/placement.rs | 28 +- .../bindings/_helpers/scopes/scope_command.rs | 34 +- .../bindings/_helpers/scopes/scope_lookup.rs | 52 +- .../bindings/_helpers/scopes/scope_objects.rs | 22 +- .../bindings/_helpers/scopes/scope_reports.rs | 93 +-- .../_helpers/scopes/scope_validation.rs | 14 +- .../_helpers/scopes/scope_visibility.rs | 8 +- .../_helpers/semantic_checks/metadata.rs | 52 +- .../_helpers/semantic_checks/resource_sql.rs | 13 +- .../_helpers/semantic_shapes/shapes.rs | 16 +- .../_helpers/sql_test_glue/assembly.rs | 67 +- .../_helpers/sql_test_glue/compiled_tests.rs | 24 +- .../_helpers/sql_test_glue/plan_rows.rs | 45 +- .../_helpers/sql_test_glue/planning.rs | 96 +-- .../sqltext/declaration_references.rs | 27 +- .../_helpers/sqltext/model_headers.rs | 26 +- .../_helpers/sqltext/sql_references.rs | 55 +- .../bindings/_helpers/sqltext/static_sql.rs | 36 +- .../_helpers/type_system/normalization.rs | 33 +- crates/sqlbuild-python/src/bindings/models.rs | 47 +- crates/sqlbuild-python/src/bindings/types.rs | 13 + .../_helpers/scanning/sql_sites.rs | 31 +- .../src/engine/_helpers/rows.rs | 32 +- .../src/engine/tests/test_sql_scanning.rs | 12 +- .../src/fix_verification/main/compare.rs | 2 +- .../src/fix_verification/models.rs | 13 +- .../src/lint_text/_helpers/sql_diagnostics.rs | 27 +- .../src/lint_text/_helpers/suppressions.rs | 39 +- .../src/sql_lint/_helpers/preparation.rs | 18 +- crates/sqlbuild-rules/src/sql_lint/types.rs | 4 +- .../src/scope_index/_helpers/grants.rs | 26 +- .../src/scope_index/_helpers/ordering.rs | 37 +- .../src/scope_report/_helpers/declarations.rs | 31 +- .../src/scope_report/_helpers/projection.rs | 11 +- .../src/scope_text/_helpers/results.rs | 49 +- .../sql_interpolation/substitution.rs | 32 +- .../sqlbuild-sqltext/src/compiler/models.rs | 11 +- .../src/compiler/tests/helpers.rs | 15 +- .../compiler/tests/test_sql_interpolation.rs | 8 +- .../src/compiler/tests/test_types.rs | 4 +- crates/sqlbuild-sqltext/src/compiler/types.rs | 3 - fensu.toml | 26 +- 180 files changed, 4537 insertions(+), 2466 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 3b4eb35339..ce7fe35a60 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -16,7 +16,7 @@ use crate::assembly::analysis_session::models::{ LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, SessionRequest, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; const LINEAGE_NATIVE: u64 = 0; const LINEAGE_FACTS: u64 = 1; @@ -454,7 +454,16 @@ pub(crate) fn encode_outcome( encoder.flag(analysis.has_star); encoder.flag(analysis.star_resolved); encoder.uint(analysis.binding_diagnostics.len() as u64); - for (code, message, line, column, start, end, severity) in &analysis.binding_diagnostics { + for DiagnosticRow { + code, + message, + line, + column, + start, + end, + severity, + } in &analysis.binding_diagnostics + { encoder.text(code); encoder.text(message); for position in [line, column, start, end] { @@ -529,15 +538,15 @@ pub(crate) fn decode_outcome( let diagnostic_count: usize = decoder.index()?; let mut binding_diagnostics: Vec = Vec::new(); for _ in 0..diagnostic_count { - binding_diagnostics.push(( - decoder.text()?, - decoder.text()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.text()?, - )); + binding_diagnostics.push(DiagnosticRow { + code: decoder.text()?, + message: decoder.text()?, + line: decoder.optional_index()?, + column: decoder.optional_index()?, + start: decoder.optional_index()?, + end: decoder.optional_index()?, + severity: decoder.text()?, + }); } let binding_validated: bool = decoder.flag()?; decoder.finish()?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 9a274defc7..8679f4480d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -23,7 +23,8 @@ use crate::assembly::analysis_session::constants::{ use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::main::diagnostics::binding_diagnostics; -use crate::semantic_validation::types::{DiagnosticRow, NormalizationRequest, Relations}; +use crate::semantic_validation::models::DiagnosticRow; +use crate::semantic_validation::types::{NormalizationRequest, Relations}; const INVALID_BATCH: &str = "native compact query analysis returned an invalid batch response"; const INVALID_TEMPLATE: &str = "native compact query analysis returned an invalid template"; @@ -1424,15 +1425,15 @@ fn diagnostic_row(error: &Value) -> Result, String> { Some(Value::String(message)) if !message.is_empty() => message.clone(), _ => DEFAULT_BINDING_MESSAGE.to_owned(), }; - Ok(Some(( - code.to_owned(), + Ok(Some(DiagnosticRow { + code: code.to_owned(), message, - position(error, "line")?, - position(error, "column")?, - position(error, "start")?, - position(error, "end")?, - severity.to_owned(), - ))) + line: position(error, "line")?, + column: position(error, "column")?, + start: position(error, "start")?, + end: position(error, "end")?, + severity: severity.to_owned(), + })) } fn position(error: &Value, key: &str) -> Result, String> { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index b37e6c4e4b..e49ac7c85c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -192,7 +192,12 @@ fn pivot_proof( if truthy(pivot.get("unpivot")) { return failure("UNPIVOT does not produce a dynamic column family".to_owned()); } - let (pivot_columns, aggregates, dynamic, columns_valid) = components(pivot, context.dialect); + let PivotComponents { + pivot_columns, + aggregates, + dynamic, + columns_valid, + } = components(pivot, context.dialect); if !dynamic { return failure( "static pivot values must use ordinary exact column declarations".to_owned(), @@ -309,11 +314,18 @@ fn fixed_names( .collect() } +/// Python's `_pivot_components` for one pivot clause. +struct PivotComponents<'a> { + pivot_columns: Vec<&'a str>, + aggregates: Vec<&'a Value>, + /// Whether the pivot values come from a dynamic source. + dynamic: bool, + /// Whether every pivot column resolved to a name. + columns_valid: bool, +} + /// Python's `_pivot_components`: pivot columns, aggregates, dynamic values, columns valid. -fn components<'a>( - pivot: &'a Map, - dialect: &str, -) -> (Vec<&'a str>, Vec<&'a Value>, bool, bool) { +fn components<'a>(pivot: &'a Map, dialect: &str) -> PivotComponents<'a> { if SIMPLIFIED_PIVOT_DIALECTS.contains(&dialect) && truthy(pivot.get("using")) { let expressions: Vec<&Value> = dict_list(pivot.get("expressions")); let columns: Vec<&str> = expressions @@ -321,12 +333,12 @@ fn components<'a>( .filter_map(|expression| column_name(Some(expression))) .collect(); let valid: bool = columns.len() == expressions.len(); - return ( - columns, - dict_list(pivot.get("using")), - !truthy(pivot.get("fields")), - valid, - ); + return PivotComponents { + pivot_columns: columns, + aggregates: dict_list(pivot.get("using")), + dynamic: !truthy(pivot.get("fields")), + columns_valid: valid, + }; } let fields: Vec<&Value> = dict_list(pivot.get("fields")); let columns: Vec<&str> = fields @@ -335,7 +347,12 @@ fn components<'a>( .collect(); let dynamic: bool = fields.iter().any(|field| has_dynamic_values(field)); let valid: bool = columns.len() == fields.len(); - (columns, dict_list(pivot.get("expressions")), dynamic, valid) + PivotComponents { + pivot_columns: columns, + aggregates: dict_list(pivot.get("expressions")), + dynamic, + columns_valid: valid, + } } /// Python's `_has_dynamic_value_source` for one pivot field. @@ -651,13 +668,24 @@ fn failure(reason: String) -> PivotOutcome { } /// Python's `_families_equal` key for one family. -fn normalized(family: &DynamicFamily) -> (String, String, String, String, String, Option<&str>) { - ( - casefold(&family.name), - casefold(&family.pivot_column), - casefold(&family.value_column), - casefold(&family.aggregate), - casefold(&family.data_type), - family.name_pattern.as_deref(), - ) +#[derive(PartialEq, Eq)] +struct FamilyKey<'a> { + name: String, + pivot_column: String, + value_column: String, + aggregate: String, + data_type: String, + name_pattern: Option<&'a str>, +} + +/// Python's `_families_equal` key for one family. +fn normalized(family: &DynamicFamily) -> FamilyKey<'_> { + FamilyKey { + name: casefold(&family.name), + pivot_column: casefold(&family.pivot_column), + value_column: casefold(&family.value_column), + aggregate: casefold(&family.aggregate), + data_type: casefold(&family.data_type), + name_pattern: family.name_pattern.as_deref(), + } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 66f3c1196b..2451efca3d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -36,8 +36,8 @@ use crate::assembly::analysis_session::models::{ RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; -use crate::semantic_validation::types::DiagnosticRow; /// What an internal native failure of the session names. const SESSION_CONTEXT: &str = "native model analysis"; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 3a625a8170..da66105b17 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -11,7 +11,7 @@ use sqlbuild_cache::store::models::NativeStore; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::types::{NullabilityRuleFn, OutputSources, Pairs, Shapes}; -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index cef7394585..321905eb3e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -230,7 +230,7 @@ pub(crate) fn described(outcome: &ModelOutcome) -> Vec { analysis .binding_diagnostics .iter() - .map(|row| row.0.as_str()) + .map(|row| row.code.as_str()) .collect::>() )); lines diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs index 80221f225b..71738049d7 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -3,9 +3,11 @@ use std::collections::HashSet; use crate::binding_locations::constants::{ERROR_SEVERITY, UNKNOWN_TABLE_INTERNAL_CODE}; -use crate::binding_locations::models::{AuthoredSpan, BindingSource, LocatedDiagnostic}; +use crate::binding_locations::models::{ + AuthoredSpan, BindingSource, LocatedDiagnostic, SeenDiagnostic, +}; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::{BindingPositions, PositionInput}; -use crate::semantic_validation::types::DiagnosticRow; /// Each reported diagnostic in order; an error mapping a position is the mapper's message. pub fn locate_binding_diagnostics( @@ -17,9 +19,18 @@ pub fn locate_binding_diagnostics( .find(source.query) .map(|start| source.authored[..start].matches('\n').count()); let mut positions: Option = None; - let mut seen: HashSet<(String, String, Option, Option)> = HashSet::new(); + let mut seen: HashSet = HashSet::new(); let mut located: Vec = Vec::new(); - for (code, message, line, column, start, end, severity) in rows { + for DiagnosticRow { + code, + message, + line, + column, + start, + end, + severity, + } in rows + { if code == UNKNOWN_TABLE_INTERNAL_CODE { continue; } @@ -39,7 +50,12 @@ pub fn locate_binding_diagnostics( *column, ), }; - let key = (code.clone(), message.clone(), line, column); + let key = SeenDiagnostic { + code: code.clone(), + message: message.clone(), + line, + column, + }; if seen.contains(&key) && start.is_some() && severity == ERROR_SEVERITY { continue; } diff --git a/crates/sqlbuild-analysis/src/binding_locations/models.rs b/crates/sqlbuild-analysis/src/binding_locations/models.rs index 292ed5690a..efd854bf90 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/models.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/models.rs @@ -1,6 +1,15 @@ //! A model's authored SQL texts and the located diagnostics read from them. -use crate::semantic_validation::types::Expansion; +use crate::semantic_validation::models::Expansion; + +/// A reported diagnostic's identity: an error repeating it at a known start is dropped. +#[derive(Debug, PartialEq, Eq, Hash)] +pub(crate) struct SeenDiagnostic { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) line: Option, + pub(crate) column: Option, +} /// The texts one model's binding diagnostics are mapped back through. #[derive(Clone, Copy, Debug)] diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs index a09c0b76aa..e7dcf94764 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; pub(super) const MODEL: &str = "MODEL (\n description \"Orders\"\n);\n\nSELECT o.amount, o.missing\nFROM orders AS o"; @@ -11,15 +11,15 @@ fn row( span: (Option, Option), severity: &str, ) -> DiagnosticRow { - ( - code.to_owned(), - message.to_owned(), - position.0, - position.1, - span.0, - span.1, - severity.to_owned(), - ) + DiagnosticRow { + code: code.to_owned(), + message: message.to_owned(), + line: position.0, + column: position.1, + start: span.0, + end: span.1, + severity: severity.to_owned(), + } } fn missing_column() -> DiagnosticRow { diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs index 300b0e43f8..336b4dc4d1 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; /// `(code, line, column, (line, column, end line, end column))` of one reported diagnostic. pub(super) type LocatedCase = ( diff --git a/crates/sqlbuild-analysis/src/lineage/models.rs b/crates/sqlbuild-analysis/src/lineage/models.rs index f660fbad29..d8d4a2657b 100644 --- a/crates/sqlbuild-analysis/src/lineage/models.rs +++ b/crates/sqlbuild-analysis/src/lineage/models.rs @@ -149,15 +149,41 @@ pub enum FastLineageOutcome { UnknownDialect(String), } +/// A lineage outcome flattened into its status, columns, star flag and detail. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LineageOutcomeParts { + pub status: &'static str, + pub columns: Vec, + pub has_star: bool, + pub detail: Option, +} + +impl LineageOutcomeParts { + fn new(status: &'static str, columns: Vec, has_star: bool, detail: Option) -> Self { + Self { + status, + columns, + has_star, + detail, + } + } +} + impl FastLineageOutcome { - /// `(status, columns, has_star, detail)`: detail is the parse error or the unknown dialect. - pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + /// The status, columns and star flag; detail is the parse error or the unknown dialect. + pub fn into_parts(self) -> LineageOutcomeParts { match self { - Self::StarColumns(columns) => ("star", columns, true, None), - Self::Built { columns, has_star } => ("built", columns, has_star, None), - Self::Omitted => ("omitted", Vec::new(), false, None), - Self::Unparsed(message) => ("unparsed", Vec::new(), false, Some(message)), - Self::UnknownDialect(name) => ("unknown_dialect", Vec::new(), false, Some(name)), + Self::StarColumns(columns) => LineageOutcomeParts::new("star", columns, true, None), + Self::Built { columns, has_star } => { + LineageOutcomeParts::new("built", columns, has_star, None) + } + Self::Omitted => LineageOutcomeParts::new("omitted", Vec::new(), false, None), + Self::Unparsed(message) => { + LineageOutcomeParts::new("unparsed", Vec::new(), false, Some(message)) + } + Self::UnknownDialect(name) => { + LineageOutcomeParts::new("unknown_dialect", Vec::new(), false, Some(name)) + } } } } @@ -217,11 +243,15 @@ pub enum RichLineageOutcome { } impl RichLineageOutcome { - /// `(status, columns, has_star, detail)`: detail is the polyglot error. - pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + /// The status, columns and star flag; detail is the polyglot error. + pub fn into_parts(self) -> LineageOutcomeParts { match self { - Self::Built { columns, has_star } => ("built", columns, has_star, None), - Self::Skipped(message) => ("skipped", Vec::new(), false, Some(message)), + Self::Built { columns, has_star } => { + LineageOutcomeParts::new("built", columns, has_star, None) + } + Self::Skipped(message) => { + LineageOutcomeParts::new("skipped", Vec::new(), false, Some(message)) + } } } } diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index 73e206171a..bf332a89da 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -4,8 +4,9 @@ use crate::lineage::main::build_fast_lineage::build_fast_lineage; use crate::lineage::main::build_rich_lineage::build_rich_lineage; use crate::lineage::models::RelationFingerprint; use crate::lineage::models::{ - FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageResourceType, - LineageSchemaResource, RichLineageColumn, RichLineageRequest, RichSchemaResource, + FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageOutcomeParts, + LineageResourceType, LineageSchemaResource, RichLineageColumn, RichLineageRequest, + RichSchemaResource, }; use crate::semantic_validation::models::ProjectCatalog; @@ -64,7 +65,12 @@ fn catalog() -> ProjectCatalog { fn described_outcome( outcome: FastLineageOutcome, ) -> (&'static str, Vec, bool, Option) { - let (status, columns, has_star, detail) = outcome.into_parts(); + let LineageOutcomeParts { + status, + columns, + has_star, + detail, + } = outcome.into_parts(); (status, described(&columns), has_star, detail) } @@ -139,7 +145,12 @@ pub(crate) fn rich_outcome( }; let mut outcomes = build_rich_lineage(&request).expect("the rich lineage pool builds"); assert_eq!(outcomes.len(), 1); - let (status, columns, has_star, detail) = outcomes.remove(0).into_parts(); + let LineageOutcomeParts { + status, + columns, + has_star, + detail, + } = outcomes.remove(0).into_parts(); ( status, columns.iter().map(described_rich_column).collect(), diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs index b4a0944961..1f4d748a01 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs @@ -55,7 +55,7 @@ fn given_an_injected_panic_when_building_rich_lineage_then_the_request_fails() { let answered: Result, String> = outcomes.map(|outcomes| { outcomes .into_iter() - .map(|outcome| outcome.into_parts().3.unwrap_or_default()) + .map(|outcome| outcome.into_parts().detail.unwrap_or_default()) .collect() }); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs index 5e70d90bcc..8ab1fbca56 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs @@ -25,9 +25,9 @@ pub(crate) fn resolved_reads( let references: Vec = key .references .iter() - .map(|(kind, name, _, _)| ReferenceInput { - kind: kind.clone(), - name: name.clone(), + .map(|reference| ReferenceInput { + kind: reference.kind.clone(), + name: reference.name.clone(), }) .collect(); cache.misses += 1; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs index d2a2c86b91..4134ac1029 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs @@ -26,14 +26,14 @@ pub fn required_columns( let mut result = FixtureReadRequirements::default(); for model in models { let mut reads = model.high_confidence.clone(); - if model.key.references.iter().any(|(kind, name, _, _)| { - let resource_type = match kind.as_str() { + if model.key.references.iter().any(|reference| { + let resource_type = match reference.kind.as_str() { "ref" => "model", "source" => "source", "seed" => "seed", _ => return false, }; - fixture_keys.contains(&(resource_type.to_owned(), name.clone())) + fixture_keys.contains(&(resource_type.to_owned(), reference.name.clone())) }) { let (updated, resolved) = resolved_reads(cache, &model.key); cache = updated; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs index 42b1da1020..b010d05a00 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs @@ -7,4 +7,3 @@ pub mod main; pub mod models; #[cfg(test)] mod tests; -pub mod types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index 703a4defa1..944b845040 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -12,7 +12,14 @@ pub struct FixtureReadSchemas<'a> { use crate::required_columns::models::ColumnRead; -use crate::planner_fixtures::types::FixtureReadReference; +/// The complete reference identity retained by the fixture read cache. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct FixtureReadReference { + pub kind: String, + pub name: String, + pub package: Option, + pub argument_count: Option, +} /// The original SQL, complete references and dialect form the read cache key. #[derive(Clone, Debug, Eq, Hash, PartialEq)] diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs index 8d19e9e57e..b7832b9526 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs @@ -1,6 +1,6 @@ use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; -use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey, FixtureReadReference}; use crate::planner_fixtures::tests::test_types::{FixtureMarkerTestCase, FixtureReadCacheTestCase}; #[test] @@ -50,7 +50,12 @@ fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reu for test_case in test_cases { let key = FixtureReadKey { sql: "SELECT order_id FROM __source(\"orders\")".to_owned(), - references: vec![("source".into(), "orders".into(), None, None)], + references: vec![FixtureReadReference { + kind: "source".into(), + name: "orders".into(), + package: None, + argument_count: None, + }], dialect: Some("duckdb".into()), }; let (cache, initial) = resolved_reads(FixtureReadCache::default(), &key); @@ -59,12 +64,12 @@ fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reu assert_eq!(cache.misses, 1, "{}", test_case.description); let changed = FixtureReadKey { sql: test_case.sql.to_owned(), - references: vec![( - "source".into(), - "orders".into(), - test_case.package.map(str::to_owned), - test_case.argument_count, - )], + references: vec![FixtureReadReference { + kind: "source".into(), + name: "orders".into(), + package: test_case.package.map(str::to_owned), + argument_count: test_case.argument_count, + }], dialect: test_case.dialect.map(str::to_owned), }; let (cache, reads) = resolved_reads(cache, &changed); diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs deleted file mode 100644 index ec569300e0..0000000000 --- a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Fixture analysis identities. - -/// The complete reference identity retained by the fixture read cache. -pub type FixtureReadReference = (String, String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index b5def1c23c..bcd4228bfe 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -7,7 +7,8 @@ use polyglot_sql::{ ValidationSchema, analyze_query, analyze_query_for_project_projections, }; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeTuple; +use serde::{Deserialize, Serialize, Serializer}; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, PoisonError}; @@ -1128,7 +1129,29 @@ fn analyze_project_result(request: ProjectAnalysisRequest) -> Result, u8, u8, u8, Vec<(usize, usize, usize)>); +/// One interned column fact; it serializes as the positional array the batch reader expects. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct CompactProjectColumn { + name: usize, + data_type: Option, + nullability: u8, + transform: u8, + confidence: u8, + upstream: Vec<(usize, usize, usize)>, +} + +impl Serialize for CompactProjectColumn { + fn serialize(&self, serializer: S) -> Result { + let mut fields = serializer.serialize_tuple(6)?; + fields.serialize_element(&self.name)?; + fields.serialize_element(&self.data_type)?; + fields.serialize_element(&self.nullability)?; + fields.serialize_element(&self.transform)?; + fields.serialize_element(&self.confidence)?; + fields.serialize_element(&self.upstream)?; + fields.end() + } +} #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -1222,14 +1245,14 @@ impl CompactProjectAccumulator { ) }) .collect(); - let fact = ( - self.interner.intern(column.name), - column.data_type.map(|value| self.interner.intern(value)), - nullability_code(column.nullability), - transform_code(lineage.transform_kind), - confidence_code(lineage.confidence), + let fact = CompactProjectColumn { + name: self.interner.intern(column.name), + data_type: column.data_type.map(|value| self.interner.intern(value)), + nullability: nullability_code(column.nullability), + transform: transform_code(lineage.transform_kind), + confidence: confidence_code(lineage.confidence), upstream, - ); + }; let next_index = self.facts.len(); let fact_index = *self.fact_indexes.entry(fact.clone()).or_insert_with(|| { self.facts.push(fact); diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs index 748ea8a336..c6d39d9a39 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs @@ -13,8 +13,8 @@ use crate::semantic_checks::models::{ }; use crate::semantic_validation::main::diagnostics::binding_diagnostics; use crate::semantic_validation::main::validations_json; -use crate::semantic_validation::models::{CatalogInput, Columns, ProjectCatalog}; -use crate::semantic_validation::types::{DiagnosticRow, Relations}; +use crate::semantic_validation::models::{CatalogInput, Columns, DiagnosticRow, ProjectCatalog}; +use crate::semantic_validation::types::Relations; const EXPRESSION_AUDIT: &str = "expression_is_true"; const RELATIONSHIPS_AUDIT: &str = "relationships"; @@ -107,15 +107,15 @@ fn response_rows(response: &Value) -> Result, SemanticFailure Some(Value::String(text)) if !text.is_empty() => text.clone(), _ => DEFAULT_BINDING_MESSAGE.to_owned(), }; - rows.push(( - code.to_owned(), + rows.push(DiagnosticRow { + code: code.to_owned(), message, - optional_index(error.get("line")), - optional_index(error.get("column")), - optional_index(error.get("start")), - optional_index(error.get("end")), - severity.to_owned(), - )); + line: optional_index(error.get("line")), + column: optional_index(error.get("column")), + start: optional_index(error.get("start")), + end: optional_index(error.get("end")), + severity: severity.to_owned(), + }); } Ok(rows) } @@ -162,8 +162,13 @@ fn standalone_validation( binding_diagnostics(sql, &dialect, response_rows(response)?).map_err(internal) } -/// One `(code, message, severity)` finding before it is located. -type Unlocated = (String, String, String, String); +/// One audit finding before it is located at the named column or field. +struct Unlocated { + code: String, + message: String, + severity: String, + name: String, +} fn expression_findings( expression: &str, @@ -188,14 +193,12 @@ fn expression_findings( .unwrap_or_default(); Ok(rows .into_iter() - .filter(|row| row.0 != UNKNOWN_TABLE_CODE) - .map(|row| { - ( - row.0, - format!("{EXPRESSION_AUDIT}: {}", row.1), - row.6, - expression.to_owned(), - ) + .filter(|row| row.code != UNKNOWN_TABLE_CODE) + .map(|row| Unlocated { + code: row.code, + message: format!("{EXPRESSION_AUDIT}: {}", row.message), + severity: row.severity, + name: expression.to_owned(), }) .collect()) } @@ -203,17 +206,17 @@ fn expression_findings( fn typed_findings(rows: Vec, label: &str, column: &str) -> Vec { rows.into_iter() .map(|row| { - let code = if row.6 == ERROR_SEVERITY { + let code = if row.severity == ERROR_SEVERITY { AUDIT_TYPE_CODE.to_owned() } else { - row.0 + row.code }; - ( + Unlocated { code, - format!("{label}: {}", row.1), - row.6, - column.to_owned(), - ) + message: format!("{label}: {}", row.message), + severity: row.severity, + name: column.to_owned(), + } }) .collect() } @@ -243,12 +246,12 @@ fn relationship_findings( if target_shape.is_empty() { return Ok(Vec::new()); } - return Ok(vec![( - MISSING_TARGET_COLUMN_CODE.to_owned(), - format!("relationships target has no column '{}'", audit.field), - ERROR_SEVERITY.to_owned(), - audit.field.clone(), - )]); + return Ok(vec![Unlocated { + code: MISSING_TARGET_COLUMN_CODE.to_owned(), + message: format!("relationships target has no column '{}'", audit.field), + severity: ERROR_SEVERITY.to_owned(), + name: audit.field.clone(), + }]); }; let Some((_, left)) = shape.iter().find(|(name, _)| name == column) else { return Ok(Vec::new()); @@ -337,7 +340,13 @@ pub(crate) fn audit_errors( ) -> Result, SemanticFailure> { let mut findings: Vec = Vec::new(); for audit in &model.audits { - for (code, message, severity, name) in audit_findings(audit, shape, context)? { + for Unlocated { + code, + message, + severity, + name, + } in audit_findings(audit, shape, context)? + { let (line, column) = text_position(&model.authored_sql, &name, 0)?; findings.push(AuditFinding { code, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs index 5df90d2dcc..33ced0fba8 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -16,7 +16,8 @@ use crate::semantic_checks::_helpers::resource_sql::resources::{ MissingColumnAudit, ResourceSql, audit_label, missing_column_audits, resources, }; use crate::semantic_checks::models::{ - ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SemanticFailure, + ResourceSpan, ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, + SemanticFailure, }; use crate::semantic_validation::main::normalize::normalize_analysis_sql; use crate::semantic_validation::models::{Columns, NormalizationInput, ProjectCatalog}; @@ -32,7 +33,17 @@ const CURSOR_CONTEXT: &str = "Model SQL"; const ANALYSIS_CURSOR_LITERAL: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; const GENERIC_DIALECT: &str = "generic"; -type Key = (String, String, String, i64, i64, i64); +/// A finding's identity: its path, code, message, start and native order when unlocated. +#[derive(Clone, PartialEq, Eq, Hash)] +struct Key { + path: String, + code: String, + message: String, + line: i64, + column: i64, + /// The finding's native index when it has no start, else `-1`. + order: i64, +} fn kind_label(kind: ResourceSqlKind) -> &'static str { match kind { @@ -88,26 +99,26 @@ fn binding_request( /// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. fn reported(resource: &ResourceSql, finding: &Finding) -> Option { - if finding.0 != UNKNOWN_TABLE_CODE { + if finding.code != UNKNOWN_TABLE_CODE { return Some(finding.clone()); } - if finding.1.starts_with(UNKNOWN_TABLE_PREFIX) { + if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) { return None; } - Some(( - SYNTAX_ERROR_CODE.to_owned(), - format!( + Some(Finding { + code: SYNTAX_ERROR_CODE.to_owned(), + message: format!( "SQL syntax error in {} '{}': {}", kind_label(resource.kind), resource.name, - finding.1 + finding.message ), - finding.2, - finding.3, - finding.4, - finding.5, - ERROR_SEVERITY.to_owned(), - )) + line: finding.line, + column: finding.column, + start: finding.start, + end: finding.end, + severity: ERROR_SEVERITY.to_owned(), + }) } /// What one resource's findings are checked against. @@ -128,31 +139,31 @@ fn resource_diagnostics( continue; }; let span = location(resource, checking.cleaned_sql, &finding)?; - let key: Key = ( - resource.path.clone(), - finding.0.clone(), - finding.1.clone(), - span.0, - span.1, - if finding.4.is_none() { + let key: Key = Key { + path: resource.path.clone(), + code: finding.code.clone(), + message: finding.message.clone(), + line: span.line, + column: span.column, + order: if finding.start.is_none() { index as i64 } else { -1 }, - ); + }; if found.iter().any(|(existing, _)| *existing == key) { continue; } let authored_text = text_from(resource, span); let authored_line = line_at(resource, span); let help = resource_sql_help( - &finding.0, - &finding.1, + &finding.code, + &finding.message, &HelpContext { kind: resource.kind, authored_text: &authored_text, authored_line: &authored_line, - column: span.1, + column: span.column, authored_body: &resource.authored_body, audit_definition: resource.audit_definition.as_deref(), shapes: &checking.request.shapes, @@ -162,16 +173,18 @@ fn resource_diagnostics( ) .map_err(semantic)?; let message = match &resource.label { - Some(label) if finding.0 != SYNTAX_ERROR_CODE => format!("{label}: {}", finding.1), - _ => finding.1.clone(), + Some(label) if finding.code != SYNTAX_ERROR_CODE => { + format!("{label}: {}", finding.message) + } + _ => finding.message.clone(), }; found.push(( key, ResourceSqlDiagnostic { kind: resource.kind, resource_name: resource.name.clone(), - is_error: finding.6 == ERROR_SEVERITY, - code: finding.0.clone(), + is_error: finding.severity == ERROR_SEVERITY, + code: finding.code.clone(), message, path: resource.path.clone(), span, @@ -199,7 +212,7 @@ fn opt_out_diagnostic( code: OPT_OUT_CODE.to_owned(), message, path: resource.path.clone(), - span: (line, column, None, None), + span: ResourceSpan::start(line, column), note: Some(note), help: Some(help), }) @@ -230,7 +243,12 @@ pub(crate) fn missing_column_diagnostic( model.name ), path: model.path.clone(), - span: (line, start, Some(line), Some(end)), + span: ResourceSpan { + line, + column: start, + end_line: Some(line), + end_column: Some(end), + }, note: None, help: Some(format!( "add {column} to the SELECT list of model '{}' or attach the {definition} audit to \ diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs index 10693cdcef..bcafbf2275 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs @@ -87,14 +87,27 @@ fn empty_fixture_call_body(text: &[char], index: usize) -> Option { literal(text, after, ")", false) } -/// Every empty fixture CTE in `text`: start, prefix end, name and end. -fn empty_fixture_ctes(text: &[char]) -> Vec<(usize, usize, String, usize)> { - let mut found: Vec<(usize, usize, String, usize)> = Vec::new(); +/// One empty fixture CTE: where it starts, where its prefix ends, its name and its end. +struct EmptyFixtureCte { + start: usize, + prefix_end: usize, + name: String, + end: usize, +} + +/// Every empty fixture CTE in `text`. +fn empty_fixture_ctes(text: &[char]) -> Vec { + let mut found: Vec = Vec::new(); let mut index = 0; while index < text.len() { match empty_fixture_cte(text, index) { Some((prefix_end, name, end)) => { - found.push((index, prefix_end, name, end)); + found.push(EmptyFixtureCte { + start: index, + prefix_end, + name, + end, + }); index = end; } None => index += 1, @@ -109,11 +122,11 @@ pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { let ctes = empty_fixture_ctes(&body); let mut opened: Vec = Vec::with_capacity(body.len()); let mut copied = 0; - for (start, prefix_end, name, end) in &ctes { - opened.extend_from_slice(&body[copied..*start]); - opened.extend_from_slice(&body[*start..*prefix_end]); - opened.extend(name.chars()); - copied = *end; + for cte in &ctes { + opened.extend_from_slice(&body[copied..cte.start]); + opened.extend_from_slice(&body[cte.start..cte.prefix_end]); + opened.extend(cte.name.chars()); + copied = cte.end; } opened.extend_from_slice(&body[copied..]); let mut sql = String::with_capacity(opened.len()); @@ -128,6 +141,6 @@ pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { } } let mut relations = reference_names(&body); - relations.extend(ctes.into_iter().map(|(_, _, name, _)| name)); + relations.extend(ctes.into_iter().map(|cte| cte.name)); (sql, relations) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs index 0cc89868c2..85eadd87bb 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs @@ -4,25 +4,14 @@ use sqlbuild_core::text::main::ignorecase_key::ignorecase_key; use crate::semantic_checks::_helpers::resource_sql::resources::{ResourceSql, find_code_points}; use crate::semantic_checks::_helpers::resource_sql::scanning::word_at; -use crate::semantic_checks::models::ResourceSqlFailure; +use crate::semantic_checks::models::{ResourceSpan, ResourceSqlFailure}; use crate::semantic_validation::main::truth_test_spans::truth_test_spans; -use crate::semantic_validation::models::{BindingPositions, PositionInput}; - -/// A binding finding: code, message, line, column, start, end and severity. -pub(crate) type Finding = ( - String, - String, - Option, - Option, - Option, - Option, - String, -); +use crate::semantic_validation::models::{BindingPositions, DiagnosticRow, PositionInput}; -const UNSUPPORTED_TRUTH_TEST_CODE: &str = "B103"; +/// A binding finding: code, message, cleaned position and severity. +pub(crate) type Finding = DiagnosticRow; -/// `(line, column, end line, end column)`. -pub(crate) type Span = (i64, i64, Option, Option); +const UNSUPPORTED_TRUTH_TEST_CODE: &str = "B103"; /// Python's `_offset_location`: the 1-based line and column of a code-point offset. pub(crate) fn offset_location(text: &str, offset: usize) -> (i64, i64) { @@ -89,7 +78,7 @@ fn mapped_span( resource: &ResourceSql, cleaned_sql: &str, finding: &Finding, -) -> Result, ResourceSqlFailure> { +) -> Result, ResourceSqlFailure> { let mapper = BindingPositions::new(PositionInput { authored: resource.contents.to_string(), query: resource.authored_body.clone(), @@ -99,27 +88,36 @@ fn mapped_span( }) .map_err(ResourceSqlFailure::Normalization)?; let (line, column) = mapper - .position(finding.4, finding.2, finding.3, &finding.1) + .position( + finding.start, + finding.line, + finding.column, + &finding.message, + ) .map_err(ResourceSqlFailure::Normalization)?; let (Some(line), Some(column)) = (line, column) else { return Ok(None); }; - let (end_line, end_column) = match finding.5 { + let (end_line, end_column) = match finding.end { Some(end) => mapper .position(Some(end), None, None, "") .map_err(ResourceSqlFailure::Normalization)?, None => (None, None), }; - Ok(Some(( - line as i64, - column as i64, - usize_line(end_line), - usize_line(end_column), - ))) + Ok(Some(ResourceSpan { + line: line as i64, + column: column as i64, + end_line: usize_line(end_line), + end_column: usize_line(end_column), + })) } /// The authored span of the truth-value test a B103 finding starts at, matched by its order. -fn truth_test_span(resource: &ResourceSql, cleaned_sql: &str, start: usize) -> Option { +fn truth_test_span( + resource: &ResourceSql, + cleaned_sql: &str, + start: usize, +) -> Option { let cleaned: Vec<(usize, usize)> = truth_test_spans(cleaned_sql); let order: usize = cleaned .iter() @@ -134,7 +132,12 @@ fn truth_test_span(resource: &ResourceSql, cleaned_sql: &str, start: usize) -> O let length: usize = resource.authored_body[test_start..test_end].chars().count(); let (line, column) = offset_location(&resource.contents, offset); let (end_line, end_column) = offset_location(&resource.contents, offset + length); - Some((line, column, Some(end_line), Some(end_column))) + Some(ResourceSpan { + line, + column, + end_line: Some(end_line), + end_column: Some(end_column), + }) } /// Python's `_location`: the offending authored text, falling back to the body start. @@ -142,18 +145,18 @@ pub(crate) fn location( resource: &ResourceSql, cleaned_sql: &str, finding: &Finding, -) -> Result { +) -> Result { let body_offset = find_code_points(&resource.contents, &resource.authored_body).unwrap_or(0); - if finding.4.is_none() { + if finding.start.is_none() { let text: Vec = resource.contents.chars().collect(); - if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.1) { + if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.message) { let (line, column) = offset_location(&resource.contents, offset); - return Ok((line, column, None, None)); + return Ok(ResourceSpan::start(line, column)); } } - if finding.0 == UNSUPPORTED_TRUTH_TEST_CODE + if finding.code == UNSUPPORTED_TRUTH_TEST_CODE && let Some(span) = finding - .4 + .start .and_then(|start| truth_test_span(resource, cleaned_sql, start)) { return Ok(span); @@ -162,7 +165,7 @@ pub(crate) fn location( return Ok(span); } let (line, column) = offset_location(&resource.contents, body_offset); - Ok((line, column, None, None)) + Ok(ResourceSpan::start(line, column)) } /// Code-point starts of each line of `text`. @@ -187,12 +190,12 @@ fn offset(starts: &[usize], line: i64, column: i64) -> Option { } /// Python's `_text_from`: the authored text of a span, or from its start. -pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { +pub(crate) fn text_from(resource: &ResourceSql, span: ResourceSpan) -> String { let starts = line_starts(&resource.contents); - let Some(start) = offset(&starts, span.0, span.1) else { + let Some(start) = offset(&starts, span.line, span.column) else { return String::new(); }; - let end = match (span.2, span.3) { + let end = match (span.end_line, span.end_column) { (Some(line), Some(column)) => offset(&starts, line, column), _ => None, }; @@ -204,12 +207,12 @@ pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { } /// Python's `_line_at`: the authored line of a span, or empty text. -pub(crate) fn line_at(resource: &ResourceSql, span: Span) -> String { +pub(crate) fn line_at(resource: &ResourceSql, span: ResourceSpan) -> String { let lines: Vec<&str> = resource.contents.lines().collect(); - if span.0 < 1 { + if span.line < 1 { return String::new(); } lines - .get((span.0 - 1) as usize) + .get((span.line - 1) as usize) .map_or_else(String::new, |line| (*line).to_owned()) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/models.rs b/crates/sqlbuild-analysis/src/semantic_checks/models.rs index d12faf7325..7674a801ce 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/models.rs @@ -445,6 +445,27 @@ pub struct ResourceSqlRequest { pub hook_files: Vec<(String, Arc)>, } +/// A resource SQL finding's authored start and, when it covers text, its end. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ResourceSpan { + pub line: i64, + pub column: i64, + pub end_line: Option, + pub end_column: Option, +} + +impl ResourceSpan { + /// A span that marks only its start. + pub fn start(line: i64, column: i64) -> Self { + Self { + line, + column, + end_line: None, + end_column: None, + } + } +} + /// A located resource SQL finding, or a rejected `sql_analysis false` with its note. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ResourceSqlDiagnostic { @@ -454,8 +475,7 @@ pub struct ResourceSqlDiagnostic { pub code: String, pub message: String, pub path: String, - /// `(line, column, end line, end column)`. - pub span: (i64, i64, Option, Option), + pub span: ResourceSpan, pub note: Option, pub help: Option, } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs index 53b07ab184..f91795ba59 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs @@ -120,9 +120,9 @@ fn given_built_in_audit_on_missing_column_when_checking_then_reports_once_at_its ); ( diagnostic.message, - diagnostic.span.0, - diagnostic.span.1, - diagnostic.span.3.unwrap_or_default(), + diagnostic.span.line, + diagnostic.span.column, + diagnostic.span.end_column.unwrap_or_default(), ) }); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/semantic_usage/main.rs b/crates/sqlbuild-analysis/src/semantic_usage/main.rs index 37a3d7e161..beeedb31e0 100644 --- a/crates/sqlbuild-analysis/src/semantic_usage/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/main.rs @@ -227,16 +227,26 @@ fn physical_source_name(expression: &Expression) -> Option { } } +/// The identity two usage facts share when one repeats the other. +#[derive(PartialEq, Eq, Hash)] +struct UsageKey { + context: String, + expression_sql: String, + source_name: String, + column: String, + start: Option, +} + fn dedupe(mut uses: Vec) -> Vec { - let mut seen: HashSet<(String, String, String, String, Option)> = HashSet::new(); + let mut seen: HashSet = HashSet::new(); uses.retain(|fact| { - seen.insert(( - fact.context.clone(), - fact.expression_sql.clone(), - fact.source_name.clone(), - fact.column.clone(), - fact.start, - )) + seen.insert(UsageKey { + context: fact.context.clone(), + expression_sql: fact.expression_sql.clone(), + source_name: fact.source_name.clone(), + column: fact.column.clone(), + start: fact.start, + }) }); uses } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index a771ef8731..7f892c622e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -6,10 +6,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use crate::semantic_validation::models::{ - CatalogInput, Columns, CompactAnalysisJob, FunctionProbes, ProjectCatalog, + CatalogInput, Columns, CompactAnalysisJob, DiagnosticRow, FunctionProbes, ProjectCatalog, }; use crate::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, PreparedCompactAnalysis, Relations, + BindingRequest, NormalizationRequest, PreparedCompactAnalysis, Relations, }; use crate::semantic_validation::{ _helpers::{diagnostics, identifiers}, @@ -616,8 +616,8 @@ impl ProjectCatalog { /// The diagnostic row of `error`, naming the failure when the parser gives no message. fn messaged_row(error: ValidationError) -> DiagnosticRow { let mut row: DiagnosticRow = diagnostic_row(error); - if row.1.is_empty() { - DEFAULT_BINDING_MESSAGE.clone_into(&mut row.1); + if row.message.is_empty() { + DEFAULT_BINDING_MESSAGE.clone_into(&mut row.message); } row } @@ -627,15 +627,15 @@ pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { polyglot_sql::ValidationSeverity::Error => "error", _ => "warning", }; - ( - error.code, - error.message, - error.line, - error.column, - error.start, - error.end, - severity.to_owned(), - ) + DiagnosticRow { + code: error.code, + message: error.message, + line: error.line, + column: error.column, + start: error.start, + end: error.end, + severity: severity.to_owned(), + } } fn has_exact_columns(table: &SchemaTable) -> bool { diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index 0983e58f70..17175e9868 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -4,8 +4,8 @@ use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; use crate::semantic_validation::_helpers::truth_tests; use crate::semantic_validation::_helpers::type_locations::locate_type_findings; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::FunctionProbes; -use crate::semantic_validation::types::DiagnosticRow; use polyglot_sql::expressions::Select; use polyglot_sql::{ Dialect, DialectType, Expression, ExpressionWalk, ValidationError, ValidationResult, @@ -393,21 +393,19 @@ pub(crate) fn binding_diagnostics( ) -> Result, String> { let errors: Vec = rows .into_iter() - .map( - |(code, message, line, column, start, end, severity)| ValidationError { - code, - message, - line, - column, - start, - end, - severity: if severity == ERROR { - ValidationSeverity::Error - } else { - ValidationSeverity::Warning - }, + .map(|row| ValidationError { + code: row.code, + message: row.message, + line: row.line, + column: row.column, + start: row.start, + end: row.end, + severity: if row.severity == ERROR { + ValidationSeverity::Error + } else { + ValidationSeverity::Warning }, - ) + }) .collect(); let result = map_diagnostics( sql, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs index d59718854b..0cf001abe2 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs @@ -2,7 +2,7 @@ use crate::semantic_validation::_helpers::diagnostics::column_pattern; use crate::semantic_validation::_helpers::normalization::normalize; -use crate::semantic_validation::models::{BindingPositions, PositionInput}; +use crate::semantic_validation::models::{BindingPositions, Expansion, PositionInput}; use regex::Regex; use std::collections::HashMap; use std::sync::{LazyLock, OnceLock}; @@ -10,7 +10,6 @@ use std::sync::{LazyLock, OnceLock}; const POPULAR_TOKEN_THRESHOLD: usize = 200; static TOKENS: LazyLock> = LazyLock::new(|| Regex::new(r"\w+|[^\w\s]").map_err(|error| error.to_string())); -type Mapping = (usize, usize, usize, usize); impl BindingPositions { pub fn new(request: PositionInput) -> Result { @@ -75,7 +74,13 @@ impl BindingPositions { .unwrap_or_else(|| *self.offsets.last().unwrap_or(&0)); for pass in self.passes.iter().rev() { let mut adjusted = mapped as isize; - for &(source_start, source_end, output_start, output_end) in pass { + for &Expansion { + source_start, + source_end, + output_start, + output_end, + } in pass + { if mapped < output_start { break; } @@ -227,28 +232,31 @@ fn align_offsets(original: &str, normalized: &str) -> Result, String> blocks.sort(); let ac = byte_char_map(original); let bc = byte_char_map(normalized); - let mut mappings: Vec = Vec::new(); + let mut mappings: Vec = Vec::new(); for (ai, bi, size) in blocks { for i in 0..size { - mappings.push(( - ac[a[ai + i].start()], - ac[a[ai + i].end()], - bc[b[bi + i].start()], - bc[b[bi + i].end()], - )); + mappings.push(Expansion { + source_start: ac[a[ai + i].start()], + source_end: ac[a[ai + i].end()], + output_start: bc[b[bi + i].start()], + output_end: bc[b[bi + i].end()], + }); } } let mut offsets = Vec::with_capacity(normalized.chars().count() + 1); let original_len = original.chars().count(); let mut index = 0; for offset in 0..=normalized.chars().count() { - while index < mappings.len() && offset > mappings[index].3 { + while index < mappings.len() && offset > mappings[index].output_end { index += 1; } offsets.push(match mappings.get(index) { - Some(&(start, end, target, _)) => { - start + offset.saturating_sub(target).min(end - start) - } + Some(&Expansion { + source_start: start, + source_end: end, + output_start: target, + .. + }) => start + offset.saturating_sub(target).min(end - start), None => original_len, }); } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs index 2efcff0313..ae7f192632 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; pub fn binding_diagnostics( sql: &str, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/models.rs b/crates/sqlbuild-analysis/src/semantic_validation/models.rs index 7f1805efcf..a24345c77a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/models.rs @@ -1,6 +1,6 @@ //! Compile-owned native catalog, mapping state and boundary requests. -use crate::semantic_validation::types::{Expansion, PreparedCompactAnalysis, ProbeKey, Relations}; +use crate::semantic_validation::types::{PreparedCompactAnalysis, ProbeKey, Relations}; use polyglot_sql::validation::SchemaTable; use polyglot_sql::{DialectType, SchemaValidationOptions}; use rayon::ThreadPool; @@ -8,6 +8,27 @@ use std::collections::HashMap; use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex}; +/// One schema-validation diagnostic with its cleaned-SQL position. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticRow { + pub code: String, + pub message: String, + pub line: Option, + pub column: Option, + pub start: Option, + pub end: Option, + pub severity: String, +} + +/// One macro-expansion edit: the authored char span and the span it expanded to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Expansion { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + #[derive(Default, Debug)] pub struct Columns(pub Vec<(String, Option)>); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs index 06c47b5f23..7bbe92b98e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs @@ -1,4 +1,5 @@ use crate::semantic_validation::main::diagnostics::binding_diagnostics; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::tests::test_types::TypeLocationTestCase; #[test] @@ -68,16 +69,14 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex let rows = test_case .findings .iter() - .map(|(code, message)| { - ( - (*code).to_owned(), - (*message).to_owned(), - None, - None, - None, - None, - "error".to_owned(), - ) + .map(|(code, message)| DiagnosticRow { + code: (*code).to_owned(), + message: (*message).to_owned(), + line: None, + column: None, + start: None, + end: None, + severity: "error".to_owned(), }) .collect(); let mapped = binding_diagnostics(test_case.sql, test_case.dialect, rows)?; @@ -85,7 +84,7 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex .iter() .zip(test_case.expected_starts) .map(|(row, expected)| { - row.4.map_or_else(String::new, |start| { + row.start.map_or_else(String::new, |start| { test_case .sql .chars() @@ -107,7 +106,9 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex test_case.description ); assert!( - mapped.iter().all(|row| row.2.is_some() && row.3.is_some()), + mapped + .iter() + .all(|row| row.line.is_some() && row.column.is_some()), "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/types.rs b/crates/sqlbuild-analysis/src/semantic_validation/types.rs index a6a8770c55..8ab94dd7a8 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/types.rs @@ -5,16 +5,6 @@ pub type Relations = HashMap; pub type BindingRequest = (String, Vec<(String, bool)>, Relations); /// One analysis SQL with its relation stubs and placeholder defaults. pub type NormalizationRequest = (String, HashMap, HashMap); -pub type DiagnosticRow = ( - String, - String, - Option, - Option, - Option, - Option, - String, -); -pub type Expansion = (usize, usize, usize, usize); pub(crate) type ProbeKey = (polyglot_sql::DialectType, String, usize); /// Compact analysis whose catalog schemas are resolved; it runs with an options-only catalog. pub(crate) type PreparedCompactAnalysis = diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs index 8a302d80e9..958cf3bd53 100644 --- a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs +++ b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs @@ -1,13 +1,14 @@ //! One of database or schema of a function namespace. -pub(crate) type Part = ( - Option, - Option, - Option, - Option, -); +/// The physical, logical and fingerprint names of one of database or schema. +pub(crate) struct Part { + pub(crate) physical: Option, + pub(crate) logical: Option, + pub(crate) fingerprint: Option, + pub(crate) fingerprint_logical: Option, +} -/// `(physical, logical, fingerprint, fingerprint logical)` for one of database or schema. +/// The physical, logical, fingerprint and fingerprint logical names of one namespace part. pub(crate) fn part( header: Option<&String>, default: Option<&String>, @@ -26,5 +27,10 @@ pub(crate) fn part( }; let fingerprint_logical: Option = header.or(default).cloned(); let fingerprint: Option = target.cloned().or_else(|| fingerprint_logical.clone()); - (physical, logical, fingerprint, fingerprint_logical) + Part { + physical, + logical, + fingerprint, + fingerprint_logical, + } } diff --git a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs index ae40e9d40b..74bff9d485 100644 --- a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs +++ b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs @@ -8,26 +8,26 @@ use crate::functions::models::{FunctionLanguage, FunctionNamespace, NamespaceInp pub fn resolve_function_namespace(inputs: &NamespaceInputs) -> FunctionNamespace { let inherit: bool = inputs.language == FunctionLanguage::Sql || inputs.inherit_default_namespace; - let (database, logical_database, fingerprint_database, fingerprint_logical_database) = part( + let database = part( inputs.header_database.as_ref(), inputs.default_database.as_ref(), inputs.target_database.as_ref(), inherit, ); - let (schema, logical_schema, fingerprint_schema, fingerprint_logical_schema) = part( + let schema = part( inputs.header_schema.as_ref(), inputs.default_schema.as_ref(), inputs.target_schema.as_ref(), inherit, ); FunctionNamespace { - database, - schema, - logical_database, - logical_schema, - fingerprint_database, - fingerprint_schema, - fingerprint_logical_database, - fingerprint_logical_schema, + database: database.physical, + schema: schema.physical, + logical_database: database.logical, + logical_schema: schema.logical, + fingerprint_database: database.fingerprint, + fingerprint_schema: schema.fingerprint, + fingerprint_logical_database: database.fingerprint_logical, + fingerprint_logical_schema: schema.fingerprint_logical, } } diff --git a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs index 6c7719b1ad..251dda70cb 100644 --- a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs +++ b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs @@ -14,7 +14,10 @@ pub fn render_dbt_execution_summary( if results.is_empty() { return None; } - let (mut passed, mut warned, mut failed, mut skipped): (i64, i64, i64, i64) = (0, 0, 0, 0); + let mut passed: i64 = 0; + let mut warned: i64 = 0; + let mut failed: i64 = 0; + let mut skipped: i64 = 0; let mut elapsed: f64 = 0.0; for result in results { let status: String = result.status.to_lowercase(); diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index fd5d643514..70e6a04f84 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -1,6 +1,6 @@ //! The versioned lifecycle event catalogue and the payload value contracts it validates. -use crate::lifecycle_events::types::CatalogEntry; +use crate::lifecycle_events::models::EventDefinition; /// The lifecycle event schema version SQLBuild produces. pub const CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: i64 = 2; @@ -277,45 +277,51 @@ pub const OPERATION_ADAPTERS: [&str; 10] = [ ]; /// The schema version 1 catalogue in declaration order. -pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ - ("invocation_started", &[], &[], &["command"], false), - ( - "invocation_completed", - &[], - &[], - &["command", "duration_ms", "exit_code"], - true, - ), - ( - "invocation_failed", - &[], - &[], - &[ +pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [EventDefinition; 20] = [ + EventDefinition { + event_type: "invocation_started", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command"], + terminal: false, + }, + EventDefinition { + event_type: "invocation_completed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command", "duration_ms", "exit_code"], + terminal: true, + }, + EventDefinition { + event_type: "invocation_failed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &[ "command", "duration_ms", "error_code", "error_type", "exit_code", ], - true, - ), - ( - "run_started", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_started", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "configured_concurrency", "run_kind", "selected_count", "worker_count", ], - false, - ), - ( - "run_completed", - &["run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "run_completed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "fail_count", "failed_count", @@ -325,13 +331,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "run_failed", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_failed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "error_code", "error_type", @@ -343,32 +349,32 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "resource_attempt_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &["attempt_number", "resource_kind", "resource_name"], - false, - ), - ( - "resource_attempt_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &["attempt_number", "resource_kind", "resource_name"], + terminal: false, + }, + EventDefinition { + event_type: "resource_attempt_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "error_code", @@ -376,18 +382,18 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_skipped", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_skipped", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "attempt_number", "resource_kind", "resource_name", "skip_code", ], - &[ + allowed_payload: &[ "attempt_number", "duration_ms", "resource_kind", @@ -395,13 +401,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "skip_code", "skip_mode", ], - true, - ), - ( - "operation_started", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_started", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "hook_index", "hook_name", @@ -415,13 +421,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - false, - ), - ( - "operation_completed", - &["operation_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "operation_completed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "exit_code", @@ -439,13 +445,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - true, - ), - ( - "operation_failed", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_failed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "error_code", @@ -465,30 +471,30 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - true, - ), - ( - "retry_scheduled", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "retry_scheduled", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "delay_ms", "error_type", "failed_attempt_number", "next_attempt_number", ], - &[ + allowed_payload: &[ "delay_ms", "error_code", "error_type", "failed_attempt_number", "next_attempt_number", ], - false, - ), - ( - "audit_completed", - &["run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "audit_completed", + required_correlations: &["run_id"], + required_payload: &[ "attachment_kind", "audit_definition_name", "audit_name", @@ -497,7 +503,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "run_scope_phase", "severity", ], - &[ + allowed_payload: &[ "attached_column_name", "attached_target_kind", "attached_target_name", @@ -534,13 +540,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "thresholds", "violation_count", ], - true, - ), - ( - "statement_started", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_started", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -552,13 +558,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_submitted", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_submitted", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -569,13 +575,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_heartbeat", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_heartbeat", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -588,13 +594,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_completed", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_completed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -607,13 +613,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "statement_failed", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_failed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -628,55 +634,55 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - true, - ), + terminal: true, + }, ]; /// The schema version 2 catalogue in declaration order. -pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ - ( - "invocation_started", - &[], - &[], - &["adapter", "command", "selection_count", "target"], - false, - ), - ( - "invocation_completed", - &[], - &[], - &["command", "duration_ms", "exit_code"], - true, - ), - ( - "invocation_failed", - &[], - &[], - &[ +pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ + EventDefinition { + event_type: "invocation_started", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["adapter", "command", "selection_count", "target"], + terminal: false, + }, + EventDefinition { + event_type: "invocation_completed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command", "duration_ms", "exit_code"], + terminal: true, + }, + EventDefinition { + event_type: "invocation_failed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &[ "command", "duration_ms", "error_code", "error_type", "exit_code", ], - true, - ), - ( - "run_started", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_started", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "configured_concurrency", "run_kind", "selected_count", "worker_count", ], - false, - ), - ( - "run_completed", - &["run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "run_completed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "fail_count", "failed_count", @@ -686,13 +692,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "run_failed", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_failed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "error_code", "error_type", @@ -704,26 +710,26 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "resource_attempt_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "plan_action", "plan_reason_code", "resource_kind", "resource_name", ], - false, - ), - ( - "resource_attempt_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "resource_attempt_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "plan_action", @@ -731,13 +737,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "error_code", @@ -747,18 +753,18 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_skipped", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_skipped", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "attempt_number", "resource_kind", "resource_name", "skip_code", ], - &[ + allowed_payload: &[ "attempt_number", "duration_ms", "plan_action", @@ -768,13 +774,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "skip_code", "skip_mode", ], - true, - ), - ( - "operation_started", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_started", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "hook_index", "hook_name", @@ -788,13 +794,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - false, - ), - ( - "operation_completed", - &["operation_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "operation_completed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "exit_code", @@ -812,13 +818,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - true, - ), - ( - "operation_failed", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_failed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "error_code", @@ -838,30 +844,30 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - true, - ), - ( - "retry_scheduled", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "retry_scheduled", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "delay_ms", "error_type", "failed_attempt_number", "next_attempt_number", ], - &[ + allowed_payload: &[ "delay_ms", "error_code", "error_type", "failed_attempt_number", "next_attempt_number", ], - false, - ), - ( - "audit_completed", - &["run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "audit_completed", + required_correlations: &["run_id"], + required_payload: &[ "attachment_kind", "audit_definition_name", "audit_name", @@ -870,7 +876,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "run_scope_phase", "severity", ], - &[ + allowed_payload: &[ "attached_column_name", "attached_target_kind", "attached_target_name", @@ -907,13 +913,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "thresholds", "violation_count", ], - true, - ), - ( - "statement_started", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_started", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -926,13 +932,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_submitted", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_submitted", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -944,13 +950,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_heartbeat", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_heartbeat", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -964,13 +970,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_completed", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_completed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -984,13 +990,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "statement_failed", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_failed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -1006,12 +1012,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "microbatch_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "microbatch_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1024,7 +1030,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1037,12 +1043,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - false, - ), - ( - "microbatch_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "microbatch_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1055,7 +1061,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "affected_rows", "batch_count", "batch_index", @@ -1070,12 +1076,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - true, - ), - ( - "microbatch_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "microbatch_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1089,7 +1095,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1105,6 +1111,6 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - true, - ), + terminal: true, + }, ]; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs index 74d374d42a..3d8922df03 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs @@ -2,26 +2,14 @@ use crate::lifecycle_events::constants::{LIFECYCLE_EVENT_CATALOG_V1, LIFECYCLE_EVENT_CATALOG_V2}; use crate::lifecycle_events::models::EventDefinition; -use crate::lifecycle_events::types::CatalogEntry; - -fn definition(entry: &CatalogEntry) -> EventDefinition { - let (event_type, required_correlations, required_payload, allowed_payload, terminal) = *entry; - EventDefinition { - event_type, - required_correlations, - required_payload, - allowed_payload, - terminal, - } -} /// Every definition of `schema_version`, or `None` for an uncatalogued version. #[must_use] pub fn catalog(schema_version: i64) -> Option> { - let entries: &[CatalogEntry] = match schema_version { + let entries: &[EventDefinition] = match schema_version { 1 => &LIFECYCLE_EVENT_CATALOG_V1, 2 => &LIFECYCLE_EVENT_CATALOG_V2, _ => return None, }; - Some(entries.iter().map(definition).collect()) + Some(entries.to_vec()) } diff --git a/crates/sqlbuild-core/src/lifecycle_events/types.rs b/crates/sqlbuild-core/src/lifecycle_events/types.rs index 2af438e15f..7719efd781 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/types.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/types.rs @@ -1,13 +1,4 @@ -//! The shape of one catalogue entry. - -/// One catalogued event: type, required correlations, required and allowed payload, terminal. -pub(crate) type CatalogEntry = ( - &'static str, - &'static [&'static str], - &'static [&'static str], - &'static [&'static str], - bool, -); +//! The shape of one event-family payload rule. /// One event-family payload rule: the first violation in a payload, if any. pub(crate) type PayloadCheck = diff --git a/crates/sqlbuild-discovery/src/_helpers/locations.rs b/crates/sqlbuild-discovery/src/_helpers/locations.rs index e74876e9b0..14e7d4fa22 100644 --- a/crates/sqlbuild-discovery/src/_helpers/locations.rs +++ b/crates/sqlbuild-discovery/src/_helpers/locations.rs @@ -18,7 +18,13 @@ pub(crate) fn header_column_locations( let header: Vec = contents[header_start..header_end].chars().collect(); relative_locations(&header, offsets) .into_iter() - .map(|(name, relative_line, relative_column, length)| { + .map(|location| { + let RelativeLocation { + name, + line: relative_line, + column: relative_column, + length, + } = location; let line = header_line + relative_line - 1; let column = relative_column + if relative_line == 1 { @@ -39,11 +45,19 @@ pub(crate) fn header_column_locations( .collect() } +/// One header column key's line and column relative to the header, and its length. +struct RelativeLocation { + name: String, + line: usize, + column: usize, + length: usize, +} + /// Python's `_header_column_relative_locations`, including its forward-only cursor. fn relative_locations( header: &[char], offsets: &[(String, usize, usize)], -) -> Vec<(String, usize, usize, usize)> { +) -> Vec { let mut locations = Vec::with_capacity(offsets.len()); let (mut line, mut column, mut cursor) = (1, 1, 0); for (name, position, length) in offsets { @@ -63,7 +77,12 @@ fn relative_locations( } } } - locations.push((name.clone(), line, column, *length)); + locations.push(RelativeLocation { + name: name.clone(), + line, + column, + length: *length, + }); } locations } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 00b92ba4c4..a7db77b4eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -9,7 +9,10 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyDict, PyMapping, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::types::{BindingRequest, DiagnosticRow, Relations}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; + +use crate::bindings::types::IntoPositional; +use sqlbuild_analysis::semantic_validation::types::{BindingRequest, Relations}; use crate::bindings::_helpers::analysis::conversions; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -57,7 +60,13 @@ pub(crate) struct BindingCatalog { } /// A catalog's dialect, case mode, known functions and known types. -pub(crate) type CatalogKey = (String, bool, Vec, Vec); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct CatalogKey { + pub(crate) dialect: String, + pub(crate) quoted_ignore_case: bool, + pub(crate) known_functions: Vec, + pub(crate) known_types: Vec, +} impl BindingCatalog { /// Whether `schemas` holds exactly the relations it was created with and later given. @@ -68,9 +77,15 @@ impl BindingCatalog { /// A catalog for `key` that starts with `relations`. pub(crate) fn create( py: Python<'_>, - (dialect, quoted_ignore_case, known_functions, known_types): CatalogKey, + key: CatalogKey, relations: &Bound<'_, PyAny>, ) -> PyResult { + let CatalogKey { + dialect, + quoted_ignore_case, + known_functions, + known_types, + } = key; let input: CatalogInput = CatalogInput { dialect: dialect.clone(), quoted_ignore_case, @@ -160,12 +175,12 @@ impl BindingCatalog { )) }) }; - let key: CatalogKey = ( - field("dialect")?.extract()?, - field("quoted_ignore_case")?.extract()?, - field("known_functions")?.extract()?, - field("known_types")?.extract()?, - ); + let key: CatalogKey = CatalogKey { + dialect: field("dialect")?.extract()?, + quoted_ignore_case: field("quoted_ignore_case")?.extract()?, + known_functions: field("known_functions")?.extract()?, + known_types: field("known_types")?.extract()?, + }; Self::create(py, key, &field("relations")?) } @@ -309,12 +324,13 @@ pub(crate) fn schema_validations<'py>( let mut created: Vec<(CatalogKey, Py)> = Vec::new(); let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); for (index, request) in requests.iter().enumerate() { - let key: CatalogKey = ( - native_dialect(&native_field(request, "dialect")?)?, - native_field(request, "quoted_identifiers_ignore_case")?.extract()?, - native_field(request, "known_functions")?.extract()?, - native_field(request, "known_types")?.extract()?, - ); + let key: CatalogKey = CatalogKey { + dialect: native_dialect(&native_field(request, "dialect")?)?, + quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(request, "known_functions")?.extract()?, + known_types: native_field(request, "known_types")?.extract()?, + }; let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; let own: bool = !explicit.is_none(); let catalog: Py = if own { @@ -363,7 +379,14 @@ pub(crate) fn schema_validations<'py>( for (index, diagnostics) in members.into_iter().zip(rows) { let built: Vec> = diagnostics .into_iter() - .map(|row| new_record(py, RecordId::SqlBindingDiagnostic, row, None)) + .map(|row| { + new_record( + py, + RecordId::SqlBindingDiagnostic, + row.into_positional(py)?, + None, + ) + }) .collect::>()?; let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs index 4ac680f26c..52f7e0d07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs @@ -4,27 +4,44 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; -use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::binding_locations::models::{ + AuthoredSpan, BindingSource, LocatedDiagnostic, +}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; -/// `(code, message, severity, line, column, (line, column, end line, end column))`. -type LocatedRow = ( - String, - String, - String, - Option, - Option, - Option<(usize, usize, Option, Option)>, -); +use crate::bindings::models::Positional; + +positional_record! { + into; + /// One located diagnostic as Python reads it, its span `(line, column, end line, end column)`. + pub(crate) struct LocatedRow { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) severity: String, + pub(crate) line: Option, + pub(crate) column: Option, + pub(crate) span: Option>, + } +} + +/// The expanded SQL and the expansion passes Python sends for a located model. +pub(crate) type ExpansionRow = Option<(String, Vec>>)>; /// Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. #[pyfunction] pub(crate) fn locate_model_binding_diagnostics( - rows: Vec, + rows: Vec>, texts: (String, String, String), - expansion: Option<(String, Vec>)>, + expansion: ExpansionRow, ) -> PyResult> { let (authored, query, cleaned) = texts; + let rows: Vec = Positional::unwrap_all(rows); + let expansion: Option<(String, Vec>)> = expansion.map(|(expanded, passes)| { + ( + expanded, + passes.into_iter().map(Positional::unwrap_all).collect(), + ) + }); let source: BindingSource<'_> = BindingSource { authored: &authored, query: &query, @@ -39,17 +56,14 @@ pub(crate) fn locate_model_binding_diagnostics( } fn located_row(item: LocatedDiagnostic) -> LocatedRow { - let span = item - .location - .map(|span| (span.line, span.column, span.end_line, span.end_column)); - ( - item.code, - item.message, - item.severity, - item.line, - item.column, - span, - ) + LocatedRow { + code: item.code, + message: item.message, + severity: item.severity, + line: item.line, + column: item.column, + span: item.location.map(Positional), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs index 37314dacda..52c663d6b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::analysis::conversions; use crate::bindings::_helpers::analysis::normalization_results::normalization_results; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::models::Positional; use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; use crate::bindings::types::CompilerDetach; use pyo3::exceptions::PyValueError; @@ -10,8 +11,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::PyBytes; use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_analysis::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, Relations, + BindingRequest, NormalizationRequest, Relations, }; use std::collections::HashMap; @@ -90,9 +92,11 @@ impl ProjectCatalog { &self, py: Python<'_>, #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, - ) -> PyResult>> { - py.compiler_detach(|| self.inner.binding_results(requests)) - .map_err(compiler_error) + ) -> PyResult>>> { + let rows: Vec> = py + .compiler_detach(|| self.inner.binding_results(requests)) + .map_err(compiler_error)?; + Ok(rows.into_iter().map(Positional::wrap_all).collect()) } /// Normalize a preparation batch on the idle analysis pool, else in turn, in one detached call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 47c47892ed..221a27d6de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -423,16 +423,23 @@ pub(crate) fn old_name_reference_message( old_name_message(owner_label, written, method, model_name) } -/// `(audit name, target, gated, read)` for one attached-audit ordering edge. -pub(crate) type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); -/// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. -pub(crate) type HookRow = ( - String, - ResourceKey, - ResourceKey, - &'static str, - Option, -); +/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. +pub(crate) struct GateRow { + pub(crate) audit_name: String, + pub(crate) target: ResourceKey, + pub(crate) gated: ResourceKey, + pub(crate) read: ResourceKey, +} + +/// One hook-read ordering edge: the hook, the gated and read resources, its type and path. +pub(crate) struct HookRow { + pub(crate) hook_name: String, + pub(crate) gated: ResourceKey, + pub(crate) read: ResourceKey, + /// The `HookReadType` value. + pub(crate) hook_type: &'static str, + pub(crate) path: Option, +} /// The attached-audit gate edges and hook-read edges of one project, in Python's order. pub(crate) fn native_execution_gate_edges( @@ -452,7 +459,12 @@ pub(crate) fn native_execution_gate_edges( } fn gate_row(edge: GateEdge) -> GateRow { - (edge.audit_name, edge.target, edge.gated, edge.read) + GateRow { + audit_name: edge.audit_name, + target: edge.target, + gated: edge.gated, + read: edge.read, + } } fn hook_row(edge: HookEdge) -> HookRow { @@ -461,7 +473,13 @@ fn hook_row(edge: HookEdge) -> HookRow { HookType::Sql => "SQL", HookType::InlineSql => "inline SQL", }; - (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) + HookRow { + hook_name: edge.hook_name, + gated: edge.gated, + read: edge.read, + hook_type, + path: edge.path, + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index e1012641f5..9c449309a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -7,8 +7,14 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ }; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(output, transform kind, confidence, [(resource type, resource, column)])` of one lineage output. -pub(crate) type LineageFactRow = (String, String, String, Vec<(String, String, String)>); +/// One lineage output with its transform kind and confidence values, and its sources. +pub(crate) struct LineageFactRow { + pub(crate) output_column: String, + pub(crate) transform: String, + pub(crate) confidence: String, + /// `(resource type, resource, column)` of each source. + pub(crate) sources: Vec<(String, String, String)>, +} const TRANSFORM_VALUES: [&str; 6] = [ "direct", @@ -139,12 +145,12 @@ pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult = Vec::with_capacity(rows.len()); for row in rows { - facts.push(( - row.output_column, - coded(&TRANSFORM_VALUES, row.transform_code)?, - coded(&CONFIDENCE_VALUES, row.confidence_code)?, - row.sources, - )); + facts.push(LineageFactRow { + output_column: row.output_column, + transform: coded(&TRANSFORM_VALUES, row.transform_code)?, + confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, + sources: row.sources, + }); } return Ok(Some(facts)); } @@ -164,12 +170,12 @@ pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult, sql: &str, dialect: &str, - rows: Vec, -) -> PyResult> { + rows: Vec>, +) -> PyResult>> { + let rows: Vec = Positional::unwrap_all(rows); py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics( sql, dialect, rows, ) }) + .map(Positional::wrap_all) .map_err(value_error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 2610876a8a..840a961495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -19,11 +19,14 @@ use crate::bindings::_helpers::analysis_session::session::{ use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; -use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::compile_driver::analysis_cache::{ + AnalysisCacheCounts, record_analysis_cache_metrics, +}; use crate::bindings::_helpers::compiled_project::model_input_view::{ AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, input_reference_rows, }; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::enum_member; @@ -106,36 +109,46 @@ fn session_request<'py>( let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; families.push((name, family_rows(&declared)?)); } - let (types, nullability, complete, schemas): (Shapes, Shapes, Shapes, Shapes) = - match declared_rows(request, catalog)? { - Some(rows) => rows, - None => ( - shape_rows(&request.column_types)?, - shape_rows(&request.nullability)?, - shape_rows(&request.complete_schemas)?, - shape_rows(&native_field(catalog, "schemas")?)?, - ), - }; - Ok(( - profile_dialect(profile)?, - case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)?, - return_types(profile)?, - Some(rules), - request.rich_type_inference, - types, - nullability, - complete, - schemas, - families, - model_rows(profile.py(), &request.model_inputs)?, - )) + let shapes: RelationShapes = match declared_rows(request, catalog)? { + Some(rows) => rows, + None => RelationShapes { + types: shape_rows(&request.column_types)?, + nullability: shape_rows(&request.nullability)?, + complete: shape_rows(&request.complete_schemas)?, + schemas: shape_rows(&native_field(catalog, "schemas")?)?, + }, + }; + Ok(RequestRow { + dialect: profile_dialect(profile)?, + case_sensitive_shapes: case_sensitive_shapes( + profile, + &native_field(profile, "sql_analysis_dialect")?, + )?, + function_return_types: return_types(profile)?, + nullability_rules: Some(rules), + rich_type_inference: request.rich_type_inference, + column_types: shapes.types, + column_nullability: shapes.nullability, + complete_schemas: shapes.complete, + catalog_schemas: shapes.schemas, + dynamic_families_by_table: families, + models: model_rows(profile.py(), &request.model_inputs)?, + }) +} + +/// The column types, nullability, complete schemas and catalog schemas a session reads. +struct RelationShapes { + types: Shapes, + nullability: Shapes, + complete: Shapes, + schemas: Shapes, } /// The caller's native declaration rows when they still match the maps and catalog they mirror. fn declared_rows( request: &ModelAnalysisRequest<'_, '_>, catalog: &Bound<'_, PyAny>, -) -> PyResult> { +) -> PyResult> { let Some(declared) = request.declared else { return Ok(None); }; @@ -151,12 +164,12 @@ fn declared_rows( if !unchanged { return Ok(None); } - Ok(Some(( - declared.types.rows().clone(), - declared.nullability.rows().clone(), - complete.clone(), - complete.clone(), - ))) + Ok(Some(RelationShapes { + types: declared.types.rows().clone(), + nullability: declared.nullability.rows().clone(), + complete: complete.clone(), + schemas: complete.clone(), + })) } /// The analysis store's path and environment, or None where Python bypasses its cache. @@ -276,7 +289,13 @@ fn record_cache( python: &[Bound<'_, PyAny>], stats: Option, ) -> PyResult<()> { - let Some((hits, misses, _, failure)) = stats else { + let Some(CacheStatsRow { + hits, + misses, + failure, + .. + }) = stats + else { return record_metrics((0, 0, python.len())); }; if let Some(failure) = failure { @@ -294,7 +313,12 @@ fn record_cache( /// Record the analysis cache entries a native session read, missed and bypassed. fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { - record_analysis_cache_metrics((0, hits, misses, bypasses)); + record_analysis_cache_metrics(AnalysisCacheCounts { + batch_hits: 0, + entry_hits: hits, + misses, + bypasses, + }); Ok(()) } @@ -334,15 +358,15 @@ fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyRes Some(schema) => family_rows(&schema.dynamic_columns)?, None => Vec::new(), }; - rows.push(( - input_model_name(model_input)?, + rows.push(ModelRow { + name: input_model_name(model_input)?, query_sql, placeholders, - references, - recovers, + reference_rows: references, + recover_cte_facts: recovers, snapshot_columns, - families, - )); + dynamic_families: families, + }); } Ok(rows) } @@ -352,7 +376,7 @@ fn analysed_references(model_input: &Bound<'_, PyAny>) -> PyResult() { return Ok(input_reference_rows(model_input)? .into_iter() - .map(|(kind, name, _, _)| (kind, name)) + .map(|ReferenceRow { kind, name, .. }| (kind, name)) .collect()); } let mut references: Vec<(String, String)> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs index 5d597675aa..e195edbd4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs @@ -10,15 +10,36 @@ use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -/// `(cleaned SQL, dialect, input schemas, return types, nullability rules, recover, null filter)`. -type RecoveryRow = (String, String, Shapes, Pairs, Option, bool, bool); -/// `(deferral, types, nullability, sorted direct outputs, sorted non-null outputs)`. -type FactsRow = (Option, Pairs, Pairs, Vec, Vec); +positional_record! { + extract; + /// One query's CTE fact recovery request, as the oracle tests send it. + struct RecoveryRow { + cleaned_sql: String, + dialect: String, + input_schemas: Shapes, + function_return_types: Pairs, + nullability_rules: Option, + recover: bool, + null_filter: bool, + } +} + +positional_record! { + into; + /// The deferral, types, nullability, sorted direct outputs and sorted non-null outputs. + struct FactsRow { + deferral: Option, + types: Pairs, + nullability: Pairs, + direct_outputs: Vec, + non_null_outputs: Vec, + } +} /// The facts native enrichment recovers for one query, or the reason it defers to Python. #[pyfunction] fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult { - let ( + let RecoveryRow { cleaned_sql, dialect, input_schemas, @@ -26,7 +47,7 @@ fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult, request: RecoveryRow) -> PyResult ( - None, - facts.types, - facts.nullability, - facts.direct_outputs, - facts.non_null_outputs, - ), - Err(deferral) => ( - Some(deferral), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - ), + Ok(facts) => FactsRow { + deferral: None, + types: facts.types, + nullability: facts.nullability, + direct_outputs: facts.direct_outputs, + non_null_outputs: facts.non_null_outputs, + }, + Err(deferral) => FactsRow { + deferral: Some(deferral), + types: Vec::new(), + nullability: Vec::new(), + direct_outputs: Vec::new(), + non_null_outputs: Vec::new(), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index 02ae2be7b0..4143d02422 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -15,27 +15,45 @@ use crate::bindings::_helpers::records::ids::RecordId; const LINEAGE_NATIVE: &str = "native"; type Object<'py> = Bound<'py, PyAny>; -/// `(succeeded, columns, lineage source, lineage rows, star, resolved, bindings, validated, SQL)`. -type OutcomeObjects<'py> = ( - Object<'py>, - Object<'py>, - String, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, -); -/// `(output proven, fixed columns, families, input relations, failure reason, bare pivot)`. -type ProofObjects<'py> = ( - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, -); +positional_record! { + extract; + /// A session outcome row: its analysis, lineage, binding diagnostics and cleaned SQL. + struct OutcomeObjects<'py> { + succeeded: Object<'py>, + columns: Object<'py>, + source: String, + rows: Object<'py>, + has_star: Object<'py>, + star_resolved: Object<'py>, + raw: Object<'py>, + validated: Object<'py>, + cleaned: Object<'py>, + } +} + +positional_record! { + extract; + /// A dynamic pivot proof row. + struct ProofObjects<'py> { + output_proven: Object<'py>, + columns: Object<'py>, + families: Object<'py>, + inputs: Object<'py>, + failure_reason: Object<'py>, + bare: Object<'py>, + } +} + +positional_record! { + extract; + /// A lineage row: output column, transform and confidence codes and its sources. + struct LineageObjects<'py> { + output_column: Object<'py>, + transform_code: Object<'py>, + confidence_code: Object<'py>, + sources: Object<'py>, + } +} /// The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. #[pyfunction] @@ -49,7 +67,17 @@ pub(crate) fn session_outcome_analysis<'py>( .get("session")? .call_method1("outcome_rows", (request.get("position")?,))? .extract()?; - let (succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned): OutcomeObjects<'py> = outcome.extract()?; + let OutcomeObjects { + succeeded, + columns, + source, + rows, + has_star, + star_resolved, + raw, + validated, + cleaned, + } = outcome.extract()?; let lineage: Bound<'py, PyAny> = if source != LINEAGE_NATIVE { lineage_facts(&rows)? } else if succeeded.is_truthy()? { @@ -101,8 +129,14 @@ fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { if row.is_none() { return Ok(row.clone()); } - let (output_proven, columns, families, inputs, failure_reason, bare): ProofObjects<'py> = - row.extract()?; + let ProofObjects { + output_proven, + columns, + families, + inputs, + failure_reason, + bare, + } = row.extract()?; let family_class: Bound<'py, PyAny> = record_class(py, RecordId::DynamicColumnFamilyProof)?; let mut proofs: Vec> = Vec::new(); for family in families.try_iter()? { @@ -174,12 +208,12 @@ fn compact_lineage<'py>(rows: &Bound<'py, PyAny>) -> PyResult> }; let mut compact: Vec> = Vec::new(); for row in rows.try_iter()? { - let (output_column, transform_code, confidence_code, sources): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = row?.extract()?; + let LineageObjects { + output_column, + transform_code, + confidence_code, + sources, + } = row?.extract()?; let mut source_indexes: Vec> = Vec::new(); for source in sources.try_iter()? { let (resource_type, resource_name, column_name): ( @@ -225,12 +259,12 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { let source_class: Bound<'py, PyAny> = record_class(py, RecordId::CompiledLineageSourceFact)?; let mut facts: Vec> = Vec::new(); for row in rows.try_iter()? { - let (output_column, transform_code, confidence_code, sources): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = row?.extract()?; + let LineageObjects { + output_column, + transform_code, + confidence_code, + sources, + } = row?.extract()?; let mut upstream: Vec> = Vec::new(); for source in sources.try_iter()? { let (resource_type, resource_name, column_name): ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 7348587e6b..bdf1ddf33d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, + BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, NATIVE_DIALECT_ALIASES, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -135,12 +135,13 @@ pub(crate) fn profile_catalog<'py>( let py: Python<'py> = profile.py(); let catalog: BindingCatalog = BindingCatalog::create( py, - ( - dialect.to_owned(), - native_field(profile, "quoted_identifiers_ignore_case")?.extract()?, - Vec::new(), - Vec::new(), - ), + CatalogKey { + dialect: dialect.to_owned(), + quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: Vec::new(), + known_types: Vec::new(), + }, PyDict::new(py).as_any(), )?; Ok(Bound::new(py, catalog)?.into_any()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs index 13b2eecf79..e54f8b3274 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs @@ -23,17 +23,32 @@ use crate::bindings::types::CompilerDetach; const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; -/// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. -pub(crate) type QueryRow = ( - String, - Pairs, - Vec, - Vec<(String, String, String)>, - bool, - String, -); -/// `(dialect, return types, rules, types, nullability, queries, normalization dialect)`. -type RequestRow = (String, Pairs, Pairs, Shapes, Shapes, Vec, String); +positional_record! { + extract; + /// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. + pub(crate) struct QueryRow { + pub(crate) sql: String, + pub(crate) placeholders: Pairs, + pub(crate) analysis_names: Vec, + pub(crate) lineage_references: Vec<(String, String, String)>, + pub(crate) recover_cte_facts: bool, + pub(crate) mode: String, + } +} + +positional_record! { + extract; + /// `(dialect, return types, rules, types, nullability, queries, normalization dialect)`. + pub(crate) struct QueryRequestRow { + pub(crate) dialect: String, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Pairs, + pub(crate) types: Shapes, + pub(crate) nullability: Shapes, + pub(crate) queries: Vec, + pub(crate) normalization_dialect: String, + } +} /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); /// `(succeeded, [(name, type, nullability)] or None, has star)`. @@ -45,10 +60,10 @@ pub(crate) type ColumnsRow = (bool, Option, String)> pub(crate) fn infer_query_columns( py: Python<'_>, catalog: Option>, - request: RequestRow, + request: QueryRequestRow, adapter_rules: Option<(Py, Py)>, ) -> PyResult> { - let ( + let QueryRequestRow { dialect, function_return_types, nullability_rules, @@ -56,7 +71,7 @@ pub(crate) fn infer_query_columns( nullability, queries, normalization_dialect, - ) = request; + } = request; let queries: Vec = queries .into_iter() .map(query_input) @@ -99,7 +114,14 @@ pub(crate) fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { } fn query_input(row: QueryRow) -> Result { - let (sql, placeholders, analysis_names, lineage_references, recover_cte_facts, mode) = row; + let QueryRow { + sql, + placeholders, + analysis_names, + lineage_references, + recover_cte_facts, + mode, + } = row; let mode: QueryColumnsMode = match mode.as_str() { "batch" => QueryColumnsMode::Batch, "reanalysis" => QueryColumnsMode::Reanalysis, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index daffde72a8..30dd79b759 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -23,7 +23,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{ SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; -use sqlbuild_analysis::semantic_validation::types::DiagnosticRow; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlbuild_cache::store::models::NativeStore; @@ -38,77 +38,138 @@ use crate::bindings::_helpers::analysis_session::nullability_rules::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; +use crate::bindings::models::Positional; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; /// `(name, type, nullability)`. type ColumnRow = (String, Option, String); -/// `(output column, transform code, confidence code, [(resource type, resource, column)])`. -type LineageItem = (String, u8, u8, Vec<(String, String, String)>); -/// `(name, pivot column, value column, aggregate, type, name pattern)`. -pub(crate) type FamilyRow = (String, String, String, String, String, Option); +positional_record! { + into; + /// One lineage column: output column, transform and confidence codes and its sources. + struct LineageItem { + output_column: String, + transform_code: u8, + confidence_code: u8, + /// `(resource type, resource, column)` of each source. + sources: Vec<(String, String, String)>, + } +} + +positional_record! { + extract; + /// One declared dynamic column family. + pub(crate) struct FamilyRow { + pub(crate) name: String, + pub(crate) pivot_column: String, + pub(crate) value_column: String, + pub(crate) aggregate: String, + pub(crate) data_type: String, + pub(crate) name_pattern: Option, + } +} + /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -pub(crate) type ModelRow = ( - String, - String, - Pairs, - Vec<(String, String)>, - bool, - Option<(String, String)>, - Vec, -); +pub(crate) struct ModelRow { + pub(crate) name: String, + pub(crate) query_sql: String, + pub(crate) placeholders: Pairs, + pub(crate) reference_rows: Vec<(String, String)>, + pub(crate) recover_cte_facts: bool, + pub(crate) snapshot_columns: Option<(String, String)>, + pub(crate) dynamic_families: Vec, +} + /// The profile settings, relation shapes and models one session analyses. -pub(crate) type RequestRow = ( - String, - bool, - Pairs, - Option, - bool, - Shapes, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, -); +pub(crate) struct RequestRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) rich_type_inference: bool, + pub(crate) column_types: Shapes, + pub(crate) column_nullability: Shapes, + pub(crate) complete_schemas: Shapes, + pub(crate) catalog_schemas: Shapes, + pub(crate) dynamic_families_by_table: Vec<(String, Vec)>, + pub(crate) models: Vec, +} + /// `(publications, failures)`. pub(crate) type StepRow = (Shapes, Vec); -/// One model's analysis, lineage source and cleaned SQL. -type OutcomeRow = ( - bool, - Option>, - &'static str, - Vec, - bool, - bool, - Vec, - bool, - String, -); -/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. -type ProofRow = ( - bool, - Vec, - Vec<(String, Option)>, - Vec, - Option, - bool, -); + +positional_record! { + into; + /// One model's analysis, lineage source and cleaned SQL. + struct OutcomeRow { + analysis_succeeded: bool, + columns: Option>, + lineage_source: &'static str, + lineage: Vec, + has_star: bool, + star_resolved: bool, + binding_diagnostics: Vec>, + binding_validated: bool, + cleaned_sql: String, + } +} + +positional_record! { + into; + /// One dynamic pivot proof. + pub(crate) struct ProofRow { + output_proven: bool, + fixed_columns: Vec, + /// `(family, inferred type)` of each family. + families: Vec<(String, Option)>, + input_relations: Vec, + failure_reason: Option, + bare_dynamic_pivot: bool, + } +} + /// One model's pivot SQL and declared families. pub(crate) type PivotModelRow = (String, Vec); -/// The dialect, relation facts and families by table every proof reads, and the models. -type PivotRequestRow = ( - String, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, -); + +positional_record! { + extract; + /// The dialect, relation facts and families by table every proof reads, and the models. + pub(crate) struct PivotRequestRow { + pub(crate) dialect: String, + pub(crate) column_types: Shapes, + pub(crate) authoritative_types: Shapes, + pub(crate) column_nullability: Shapes, + pub(crate) families_by_table: Vec<(String, Vec)>, + pub(crate) models: Vec, + } +} + /// `("absent" | "deferred" | "proof", proof)`. pub(crate) type ContractRow = (&'static str, Option); -/// `(hits, misses, stored, why the store was not read or saved)`. -pub(crate) type CacheStatsRow = (usize, usize, usize, Option); + +positional_record! { + into; + /// The analysis cache's hits, misses, stored entries and why it was not read or saved. + #[derive(Clone)] + pub(crate) struct CacheStatsRow { + pub(crate) hits: usize, + pub(crate) misses: usize, + pub(crate) stored: usize, + pub(crate) failure: Option, + } +} + +positional_record! { + extract; + /// The settings and expressions one expression-source shape request infers. + pub(crate) struct ExpressionShapesRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) expressions: Vec, + } +} /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. pub(crate) type FinishRow = (Shapes, Vec, Vec); @@ -188,7 +249,12 @@ impl NativeModelAnalysisSession { let saved: Option = save_store(py, &store, path, &[]) .err() .map(|error| error.to_string()); - self.cache_stats = Some((stats.hits, stats.misses, stats.stored, saved)); + self.cache_stats = Some(CacheStatsRow { + hits: stats.hits, + misses: stats.misses, + stored: stats.stored, + failure: saved, + }); } let SessionOutcome { models, @@ -217,10 +283,10 @@ impl NativeModelAnalysisSession { } /// Each requested model's raw binding diagnostic rows, in request order. - pub(crate) fn binding_rows(&self) -> Vec> { + pub(crate) fn binding_rows(&self) -> Vec>> { self.outcomes .iter() - .map(|(outcome, _)| outcome.analysis.binding_diagnostics.clone()) + .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) .collect() } @@ -295,7 +361,14 @@ pub(crate) fn start_model_analysis_session( attach_analysis_cache(&mut session, store); cache_path = Some(path); } - Err(error) => cache_stats = Some((0, 0, 0, Some(error.to_string()))), + Err(error) => { + cache_stats = Some(CacheStatsRow { + hits: 0, + misses: 0, + stored: 0, + failure: Some(error.to_string()), + }); + } } } Ok(NativeModelAnalysisSession { @@ -315,11 +388,16 @@ pub(crate) fn start_model_analysis_session( pub(crate) fn infer_expression_source_shapes( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, - request: (String, bool, Pairs, Option, Vec), + request: ExpressionShapesRow, adapter_rules: Option<(Py, Py)>, ) -> PyResult>> { - let (dialect, case_sensitive_shapes, function_return_types, nullability_rules, expressions) = - request; + let ExpressionShapesRow { + dialect, + case_sensitive_shapes, + function_return_types, + nullability_rules, + expressions, + } = request; let rule_failure: RuleFailure = RuleFailure::default(); let request = ExpressionShapeRequest { dialect, @@ -347,7 +425,7 @@ pub(crate) fn infer_expression_source_shapes( } fn session_request(request: RequestRow) -> SessionRequest { - let ( + let RequestRow { dialect, case_sensitive_shapes, function_return_types, @@ -359,7 +437,7 @@ fn session_request(request: RequestRow) -> SessionRequest { catalog_schemas, dynamic_families_by_table, models, - ) = request; + } = request; SessionRequest { dialect, case_sensitive_shapes, @@ -379,7 +457,7 @@ fn session_request(request: RequestRow) -> SessionRequest { } fn model_request(row: ModelRow) -> ModelRequest { - let ( + let ModelRow { name, query_sql, placeholders, @@ -387,7 +465,7 @@ fn model_request(row: ModelRow) -> ModelRequest { recover_cte_facts, snapshot_columns, dynamic_families, - ) = row; + } = row; let has_set_operation = names_set_operation(&query_sql); let facts = model_reference_facts(&reference_rows); let pivot_sql = if placeholders.is_empty() { @@ -420,13 +498,11 @@ fn column_row(column: ColumnFact) -> ColumnRow { fn lineage_items(rows: Vec) -> Vec { rows.into_iter() - .map(|row| { - ( - row.output_column, - row.transform_code, - row.confidence_code, - row.sources, - ) + .map(|row| LineageItem { + output_column: row.output_column, + transform_code: row.transform_code, + confidence_code: row.confidence_code, + sources: row.sources, }) .collect() } @@ -438,7 +514,14 @@ fn families_by_table(rows: Vec<(String, Vec)>) -> Vec<(String, Vec DynamicFamily { - let (name, pivot_column, value_column, aggregate, data_type, name_pattern) = row; + let FamilyRow { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + } = row; DynamicFamily { name, pivot_column, @@ -454,14 +537,14 @@ fn contract_row(outcome: PivotOutcome) -> ContractRow { PivotOutcome::Absent => ("absent", None), PivotOutcome::Proof(proof) => ( "proof", - Some(( - proof.output_proven, - proof.fixed_columns.into_iter().map(column_row).collect(), - proof.families, - proof.input_relations, - proof.failure_reason, - proof.bare_dynamic_pivot, - )), + Some(ProofRow { + output_proven: proof.output_proven, + fixed_columns: proof.fixed_columns.into_iter().map(column_row).collect(), + families: proof.families, + input_relations: proof.input_relations, + failure_reason: proof.failure_reason, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + }), ), } } @@ -472,19 +555,19 @@ fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { LineageFacts::Native(rows) => ("native", lineage_items(rows)), LineageFacts::NativeFacts(rows) => ("facts", lineage_items(rows)), }; - ( - analysis.analysis_succeeded, - analysis + OutcomeRow { + analysis_succeeded: analysis.analysis_succeeded, + columns: analysis .columns .map(|columns| columns.into_iter().map(column_row).collect()), - source, + lineage_source: source, lineage, - analysis.has_star, - analysis.star_resolved, - analysis.binding_diagnostics, - analysis.binding_validated, + has_star: analysis.has_star, + star_resolved: analysis.star_resolved, + binding_diagnostics: Positional::wrap_all(analysis.binding_diagnostics), + binding_validated: analysis.binding_validated, cleaned_sql, - ) + } } /// Each model's dynamic pivot proof outside any session. @@ -493,8 +576,14 @@ pub(crate) fn prove_dynamic_column_contracts( py: Python<'_>, request: PivotRequestRow, ) -> PyResult> { - let (dialect, column_types, authoritative_types, column_nullability, by_table, models) = - request; + let PivotRequestRow { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: by_table, + models, + } = request; let request = PivotBatchRequest { tables: PivotTables { dialect, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 34104f8412..d4025bf278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -13,11 +13,11 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::query_columns::{ - ColumnsRow, QueryRow, infer_query_columns, + ColumnsRow, QueryRequestRow, QueryRow, infer_query_columns, }; use crate::bindings::_helpers::analysis_session::session::{ - ContractRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, - infer_expression_source_shapes, prove_dynamic_column_contracts, + ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, + PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::stage_work::{ @@ -154,13 +154,13 @@ pub(crate) fn expression_source_shapes<'py>( let shapes: Vec> = infer_expression_source_shapes( py, native, - ( + ExpressionShapesRow { dialect, - case_sensitive, - return_types(&profile)?, - Some(rules), - pending_names, - ), + case_sensitive_shapes: case_sensitive, + function_return_types: return_types(&profile)?, + nullability_rules: Some(rules), + expressions: pending_names, + }, adapter_rules, )?; for (expression, shape) in pending.keys().iter().zip(shapes) { @@ -223,17 +223,20 @@ pub(crate) fn dynamic_contract_proofs<'py>( } prove_dynamic_column_contracts( py, - ( - dialect_or_generic(&native_field(tables, "dialect")?)?, - shape_rows(&native_field(tables, "column_types_by_table")?)?, - shape_rows(&native_field( + PivotRequestRow { + dialect: dialect_or_generic(&native_field(tables, "dialect")?)?, + column_types: shape_rows(&native_field(tables, "column_types_by_table")?)?, + authoritative_types: shape_rows(&native_field( tables, "authoritative_column_types_by_table", )?)?, - shape_rows(&native_field(tables, "column_nullability_by_table")?)?, - families, - rows, - ), + column_nullability: shape_rows(&native_field( + tables, + "column_nullability_by_table", + )?)?, + families_by_table: families, + models: rows, + }, )? } else { let session: PyRef<'py, NativeModelAnalysisSession> = session.extract()?; @@ -251,14 +254,14 @@ pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult = Vec::new(); for family in families.try_iter()? { let family: Bound<'_, PyAny> = family?; - rows.push(( - native_field(&family, "name")?.extract()?, - native_field(&family, "pivot_column")?.extract()?, - native_field(&family, "value_column")?.extract()?, - native_field(&family, "aggregate")?.extract()?, - native_field(&family, "type")?.extract()?, - native_field(&family, "name_pattern")?.extract()?, - )); + rows.push(FamilyRow { + name: native_field(&family, "name")?.extract()?, + pivot_column: native_field(&family, "pivot_column")?.extract()?, + value_column: native_field(&family, "value_column")?.extract()?, + aggregate: native_field(&family, "aggregate")?.extract()?, + data_type: native_field(&family, "type")?.extract()?, + name_pattern: native_field(&family, "name_pattern")?.extract()?, + }); } Ok(rows) } @@ -310,15 +313,15 @@ pub(crate) fn query_column_rows( infer_query_columns( py, catalog, - ( + QueryRequestRow { dialect, - return_types(profile)?, - rules, - shape_rows(types)?, - shape_rows(nullability)?, - rows, + function_return_types: return_types(profile)?, + nullability_rules: rules, + types: shape_rows(types)?, + nullability: shape_rows(nullability)?, + queries: rows, normalization_dialect, - ), + }, adapter_rules, ) } @@ -359,14 +362,14 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { } } } - Ok(( - native_field(query, "sql")?.extract()?, + Ok(QueryRow { + sql: native_field(query, "sql")?.extract()?, placeholders, analysis_names, - lineage, - native_field(query, "recover_cte_facts")?.extract()?, - native_field(query, "mode")?.extract()?, - )) + lineage_references: lineage, + recover_cte_facts: native_field(query, "recover_cte_facts")?.extract()?, + mode: native_field(query, "mode")?.extract()?, + }) } trait OrderedDict { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 56be30d4df..ae146797c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,6 +1,10 @@ +#[macro_use] +mod positional; + mod allocator; pub(crate) mod config_values; pub(crate) mod expected_errors; pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; +mod positional_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs new file mode 100644 index 0000000000..0485f9623e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs @@ -0,0 +1,114 @@ +//! Macros declaring named records that cross the Python boundary as positional tuples. + +/// Implement the positional traits for another crate's struct, its fields in tuple order. +macro_rules! positional_fields { + ($($mode:ident),+; $name:ident $body:tt) => { + $( + positional_fields!( + @$mode $name $body + ); + )+ + }; + (@extract $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => { + impl $crate::bindings::types::FromPositional for $name { + fn from_positional(value: &pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok($name { $($field),+ }) + } + } + }; + (@into $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => { + impl $crate::bindings::types::IntoPositional for $name { + fn into_positional<'py>( + self, + py: pyo3::Python<'py>, + ) -> pyo3::PyResult> { + let $name { $($field),+ } = self; + pyo3::IntoPyObject::into_pyobject(($($field,)+), py) + } + } + }; +} + +/// Declare a struct naming the positions of a tuple Python sends (`extract`) or receives (`into`). +macro_rules! positional_record { + ( + $($mode:ident),+; + $(#[$meta:meta])* + $vis:vis struct $name:ident <$lifetime:lifetime> $body:tt + ) => { + $(#[$meta])* + $vis struct $name<$lifetime> $body + + $( + positional_record!( + @$mode $name <$lifetime> $body + ); + )+ + }; + ( + $($mode:ident),+; + $(#[$meta:meta])* + $vis:vis struct $name:ident $body:tt + ) => { + $(#[$meta])* + $vis struct $name $body + + $( + positional_record!( + @$mode $name $body + ); + )+ + }; + (@extract $name:ident { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<'py> pyo3::FromPyObject<'py> for $name { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { $($field),+ }) + } + } + }; + (@extract $name:ident <$lifetime:lifetime> { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<$lifetime> pyo3::FromPyObject<$lifetime> for $name<$lifetime> { + fn extract_bound( + value: &pyo3::Bound<$lifetime, pyo3::PyAny>, + ) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { $($field),+ }) + } + } + }; + (@into $name:ident { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<'py> pyo3::IntoPyObject<'py> for $name { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject(($(self.$field,)+), py) + } + } + }; + (@into $name:ident <$lifetime:lifetime> { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<$lifetime> pyo3::IntoPyObject<$lifetime> for $name<$lifetime> { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<$lifetime, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject( + self, + py: pyo3::Python<$lifetime>, + ) -> Result { + pyo3::IntoPyObject::into_pyobject(($(self.$field,)+), py) + } + } + }; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs new file mode 100644 index 0000000000..3162c55985 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs @@ -0,0 +1,50 @@ +//! The tuple positions of records from other crates that Python exchanges as tuples. + +use sqlbuild_analysis::binding_locations::models::AuthoredSpan; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; +use sqlbuild_rules::lint_text::models::InterpolationSite; + +positional_fields! { + extract, into; + InterpolationSite { + sentinel: String, + neutralized_start: usize, + neutralized_end: usize, + original_start: usize, + original_end: usize, + original_text: String, + } +} + +positional_fields! { + extract, into; + DiagnosticRow { + code: String, + message: String, + line: Option, + column: Option, + start: Option, + end: Option, + severity: String, + } +} + +positional_fields! { + extract; + Expansion { + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, + } +} + +positional_fields! { + into; + AuthoredSpan { + line: usize, + column: usize, + end_line: Option, + end_column: Option, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index e14811cbdd..92558b69c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -17,13 +17,13 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; /// The project directory, discovered inputs, adapter context, colour and progress stream. -type InvocationContext<'py> = ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - bool, - Bound<'py, PyAny>, -); +struct InvocationContext<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + adapter_context: Bound<'py, PyAny>, + use_color: bool, + stream: Bound<'py, PyAny>, +} fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { PyBool::new(py, value).to_owned().into_any() @@ -97,13 +97,25 @@ fn invocation_context<'py>( let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if machine_output { "stderr" } else { "stdout" })?; - Ok((project_dir, discovered, adapter_context, use_color, stream)) + Ok(InvocationContext { + project_dir, + discovered, + adapter_context, + use_color, + stream, + }) } fn context_fields<'py>( context: &InvocationContext<'py>, ) -> PyResult)>> { - let (project_dir, discovered, adapter_context, use_color, stream) = context; + let InvocationContext { + project_dir, + discovered, + adapter_context, + use_color, + stream, + } = context; let py: Python<'py> = project_dir.py(); Ok(vec![ ("effective_project_dir", project_dir.clone()), @@ -139,7 +151,12 @@ pub(crate) fn resolve_reported_command_invocation<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let context: InvocationContext<'py> = invocation_context(request, require_connection)?; - let (_, _, adapter_context, use_color, stream) = &context; + let InvocationContext { + adapter_context, + use_color, + stream, + .. + } = &context; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = context_fields(&context)?; fields.push(( "connection_progress", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index 7b662cb091..68a6c51b19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -5,6 +5,7 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -86,7 +87,7 @@ pub(crate) fn resolve_cursor_input_roles<'py>( Err(_) => { let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; if cursor.is_instance_of::() { - for (_, name, _, _) in input_reference_rows(model)? { + for ReferenceRow { name, .. } in input_reference_rows(model)? { filter.set_item(name, &cursor)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 7b371a1eb9..ee385e3913 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; +use crate::bindings::_helpers::compiled_project::model_loop::{LoopSwitches, NativeModelLoop}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; @@ -275,12 +275,13 @@ pub(crate) fn build_model_inputs<'py>( native_field(&syntax, "native_mapping")?.extract()?, reserved_cursor_markers(), ( - ( - native_field(&settings, "sql_analysis")?.extract()?, + LoopSwitches { + sql_analysis: native_field(&settings, "sql_analysis")?.extract()?, no_sql_validation, - defer_model_sql_validation, - native_field(&settings, "require_sql_analysis")?.extract()?, - ), + defer_sql_validation: defer_model_sql_validation, + require_sql_analysis: native_field(&settings, "require_sql_analysis")? + .extract()?, + }, project_config_path.str()?.extract()?, ), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index dd32ec6f5e..6a44449683 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -252,13 +252,40 @@ fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { .map(|(_, bytes)| bytes.as_slice()) } -/// The diagnostics and report lines an image keeps as JSON, being small and varied. -type StoredMisc = ( - Vec, - Option>, - Vec, - Option>, -); +/// The diagnostics and report lines an image keeps as one JSON array, being small and varied. +struct StoredMisc { + diagnostics: Vec, + text_models: Option>, + environment_names: Vec, + coverage: Option>, +} + +impl<'de> serde::Deserialize<'de> for StoredMisc { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_tuple(4, StoredMiscVisitor) + } +} + +/// Reads [`StoredMisc`] from its four-element array. +struct StoredMiscVisitor; + +impl<'de> serde::de::Visitor<'de> for StoredMiscVisitor { + type Value = StoredMisc; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a tuple of size 4") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let missing = |index: usize| serde::de::Error::invalid_length(index, &self); + Ok(StoredMisc { + diagnostics: seq.next_element()?.ok_or_else(|| missing(0))?, + text_models: seq.next_element()?.ok_or_else(|| missing(1))?, + environment_names: seq.next_element()?.ok_or_else(|| missing(2))?, + coverage: seq.next_element()?.ok_or_else(|| missing(3))?, + }) + } +} fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { encoder.count(shapes.len()); @@ -458,8 +485,12 @@ impl StoredFacts { .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? .into_iter() .collect(); - let (diagnostics, text_models, environment_names, coverage): StoredMisc = - from_json(decoder.bytes()?)?; + let StoredMisc { + diagnostics, + text_models, + environment_names, + coverage, + }: StoredMisc = from_json(decoder.bytes()?)?; decoder.finish()?; Some(Self { usages, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs index 3e92952ae7..73dc556a63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -120,9 +120,22 @@ fn build_analysis_cache_context<'py>( ) } +/// Semantic analysis cache outcome counts for one invocation. +pub(crate) struct AnalysisCacheCounts { + pub(crate) batch_hits: usize, + pub(crate) entry_hits: usize, + pub(crate) misses: usize, + pub(crate) bypasses: usize, +} + /// Record semantic analysis cache outcomes for one invocation. pub(crate) fn record_analysis_cache_metrics( - (batch_hits, entry_hits, misses, bypasses): (usize, usize, usize, usize), + AnalysisCacheCounts { + batch_hits, + entry_hits, + misses, + bypasses, + }: AnalysisCacheCounts, ) { for (metric, value) in [ ("analysis_batch_cache_hits", batch_hits), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 60ec1d1a43..0f447a7cf2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -7,7 +7,9 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; +use crate::bindings::_helpers::analysis::binding_catalog::{ + BindingCatalog, CatalogKey, ignores_quoted_case, +}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; @@ -93,8 +95,12 @@ pub(crate) fn analysis_context<'py>( &native_field(inputs, "effective_settings")?.getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let (column_types, nullability, complete_schemas, declared): DeclaredRelations<'py> = - declared_relations(inputs.clone())?; + let DeclaredRelations { + column_types, + nullability, + complete_schemas, + declared, + }: DeclaredRelations<'py> = declared_relations(inputs.clone())?; let (column_types, nullability, complete_schemas): ( Bound<'py, PyAny>, Bound<'py, PyAny>, @@ -218,12 +224,13 @@ fn profile_with_catalog<'py>( py, BindingCatalog::create( py, - ( + CatalogKey { dialect, - native_field(&profile, "quoted_identifiers_ignore_case")?.extract()?, - native_field(&profile, "semantic_known_functions")?.extract()?, - native_field(&profile, "semantic_known_types")?.extract()?, - ), + quoted_ignore_case: native_field(&profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(&profile, "semantic_known_functions")?.extract()?, + known_types: native_field(&profile, "semantic_known_types")?.extract()?, + }, PyDict::new(py).as_any(), )?, )? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 02f990d47b..5caef70872 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -270,7 +270,7 @@ pub(crate) fn binding_rows<'py>( .flatten() { let position: usize = position.extract()?; - rows_by_name.set_item(name, rows.get(position).into_pyobject(py)?)?; + rows_by_name.set_item(name, rows.get(position).cloned().into_pyobject(py)?)?; } Ok(rows_by_name) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 2c61cac5ed..6563157e15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -24,7 +24,17 @@ use sqlbuild_cache::project_snapshot::models::SnapshotRules; use crate::bindings::_helpers::boundary::panics::compiler_guard; -type SettingsRow = (bool, Vec, Vec, Vec, Vec); +positional_record! { + extract; + /// Settings case sensitivity, names, prefixes, env files and secrets directories. + struct SettingsRow { + case_sensitive: bool, + names: Vec, + prefixes: Vec, + env_files: Vec, + secrets_dirs: Vec, + } +} type ReplayRow = (String, Vec, i64); type LaunchIdentityRow = (String, BTreeMap, Vec); /// A written artifact's digest, or a kept artifact's size and mtime. @@ -254,7 +264,13 @@ impl NativeReuseAttempt { } fn settings_inputs(row: SettingsRow) -> SettingsInputs { - let (case_sensitive, names, prefixes, env_files, secrets_dirs) = row; + let SettingsRow { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + } = row; SettingsInputs { case_sensitive, names, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 1b899e2c31..e733e70446 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -28,7 +28,13 @@ const PYTHON_CHECK_KIND: &str = "python_check"; const JSON_INDENT: i64 = 2; /// One test's identity as the test runtime target matches entries to results. -type TestKey = (String, Option, Option, Option); +#[derive(Hash, PartialEq, Eq)] +struct TestKey { + name: String, + source_path: Option, + block_index: Option, + case_name: Option, +} fn posix(path: &Bound<'_, PyAny>) -> PyResult> { if path.is_none() { @@ -42,16 +48,21 @@ fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> } fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { - Ok(( - native_field(value, name_field)?.extract()?, - posix(&native_field(value, "source_path")?)?, - native_field(value, "block_index")?.extract()?, - native_field(value, "case_name")?.extract()?, - )) + Ok(TestKey { + name: native_field(value, name_field)?.extract()?, + source_path: posix(&native_field(value, "source_path")?)?, + block_index: native_field(value, "block_index")?.extract()?, + case_name: native_field(value, "case_name")?.extract()?, + }) } fn path_facts(test: &Bound<'_, PyAny>, model_names: Vec) -> PyResult { - let (name, source_path, block_index, case_name) = test_key(test, "name")?; + let TestKey { + name, + source_path, + block_index, + case_name, + } = test_key(test, "name")?; Ok(SqlTestPathFacts { name, source_path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 5adc9d2492..ea17c67eb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::audit_loop::{AuditBlockRow, AuditFileRow}; -use crate::bindings::_helpers::compiled_project::function_loop::FunctionRow; +use crate::bindings::_helpers::compiled_project::function_loop::{FunctionRow, FunctionSettings}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::resource_diagnostics::{ DiagnosticSettings, LoaderRow, resource_diagnostics, @@ -92,19 +92,19 @@ fn function_rows( items(discovered, field)? .into_iter() .map(|file| { - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "file_path")?, - file.clone().unbind(), - attr(&file, "header_values")?, - attr(&file, body)?, - if python { + Ok(FunctionRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + name: stem(&file, "file_path")?, + function_file: file.clone().unbind(), + header_values: attr(&file, "header_values")?, + body: attr(&file, body)?, + contents: if python { None } else { Some(attr(&file, "contents")?) }, - )) + }) }) .collect() } @@ -187,17 +187,17 @@ pub(crate) fn attach_resources<'py>( true, )?, ), - ( + FunctionSettings { adapter_name, validate, - ( + defaults: ( either(&defaults, ("function_database", "database"))?, either(&defaults, ("function_schema", "schema"))?, ), - target_namespace(&target_config)?, - python_functions_inherit_default_namespace, - function_kind.unbind(), - ), + target: target_namespace(&target_config)?, + inherit: python_functions_inherit_default_namespace, + kind: function_kind.unbind(), + }, )?; let mut source_rows: Vec = Vec::new(); for source_file in items(discovered_inputs, "source_files")? { @@ -241,46 +241,46 @@ fn test_case_row(case: Bound<'_, PyAny>) -> PyResult { } fn test_block_row(block: Bound<'_, PyAny>) -> PyResult { - Ok(( - block.clone().unbind(), - attr(&block, "name")?, - attr(&block, "sql_body")?, - attr(&native_field(&block, "mode")?, "value")?, - items(&block, "parameters")? + Ok(TestBlockRow { + block: block.clone().unbind(), + name: attr(&block, "name")?, + sql_body: attr(&block, "sql_body")?, + mode: attr(&native_field(&block, "mode")?, "value")?, + parameters: items(&block, "parameters")? .iter() .map(|parameter| attr(parameter, "name")) .collect::>()?, - items(&block, "cases")? + cases: items(&block, "cases")? .into_iter() .map(test_case_row) .collect::>()?, - attr(&block, "sql_body_span")?, - )) + sql_body_span: attr(&block, "sql_body_span")?, + }) } fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "relative_path")?, - file.clone().unbind(), - attr(&file, "contents")?, - items(&file, "blocks")? + Ok(TestFileRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + stem: stem(&file, "relative_path")?, + file: file.clone().unbind(), + contents: attr(&file, "contents")?, + blocks: items(&file, "blocks")? .into_iter() .map(test_block_row) .collect::>()?, - )) + }) } fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { - Ok(( - text(&scenario, "file_path")?, - text(&scenario, "relative_path")?, - scenario.clone().unbind(), - attr(&scenario, "contents")?, - attr(&scenario, "sql_body")?, - attr(&scenario, "name")?, - )) + Ok(ScenarioRow { + file_path: text(&scenario, "file_path")?, + relative_path: text(&scenario, "relative_path")?, + scenario_file: scenario.clone().unbind(), + contents: attr(&scenario, "contents")?, + sql_body: attr(&scenario, "sql_body")?, + name: attr(&scenario, "name")?, + }) } /// The adapter's typed scalar renderer as `(text, None)`, or `(None, error)` for a bad value. @@ -378,31 +378,31 @@ fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = block.py(); let measurement: Bound<'_, PyAny> = host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?; - Ok(( - block.clone().unbind(), - attr(&block, "name")?, - attr(&block, "sql_body")?, - attr(&block, "evidence_sql")?, - native_field(&block, "evaluation_mode")?.eq(measurement)?, - attr(&block, "header_values")?, - )) + Ok(AuditBlockRow { + block: block.clone().unbind(), + name: attr(&block, "name")?, + sql_body: attr(&block, "sql_body")?, + evidence_sql: attr(&block, "evidence_sql")?, + measurement: native_field(&block, "evaluation_mode")?.eq(measurement)?, + header: attr(&block, "header_values")?, + }) } fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = file.py(); let generic: Bound<'_, PyAny> = enum_member(py, RecordId::DeclarationKind, "AUDIT")?; - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "file_path")?, - file.clone().unbind(), - native_field(&file, "declaration_kind")?.is(&generic), - attr(&file, "contents")?, - items(&file, "blocks")? + Ok(AuditFileRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + stem: stem(&file, "file_path")?, + file: file.clone().unbind(), + generic: native_field(&file, "declaration_kind")?.is(&generic), + contents: attr(&file, "contents")?, + blocks: items(&file, "blocks")? .into_iter() .map(audit_block_row) .collect::>()?, - )) + }) } /// The discovered file of one built-in generic audit, as project discovery would describe it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index f748dffb8e..22a82e9e5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -61,26 +61,32 @@ const MODEL_TARGET: &str = "model"; const SOURCE_TARGET: &str = "source"; const SEED_TARGET: &str = "seed"; -/// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. -pub(crate) type AuditBlockRow = ( - Py, - Option, - String, - Option, - bool, - Py, -); - -/// An audit file's paths, stem, discovered file, generic flag, contents and blocks. -pub(crate) type AuditFileRow = ( - String, - String, - String, - Py, - bool, - String, - Vec, -); +positional_record! { + extract, into; + /// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. + pub(crate) struct AuditBlockRow { + pub(crate) block: Py, + pub(crate) name: Option, + pub(crate) sql_body: String, + pub(crate) evidence_sql: Option, + pub(crate) measurement: bool, + pub(crate) header: Py, + } +} + +positional_record! { + extract, into; + /// An audit file's paths, stem, discovered file, generic flag, contents and blocks. + pub(crate) struct AuditFileRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) stem: String, + pub(crate) file: Py, + pub(crate) generic: bool, + pub(crate) contents: String, + pub(crate) blocks: Vec, + } +} /// What every audit in one compile resolves against, and the Python values it builds with. #[derive(FromPyObject)] @@ -132,8 +138,8 @@ pub(crate) fn attached_audits<'py>( settings, }; let mut audits: Vec = Vec::new(); - for file in files.iter().filter(|file| !file.4) { - for block in &file.6 { + for file in files.iter().filter(|file| !file.generic) { + for block in &file.blocks { audits.push(scope.singular(file, block)?); } } @@ -158,41 +164,41 @@ fn generic_definitions<'a, 'py>( ) -> PyResult> { let py: Python<'py> = context.host.py(); let mut project: BTreeMap> = BTreeMap::new(); - for file in files.iter().filter(|file| file.4) { - let [block] = file.6.as_slice() else { + for file in files.iter().filter(|file| file.generic) { + let [block] = file.blocks.as_slice() else { let message: String = format!( "Generic audit definition {} must contain exactly one AUDIT block", - file.1 + file.relative_path ); return Err(compile_error(context.classes, &message, None)); }; - let header: &Bound<'_, PyDict> = block.5.bind(py); - if block.4 + let header: &Bound<'_, PyDict> = block.header.bind(py); + if block.measurement && (header.contains(MEASUREMENT_HEADER_KEYS[0])? || header.contains(MEASUREMENT_HEADER_KEYS[1])?) { let message: String = format!( "Generic measurement audit definition {} must not define thresholds or \ minimum_samples in its AUDIT header; attachment owns policy", - file.1 + file.relative_path ); return Err(compile_error(context.classes, &message, None)); } - if let Some(existing) = project.get(&file.2) { + if let Some(existing) = project.get(&file.stem) { let message: String = format!( "Duplicate generic audit definition found for '{}' in {} and {}", - file.2, existing.file.1, file.1 + file.stem, existing.file.relative_path, file.relative_path ); return Err(compile_error(context.classes, &message, None)); } - project.insert(file.2.clone(), Definition { file, block }); + project.insert(file.stem.clone(), Definition { file, block }); } let mut shadows: Vec> = Vec::new(); let mut definitions: HashMap> = HashMap::new(); for builtin in builtins { - if let Some(block) = builtin.6.first() { + if let Some(block) = builtin.blocks.first() { definitions.insert( - builtin.2.clone(), + builtin.stem.clone(), Definition { file: builtin, block, @@ -202,7 +208,11 @@ fn generic_definitions<'a, 'py>( } for (name, definition) in project { if definitions.contains_key(&name) { - shadows.push(shadow_warning(context, &name, &definition.file.1)?); + shadows.push(shadow_warning( + context, + &name, + &definition.file.relative_path, + )?); } definitions.insert(name, definition); } @@ -251,8 +261,20 @@ impl<'py> AuditScope<'_, 'py> { /// One singular audit block, as `_build_singular_audit_inputs` builds it. fn singular(&self, file: &AuditFileRow, block: &AuditBlockRow) -> PyResult { let py: Python<'py> = self.context.host.py(); - let (_, relative_path, stem, audit_file, _, _, _) = file; - let (audit_block, block_name, sql_body, evidence_sql, measurement, header) = block; + let AuditFileRow { + relative_path, + stem, + file: audit_file, + .. + } = file; + let AuditBlockRow { + block: audit_block, + name: block_name, + sql_body, + evidence_sql, + measurement, + header, + } = block; let audit_name: String = block_name.clone().unwrap_or_else(|| stem.clone()); let identity: Bound<'py, PyAny> = self .settings @@ -394,7 +416,7 @@ impl<'py> AuditScope<'_, 'py> { let minimum_samples: Bound<'py, PyAny> = native_field(instance, "minimum_samples")?; let instance_severity: Option = native_field(instance, "severity")?.extract()?; let label: String = format!("{owner_file} audit '{definition_name}'"); - let measurement: bool = definition.block.4; + let measurement: bool = definition.block.measurement; if measurement { if instance_severity.is_some() { let message: String = format!( @@ -425,8 +447,8 @@ impl<'py> AuditScope<'_, 'py> { let rendering: AuditRendering = render_attached_audit(&AuditAttachment { owner_label: owner_file.to_owned(), definition_name: definition_name.clone(), - sql_body: definition.block.2.clone(), - evidence_sql: definition.block.3.clone(), + sql_body: definition.block.sql_body.clone(), + evidence_sql: definition.block.evidence_sql.clone(), implicit_arguments: implicit, explicit_arguments: argument_pairs(native_field(instance, "arguments")?.downcast()?)?, has_thresholds: !thresholds.is_none(), @@ -455,7 +477,7 @@ impl<'py> AuditScope<'_, 'py> { &format!("Audit '{definition_name}'"), )?; let calls: Vec<(&str, &str)> = reference_calls(&expanded.references); - self.checked_references(&definition.file.1, &calls)?; + self.checked_references(&definition.file.relative_path, &calls)?; let (severity, source) = rendered .policies .map_err(|message| compile_error(self.context.classes, &message, None))?; @@ -471,13 +493,13 @@ impl<'py> AuditScope<'_, 'py> { } usages.extend(expanded.usages); Ok(AttachedAudit { - audit_file: definition.file.3.clone_ref(py), - audit_block: definition.block.0.clone_ref(py), + audit_file: definition.file.file.clone_ref(py), + audit_block: definition.block.block.clone_ref(py), definition_name: definition .block - .1 + .name .clone() - .unwrap_or_else(|| definition.file.2.clone()), + .unwrap_or_else(|| definition.file.stem.clone()), sql_body: expanded.sql_body, measurement, evidence_sql: expanded.evidence_sql, @@ -508,7 +530,12 @@ impl<'py> AuditScope<'_, 'py> { identity: &Bound<'py, PyAny>, label: &str, ) -> PyResult> { - let (file_path, relative_path, _, _, _, contents, _) = file; + let AuditFileRow { + file_path, + relative_path, + contents, + .. + } = file; let declarations: Bound<'py, PyAny> = file_declarations(self.context, file_path, identity)?; let body: AuthoredExpansion<'py> = expanded_authored_sql(self.context, (sql.0, file_path), &declarations)?; @@ -682,7 +709,7 @@ impl<'py> AuditScope<'_, 'py> { fn reference_calls(references: &[ReferenceRow]) -> Vec<(&str, &str)> { references .iter() - .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .map(|ReferenceRow { kind, name, .. }| (kind.as_str(), name.as_str())) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs index 98af5da878..7ddaf506dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -331,7 +331,14 @@ audit_getters!(NativeAuditInput, { attached .references .iter() - .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind.clone(), name.clone(), package.clone()), + ) .collect(), attached .target diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 8871aa9d1d..f42801729b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{ }; use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -30,15 +30,18 @@ use crate::bindings::_helpers::record_behaviours::lineage_facts::{ const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; -/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. -pub(crate) type ProofArgument = ( - bool, - Vec, - Vec<(String, Option)>, - Vec, - Option, - bool, -); +positional_record! { + extract; + /// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. + pub(crate) struct ProofArgument { + output_proven: bool, + fixed: Vec, + families: Vec<(String, Option)>, + input_relations: Vec, + failure_reason: Option, + bare_dynamic_pivot: bool, + } +} /// The session analysis a model was assembled from: the session and its request position. pub(crate) type SessionArgument = (Py, usize); @@ -81,8 +84,14 @@ fn inferred_row(column: &ColumnFact) -> InferredRow { } fn proof_fact(argument: ProofArgument) -> ContractProof { - let (output_proven, fixed, families, input_relations, failure_reason, bare_dynamic_pivot) = - argument; + let ProofArgument { + output_proven, + fixed, + families, + input_relations, + failure_reason, + bare_dynamic_pivot, + } = argument; ContractProof { output_proven, fixed_columns: fixed @@ -157,8 +166,18 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> = vec![ - expansion.declaration_spans.clone(), - expansion.macro_spans.clone(), + expansion + .declaration_spans + .iter() + .copied() + .map(Expansion::from) + .collect(), + expansion + .macro_spans + .iter() + .copied() + .map(Expansion::from) + .collect(), ]; let source: BindingSource<'_> = BindingSource { authored, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index cd7cf04bb6..fc157b5c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -161,6 +161,14 @@ impl Default for ViewProjections { } } +/// A model's name, compiled query, authored file, and whether it calls a function. +pub(crate) struct ModelMetadataTexts { + pub(crate) name: String, + pub(crate) query_sql: String, + pub(crate) authored_sql: String, + pub(crate) calls_functions: bool, +} + /// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. #[derive(FromPyObject)] pub(crate) struct CompiledArguments( @@ -444,17 +452,17 @@ impl NativeCompiledModel { &self, py: Python<'_>, function_kinds: &[&str], - ) -> PyResult<(String, String, String, bool)> { + ) -> PyResult { self.read(py, |record, _| { - Ok(( - model_name(record), - record.query_sql.to_string(), - record.model_file.get().contents_text().to_string(), - record + Ok(ModelMetadataTexts { + name: model_name(record), + query_sql: record.query_sql.to_string(), + authored_sql: record.model_file.get().contents_text().to_string(), + calls_functions: record .references .iter() - .any(|(kind, _, _, _)| function_kinds.contains(&kind.as_str())), - )) + .any(|reference| function_kinds.contains(&reference.kind.as_str())), + }) }) } @@ -904,12 +912,12 @@ pub(crate) fn model_graph_row( } } let folder: String = record.model_file.get().folder(py)?; - Ok(( - (MODEL_RESOURCE.to_owned(), model_name(record)), - compiled.deps.clone(), + Ok(ResourceRow { + key: (MODEL_RESOURCE.to_owned(), model_name(record)), + deps: compiled.deps.clone(), tags, - Some(folder), - )) + folder: Some(folder), + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs index ea653300d5..31748115b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs @@ -38,7 +38,9 @@ use crate::bindings::_helpers::compiled_project::function_views::{ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, compile_error, }; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; use crate::bindings::_helpers::compiled_project::source_loop::{ @@ -55,26 +57,32 @@ const SCHEMA_KEY: &str = "schema"; /// The reference kind of a source. const SOURCE_KIND: &str = "source"; -/// A function's paths, stem, discovered file, header values, body and SQL file contents. -pub(crate) type FunctionRow = ( - String, - String, - String, - Py, - Py, - String, - Option, -); +positional_record! { + extract; + /// A function's paths, stem, discovered file, header values, body and SQL file contents. + pub(crate) struct FunctionRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) name: String, + pub(crate) function_file: Py, + pub(crate) header_values: Py, + pub(crate) body: String, + pub(crate) contents: Option, + } +} -/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. -pub(crate) type FunctionSettings = ( - String, - bool, - (Option, Option), - Option<(Option, Option)>, - bool, - Py, -); +positional_record! { + extract; + /// Adapter, validation, raw default and target namespaces, inheritance and resource kind. + pub(crate) struct FunctionSettings { + pub(crate) adapter_name: String, + pub(crate) validate: bool, + pub(crate) defaults: (Option, Option), + pub(crate) target: Option<(Option, Option)>, + pub(crate) inherit: bool, + pub(crate) kind: Py, + } +} /// What every function in one compile resolves against. struct FunctionScope<'a, 'py> { @@ -97,8 +105,14 @@ pub(crate) fn attached_functions<'py>( settings: FunctionSettings, ) -> PyResult> { let py: Python<'py> = context.host.py(); - let (adapter_name, validate, (default_database, default_schema), target, inherit, kind) = - settings; + let FunctionSettings { + adapter_name, + validate, + defaults: (default_database, default_schema), + target, + inherit, + kind, + } = settings; let defaults = ( optional_template(context, default_database, "default function database")?, optional_template(context, default_schema, "default function schema")?, @@ -130,15 +144,15 @@ pub(crate) fn attached_functions<'py>( let mut attached: Vec = Vec::with_capacity(sql_rows.len() + python_rows.len()); for row in sql_rows { - if !names.insert(row.2.clone()) { - let message: String = format!("Duplicate SQL function name '{}'", row.2); + if !names.insert(row.name.clone()) { + let message: String = format!("Duplicate SQL function name '{}'", row.name); return Err(compile_error(context.classes, &message, None)); } attached.push(sql_function(&scope, row)?); } for row in python_rows { - if !names.insert(row.2.clone()) { - let message: String = format!("Duplicate function name '{}'", row.2); + if !names.insert(row.name.clone()) { + let message: String = format!("Duplicate function name '{}'", row.name); return Err(compile_error(context.classes, &message, None)); } attached.push(python_function(&scope, row)?); @@ -150,7 +164,15 @@ pub(crate) fn attached_functions<'py>( fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult { let context: &LoopContext<'_, '_> = scope.context; let py: Python<'_> = context.host.py(); - let (file_path, relative_path, name, function_file, header_values, body, contents) = row; + let FunctionRow { + file_path, + relative_path, + name, + function_file, + header_values, + body, + contents, + } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("SQL function {relative_path}"); let header: FunctionHeader = @@ -238,7 +260,14 @@ fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult, row: FunctionRow) -> PyResult { let context: &LoopContext<'_, '_> = scope.context; let py: Python<'_> = context.host.py(); - let (_, relative_path, name, function_file, header_values, body, _) = row; + let FunctionRow { + relative_path, + name, + function_file, + header_values, + body, + .. + } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("Python function {relative_path}"); let header: FunctionHeader = @@ -505,7 +534,12 @@ fn function_references( context, syntax, &scanned.invalid_calls, - (None, relative_path, file_path, contents), + CallOrigin { + name: None, + relative_path, + file_path, + contents, + }, )?; let references: Vec = merge_call_site_references( scanned.references.iter().map(reference_row).collect(), @@ -520,7 +554,7 @@ fn function_references( let known: KnownResources<'_> = validator.get().known_resources(); let calls: Vec<(&str, &str)> = references .iter() - .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .map(|ReferenceRow { kind, name, .. }| (kind.as_str(), name.as_str())) .collect(); record_source_reads( calls @@ -585,7 +619,14 @@ pub(crate) fn checked_table_function_arities( for (owner, references) in owners { let calls: Vec<(&str, &str, Option)> = references .iter() - .map(|(kind, name, _, count)| (kind.as_str(), name.as_str(), *count)) + .map( + |ReferenceRow { + kind, + name, + call_argument_count: count, + .. + }| (kind.as_str(), name.as_str(), *count), + ) .collect(); if let Some(message) = table_function_arity_error(&owner, &calls, &counts) { return Err(compile_error(context.classes, &message, None)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs index ca5bc2aeb9..80158fa8b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -38,13 +38,16 @@ pub(crate) struct FunctionClasses<'py> { /// `(name, type)` of an argument or return column, its types template-expanded. pub(crate) type NamedTypeRow = (String, String); -/// `(node type, name, tags, references as (kind, name, package))`. -pub(crate) type FunctionAssemblyRow = ( - &'static str, - String, - Vec, - Vec<(String, String, Option)>, -); +positional_record! { + into; + /// `(node type, name, tags, references as (kind, name, package))`. + pub(crate) struct FunctionAssemblyRow { + node_type: &'static str, + name: String, + tags: Vec, + references: Vec<(String, String, Option)>, + } +} /// `(kind, name)` of a compiled object key. pub(crate) type ObjectKeyRow = (String, String); @@ -396,16 +399,23 @@ function_getters!(NativeFunctionInput, { pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { read_function(&self.owner, self.index, py, |function, _| { let attached: &AttachedFunction = &function.attached; - Ok(( - attached.node_type(), - attached.name.clone(), - attached.tags.clone(), - attached + Ok(FunctionAssemblyRow { + node_type: attached.node_type(), + name: attached.name.clone(), + tags: attached.tags.clone(), + references: attached .references .iter() - .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind.clone(), name.clone(), package.clone()), + ) .collect(), - )) + }) }) } }); @@ -433,15 +443,15 @@ impl NativeCompiledFunction { .deps .as_deref() .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; - Ok(( - ( + Ok(ResourceRow { + key: ( function.attached.node_type().to_owned(), function.attached.name.clone(), ), - deps.to_vec(), - function.attached.tags.clone(), - None, - )) + deps: deps.to_vec(), + tags: function.attached.tags.clone(), + folder: None, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index bc37f09b55..9e37af932f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fac use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; const MODEL_RELATION_KIND: &str = "ref"; @@ -382,7 +382,13 @@ pub(crate) fn reference_tuple( cache .get_or_try_init(py, || { let mut references: Vec> = Vec::with_capacity(rows.len()); - for (kind, name, package, count) in rows { + for ReferenceRow { + kind, + name, + package, + call_argument_count: count, + } in rows + { let fields: Bound<'_, PyDict> = PyDict::new(py); fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; fields.set_item("ref_name", name)?; @@ -561,10 +567,10 @@ pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - Ok(( - native_field(reference, "ref_kind")?.str()?.extract()?, - native_field(reference, "ref_name")?.extract()?, - native_field(reference, "ref_package")?.extract()?, - native_field(reference, "call_argument_count")?.extract()?, - )) + Ok(ReferenceRow { + kind: native_field(reference, "ref_kind")?.str()?.extract()?, + name: native_field(reference, "ref_name")?.extract()?, + package: native_field(reference, "ref_package")?.extract()?, + call_argument_count: native_field(reference, "call_argument_count")?.extract()?, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index bdea620787..e6d8cd6bb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -9,7 +9,9 @@ use crate::bindings::_helpers::compiled_project::function_loop::{ }; use crate::bindings::_helpers::compiled_project::function_views::AttachedFunction; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, ModelConfigs}; use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; use crate::bindings::_helpers::compiled_project::model_loop_header::{ @@ -63,10 +65,11 @@ use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, expand_declarations, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -118,8 +121,19 @@ const COMPILE_ERROR_KEY: &str = "compile_error"; /// The native validator, header contract classes, and the loop's Python classes. type LoopChecks = (Py, Py, Py); +positional_record! { + extract; + /// Whether SQL analysis runs, is skipped, is deferred, and is required. + pub(crate) struct LoopSwitches { + pub(crate) sql_analysis: bool, + pub(crate) no_sql_validation: bool, + pub(crate) defer_sql_validation: bool, + pub(crate) require_sql_analysis: bool, + } +} + /// `(analysis switches, project config path)`. -type LoopSettingsInput = ((bool, bool, bool, bool), String); +type LoopSettingsInput = (LoopSwitches, String); /// The Python classes and error factory the loop builds usages, hooks and errors with. #[derive(FromPyObject)] @@ -216,7 +230,12 @@ impl NativeModelLoop { ) -> PyResult { let (validator, contract_classes, loop_classes) = checks; let ( - (sql_analysis, no_sql_validation, defer_sql_validation, require_sql_analysis), + LoopSwitches { + sql_analysis, + no_sql_validation, + defer_sql_validation, + require_sql_analysis, + }, project_config_path, ) = settings; Python::attach(|py| index_sql_hook_definitions(loop_classes.bind(py)))?; @@ -291,19 +310,30 @@ fn prepared_rows<'py>( &sqls, )?; let mut rows: Vec = Vec::with_capacity(model_files.len()); - for (((model_file, paths), (authored, _)), (rendered, spans, reads, error)) in model_files + for (((model_file, paths), (authored, _)), interpolated) in model_files .into_iter() .zip(paths) .zip(sqls) .zip(interpolated) { + let SqlInterpolationRow { + sql: rendered, + spans, + reads, + error, + } = interpolated; let sql: String = rendered.unwrap_or(authored); let scan: Option = error.is_none().then(|| declaration_scan(&sql)); let reads: Vec<(String, String)> = reads .into_iter() .map(|(kind, name)| (kind.to_owned(), name)) .collect(); - let interpolation: InterpolationRow = (sql, reads, error, spans); + let interpolation: InterpolationRow = InterpolationRow { + sql, + reads, + error, + spans, + }; rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); } Ok(rows) @@ -673,7 +703,12 @@ impl NativeModelLoop { context, &self.syntax, &scanned.invalid_calls, - (Some(name), relative_path, file_path, contents), + CallOrigin { + name: Some(name), + relative_path, + file_path, + contents, + }, )?; let analysis: (bool, Option>) = ( analysed && scanned.invalid_calls.is_empty(), @@ -807,7 +842,9 @@ impl NativeModelLoop { let rows: Vec<(&str, &str, bool)> = references .iter() .zip(&external) - .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .map(|(ReferenceRow { kind, name, .. }, error)| { + (kind.as_str(), name.as_str(), error.is_some()) + }) .collect(); let unmanaged: Bound<'_, PyAny> = native_field( &native_field(config, RETENTION_ATTRIBUTE)?, @@ -851,7 +888,9 @@ fn applied_interpolation( context: &LoopContext<'_, '_>, interpolation: &InterpolationRow, ) -> PyResult { - let (sql, reads, error, _) = interpolation; + let InterpolationRow { + sql, reads, error, .. + } = interpolation; record_reads(context, reads)?; match error { Some(message) => Err(compile_error(context.classes, message, None)), @@ -919,13 +958,11 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { /// Span rows as the spliced spans the scan-fault locator maps through. fn spliced_spans(rows: &[SpanRow]) -> Vec { rows.iter() - .map( - |&(source_start, source_end, output_start, output_end)| SplicedSpan { - source_start, - source_end, - output_start, - output_end, - }, - ) + .map(|row| SplicedSpan { + source_start: row.source_start, + source_end: row.source_end, + output_start: row.output_start, + output_end: row.output_end, + }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs index f80603a5f8..0403d11316 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs @@ -19,8 +19,14 @@ use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compi const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; const MODEL_RESOURCE: &str = "model"; -/// `(model name or none for a function, relative path, file path, contents)` of a file. -pub(crate) type CallOrigin<'a> = (Option<&'a str>, &'a str, &'a str, &'a str); +/// The model name, or none for a function, and the relative path, file path and contents of a file. +#[derive(Clone, Copy)] +pub(crate) struct CallOrigin<'a> { + pub(crate) name: Option<&'a str>, + pub(crate) relative_path: &'a str, + pub(crate) file_path: &'a str, + pub(crate) contents: &'a str, +} /// Report each rejected call in order, at its authored location when the authored text has it. pub(crate) fn report_invalid_calls( @@ -32,7 +38,7 @@ pub(crate) fn report_invalid_calls( if invalid_calls.is_empty() { return Ok(()); } - let (_, _, _, contents) = origin; + let CallOrigin { contents, .. } = origin; let authored: HashMap> = match extract_sql_references(contents, syntax) { ReferenceExtraction::Extracted(scan) => { let mut starts: HashMap> = HashMap::new(); @@ -62,7 +68,12 @@ pub(crate) fn report_invalid_call( start: Option, origin: CallOrigin<'_>, ) -> PyResult<()> { - let (name, relative_path, file_path, contents) = origin; + let CallOrigin { + name, + relative_path, + file_path, + contents, + } = origin; let py: Python<'_> = context.host.py(); let path: Bound<'_, PyAny> = context.classes.path_class.call1((relative_path,))?; let fields: Bound<'_, PyDict> = PyDict::new(py); @@ -152,7 +163,12 @@ pub(crate) fn origin_references( context, syntax, &scanned.invalid_calls, - (None, relative_path, file_path, contents), + CallOrigin { + name: None, + relative_path, + file_path, + contents, + }, )?; Ok(scanned.references) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs index d968d3fae7..e6b96c419c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs @@ -20,7 +20,13 @@ pub(crate) fn external_errors( relative_path: &str, ) -> PyResult>> { let mut errors: Vec> = Vec::with_capacity(references.len()); - for (kind, name, package, _) in references { + for ReferenceRow { + kind, + name, + package, + .. + } in references + { errors.push(if kind == DBT_REF_KIND { external_error(context, (kind, name, package.as_deref()), relative_path)? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index 2f27b2dd4e..eb86febd9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -23,7 +23,9 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, validation_outcome, }; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ @@ -145,7 +147,9 @@ pub(crate) fn expanded_hooks<'py>( let rows: Vec<(&str, &str, bool)> = references .iter() .zip(&external) - .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .map(|(ReferenceRow { kind, name, .. }, error)| { + (kind.as_str(), name.as_str(), error.is_some()) + }) .collect(); let outcome: Bound<'py, PyAny> = model_loop .validator @@ -448,12 +452,12 @@ impl<'py> HookRun<'_, '_, 'py> { self.context, &self.model_loop.syntax, &scanned.invalid_calls, - ( - Some(self.model.name), - self.model.relative_path, - self.model.file_path, - self.model.contents, - ), + CallOrigin { + name: Some(self.model.name), + relative_path: self.model.relative_path, + file_path: self.model.file_path, + contents: self.model.contents, + }, )?; self.references .borrow_mut() @@ -496,7 +500,7 @@ impl<'py> HookRun<'_, '_, 'py> { let py: Python<'py> = hooks.explicit_kinds.py(); let mut seen: Vec<(String, String)> = Vec::new(); let mut reads: Vec> = Vec::new(); - for (kind, name, _, _) in self.references.borrow().iter().skip(start) { + for ReferenceRow { kind, name, .. } in self.references.borrow().iter().skip(start) { let Some(resource_kind) = hooks.explicit_kinds.get_item(kind)? else { continue; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 38a437789d..85574127c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -25,11 +25,10 @@ use crate::bindings::_helpers::compiled_project::project::{ }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; -use crate::bindings::_helpers::render::declaration_expansion::{ - ExpandedDeclarations, ScannedReference, span_rows, -}; +use crate::bindings::_helpers::render::declaration_expansion::{ExpandedDeclarations, span_rows}; use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; +use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; use sqlbuild_discovery::model_files::models::HeaderFacts; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; @@ -77,10 +76,25 @@ impl ModelLoopRow { } } } -/// `(interpolated query, template reads, error, substitution spans)`. -pub(crate) type InterpolationRow = (String, Vec<(String, String)>, Option, Vec); -/// `(kind, name, package, call argument count)`. -pub(crate) type ReferenceRow = (String, String, Option, Option); +/// A model query's interpolation: the interpolated query, template reads, error and spans. +pub(crate) struct InterpolationRow { + pub(crate) sql: String, + pub(crate) reads: Vec<(String, String)>, + pub(crate) error: Option, + /// The substitution spans. + pub(crate) spans: Vec, +} +positional_record! { + extract, into; + /// `(kind, name, package, call argument count)`. + #[derive(Clone, Debug, PartialEq, Eq)] + pub(crate) struct ReferenceRow { + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) package: Option, + pub(crate) call_argument_count: Option, + } +} /// One model the native loop finished, before the host builds its Python view. pub(crate) struct FinishedModel<'py> { @@ -130,7 +144,12 @@ pub(crate) fn finished_record<'py>( }, authored_query: authored_query_sql, contents: authored_sql, - interpolation: (interpolated_sql, _, _, interpolation_spans), + interpolation: + InterpolationRow { + sql: interpolated_sql, + spans: interpolation_spans, + .. + }, model_file, .. } = row; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs index 8574f80e70..82a3173139 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs @@ -68,15 +68,22 @@ pub(crate) fn stripped_values<'py>( } pub(crate) fn reference_row(reference: &SqlReference) -> ReferenceRow { - ( - reference.kind.to_owned(), - reference.name.clone(), - reference.package.clone(), - reference.call_argument_count, - ) + ReferenceRow { + kind: reference.kind.to_owned(), + name: reference.name.clone(), + package: reference.package.clone(), + call_argument_count: reference.call_argument_count, + } } -pub(crate) fn reference_fact((kind, name, package, _): ReferenceRow) -> ModelReferenceFact { +pub(crate) fn reference_fact( + ReferenceRow { + kind, + name, + package, + .. + }: ReferenceRow, +) -> ModelReferenceFact { ModelReferenceFact { kind, name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs index 64a85b3c40..56eef6ba1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, expanded, + ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -112,13 +112,13 @@ pub(crate) fn expanded_with<'py>( let py: Python<'py> = consumer.py(); let expansion: NativeMacroExpansion = NativeMacroExpansion::new( &context.classes.path_class.call1((file_path,))?, - ( - classes.loaded_macros.clone_ref(py), - PyDict::new(py).unbind(), - classes.macro_context.clone().unbind(), - declarations.clone().unbind(), - consumer.clone().unbind(), - ), + StateInput { + loaded_macros: classes.loaded_macros.clone_ref(py), + overrides: PyDict::new(py).unbind(), + macro_context: classes.macro_context.clone().unbind(), + declarations: declarations.clone().unbind(), + consumer: consumer.clone().unbind(), + }, native_field(&classes.macro_context, ENFORCE_ATTRIBUTE)?.is_truthy()?, classes.expansion.clone_ref(py), )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index ef803053de..6189299d1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -49,7 +49,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_project::seed_views::{ - NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, paired_seeds, + NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedParts, SeedRecord, paired_seeds, }; use crate::bindings::_helpers::compiled_project::source_loop::{SourceRow, SourceSettings}; use crate::bindings::_helpers::compiled_project::source_views::{ @@ -70,11 +70,42 @@ const MATERIALIZED_KEY: &str = "materialized"; const TABLE_MATERIALIZATION: &str = "table"; /// `(name, type)`. -type NamedType = (String, Option); -/// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. -type ProofPy = (bool, bool, Vec, Option, Vec); -/// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. -type AnalysisPy = (bool, Vec, Vec, bool, Vec); +pub(crate) type NamedType = (String, Option); +positional_record! { + extract; + /// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. + pub(crate) struct ProofPy { + pub(crate) output_proven: bool, + pub(crate) bare_dynamic_pivot: bool, + pub(crate) families: Vec, + pub(crate) failure_reason: Option, + pub(crate) named_families: Vec, + } +} + +positional_record! { + extract; + /// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. + pub(crate) struct AnalysisPy { + pub(crate) inferred: bool, + pub(crate) inferred_columns: Vec, + pub(crate) nullable_columns: Vec, + pub(crate) fast_lineage_has_star: bool, + pub(crate) unchecked: Vec, + } +} + +positional_record! { + extract; + /// A model's name, relative path, compiled and authored queries, and authored SQL file. + pub(crate) struct ModelTextsPy { + name: String, + relative_path: String, + query_sql: String, + authored_query_sql: String, + authored_sql: String, + } +} /// Encoded config values and the encoder's error, if any. pub(crate) type ConfigValues = (BTreeMap, Option); @@ -82,7 +113,7 @@ pub(crate) type ConfigValues = (BTreeMap, Option); /// One model as the compile-input stage produced it. #[derive(FromPyObject)] pub(crate) struct ModelInputPy<'py>( - (String, String, String, String, String), + ModelTextsPy, (Bound<'py, PyDict>, Vec, Option), Vec<(String, String, Option)>, Vec, @@ -194,7 +225,12 @@ impl NativeCompiledProject { )?); } for seed in &self.seeds { - let (name, seed_file, schema_entry, schema_file) = seed.parts(); + let SeedParts { + name, + seed_file, + schema_entry, + schema_file, + } = seed.parts(); let schema_entry: &Bound<'py, PyAny> = schema_entry.bind(py); owners.push(resource_owner( SEED_OWNER, @@ -335,19 +371,28 @@ impl NativeCompiledProject { columns: AnalysisPy, dynamic_proof: Option, ) -> bool { - let (inferred, inferred_columns, nullable_columns, fast_lineage_has_star, unchecked) = - columns; + let AnalysisPy { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + } = columns; let analysis: ModelAnalysisFact = ModelAnalysisFact { inferred_columns, dynamic_proof: dynamic_proof.map( - |(output_proven, bare_dynamic_pivot, families, failure_reason, named_families)| { - DynamicProofFact { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - } + |ProofPy { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }| DynamicProofFact { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, }, ), inferred, @@ -620,7 +665,13 @@ impl NativeCompiledProject { pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { let ModelInputPy( - (name, relative_path, query_sql, authored_query_sql, authored_sql), + ModelTextsPy { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + }, (values, header_keys, layer_schema), references, enum_columns, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index a7bf443dcf..966d655754 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -576,12 +576,12 @@ fn resource_row( tag_names.push(tag?.str()?.extract()?); } } - Ok(( - key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, + Ok(ResourceRow { + key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, deps, - tag_names, - None, - )) + tags: tag_names, + folder: None, + }) } /// `(resource type, name)` of a `CompiledObjectKey`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 503d7026a5..4d095ca181 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -13,7 +13,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::project::{NativeCompiledProject, model_facts}; +use crate::bindings::_helpers::compiled_project::project::{ + AnalysisPy, NativeCompiledProject, ProofPy, model_facts, +}; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; use crate::bindings::_helpers::records::construct::{enum_member, new_record, record_class}; @@ -397,28 +399,30 @@ pub(crate) fn record_model_analyses<'py>( None } else { let families: Bound<'py, PyAny> = native_field(&proof, "families")?; - Some(( - native_field(&proof, "output_proven")?.extract()?, - native_field(&proof, "bare_dynamic_pivot")?.extract()?, - family_types(&families, true)?, - native_field(&proof, "failure_reason")?.extract()?, - family_types(&families, false)?, - )) + Some(ProofPy { + output_proven: native_field(&proof, "output_proven")?.extract()?, + bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, + families: family_types(&families, true)?, + failure_reason: native_field(&proof, "failure_reason")?.extract()?, + named_families: family_types(&families, false)?, + }) }; - let analysis = ( - !inferred.is_none(), - rows.iter() + let analysis = AnalysisPy { + inferred: !inferred.is_none(), + inferred_columns: rows + .iter() .map(|(name, column_type, _)| (name.clone(), column_type.clone())) .collect(), - rows.iter() + nullable_columns: rows + .iter() .map(|(_, _, nullability)| nullability.eq(&nullable)) .collect::>()?, - native_field(model, "fast_lineage_has_star")?.is_truthy()?, - native_field(model, "unchecked_output_columns")? + fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, + unchecked: native_field(model, "unchecked_output_columns")? .try_iter()? .map(|column| column?.extract()) .collect::>()?, - ); + }; let name: String = native_field(model, "name")?.extract()?; transferred += usize::from( project diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index efd61f55c2..9dd9ba24b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -21,7 +21,9 @@ use sqlbuild_analysis::resource_diagnostics::main::missing_description::missing_ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::seed_views::SeedParts; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; @@ -36,8 +38,39 @@ const PYTHON_UDF_DECORATOR: &str = "udf"; /// A loader's description, relative path and whether that path is a YAML file. pub(crate) type LoaderRow = (Option, String, bool); -/// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. -type PythonResourceRow = (String, String, Option>, Py, String); +positional_record! { + extract; + /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. + struct PythonResourceRow { + kind: String, + name: String, + description: Option>, + location: Py, + help: String, + } +} + +positional_record! { + extract; + /// A scenario's name, relative path, contents and authored header description. + struct ScenarioDescriptionRow { + name: String, + path: String, + contents: String, + raw: Py, + } +} + +positional_record! { + extract; + /// A SQL hook file's name, relative path, contents and description. + struct HookDescriptionRow { + name: String, + path: String, + contents: String, + description: Option, + } +} /// What the resource diagnostics read beyond the owner, and the Python values they build. #[derive(FromPyObject)] @@ -53,9 +86,9 @@ pub(crate) struct DiagnosticSettings { /// Each discovered audit factory's name, relative path and line. audit_factories: Vec<(String, String, usize)>, /// Each scenario's name, relative path, contents and authored header description. - scenarios: Vec<(String, String, String, Py)>, + scenarios: Vec, /// Each SQL hook file's name, relative path, contents and description. - sql_hooks: Vec<(String, String, String, Option)>, + sql_hooks: Vec, /// Each loader by name. loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. @@ -277,7 +310,13 @@ fn missing_descriptions<'py>( &help, )?); } - for (name, path, contents, raw) in &diagnostics.settings.scenarios { + for ScenarioDescriptionRow { + name, + path, + contents, + raw, + } in &diagnostics.settings.scenarios + { let raw: &Bound<'py, PyAny> = raw.bind(py); let description: Option = if raw.is_instance_of::() { Some(raw.extract()?) @@ -301,7 +340,11 @@ fn missing_descriptions<'py>( )?); } for seed in project.seeds() { - let (_, _, entry, schema_file) = seed.parts(); + let SeedParts { + schema_entry: entry, + schema_file, + .. + } = seed.parts(); let entry: &Bound<'py, PyAny> = entry.bind(py); let schema_file: &Bound<'py, PyAny> = schema_file.bind(py); let name: String = native_field(entry, "name")?.extract()?; @@ -324,7 +367,13 @@ fn missing_descriptions<'py>( for function in project.functions() { found.push(function_description(diagnostics, function)?); } - for (name, path, contents, description) in &diagnostics.settings.sql_hooks { + for HookDescriptionRow { + name, + path, + contents, + description, + } in &diagnostics.settings.sql_hooks + { let entry: String = format!("description \"What {name} does\""); let help: String = header_help(&HeaderHelp { kind: "hook", @@ -341,7 +390,14 @@ fn missing_descriptions<'py>( &help, )?); } - for (kind, name, description, location, help) in &diagnostics.settings.python_resources { + for PythonResourceRow { + kind, + name, + description, + location, + help, + } in &diagnostics.settings.python_resources + { let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); @@ -484,7 +540,7 @@ fn cursor_without_inputs<'py>( let kinds: Vec<&str> = record .references .iter() - .map(|(kind, _, _, _)| kind.as_str()) + .map(|ReferenceRow { kind, .. }| kind.as_str()) .collect(); let materialized: Option = config_text(&values, "materialized")?; let cursor: Option = config_text(&values, CURSOR_KEY)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs index 11af682292..2e8c4f2db4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs @@ -22,8 +22,18 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, }; -/// A scenario file's paths, discovered file, contents, SQL body and name. -pub(crate) type ScenarioRow = (String, String, Py, String, String, String); +positional_record! { + extract; + /// A scenario file's paths, discovered file, contents, SQL body and name. + pub(crate) struct ScenarioRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) scenario_file: Py, + pub(crate) contents: String, + pub(crate) sql_body: String, + pub(crate) name: String, + } +} impl<'py> TestScope<'_, 'py> { /// Every scenario expanded, classified and checked, in discovery order. @@ -33,7 +43,14 @@ impl<'py> TestScope<'_, 'py> { fn scenario(&self, row: &ScenarioRow) -> PyResult> { let py: Python<'py> = self.context.host.py(); - let (file_path, relative_path, scenario_file, contents, sql_body, name) = row; + let ScenarioRow { + file_path, + relative_path, + scenario_file, + contents, + sql_body, + name, + } = row; let resource: Bound<'py, PyAny> = self .context .classes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs index 4e775cf3cb..52486ab28b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -24,13 +24,24 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; const STEM_ATTRIBUTE: &str = "stem"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; -/// `(database, schema, logical database, logical schema)` of a seed's destination. -pub(crate) type SeedNamespace = ( - Option, - Option, - Option, - Option, -); +positional_record! { + extract; + /// `(database, schema, logical database, logical schema)` of a seed's destination. + pub(crate) struct SeedNamespace { + database: Option, + schema: Option, + logical_database: Option, + logical_schema: Option, + } +} + +/// A seed's name, CSV file, declaration and declaring schema file. +pub(crate) struct SeedParts<'a> { + pub(crate) name: &'a str, + pub(crate) seed_file: &'a Py, + pub(crate) schema_entry: &'a Py, + pub(crate) schema_file: &'a Py, +} /// One seed CSV paired with its declaration; the discovered objects are discovery's. pub(crate) struct SeedRecord { @@ -100,18 +111,23 @@ pub(crate) fn paired_seeds( impl SeedRecord { /// The seed's name, CSV file, declaration and declaring schema file. - pub(crate) fn parts(&self) -> (&str, &Py, &Py, &Py) { - ( - &self.name, - &self.seed_file, - &self.schema_entry, - &self.schema_file, - ) + pub(crate) fn parts(&self) -> SeedParts<'_> { + SeedParts { + name: &self.name, + seed_file: &self.seed_file, + schema_entry: &self.schema_entry, + schema_file: &self.schema_file, + } } /// Keep the destination native project assembly resolved for the seed. pub(crate) fn set_namespace(&mut self, namespace: SeedNamespace) { - let (database, schema, logical_database, logical_schema) = namespace; + let SeedNamespace { + database, + schema, + logical_database, + logical_schema, + } = namespace; self.destination = Some(DestinationFact { database, schema, @@ -201,12 +217,12 @@ impl NativeCompiledSeed { for tag in native_field(seed.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { tags.push(tag?.str()?.extract()?); } - Ok(( - (SEED_RESOURCE.to_owned(), seed.name.clone()), - Vec::new(), + Ok(ResourceRow { + key: (SEED_RESOURCE.to_owned(), seed.name.clone()), + deps: Vec::new(), tags, - None, - )) + folder: None, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 48690285f0..e3537d16b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -20,6 +20,17 @@ use crate::bindings::_helpers::rules::relation_keys::audit_key_rows; const SOURCE_RESOURCE: &str = "source"; +positional_record! { + into; + /// `(name, has a loader, database, schema)` of one source. + pub(crate) struct SourceAssemblyRow { + name: String, + managed: bool, + database: Option, + schema: Option, + } +} + /// `(database, schema)` native project assembly resolved for a source, or `None`. pub(crate) type SourceNamespace = Option<(Option, Option)>; @@ -210,18 +221,15 @@ impl NativeSourceInput { /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. #[getter] - pub(crate) fn assembly_row( - &self, - py: Python<'_>, - ) -> PyResult<(String, bool, Option, Option)> { + pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { read_source(&self.owner, self.index, py, |source, py| { let attached: &AttachedSource = &source.attached; - Ok(( - attached.name.clone(), - !native_field(attached.entry.bind(py), "loader")?.is_none(), - attached.database.clone(), - attached.schema.clone(), - )) + Ok(SourceAssemblyRow { + name: attached.name.clone(), + managed: !native_field(attached.entry.bind(py), "loader")?.is_none(), + database: attached.database.clone(), + schema: attached.schema.clone(), + }) }) } } @@ -241,12 +249,12 @@ impl NativeCompiledSource { /// The source's project graph row: its key, with no dependencies, tags or folder. pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { read_source(&self.owner, self.index, py, |source, _| { - Ok(( - (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), - Vec::new(), - Vec::new(), - None, - )) + Ok(ResourceRow { + key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }) }) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index 07563842eb..40aa64302a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -42,7 +42,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; use crate::bindings::_helpers::compiled_project::model_loop_calls::{ - origin_references, report_invalid_call, + CallOrigin, origin_references, report_invalid_call, }; use crate::bindings::_helpers::compiled_project::source_loop::{ AuthoredExpansion, expanded_authored_sql, scoped_declarations, @@ -70,19 +70,32 @@ const MACRO_VISIBILITY: &str = "macro_visibility"; /// A test case's discovered case, name and parameter values in authored order. pub(crate) type TestCaseRow = (Py, String, Vec<(String, Py)>); -/// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. -pub(crate) type TestBlockRow = ( - Py, - Option, - String, - String, - Vec, - Vec, - Option<(usize, usize)>, -); +positional_record! { + extract; + /// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. + pub(crate) struct TestBlockRow { + pub(crate) block: Py, + pub(crate) name: Option, + pub(crate) sql_body: String, + pub(crate) mode: String, + pub(crate) parameters: Vec, + pub(crate) cases: Vec, + pub(crate) sql_body_span: Option<(usize, usize)>, + } +} -/// A test file's paths, stem, discovered file, contents and blocks. -pub(crate) type TestFileRow = (String, String, String, Py, String, Vec); +positional_record! { + extract; + /// A test file's paths, stem, discovered file, contents and blocks. + pub(crate) struct TestFileRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) stem: String, + pub(crate) file: Py, + pub(crate) contents: String, + pub(crate) blocks: Vec, + } +} /// What every SQL test and scenario of one compile resolves and records with. #[derive(FromPyObject)] @@ -151,11 +164,11 @@ impl<'py> TestScope<'_, 'py> { pub(crate) fn attached(&self, files: &[TestFileRow]) -> PyResult>> { let mut expanded: Vec> = Vec::new(); for file in files { - for block in &file.5 { - if block.5.is_empty() { + for block in &file.blocks { + if block.cases.is_empty() { expanded.push(self.expanded(file, block, None)?); } - for case in &block.5 { + for case in &block.cases { expanded.push(self.expanded(file, block, Some(case))?); } } @@ -178,8 +191,20 @@ impl<'py> TestScope<'_, 'py> { case: Option<&'a TestCaseRow>, ) -> PyResult> { let py: Python<'py> = self.py(); - let (file_path, relative_path, stem, _, contents, _) = file; - let (_, name, sql_body, mode, parameters, _, _) = block; + let TestFileRow { + file_path, + relative_path, + stem, + contents, + .. + } = file; + let TestBlockRow { + name, + sql_body, + mode, + parameters, + .. + } = block; let test_name: &str = name.as_deref().unwrap_or(stem); let resource: Bound<'py, PyAny> = self .context @@ -329,7 +354,7 @@ impl<'py> TestScope<'_, 'py> { let (file, block) = test; let request: SqlTestRequest = SqlTestRequest { sql: sql.to_owned(), - file_label: file.1.clone(), + file_label: file.relative_path.clone(), mode: mode.to_owned(), raw: true, authored: false, @@ -348,7 +373,7 @@ impl<'py> TestScope<'_, 'py> { .. } = &classification { - let origin: FileOrigin<'_> = (&file.0, &file.1, &file.4); + let origin: FileOrigin<'_> = (&file.file_path, &file.relative_path, &file.contents); if self.authored_invalid_calls(file, block, &actual.0)? == 0 { for cte in helpers.iter().chain([expected]) { origin_references(self.context, &self.model_loop.syntax, &cte.1, origin)?; @@ -367,8 +392,8 @@ impl<'py> TestScope<'_, 'py> { ) -> PyResult { let syntax = &self.model_loop.syntax; let request: SqlTestRequest = SqlTestRequest { - sql: block.2.clone(), - file_label: file.1.clone(), + sql: block.sql_body.clone(), + file_label: file.relative_path.clone(), mode: MODEL_MODE.to_owned(), raw: false, authored: true, @@ -394,16 +419,24 @@ impl<'py> TestScope<'_, 'py> { ReferenceExtraction::Failed(failure) => return Err(self.error(&failure.message)), }; for invalid in &scan.invalid_calls { - let Some(offset) = - authored_test_offset(&file.4, &block.2, block.6, body_start + invalid.start) - else { + let Some(offset) = authored_test_offset( + &file.contents, + &block.sql_body, + block.sql_body_span, + body_start + invalid.start, + ) else { return Ok(reported); }; report_invalid_call( self.context, invalid, Some(offset), - (None, &file.1, &file.0, &file.4), + CallOrigin { + name: None, + relative_path: &file.relative_path, + file_path: &file.file_path, + contents: &file.contents, + }, )?; reported += 1; } @@ -611,11 +644,11 @@ impl<'py> TestScope<'_, 'py> { let mut files: Vec<(&str, Vec)> = Vec::new(); let mut positions: HashMap<&str, usize> = HashMap::new(); for (index, test) in tests.iter().enumerate() { - match positions.get(test.file.1.as_str()) { + match positions.get(test.file.relative_path.as_str()) { Some(position) => files[*position].1.push(index), None => { - positions.insert(&test.file.1, files.len()); - files.push((&test.file.1, vec![index])); + positions.insert(&test.file.relative_path, files.len()); + files.push((&test.file.relative_path, vec![index])); } } } @@ -635,8 +668,8 @@ impl<'py> TestScope<'_, 'py> { let mut modes: Vec<&str> = Vec::with_capacity(indexes.len()); for index in indexes { parts.push(tests[*index].sql.clone()); - parts.push(tests[*index].block.3.clone()); - modes.push(&tests[*index].block.3); + parts.push(tests[*index].block.mode.clone()); + modes.push(&tests[*index].block.mode); } let stored: Option> = match &store { Some(store) => store @@ -662,8 +695,8 @@ impl<'py> TestScope<'_, 'py> { .iter() .map(|index| SqlTestRequest { sql: tests[*index].sql.clone(), - file_label: tests[*index].file.1.clone(), - mode: tests[*index].block.3.clone(), + file_label: tests[*index].file.relative_path.clone(), + mode: tests[*index].block.mode.clone(), raw: false, authored: false, scenario: false, @@ -713,9 +746,21 @@ impl<'py> TestScope<'_, 'py> { let offset: Option = failure .token .as_ref() - .filter(|_| sql == block.2) - .and_then(|(_, offset)| authored_test_offset(&file.4, &block.2, block.6, *offset)); - located_error(self.context, failure, (&file.1, &file.4), offset) + .filter(|_| sql == block.sql_body) + .and_then(|(_, offset)| { + authored_test_offset( + &file.contents, + &block.sql_body, + block.sql_body_span, + *offset, + ) + }); + located_error( + self.context, + failure, + (&file.relative_path, &file.contents), + offset, + ) } /// Check one classified test; `None` when a P013 problem was reported instead. @@ -725,8 +770,20 @@ impl<'py> TestScope<'_, 'py> { classification: SqlTestClassification, ) -> PyResult>> { let py: Python<'py> = self.py(); - let (file_path, relative_path, stem, test_file, contents, _) = test.file; - let (test_block, name, _, mode, _, _, _) = test.block; + let TestFileRow { + file_path, + relative_path, + stem, + file: test_file, + contents, + .. + } = test.file; + let TestBlockRow { + block: test_block, + name, + mode, + .. + } = test.block; let test_name: &str = name.as_deref().unwrap_or(stem); let origin: FileOrigin<'_> = (file_path, relative_path, contents); let payload: Bound<'py, PyAny> = match &classification { @@ -911,14 +968,14 @@ impl<'py> TestScope<'_, 'py> { ) -> PyResult<()> { let py: Python<'py> = self.py(); let located: CteLocation = sql_test_cte_location( - &file.4, - &block.2, + &file.contents, + &block.sql_body, &problem.cte_name, problem.call.as_deref(), ); let (line, column) = (located.line, located.column); let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("path", native_field(file.3.bind(py), "relative_path")?)?; + arguments.set_item("path", native_field(file.file.bind(py), "relative_path")?)?; arguments.set_item("line", line)?; arguments.set_item("column", column)?; arguments.set_item("end_line", located.end_line)?; @@ -941,7 +998,7 @@ impl<'py> TestScope<'_, 'py> { py, [ HELPER_REFERENCE_CODE.to_owned(), - file.1.replace('\\', "/"), + file.relative_path.replace('\\', "/"), format!("{line:09}:{column:09}"), problem.message.clone(), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 2dba2e534a..f9b65d4618 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -56,10 +56,16 @@ fn output_column_location<'py>( } let implicit: bool = native_field(model, "extract_implicit_alias_columns")?.is_truthy()?; let rows: Vec = location_rows(model_output_columns(&authored, implicit)); - let Some((_, line, column, end_line, end_column)) = rows + let Some(ColumnLocation { + line, + column, + end_line, + end_column, + .. + }) = rows .into_iter() .rev() - .find(|(name, ..)| name == column_name) + .find(|location| location.name == column_name) else { return Ok(None); }; @@ -126,7 +132,16 @@ fn contract_diagnostic<'py>( row: DiagnosticRow, ) -> PyResult> { let py: Python<'py> = model.py(); - let (code, is_error, message, column_name, declared_index, output_column, related, help) = row; + let DiagnosticRow { + code, + is_error, + message, + column_name, + declared_index, + output_column, + related, + help, + } = row; let severity: &str = if is_error { "error" } else { "warning" }; keyword_call( &record_class(py, RecordId::CompilerDiagnostic)?, @@ -228,7 +243,14 @@ pub(crate) fn promotion_conflict_diagnostics<'py>( let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); - for (name, relative_path, code, message, help) in conflicts { + for ConflictRow { + name, + relative_path, + code, + message, + help, + } in conflicts + { diagnostics.push(keyword_call( &diagnostic_type, vec![ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index 4b840c0e71..d3d532a8c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -20,17 +20,18 @@ use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::type_system::normalization::normalization_error; use crate::bindings::types::CompilerDetach; -/// `(code, is_error, message, column, declared_index, output_column, related, help)`. -pub(crate) type DiagnosticRow = ( - &'static str, - bool, - String, - Option, - Option, - Option, - Option<(String, String)>, - String, -); +/// One contract diagnostic: its code, severity, message, column, location, related output and help. +#[derive(Clone)] +pub(crate) struct DiagnosticRow { + pub(crate) code: &'static str, + pub(crate) is_error: bool, + pub(crate) message: String, + pub(crate) column_name: Option, + pub(crate) declared_index: Option, + pub(crate) output_column: Option, + pub(crate) related: Option<(String, String)>, + pub(crate) help: String, +} /// Evaluate the column contract of every model the compile's owner recorded, in its order. pub(crate) fn evaluate_project_model_contracts( @@ -133,8 +134,14 @@ fn fact_dynamic_proof(proof: &DynamicProofFact) -> DynamicContractProof { } } -/// `(model name, model path, code, message, help)`. -pub(crate) type ConflictRow = (String, String, &'static str, String, String); +/// One K011 conflict: the model name and path, and the code, message and help. +pub(crate) struct ConflictRow { + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) code: &'static str, + pub(crate) message: String, + pub(crate) help: String, +} /// The K011 conflicts of enforced contracts under immediate table promotion, by model name. pub(crate) fn project_promotion_conflicts( @@ -166,29 +173,29 @@ pub(crate) fn project_promotion_conflicts( .into_iter() .map(|conflict| { let model: &CompiledModelFacts = &models[conflict.model_index]; - ( - model.name.clone(), - model.relative_path.clone(), - conflict.code, - conflict.message, - conflict.help, - ) + ConflictRow { + name: model.name.clone(), + relative_path: model.relative_path.clone(), + code: conflict.code, + message: conflict.message, + help: conflict.help, + } }) .collect()) } fn diagnostic_row(diagnostic: ContractDiagnostic) -> DiagnosticRow { let (declared_index, output_column) = diagnostic.location.into_parts(); - ( - diagnostic.code, - diagnostic.severity == ContractSeverity::Error, - diagnostic.message, - diagnostic.column_name, + DiagnosticRow { + code: diagnostic.code, + is_error: diagnostic.severity == ContractSeverity::Error, + message: diagnostic.message, + column_name: diagnostic.column_name, declared_index, output_column, - diagnostic + related: diagnostic .related_output .map(|related| (related.column_name, related.message)), - diagnostic.help, - ) + help: diagnostic.help, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 405832ce4d..bd8dd810b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -162,6 +162,17 @@ fn read_only_query( Ok(normalized.to_owned()) } +positional_record! { + extract; + /// The target name, target, database and schema a raw-query diff resolves. + struct ResolvedNamespace<'py> { + target_name: Bound<'py, PyAny>, + _target: Bound<'py, PyAny>, + database: Bound<'py, PyAny>, + schema: Bound<'py, PyAny>, + } +} + /// Resolve raw SQL, the active target namespace, the adapter and its connection. pub(crate) fn prepare_query_diff<'py>( request: &Bound<'py, PyAny>, @@ -206,12 +217,12 @@ pub(crate) fn prepare_query_diff<'py>( )?), ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - let (target_name, _target, database, schema): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = namespace.extract()?; + let ResolvedNamespace { + target_name, + database, + schema, + .. + }: ResolvedNamespace<'py> = namespace.extract()?; if schema.is_none() { return Err(failure( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index 605837e2c8..623fd46ff4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -103,7 +103,14 @@ fn source_locations<'py>( ) -> PyResult> { let py: Python<'py> = relative_path.py(); let locations: Bound<'py, PyDict> = PyDict::new(py); - for (name, line, column, end_line, end_column) in rows { + for ColumnLocation { + name, + line, + column, + end_line, + end_column, + } in rows + { let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("path", relative_path)?; arguments.set_item("line", line)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index 2ebfe2d8b7..ac8620b4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -39,7 +39,13 @@ fn attached_audit_gate_edges<'py>( let cache: KeyCache<'py> = KeyCache::new(py)?; let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::AttachedAuditGateEdge)?; let mut edges: Vec> = Vec::with_capacity(gates.len()); - for (audit_name, target, gated, read) in gates { + for GateRow { + audit_name, + target, + gated, + read, + } in gates + { edges.push(keyword_call( &edge_type, &[ @@ -65,7 +71,14 @@ fn hook_read_edges<'py>( let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::HookReadEdge)?; let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; let mut edges: Vec> = Vec::with_capacity(hooks.len()); - for (hook_name, gated, read, kind, path) in hooks { + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + path, + } in hooks + { let hook_path: Bound<'py, PyAny> = match path { None => py.None().into_bound(py), Some(path) => path_type.call1((path,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index 62e7a80269..6e880c4db6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -24,17 +24,28 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::views::lineage_graph_views; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(key, deps, tags, model folder)`. -pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); +positional_record! { + extract; + /// `(key, deps, tags, model folder)`. + pub(crate) struct ResourceRow { + pub(crate) key: ObjectKey, + pub(crate) deps: Vec, + pub(crate) tags: Vec, + pub(crate) folder: Option, + } +} type Edges = Vec<(ObjectKey, Vec)>; -/// `(names, upstream, downstream, tags, paths)` in Python's dict order. -type IndexesRow = ( - Vec<(String, ObjectKey)>, - Edges, - Edges, - Vec<(String, Vec)>, - Vec<(ObjectKey, String)>, -); +positional_record! { + extract, into; + /// `(names, upstream, downstream, tags, paths)` in Python's dict order. + struct IndexesRow { + names: Vec<(String, ObjectKey)>, + upstream: Edges, + downstream: Edges, + tags: Vec<(String, Vec)>, + paths: Vec<(ObjectKey, String)>, + } +} /// `(unique id, neighbour unique ids)` in graph order. type UniqueIdMap = Vec<(String, Vec)>; /// `PlannerInputError`'s `(code, message, help)`. @@ -167,12 +178,19 @@ pub(crate) fn project_graph(resources: Vec) -> PyResult = resources .into_iter() - .map(|(key, deps, tags, folder)| GraphResource { - key, - deps, - tags, - folder, - }) + .map( + |ResourceRow { + key, + deps, + tags, + folder, + }| GraphResource { + key, + deps, + tags, + folder, + }, + ) .collect(); Ok(NativeProjectGraph::new(build_project_graph(&resources))) }) @@ -190,7 +208,13 @@ impl NativeProjectGraph { #[staticmethod] fn from_indexes(indexes: IndexesRow) -> PyResult { compiler_guard(|| { - let (names, upstream, downstream, tags, paths) = indexes; + let IndexesRow { + names, + upstream, + downstream, + tags, + paths, + } = indexes; Ok(Self::new(graph_from_indexes(GraphIndexes { names, upstream, @@ -204,13 +228,13 @@ impl NativeProjectGraph { /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. fn indexes(&self) -> IndexesRow { let graph: &ProjectGraph = &self.graph; - ( - graph.names.clone(), - graph.upstream.clone(), - graph.downstream.clone(), - graph.tags.clone(), - graph.paths.clone(), - ) + IndexesRow { + names: graph.names.clone(), + upstream: graph.upstream.clone(), + downstream: graph.downstream.clone(), + tags: graph.tags.clone(), + paths: graph.paths.clone(), + } } /// The key's direct upstream lineage keys, in authored dependency order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 44539a45cf..63aa2fd6ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -24,16 +24,28 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::ErrorId; type NameRows = Vec<(String, Vec)>; -/// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. -type InventoryRow = (String, Vec, String, Vec); -/// `(edges, upstream, downstream, tags, paths)` in Python's dict order. -type IndexesRow = ( - Vec<(String, String)>, - NameRows, - NameRows, - NameRows, - Vec<(String, String)>, -); +positional_record! { + extract; + /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. + struct InventoryRow { + name: String, + tags: Vec, + relative_path: String, + upstream: Vec, + } +} + +positional_record! { + into; + /// `(edges, upstream, downstream, tags, paths)` in Python's dict order. + struct IndexesRow { + edges: Vec<(String, String)>, + upstream: NameRows, + downstream: NameRows, + tags: NameRows, + paths: Vec<(String, String)>, + } +} /// The `PlannerInputError` Python raises for a selector failure. fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { @@ -189,12 +201,19 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { compiler_guard(|| { let nodes: Vec = nodes .into_iter() - .map(|(name, tags, relative_path, upstream)| InventoryNode { - name, - tags, - relative_path, - upstream, - }) + .map( + |InventoryRow { + name, + tags, + relative_path, + upstream, + }| InventoryNode { + name, + tags, + relative_path, + upstream, + }, + ) .collect(); let InventoryIndexes { edges, @@ -203,7 +222,13 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { tags, paths, } = graph_indexes(&nodes); - Ok((edges, upstream, downstream, tags, paths)) + Ok(IndexesRow { + edges, + upstream, + downstream, + tags, + paths, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index c950f2c2f7..d2b894de74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,15 +1,19 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. -use crate::bindings::_helpers::analysis::binding_locations::locate_model_binding_diagnostics; +use crate::bindings::_helpers::analysis::binding_locations::{ + ExpansionRow, LocatedRow, locate_model_binding_diagnostics, +}; use crate::bindings::_helpers::analysis_session::outcome_objects::dynamic_contract_proof; use crate::bindings::_helpers::analysis_session::outcome_objects::session_outcome_analysis; use crate::bindings::_helpers::records::construct::{enum_from_value, enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::models::Positional; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; type Object<'py> = Bound<'py, PyAny>; @@ -264,53 +268,43 @@ fn located<'py>( )? .into_any() }; - let located: Object<'py> = locate_model_binding_diagnostics( - PyList::new(py, rows)?.extract::>()?, + let located: Vec = locate_model_binding_diagnostics( + PyList::new(py, rows)?.extract::>>()?, texts.extract::<(String, String, String)>()?, - expansion_row.extract::>)>>()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + expansion_row.extract::()?, + )?; let diagnostic_class: Object<'py> = record_class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = record_class(py, RecordId::SourceLocation)?; let phase: Object<'py> = enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?; let model: Object<'py> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; let name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); - for row in located.try_iter()? { - let (code, message, severity, line, column, span): ( - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - ) = row?.extract()?; - let location: Object<'py> = if span.is_none() { - span - } else { - call( + for row in located { + let location: Object<'py> = match row.span { + None => py.None().into_bound(py), + Some(Positional(span)) => call( &location_class, &[ ("path", path.clone()), - ("line", span.get_item(0)?), - ("column", span.get_item(1)?), - ("end_line", span.get_item(2)?), - ("end_column", span.get_item(3)?), + ("line", span.line.into_bound_py_any(py)?), + ("column", span.column.into_bound_py_any(py)?), + ("end_line", span.end_line.into_bound_py_any(py)?), + ("end_column", span.end_column.into_bound_py_any(py)?), ], - )? + )?, }; built.push(call( &diagnostic_class, &[ ("phase", phase.clone()), - ("severity", severity), - ("code", code), - ("message", message), + ("severity", row.severity.into_bound_py_any(py)?), + ("code", row.code.into_bound_py_any(py)?), + ("message", row.message.into_bound_py_any(py)?), ("resource_type", model.clone()), ("resource_name", name.clone()), ("path", path.clone()), - ("line", line), - ("column", column), + ("line", row.line.into_bound_py_any(py)?), + ("column", row.column.into_bound_py_any(py)?), ("location", location), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index 91df11f629..e588c0c463 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -58,6 +58,17 @@ enum EdgeParts { }, } +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: i64, + confidence_code: i64, + sources: Bound<'py, PyAny>, + } +} + /// Indexed edge records, materialised edges shared between equal records. struct EdgeRecords { keys: Vec, @@ -442,12 +453,12 @@ impl RecordBuilder { || native_field(columns, "resource_name_indexes")?.is_truthy()?; let mut members = CodeMembers::default(); for row in native_field(columns, "rows")?.try_iter()? { - let (name_index, transform_code, confidence_code, sources): ( - Bound<'_, PyAny>, - i64, - i64, - Bound<'_, PyAny>, - ) = row?.extract()?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row?.extract()?; let output_column: String = pool.get_item(name_index)?.extract()?; for source in sources.try_iter()? { let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs index a83d2f1dd9..e17daf4485 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs @@ -70,8 +70,27 @@ const SCHEMA_SQL: &str = " ); "; -type ResourceRow = (String, String, Option, Option); -type EdgeRow = (String, String, String, String); +positional_record! { + extract; + /// One cached resource: its type, name, relative path and qualified name. + struct ResourceRow { + resource_type: String, + name: String, + relative_path: Option, + qualified_name: Option, + } +} + +positional_record! { + extract; + /// One cached lineage edge: its upstream and downstream types and names. + struct EdgeRow { + upstream_type: String, + upstream_name: String, + downstream_type: String, + downstream_name: String, + } +} type TripleRow = (String, String, String); fn cache_path(project_dir: &Path) -> PathBuf { @@ -178,7 +197,13 @@ fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { return Ok(None); } let mut nodes: HashMap = HashMap::new(); - for (resource_type, name, relative_path, qualified_name) in resources { + for ResourceRow { + resource_type, + name, + relative_path, + qualified_name, + } in resources + { let key: LineageKey = LineageKey { resource_type, name, @@ -201,7 +226,13 @@ fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { }; let mut upstream: Groups = Groups::new(); let mut downstream: Groups = Groups::new(); - for (upstream_type, upstream_name, downstream_type, downstream_name) in edges { + for EdgeRow { + upstream_type, + upstream_name, + downstream_type, + downstream_name, + } in edges + { let (Some(parent), Some(child)) = ( known(upstream_type, upstream_name), known(downstream_type, downstream_name), @@ -379,12 +410,15 @@ fn write_index( "INSERT INTO lookups VALUES (?, ?, ?)", PyList::new(py, lookups)?, )?; - let mut edges: Vec<(&str, &str, &str, &str)> = Vec::new(); + let mut edges: Vec> = Vec::new(); for (child, parents) in &index.upstream { for parent in parents { let ((parent_type, parent_name), (child_type, child_name)) = (key_parts(parent), key_parts(child)); - edges.push((parent_type, parent_name, child_type, child_name)); + edges.push(PyTuple::new( + py, + [parent_type, parent_name, child_type, child_name], + )?); } } insert( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 49c73bacfc..a5fca8538c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -237,7 +237,7 @@ fn fast_lineage<'py>( if model.expands_star { let star_columns: Vec = outcomes .next() - .map(|outcome| outcome.into_parts().1) + .map(|outcome| outcome.into_parts().columns) .unwrap_or_default(); let mut expanded: Vec> = columns.try_iter()?.collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs index 1447ef485b..0255c9390e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs @@ -20,13 +20,16 @@ use crate::bindings::_helpers::records::fields::mapping_get_or; const DLT_ERROR: ThirdPartyHost = ThirdPartyHost::DltIntegrationError; -/// `(factory, arguments, options, dataset_name)` of one dlt destination. -type DestinationCall<'py> = ( - &'static str, - Bound<'py, PyDict>, - Bound<'py, PyDict>, - Bound<'py, PyAny>, -); +positional_record! { + into; + /// `(factory, arguments, options, dataset_name)` of one dlt destination. + struct DestinationCall<'py> { + factory: &'static str, + arguments: Bound<'py, PyDict>, + options: Bound<'py, PyDict>, + dataset: Bound<'py, PyAny>, + } +} /// A `DltIntegrationError`, or the failure to construct one. fn dlt_error(py: Python<'_>, message: String) -> PyErr { @@ -110,12 +113,12 @@ fn dlt_destination_arguments<'py>( DltDataset::Given => dataset_name.clone(), DltDataset::Default(name) => PyString::new(py, name).into_any(), }; - Ok(( - plan.factory, - arguments(&plan.arguments, connection_config)?, + Ok(DestinationCall { + factory: plan.factory, + arguments: arguments(&plan.arguments, connection_config)?, options, dataset, - )) + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 0db8c1a73f..06dd803e7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,10 +1,12 @@ +#[macro_use] +pub(crate) mod boundary; + pub(crate) mod analysis; pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod audit_command; pub(crate) mod auditing; pub(crate) mod authored_values; -pub(crate) mod boundary; pub(crate) mod build_command; pub(crate) mod build_progress; pub(crate) mod cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 68a0ba0b93..2e4d8740f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -24,17 +24,28 @@ const RETURNS_KEY: &str = "returns"; /// The only key of a table function's `returns` mapping. const TABLE_RETURNS_KEY: &str = "table"; -/// Model, seed, source, function and table function names. -type ResourceNames = ( - HashSet, - HashSet, - HashSet, - HashSet, - HashSet, -); - -/// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. -type ModelFacts = (Vec<(String, String, bool)>, Option>, bool, bool); +positional_record! { + extract; + /// Model, seed, source, function and table function names. + struct ResourceNames { + models: HashSet, + seeds: HashSet, + sources: HashSet, + functions: HashSet, + table_functions: HashSet, + } +} + +positional_record! { + extract; + /// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. + struct ModelFacts { + references: Vec<(String, String, bool)>, + declared_columns: Option>, + retention_unmanaged: bool, + table_type_declared: bool, + } +} /// The project facts every model's validation reads, captured once per compile. #[pyclass(module = "sqlbuild._native", frozen)] @@ -83,7 +94,13 @@ fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { functions.insert(name); } functions.extend(each(inputs, "python_function_files", model_stem)?); - Ok((models, seeds, sources, functions, table_functions)) + Ok(ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + }) } /// Whether a SQL function's header `returns` is a mapping whose only key is `table`. @@ -126,7 +143,13 @@ impl NativeModelValidator { custom_materializations: HashSet, microbatch_concurrency: bool, ) -> PyResult { - let (models, seeds, sources, functions, table_functions) = names; + let ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + } = names; Ok(Self { project: ProjectValidationFacts { custom_materializations, @@ -150,7 +173,12 @@ impl NativeModelValidator { ) -> PyResult> { compiler_guard(|| { let (model_name, query_sql, relative_path) = model; - let (references, declared_columns, retention_unmanaged, table_type_declared) = facts; + let ModelFacts { + references, + declared_columns, + retention_unmanaged, + table_type_declared, + } = facts; let references: Vec = references .into_iter() .map(|(kind, name, externally_rejected)| ModelReference { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 64a3add7ec..f3d60591e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -443,13 +443,13 @@ fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = ( - String, - (String, String, String), - Bound<'py, PyAny>, - Bound<'py, PyAny>, -); +/// One provider usage: provider, `(kind, name, parameter)` consumer, annotation class and module. +struct Usage<'py> { + provider: String, + consumer: (String, String, String), + class_name: Bound<'py, PyAny>, + module: Bound<'py, PyAny>, +} fn providers_json<'py>( plan: &Bound<'py, PyAny>, @@ -458,37 +458,45 @@ fn providers_json<'py>( let py: Python<'py> = plan.py(); let mut usages: Vec> = Vec::new(); for usage in items(&native_field(plan, "provider_usages")?)? { - usages.push(( - text(&native_field(&usage, "provider_name")?)?, - ( + usages.push(Usage { + provider: text(&native_field(&usage, "provider_name")?)?, + consumer: ( text(&native_field(&usage, "consumer_kind")?)?, text(&native_field(&usage, "consumer_name")?)?, text(&native_field(&usage, "parameter_name")?)?, ), - native_field(&usage, "annotation_class_name")?, - native_field(&usage, "annotation_module")?, - )); + class_name: native_field(&usage, "annotation_class_name")?, + module: native_field(&usage, "annotation_module")?, + }); } for entry in python_entries { let kind: String = text(&enum_value(entry, "kind")?)?; let name: String = text(&native_field(entry, "name")?)?; for usage in items(&native_field(entry, "provider_usages")?)? { - usages.push(( - text(&native_field(&usage, "provider_name")?)?, - ( + usages.push(Usage { + provider: text(&native_field(&usage, "provider_name")?)?, + consumer: ( kind.clone(), name.clone(), text(&native_field(&usage, "parameter_name")?)?, ), - native_field(&usage, "annotation_class_name")?, - native_field(&usage, "annotation_module")?, - )); + class_name: native_field(&usage, "annotation_class_name")?, + module: native_field(&usage, "annotation_module")?, + }); } } - usages.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + usages.sort_by(|left, right| { + (&left.provider, &left.consumer).cmp(&(&right.provider, &right.consumer)) + }); let providers: Bound<'py, PyList> = PyList::empty(py); let mut current: Option<(String, Bound<'py, PyList>)> = None; - for (provider, (kind, name, parameter), class_name, module) in usages { + for Usage { + provider, + consumer: (kind, name, parameter), + class_name, + module, + } in usages + { if current.as_ref().is_none_or(|(open, _)| *open != provider) { let used_by: Bound<'py, PyList> = PyList::empty(py); providers.append(dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 61289d856c..d2e2734323 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -23,6 +23,9 @@ use crate::bindings::_helpers::planner_phases::identities::{ use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::bindings::_helpers::planner_phases::models::{ + PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, +}; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; @@ -158,19 +161,37 @@ fn planned_output<'py>( )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); let output = base_plan_output( - (runtime, &warehouse, &identities, overrides), - ( - &inspection_scope, - &reconciliation, + PlanOutputContext { + runtime, + warehouse: &warehouse, + identities: &identities, + overrides, + }, + PlannedModelEntries { + execution_scope: &inspection_scope, + reconciliation: &reconciliation, entries, - &source_freshness, - ), + source_freshness: &source_freshness, + }, &planned_sql_tests, )?; - let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; + let output = add_storage_policies( + output, + StoragePolicyContext { + runtime, + warehouse: &warehouse, + scopes, + policies: &policies, + }, + )?; let output = add_plan_warnings( output, - (scopes, &warehouse, &identities, &stale_warning_changes), + PlanWarningContext { + scopes, + warehouse: &warehouse, + identities: &identities, + stale_warning_changes: &stale_warning_changes, + }, (&source_freshness, &policies), )?; let plan_output = add_plan_metadata(output, &source_freshness, &policies)?.build(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index d97d516142..35ebf4b34e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -142,11 +142,11 @@ pub(crate) fn build_execution_upstream_deps<'py>( let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; let cache: KeyCache<'py> = KeyCache::new(py)?; let mut extra: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for (_, _, gated, read) in &gates { - extra.push((cache.key(gated)?, cache.key(read)?)); + for gate in &gates { + extra.push((cache.key(&gate.gated)?, cache.key(&gate.read)?)); } - for (_, gated, read, _, _) in &hooks { - extra.push((cache.key(gated)?, cache.key(read)?)); + for hook in &hooks { + extra.push((cache.key(&hook.gated)?, cache.key(&hook.read)?)); } for (gated, read) in extra { let gated_upstream: Bound<'py, PyAny> = @@ -192,7 +192,13 @@ pub(crate) fn build_execution_edge_origins<'py>( .call_method1("upstream", (gated.clone(),))? .contains(read.clone()) }; - for (audit_name, target, gated, read) in &gates { + for GateRow { + audit_name, + target, + gated, + read, + } in &gates + { if is_lineage(gated, read)? { continue; } @@ -204,7 +210,14 @@ pub(crate) fn build_execution_edge_origins<'py>( ), )?; } - for (hook_name, gated, read, kind, _) in &hooks { + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + .. + } in &hooks + { if is_lineage(gated, read)? { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index dcb5eee1d3..ced5ea9dd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -26,6 +26,42 @@ pub(crate) struct ModelEntryResults<'py> { pub(crate) warnings: Vec>, } +/// The runtime, warehouse state, identities and overrides the base plan output reads. +#[derive(Debug)] +pub(crate) struct PlanOutputContext<'a, 'py> { + pub(crate) runtime: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) identities: &'a Bound<'py, PyAny>, + pub(crate) overrides: &'a Bound<'py, PyAny>, +} + +/// The execution scope, reconciled changes, planned entries and source freshness of a plan. +#[derive(Debug)] +pub(crate) struct PlannedModelEntries<'a, 'py> { + pub(crate) execution_scope: &'a Bound<'py, PyAny>, + pub(crate) reconciliation: &'a ChangeReconciliation<'py>, + pub(crate) entries: ModelEntryResults<'py>, + pub(crate) source_freshness: &'a Bound<'py, PyAny>, +} + +/// The runtime, warehouse state, scopes and policies storage policy planning reads. +#[derive(Debug)] +pub(crate) struct StoragePolicyContext<'a, 'py> { + pub(crate) runtime: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) scopes: &'a Bound<'py, PyAny>, + pub(crate) policies: &'a Bound<'py, PyAny>, +} + +/// The scopes, warehouse state, identities and changes stale-selection warnings read. +#[derive(Debug)] +pub(crate) struct PlanWarningContext<'a, 'py> { + pub(crate) scopes: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) identities: &'a Bound<'py, PyAny>, + pub(crate) stale_warning_changes: &'a Bound<'py, PyAny>, +} + /// Resolved execution environment shared by planner phases. #[pyclass(module = "sqlbuild._native", frozen, get_all)] #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 0dfe501080..71e3e09c26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -10,7 +10,8 @@ use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ - ChangeReconciliation, ModelEntryResults, PlanOutputFields, + PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, + StoragePolicyContext, }; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ @@ -69,18 +70,18 @@ impl<'py> PlanOutputFields<'py> { /// The plan output's base fields from the planned entries, relations and source freshness. pub(crate) fn base_plan_output<'py>( - (runtime, warehouse, identities, overrides): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), - (execution_scope, reconciliation, entries, source_freshness): ( - &Bound<'py, PyAny>, - &ChangeReconciliation<'py>, - ModelEntryResults<'py>, - &Bound<'py, PyAny>, - ), + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }: PlanOutputContext<'_, 'py>, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }: PlannedModelEntries<'_, 'py>, planned_sql_tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); @@ -163,12 +164,12 @@ pub(crate) fn with_column_rename_hints<'py>( /// Add migration fingerprints, rename hints, migration, table-type and retention entries. pub(crate) fn add_storage_policies<'py>( mut output: PlanOutputFields<'py>, - (runtime, warehouse, scopes, policies): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }: StoragePolicyContext<'_, 'py>, ) -> PyResult> { let py = runtime.py(); let mut entries = output.field("model_entries")?; @@ -242,12 +243,12 @@ pub(crate) fn add_storage_policies<'py>( /// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. pub(crate) fn add_plan_warnings<'py>( mut output: PlanOutputFields<'py>, - (scopes, warehouse, identities, stale_warning_changes): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }: PlanWarningContext<'_, 'py>, (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = scopes.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 71d74dd497..bd33f206d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -6,16 +6,17 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ - FixtureReadKey, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, - FixtureStarRelation, + FixtureReadKey, FixtureReadModel, FixtureReadReference, FixtureReadRequirements, + FixtureReadSchemas, FixtureStarRelation, }; -use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; +use crate::bindings::_helpers::analysis::model_reads::{ + LineageFactRow, lineage_facts, proven_inputs, star_flags, +}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; @@ -63,12 +64,12 @@ pub(crate) fn required_fixture_columns( let mut references: Vec = Vec::new(); for reference in native_field(&model, "references")?.try_iter()? { let reference = reference?; - references.push(( - native_field(&reference, "ref_kind")?.extract()?, - native_field(&reference, "ref_name")?.extract()?, - native_field(&reference, "ref_package")?.extract()?, - native_field(&reference, "call_argument_count")?.extract()?, - )); + references.push(FixtureReadReference { + kind: native_field(&reference, "ref_kind")?.extract()?, + name: native_field(&reference, "ref_name")?.extract()?, + package: native_field(&reference, "ref_package")?.extract()?, + argument_count: native_field(&reference, "call_argument_count")?.extract()?, + }); } models.push(FixtureReadModel { name: name.clone(), @@ -102,7 +103,12 @@ pub(crate) fn required_fixture_columns( pub(crate) fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let mut reads: Vec = Vec::new(); - for (_, _, confidence, sources) in lineage_facts(model)?.unwrap_or_default() { + for LineageFactRow { + confidence, + sources, + .. + } in lineage_facts(model)?.unwrap_or_default() + { if confidence != HIGH_CONFIDENCE { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index 3a6a1dd758..1fe436cfdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -9,7 +9,7 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::analysis::model_reads::lineage_facts; +use crate::bindings::_helpers::analysis::model_reads::{LineageFactRow, lineage_facts}; use crate::bindings::_helpers::planner_strategy::constants::{ DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, }; @@ -88,7 +88,13 @@ pub(crate) fn expected_fixture_types<'a>( pub(crate) fn projections(model: &Bound<'_, PyAny>) -> PyResult> { let mut projections: Vec = Vec::new(); - for (output_column, transform, _, sources) in lineage_facts(model)?.unwrap_or_default() { + for LineageFactRow { + output_column, + transform, + sources, + .. + } in lineage_facts(model)?.unwrap_or_default() + { if transform != DIRECT_TRANSFORM { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 4059788791..16afa8f495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -31,7 +31,14 @@ const INSERT_SQL: &str = "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; -type MemoKey = (String, String, Option, Vec<(String, String)>); +/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. +#[derive(Hash, PartialEq, Eq)] +struct MemoKey { + query_sql: String, + metadata_json: String, + dialect: Option, + visible_renames: Vec<(String, String)>, +} static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); @@ -160,10 +167,10 @@ impl MigrationFingerprintCache { } fingerprint_cache_key(&FingerprintCacheKey { shared_digest: self.shared_digest.clone(), - query_sql: key.0.clone(), - metadata_json: key.1.clone(), - dialect: key.2.clone(), - visible_renames: key.3.clone(), + query_sql: key.query_sql.clone(), + metadata_json: key.metadata_json.clone(), + dialect: key.dialect.clone(), + visible_renames: key.visible_renames.clone(), }) .map_err(|error| PyValueError::new_err(format!("{error:?}"))) } @@ -339,7 +346,12 @@ impl MigrationFingerprintCache { visible.push((name, identity)); } } - let key: MemoKey = (query_sql, metadata_json, dialect, visible); + let key: MemoKey = MemoKey { + query_sql, + metadata_json, + dialect, + visible_renames: visible, + }; if let Some(found) = self.fingerprints.get(&key) { return Ok(found.clone()); } @@ -348,9 +360,14 @@ impl MigrationFingerprintCache { Some(stored) => stored, None => { count_computation(py, &key)?; - let identities: BTreeMap = key.3.iter().cloned().collect(); - let computed: Option = - fingerprint(&key.0, &key.1, &identities, key.2.as_deref())?; + let identities: BTreeMap = + key.visible_renames.iter().cloned().collect(); + let computed: Option = fingerprint( + &key.query_sql, + &key.metadata_json, + &identities, + key.dialect.as_deref(), + )?; self.remember(&cache_key, computed.as_deref())?; computed } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index effacd0eb4..f308830cb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -27,13 +27,13 @@ const UNKNOWN_MEMBER: &str = "UNKNOWN"; /// `str(None)`, the text of an exact declaration without a type. const NONE_TEXT: &str = "None"; -/// The three per-relation declaration maps the analysis starts from. -pub(crate) type DeclaredRelations<'py> = ( - Bound<'py, PyDict>, - Bound<'py, PyDict>, - Bound<'py, PyDict>, - DeclaredShapes, -); +/// The three per-relation declaration maps the analysis starts from, and the declared shapes. +pub(crate) struct DeclaredRelations<'py> { + pub(crate) column_types: Bound<'py, PyDict>, + pub(crate) nullability: Bound<'py, PyDict>, + pub(crate) complete_schemas: Bound<'py, PyDict>, + pub(crate) declared: DeclaredShapes, +} /// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. #[derive(Debug, Clone, Default)] @@ -327,5 +327,10 @@ pub(crate) fn declared_relations<'py>( declared.complete(&maps, &name, (&columns, true))?; } let (types, nulls, complete) = maps; - Ok((types, nulls, complete, declared.shapes)) + Ok(DeclaredRelations { + column_types: types, + nullability: nulls, + complete_schemas: complete, + declared: declared.shapes, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 267a7b8c1d..0fceed1307 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, @@ -43,13 +44,42 @@ const CONTEXT_READ: &str = "ctx"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; /// `(kind, name, package)`. -type ReferenceRow = (String, String, Option); -/// `(name, managed, database, schema)`. -type SourceRow = (String, bool, Option, Option); -/// `(kind, name, tags, references)`. -type FunctionRow = (String, String, Vec, Vec); +type ResourceReferenceRow = (String, String, Option); /// `(references, attached (kind, name))`. -type AuditRow = (Vec, Option<(String, String)>); +type AuditRow = (Vec, Option<(String, String)>); + +positional_record! { + extract; + /// `(name, managed, database, schema)`. + struct SourceRow { + name: String, + managed: bool, + database: Option, + schema: Option, + } +} + +positional_record! { + extract; + /// `(kind, name, tags, references)`. + struct FunctionRow { + kind: String, + name: String, + tags: Vec, + references: Vec, + } +} + +positional_record! { + into; + /// `(database, schema, logical database, logical schema)` of one seed. + struct NamespaceRow { + database: Option, + schema: Option, + logical_database: Option, + logical_schema: Option, + } +} /// The request's named variables and the Python exceptions opaque ones raise, by index. type RequestVariables<'py> = (Vec<(String, Variable)>, Vec>); @@ -143,7 +173,7 @@ fn environment_rows( .collect() } -fn references(rows: Vec) -> Vec { +fn references(rows: Vec) -> Vec { rows.into_iter() .map(|(kind, name, package)| Reference { kind, @@ -166,9 +196,16 @@ fn tags(value: &Bound<'_, PyAny>) -> PyResult> { fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { let model_file: Bound<'_, PyAny> = native_field(model_input, "model_file")?; - let reference_rows: Vec = input_reference_rows(model_input)? + let reference_rows: Vec = input_reference_rows(model_input)? .into_iter() - .map(|(kind, name, package, _)| (kind, name, package)) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind, name, package), + ) .collect(); let mut syntax_checks: Vec = Vec::new(); for check in checks.try_iter()? { @@ -280,7 +317,12 @@ fn project_request<'py>( let sources: Vec = rows("source_inputs")? .into_iter() .map(|row| { - let (name, managed, database, schema): SourceRow = row.extract()?; + let SourceRow { + name, + managed, + database, + schema, + }: SourceRow = row.extract()?; Ok(SourceFacts { name, managed, @@ -292,7 +334,12 @@ fn project_request<'py>( let functions: Vec = rows("sql_function_inputs")? .into_iter() .map(|row| { - let (kind, name, tags, reference_rows): FunctionRow = row.extract()?; + let FunctionRow { + kind, + name, + tags, + references: reference_rows, + }: FunctionRow = row.extract()?; Ok(FunctionFacts { kind, name, @@ -383,20 +430,13 @@ fn failure_object<'py>( .into_any()) } -fn namespace_row( - namespace: Namespace, -) -> ( - Option, - Option, - Option, - Option, -) { - ( - namespace.database, - namespace.schema, - namespace.logical_database, - namespace.logical_schema, - ) +fn namespace_row(namespace: Namespace) -> NamespaceRow { + NamespaceRow { + database: namespace.database, + schema: namespace.schema, + logical_database: namespace.logical_database, + logical_schema: namespace.logical_schema, + } } /// `NativeProjectResources` for the native result, replaying its template reads first. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs index bbba79328d..32f93faf4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs @@ -70,16 +70,27 @@ pub(crate) fn facts_resource_name( Ok(resource_name(instance, &index)?.unbind()) } +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: usize, + confidence_code: usize, + sources: Bound<'py, PyAny>, + } +} + /// One compact row projected to a `CompiledLineageColumnFact`. fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = facts.py(); let pool: Bound<'_, PyAny> = native_field(facts, "string_pool")?; - let (name_index, transform_code, confidence_code, sources): ( - Bound<'_, PyAny>, - usize, - usize, - Bound<'_, PyAny>, - ) = row.extract()?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row.extract()?; let mut upstream: Vec> = Vec::new(); for source in sources.try_iter()? { let source: Bound<'_, PyTuple> = source?.cast_into::()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index 7e6704149a..a2fe347ae6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -131,12 +131,12 @@ fn resource_rows( for resource in resources.try_iter()? { let resource: Bound<'_, PyAny> = resource?; let (tags, folder) = describe(&resource)?; - rows.push(( - graph_key(&native_field(&resource, "key")?)?, - graph_keys(&native_field(&resource, "deps")?)?, + rows.push(ResourceRow { + key: graph_key(&native_field(&resource, "key")?)?, + deps: graph_keys(&native_field(&resource, "deps")?)?, tags, folder, - )); + }); } Ok(rows) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index df38754cff..666f5985c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -196,27 +196,61 @@ fn block_bodies(blocks: &Bound<'_, PyAny>) -> PyResult> { .collect() } +/// One discovered file group: its role, attribute, body field and whether it has blocks. +struct FileGroup { + role: SqlFileRole, + attribute: &'static str, + body: Option<&'static str>, + blocks: bool, +} + fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { - let groups: [(SqlFileRole, &str, Option<&str>, bool); 6] = [ - (SqlFileRole::Test, "test_files", None, true), - ( - SqlFileRole::Scenario, - "scenario_files", - Some("sql_body"), - false, - ), - (SqlFileRole::Audit, "audit_files", None, true), - (SqlFileRole::Hook, "sql_hook_files", Some("sql_body"), false), - ( - SqlFileRole::Function, - "sql_function_files", - Some("body_sql"), - false, - ), - (SqlFileRole::Schema, "model_schema_files", None, false), + let groups: [FileGroup; 6] = [ + FileGroup { + role: SqlFileRole::Test, + attribute: "test_files", + body: None, + blocks: true, + }, + FileGroup { + role: SqlFileRole::Scenario, + attribute: "scenario_files", + body: Some("sql_body"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Audit, + attribute: "audit_files", + body: None, + blocks: true, + }, + FileGroup { + role: SqlFileRole::Hook, + attribute: "sql_hook_files", + body: Some("sql_body"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Function, + attribute: "sql_function_files", + body: Some("body_sql"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Schema, + attribute: "model_schema_files", + body: None, + blocks: false, + }, ]; let mut files: Vec = Vec::new(); - for (role, attribute, body, blocks) in groups { + for FileGroup { + role, + attribute, + body, + blocks, + } in groups + { for item in native_field(discovered, attribute)?.try_iter()? { let item: Bound<'_, PyAny> = item?; let texts: Vec = match (body, blocks) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 498e99529b..2f52c18305 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -1,7 +1,7 @@ //! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, + Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -16,15 +16,18 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, expand_declarations, span_rows, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; +use crate::bindings::_helpers::sqltext::static_sql::{ + InterpolationRow, InterpolationSources, interpolation_batch, +}; const DECLARATION_FIELDS: [&str; 4] = [ "enums", @@ -53,7 +56,13 @@ fn substituted<'py>( let text: String = sql.extract()?; let path: String = file_path.str()?.extract()?; let rows = interpolation_batch(sources, &[(text.clone(), path)])?; - let Some((rendered, spans, reads, error)) = rows.into_iter().next() else { + let Some(InterpolationRow { + sql: rendered, + spans, + reads, + error, + }) = rows.into_iter().next() + else { return Ok((text, Vec::new())); }; record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; @@ -69,7 +78,14 @@ fn substituted<'py>( fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult> { let objects: Vec> = spans .iter() - .map(|span| new_record(py, RecordId::ExpansionSpan, *span, None)) + .map(|span| { + new_record( + py, + RecordId::ExpansionSpan, + IntoPyObject::into_pyobject(*span, py)?, + None, + ) + }) .collect::>()?; PyTuple::new(py, objects) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 57f1fa829d..57a9ea530c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -1,5 +1,6 @@ //! Enum and constant references expanded natively, for model queries and every other caller. +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use pyo3::FromPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; @@ -12,6 +13,7 @@ use sqlbuild_analysis::compiled_project::models::{ }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; const ENUM_REFERENCE_KIND: u8 = 0; const ENUM_KIND: &str = "enum"; @@ -39,11 +41,6 @@ const THROUGH: &str = "through"; const CONSUMER: &str = "consumer"; const ENUM_MEMBER: &str = "enum_member"; -/// `(kind, name, member, start, end)` for one natively scanned reference. -pub(crate) type ScannedReference = (u8, String, Option, usize, usize); -/// `(source start, source end, output start, output end)`. -pub(crate) type SpanRow = (usize, usize, usize, usize); - /// The Python classes expansion builds usages and errors from. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -82,7 +79,14 @@ pub(crate) fn expand_declarations<'py>( ) -> PyResult> { let mut splices: Vec = Vec::with_capacity(request.references.len()); let mut usages: Vec> = Vec::new(); - for (kind, name, member, start, end) in request.references { + for ScannedReference { + kind, + name, + member, + start, + end, + } in request.references + { let (replacement, visibility) = if *kind == ENUM_REFERENCE_KIND { enum_text(request, name, member.as_deref())? } else { @@ -281,13 +285,11 @@ pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { spliced .spans .iter() - .map(|span| { - ( - span.source_start, - span.source_end, - span.output_start, - span.output_end, - ) + .map(|span| SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs index 33941bf858..fb1a6f7986 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs @@ -1,28 +1,71 @@ //! Python rows of macro call sites and splice spans. +use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; +use sqlbuild_sqltext::compiler::models::CharSpan; -/// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. -pub(crate) type SiteRow = (usize, usize, usize, String, Option>, bool); -/// One substitution span: source start and end, output start and end. -pub(crate) type SpanRow = (usize, usize, usize, usize); +positional_record! { + into; + /// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. + pub(crate) struct SiteRow { + start: usize, + args_start: usize, + end: usize, + name: String, + tree_names: Option>, + typed_reference_text: bool, + } +} +positional_record! { + extract, into; + /// One substitution span: source start and end, output start and end. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) struct SpanRow { + pub(crate) source_start: usize, + pub(crate) source_end: usize, + pub(crate) output_start: usize, + pub(crate) output_end: usize, + } +} + +impl From for SpanRow { + fn from(span: CharSpan) -> Self { + Self { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } + } +} + +impl From for Expansion { + fn from(span: SpanRow) -> Self { + Self { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } + } +} pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { - ( - site.start, - site.args_start, - site.end, - site.name, - site.tree_names, - site.typed_reference_text, - ) + SiteRow { + start: site.start, + args_start: site.args_start, + end: site.end, + name: site.name, + tree_names: site.tree_names, + typed_reference_text: site.typed_reference_text, + } } pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { - ( - span.source_start, - span.source_end, - span.output_start, - span.output_end, - ) + SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs index c44bc2db8f..c605e8c5ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -33,14 +33,17 @@ const RELATION_SUFFIX: &str = "__"; const CONSTANT_KIND: &str = "constant"; const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; -/// `(sql, spans, argument references, dependencies, usages)` of one expansion. -type ExpansionRow<'py> = ( - String, - Vec, - Vec, - Vec>, - Vec>, -); +positional_record! { + into; + /// `(sql, spans, argument references, dependencies, usages)` of one expansion. + struct ExpansionRow { + sql: String, + spans: Vec, + argument_references: Vec, + dependencies: Vec>, + usages: Vec>, + } +} /// The Python classes, enums and helpers an expansion builds facts, errors and contexts from. #[derive(FromPyObject)] @@ -87,8 +90,17 @@ pub(crate) struct NativeMacroExpansion { pub(crate) facts: RefCell, } -/// The `(macros, overrides, context, declarations, consumer)` one expansion reads. -type StateInput = (Py, Py, Py, Py, Py); +positional_record! { + extract; + /// The `(macros, overrides, context, declarations, consumer)` one expansion reads. + pub(crate) struct StateInput { + pub(crate) loaded_macros: Py, + pub(crate) overrides: Py, + pub(crate) macro_context: Py, + pub(crate) declarations: Py, + pub(crate) consumer: Py, + } +} #[pymethods] impl NativeMacroExpansion { @@ -99,7 +111,13 @@ impl NativeMacroExpansion { enforce_explicit: bool, classes: Py, ) -> PyResult { - let (loaded_macros, overrides, macro_context, declarations, consumer) = state; + let StateInput { + loaded_macros, + overrides, + macro_context, + declarations, + consumer, + } = state; Ok(Self { file_path: file_path.clone().unbind(), file_label: file_path.str()?.to_string(), @@ -119,15 +137,15 @@ impl NativeMacroExpansion { slf: &Bound<'py, Self>, sql: &Bound<'py, PyString>, bridge: &Bound<'py, PyAny>, - ) -> PyResult> { + ) -> PyResult { let output: ExpansionOutput = expanded(slf, sql.to_str()?, bridge)?; - Ok(( - output.sql, - output.spans.into_iter().map(span_row).collect(), - output.argument_references, - output.dependencies, - output.usages, - )) + Ok(ExpansionRow { + sql: output.sql, + spans: output.spans.into_iter().map(span_row).collect(), + argument_references: output.argument_references, + dependencies: output.dependencies, + usages: output.usages, + }) } /// `render_relation`: an approved call-site reference's placeholder, else its reference call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 5bdfb03605..56347e672a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -19,9 +19,9 @@ use crate::bindings::_helpers::records::construct::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, expanded, + ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::sql_references::extract_references; @@ -189,16 +189,16 @@ pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult(request: ExpansionRequest<'py>) -> PyResult< }; let mut spans: Vec> = Vec::with_capacity(output.spans.len()); for span in output.spans { - let (source_start, source_end, output_start, output_end) = span_row(span); + let SpanRow { + source_start, + source_end, + output_start, + output_end, + } = span_row(span); spans.push(new_record_with( py, RecordId::ExpansionSpan, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs index d19447b762..215f7a4a2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs @@ -14,20 +14,44 @@ use sqlbuild_pysource::macro_modules::types::MacroScopeKind; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::types::CompilerDetach; -/// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. -type SourceRow = ( - String, - String, - Vec, - Option>, - Option>, - String, - String, - Option, -); - -/// `(name, file index, parameters, line, source digest, dependency macro names)`. -type ExportRow = (String, usize, Vec, usize, String, Vec); +positional_record! { + extract; + /// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. + pub(crate) struct SourceRow { + display: String, + posix: String, + parts: Vec, + declaration_root: Option>, + owning_path: Option>, + scope_kind: String, + contents: String, + syntax_error: Option, + } +} + +positional_record! { + into; + /// `(name, file index, parameters, line, source digest, dependency macro names)`. + pub(crate) struct ExportRow { + name: String, + file_index: usize, + parameters: Vec, + line: usize, + source_digest: String, + dependencies: Vec, + } +} + +positional_record! { + extract; + /// `(name, path, contents, dependencies)` of one macro module to load. + struct ModuleSpecRow { + name: String, + file_path: String, + contents: String, + dependencies: Vec, + } +} /// `(file index, message)`. type FaultRow = (usize, String); @@ -49,8 +73,16 @@ fn scope_kind_of(value: &str) -> MacroScopeKind { } fn source(row: SourceRow) -> MacroModuleSource { - let (display, posix, parts, declaration_root, owning_path, scope_kind, contents, syntax_error) = - row; + let SourceRow { + display, + posix, + parts, + declaration_root, + owning_path, + scope_kind, + contents, + syntax_error, + } = row; MacroModuleSource { relative_path: MacroModulePath { display, @@ -66,14 +98,14 @@ fn source(row: SourceRow) -> MacroModuleSource { } fn export_row(export: MacroExport) -> ExportRow { - ( - export.name, - export.file_index, - export.parameters, - export.line, - export.source_digest, - export.dependencies, - ) + ExportRow { + name: export.name, + file_index: export.file_index, + parameters: export.parameters, + line: export.line, + source_digest: export.source_digest, + dependencies: export.dependencies, + } } fn fault_row(fault: MacroFault) -> FaultRow { @@ -112,13 +144,16 @@ pub(crate) fn inventory_macro_modules( /// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. #[pyfunction] -fn macro_module_load_plan( - modules: Vec<(String, String, String, Vec)>, -) -> (String, Vec, Vec) { +fn macro_module_load_plan(modules: Vec) -> (String, Vec, Vec) { let specs: Vec = modules .into_iter() .map( - |(name, file_path, contents, dependencies)| MacroModuleSpec { + |ModuleSpecRow { + name, + file_path, + contents, + dependencies, + }| MacroModuleSpec { name, file_path, contents, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index 52734f0837..d3a30786e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -102,13 +102,24 @@ pub(crate) fn custom_rule_implementation_fingerprint( Ok(implementation_fingerprint(&source, &closure, &root)?) } +positional_record! { + into; + /// `(path, line, column, owner)` of one public-harness case. + struct EvidenceRow { + path: String, + line: usize, + column: usize, + owner: String, + } +} + /// `(path, line, column, owner)` of each public-harness case that statically evaluates the rule. #[pyfunction] #[pyo3(signature = (*, rule, project_dir))] fn custom_rule_test_evidence<'py>( rule: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py: Python<'py> = rule.py(); let Some(source) = resolved_source(rule)? else { return Ok(Vec::new()); @@ -128,7 +139,12 @@ fn custom_rule_test_evidence<'py>( }; Ok(evidence .into_iter() - .map(|item: RuleTestEvidence| (item.path, item.line, item.column, item.owner)) + .map(|item: RuleTestEvidence| EvidenceRow { + path: item.path, + line: item.line, + column: item.column, + owner: item.owner, + }) .collect()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 03d70f111e..932c2a997c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -461,12 +461,12 @@ impl<'py> FixSession<'py> { .map(|(path, reason)| { violation( &path, - ( - "format-fix-skipped", - &format!("Rule fixes skipped this file: {}", reason.str()?), - "warning", - "Fix the reported problem, then rerun `sqb format --fix`.", - ), + ViolationDetails { + code: "format-fix-skipped", + message: &format!("Rule fixes skipped this file: {}", reason.str()?), + severity: "warning", + remediation: "Fix the reported problem, then rerun `sqb format --fix`.", + }, ) }) .collect() @@ -681,9 +681,17 @@ fn refused<'py>(report: &Bound<'py, PyAny>, reason: &str) -> PyResult { + code: &'a str, + message: &'a str, + severity: &'a str, + remediation: &'a str, +} + fn violation<'py>( path: &Bound<'py, PyAny>, - details: (&str, &str, &str, &str), + details: ViolationDetails<'_>, ) -> PyResult> { let py = path.py(); new_record_with( @@ -693,11 +701,14 @@ fn violation<'py>( ("file_path", path.clone()), ("line", 1_i64.into_pyobject(py)?.into_any()), ("column", 1_i64.into_pyobject(py)?.into_any()), - ("code", details.0.into_pyobject(py)?.into_any()), - ("message", details.1.into_pyobject(py)?.into_any()), - ("severity", details.2.into_pyobject(py)?.into_any()), + ("code", details.code.into_pyobject(py)?.into_any()), + ("message", details.message.into_pyobject(py)?.into_any()), + ("severity", details.severity.into_pyobject(py)?.into_any()), ("engine", "native".into_pyobject(py)?.into_any()), - ("remediation", details.3.into_pyobject(py)?.into_any()), + ( + "remediation", + details.remediation.into_pyobject(py)?.into_any(), + ), ], ) } @@ -705,12 +716,12 @@ fn violation<'py>( fn verification_fault<'py>(path: &Bound<'py, PyAny>, reason: &str) -> PyResult> { violation( path, - ( - "format-fix-verification-failed", - reason, - "fault", - "Restructure the SQL manually and recompile; the original file was retained.", - ), + ViolationDetails { + code: "format-fix-verification-failed", + message: reason, + severity: "fault", + remediation: "Restructure the SQL manually and recompile; the original file was retained.", + }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs index d58ccb5322..f1449ca5cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs @@ -6,7 +6,9 @@ use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_rules::fix_verification::main::compare::compare; -use sqlbuild_rules::fix_verification::models::{DiagnosticComparison, ModelComparison}; +use sqlbuild_rules::fix_verification::models::{ + DiagnosticComparison, DiagnosticIdentity, ModelComparison, +}; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; @@ -107,8 +109,39 @@ fn lineage<'py>(model: &Bound<'py, PyAny>) -> PyResult> { } } -fn counts( - value: Option>, -) -> PyResult> { - value.map_or_else(|| Ok(BTreeMap::new()), |value| value.extract()) +positional_record! { + extract; + /// The `(code, resource type, resource name, message)` key `compile_facts` counts. + #[derive(PartialEq, Eq, PartialOrd, Ord)] + struct IdentityKey { + code: String, + resource_type: String, + resource_name: String, + message: String, + } +} + +fn counts(value: Option>) -> PyResult> { + let Some(value) = value else { + return Ok(BTreeMap::new()); + }; + let counted: BTreeMap = value.extract()?; + Ok(counted + .into_iter() + .map(|(key, count)| { + let IdentityKey { + code, + resource_type, + resource_name, + message, + } = key; + let identity = DiagnosticIdentity { + code, + resource_type, + resource_name, + message, + }; + (identity, count) + }) + .collect()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 54e7413b52..c28d595f73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -2,9 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::models::InterpolationSite; +use sqlbuild_rules::sql_lint::types::PreparedSql; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::rules::relation_keys; +use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -23,20 +26,31 @@ struct LintPreparationRequest { dialect: String, } +/// [`PreparedSql`] as Python reads it, each site a positional tuple. +type PreparedLintSql = (String, Vec>, Vec); + #[pyfunction] fn prepare_lint_sql( py: Python<'_>, request: LintPreparationRequest, -) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, +) -> PyResult> { + let prepared: Option = py + .compiler_detach(|| { + sqlbuild_rules::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) + }) + .map_err(value_error)?; + Ok(prepared.map(|(text, sites, referenced)| { + ( + text, + sites.into_iter().map(Positional).collect(), + referenced, ) - }) - .map_err(value_error) + })) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 7e6a2607ee..164895dc88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ }; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; @@ -39,10 +40,7 @@ fn record_path(record: &ModelInputRecord) -> PathBuf { } /// The record's shared texts with the given passes, read only when their file is linted. -fn expansion( - expansion: &ExpansionRecord, - passes: &[&Vec<(usize, usize, usize, usize)>], -) -> CompiledSource { +fn expansion(expansion: &ExpansionRecord, passes: &[&Vec]) -> CompiledSource { CompiledSource::Shared { authored_sql: Arc::clone(&expansion.authored_sql), expanded_sql: Arc::clone(&expansion.expanded_sql), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 5c06143522..934ad63333 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -43,7 +43,7 @@ use crate::bindings::_helpers::records::construct::{new_record_with, record_clas use crate::bindings::_helpers::records::fields::field as record_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::authored_sql::{AuthoredExpansion, expand_with_inputs}; -use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::dsl_headers::header_violations; use crate::bindings::_helpers::rules::native_header_parser::{ HeaderFailure, NativeHeaderParser, external_identifiers, @@ -213,14 +213,12 @@ impl CompiledSource { /// Expansion span rows as lint spans. pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { rows.iter() - .map( - |&(source_start, source_end, output_start, output_end)| ExpansionSpan { - source_start: offset(source_start), - source_end: offset(source_end), - output_start: offset(output_start), - output_end: offset(output_end), - }, - ) + .map(|row| ExpansionSpan { + source_start: offset(row.source_start), + source_end: offset(row.source_end), + output_start: offset(row.output_start), + output_end: offset(row.output_end), + }) .collect() } @@ -809,12 +807,10 @@ pub(crate) fn prepare_single_body(request: &SingleBodyRequest<'_, '_>) -> PyResu role, dialect, } = request; - let (context, file_path, range, dialect): ( - &SqlExpansionInputs, - &Bound<'_, PyAny>, - (usize, usize), - &str, - ) = (context, file_path, *range, dialect); + let context: &SqlExpansionInputs = context; + let file_path: &Bound<'_, PyAny> = file_path; + let range: (usize, usize) = *range; + let dialect: &str = dialect; let py: Python<'_> = file_path.py(); let text: Vec = contents.chars().collect(); let planned: PlannedBody = plan_body(&text, range, neutralizer(role), None); @@ -1071,28 +1067,6 @@ fn lint_body( body: PendingBody, (lint_text, prepared_sites, externally_referenced_ctes): PreparedSql, ) -> PyResult<(LintSqlBody, Vec<(String, String)>)> { - let prepared_sites: Vec = prepared_sites - .into_iter() - .map( - |( - sentinel, - neutralized_start, - neutralized_end, - original_start, - original_end, - original_text, - )| { - InterpolationSite { - sentinel, - neutralized_start, - neutralized_end, - original_start, - original_end, - original_text, - } - }, - ) - .collect(); let dependencies = dependency_sites(&prepared_sites).map_err(value_error)?; let mut passes: Vec> = Vec::with_capacity(body.passes.len() + 2); passes.push(sentinel_spans(&body.sites)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 7efa8a6682..9f6a9d83b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -30,30 +30,14 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::lint_pipeline::{ LintViolationRow, NATIVE_ENGINE, ViolationClasses, }; +use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; -/// One interpolation site as `InterpolationSite` fields. -type SiteTuple = (String, usize, usize, usize, usize, String); - -fn site_tuples(neutralized: Neutralized) -> (String, Vec) { - let sites: Vec = neutralized - .sites - .into_iter() - .map(|site| { - ( - site.sentinel, - site.neutralized_start, - site.neutralized_end, - site.original_start, - site.original_end, - site.original_text, - ) - }) - .collect(); - (neutralized.text, sites) +fn site_tuples(neutralized: Neutralized) -> (String, Vec>) { + (neutralized.text, Positional::wrap_all(neutralized.sites)) } #[pyfunction] @@ -62,7 +46,7 @@ fn neutralize_lint_interpolation( body: &str, dialect: &str, for_formatting: bool, -) -> PyResult<(String, Vec)> { +) -> PyResult<(String, Vec>)> { compiler_guard(|| { neutralize_interpolation(body, dialect, for_formatting) .map(site_tuples) @@ -71,29 +55,26 @@ fn neutralize_lint_interpolation( } #[pyfunction] -fn neutralize_lint_context_interpolation(body: &str) -> PyResult<(String, Vec)> { +fn neutralize_lint_context_interpolation( + body: &str, +) -> PyResult<(String, Vec>)> { compiler_guard(|| Ok(site_tuples(neutralize_context_interpolation(body)))) } #[pyfunction] -fn neutralize_lint_audit_parameters(body: &str) -> PyResult<(String, Vec)> { +fn neutralize_lint_audit_parameters( + body: &str, +) -> PyResult<(String, Vec>)> { compiler_guard(|| Ok(site_tuples(neutralize_audit_parameters(body)))) } /// Restored text, or `(False, reason)` when a sentinel is missing or repeated. #[pyfunction] -fn restore_lint_interpolation(fixed: &str, sites: Vec) -> PyResult<(bool, String)> { - let sites: Vec = sites - .into_iter() - .map(|site| InterpolationSite { - sentinel: site.0, - neutralized_start: site.1, - neutralized_end: site.2, - original_start: site.3, - original_end: site.4, - original_text: site.5, - }) - .collect(); +fn restore_lint_interpolation( + fixed: &str, + sites: Vec>, +) -> PyResult<(bool, String)> { + let sites: Vec = Positional::unwrap_all(sites); match compiler_guard(|| Ok(restore_interpolation(fixed, &sites)))? { Ok(restored) => Ok((true, restored)), Err(LintTextError::Restoration(message)) => Ok((false, message)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 989f88060c..406440f71f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -23,18 +23,21 @@ const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; const RELATION_KIND_SEED: &str = "seed"; const RELATION_KIND_SOURCE: &str = "source"; -/// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. -pub(crate) type RelationKeyRow = ( - String, - String, - String, - String, - String, - bool, - Option>, - Vec<(String, Vec, bool)>, - Vec<(String, Option)>, -); +positional_record! { + extract, into; + /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. + pub(crate) struct RelationKeyRow { + kind: String, + name: String, + materialized: String, + mode: String, + strategy: String, + has_cursor: bool, + unique_key: Option>, + columns: Vec<(String, Vec, bool)>, + audits: Vec<(String, Option)>, + } +} /// The key row of one relation from its config values and declared column and audit specs. pub(crate) fn relation_key_row( @@ -55,19 +58,19 @@ pub(crate) fn relation_key_row( } audits = audit_key_rows(declared_audits)?; } - Ok(( - kind.to_owned(), + Ok(RelationKeyRow { + kind: kind.to_owned(), name, - config_text(values, MATERIALIZED_CONFIG)?, - config_text(values, INCREMENTAL_MODE_CONFIG)?, - config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, - values + materialized: config_text(values, MATERIALIZED_CONFIG)?, + mode: config_text(values, INCREMENTAL_MODE_CONFIG)?, + strategy: config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, + has_cursor: values .get_item(CURSOR_CONFIG)? .is_some_and(|value| value.is_truthy().unwrap_or(false)), - unique_key(values)?, + unique_key: unique_key(values)?, columns, audits, - )) + }) } /// Each audit spec's `(definition name, column argument when it is text)`. @@ -93,15 +96,15 @@ pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult) -> Option> { read_source_view(view, |source, py| { let facts: &SourceKeyFacts = source.key_facts(py)?; - Ok(( - RELATION_KIND_SOURCE.to_owned(), - source.attached().name.clone(), - String::new(), - String::new(), - String::new(), - false, - None, - facts + Ok(RelationKeyRow { + kind: RELATION_KIND_SOURCE.to_owned(), + name: source.attached().name.clone(), + materialized: String::new(), + mode: String::new(), + strategy: String::new(), + has_cursor: false, + unique_key: None, + columns: facts .columns .iter() .map(|column| { @@ -112,8 +115,8 @@ pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option(project: &Bound<'py, PyAny>) -> PyResult DeclaredRelation { - let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits) = row; + let RelationKeyRow { + kind, + name, + materialized, + mode, + strategy, + has_cursor, + unique_key, + columns, + audits, + } = row; DeclaredRelation { kind, name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 94c3848ffd..e90de02503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -63,6 +63,16 @@ struct DeclarationRowPy<'py>( Option, ); +/// The code, path, line, column and message that identify one fault. +#[derive(Hash, PartialEq, Eq)] +struct FaultIdentity { + code: String, + path: String, + line: u64, + column: u64, + message: String, +} + /// The request's project facts, which models, tests and scenarios are built beside. struct ProjectRows<'py> { memo: Option<(String, String)>, @@ -410,7 +420,7 @@ fn finalized<'py>( .str()? .to_str()? .to_owned(); - let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); + let mut fixable: HashSet = HashSet::new(); let mut faults: Vec = Vec::with_capacity(findings.len()); for finding in &findings { let fault: Fault = finding_fault(finding)?; @@ -518,14 +528,14 @@ fn declaration_member((name, value): (String, Bound<'_, PyAny>)) -> PyResult (String, String, u64, u64, String) { - ( - fault.code.clone(), - fault.path.clone(), - fault.line, - fault.column, - fault.message.clone(), - ) +fn fault_identity(fault: &Fault) -> FaultIdentity { + FaultIdentity { + code: fault.code.clone(), + path: fault.path.clone(), + line: fault.line, + column: fault.column, + message: fault.message.clone(), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index 7334eb244e..f9e58d2580 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -52,6 +52,17 @@ fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { Ok(settings_lint_key(&LintSettings::from_record(config)?)) } +positional_record! { + extract; + /// One model's path, authored SQL, query SQL and contract. + pub(crate) struct ModelRuleRow { + pub(crate) path: String, + pub(crate) authored_sql: String, + pub(crate) query_sql: String, + pub(crate) contract: Option, + } +} + /// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. #[pyfunction] #[pyo3(signature = (*, lint_key, codes, models))] @@ -59,7 +70,7 @@ fn sql_model_rule_keys( py: Python<'_>, lint_key: &str, codes: Vec, - models: Vec<(String, String, String, Option)>, + models: Vec, ) -> BTreeMap { model_rule_keys(py, lint_key, &codes, &models) } @@ -69,20 +80,20 @@ pub(crate) fn model_rule_keys( py: Python<'_>, lint_key: &str, codes: &[String], - models: &[(String, String, String, Option)], + models: &[ModelRuleRow], ) -> BTreeMap { let inputs: Vec> = models .iter() - .map(|(_, authored_sql, query_sql, contract)| ModelKeyInput { - authored_sql, - query_sql, - contract: contract.as_deref(), + .map(|model| ModelKeyInput { + authored_sql: &model.authored_sql, + query_sql: &model.query_sql, + contract: model.contract.as_deref(), }) .collect(); let keys: Vec = py.detach(|| parallel_model_rule_keys(lint_key, codes, &inputs)); models .iter() - .map(|(path, ..)| path.clone()) + .map(|model| model.path.clone()) .zip(keys) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index ed839bc92a..6330b98eac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -43,8 +43,8 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ }; use crate::bindings::_helpers::rules::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::rule_cache::{ - StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, settings_lint_key, - stored_entries, write_rule_cache, + ModelRuleRow, StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, + settings_lint_key, stored_entries, write_rule_cache, }; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -293,17 +293,22 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult)> = Vec::new(); + let mut models: Vec = Vec::new(); for model in sql_rule_models(&run.project, true)? { if let Some(key) = model.key { - models.push((model.path, key.authored_sql, key.query_sql, key.contract)); + models.push(ModelRuleRow { + path: model.path, + authored_sql: key.authored_sql, + query_sql: key.query_sql, + contract: key.contract, + }); } } let keys: BTreeMap = model_rule_keys(run.project.py(), lint_key, run.codes, &models); let fresh: Vec<(String, String)> = models .into_iter() - .map(|(path, ..)| { + .map(|ModelRuleRow { path, .. }| { let key: String = keys[&path].clone(); (path, key) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs index 6cc919eaf7..b3ad6ee36e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, - write_completion, + CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, + progress_relay, write_completion, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, ScenarioProject, capture_limits, capture_settings, cli_user_error, compile, @@ -101,12 +101,12 @@ pub(crate) fn run_scenario_capture_command( let relay = progress_relay( &stream, §ion, - ( - &project.project_dir, - &adapter, - CompletionKind::Capture, - style, - ), + RelayContext { + project_dir: &project.project_dir, + adapter: &adapter, + kind: CompletionKind::Capture, + style: style, + }, )?; let results: Vec> = call_with( &host::entry(py, ExecutorHost::RunScenarioCapturePipeline)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index 1d40c492fb..0762d7b65e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -233,16 +233,24 @@ pub(crate) fn begin_section<'py>( }) } +/// The project directory, adapter, completion kind and style a progress relay reports with. +pub(crate) struct RelayContext<'a, 'py> { + pub(crate) project_dir: &'a Bound<'py, PyAny>, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) kind: CompletionKind, + pub(crate) style: CliStyle, +} + /// The relay reporting one section's scenarios. pub(crate) fn progress_relay<'py>( stream: &Bound<'py, PyAny>, section: &SectionStatus<'py>, - (project_dir, adapter, kind, style): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - CompletionKind, - CliStyle, - ), + RelayContext { + project_dir, + adapter, + kind, + style, + }: RelayContext<'_, 'py>, ) -> PyResult> { Bound::new( stream.py(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs index 69f78e3f35..d125ca59c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::result_output::documents::format_scenario_snapsho use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, begin_section, connection_hooks, progress_relay, + CompletionKind, RelayContext, SectionStatus, begin_section, connection_hooks, progress_relay, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, boolean, capture_limits, capture_settings, compile, flush, optional_text, @@ -161,12 +161,12 @@ pub(crate) fn sync_local_snapshots<'py>( let relay = progress_relay( &run.stream, §ion, - ( - &run.project.project_dir, - &run.project_adapter, - CompletionKind::Sync { refresh }, - run.style, - ), + RelayContext { + project_dir: &run.project.project_dir, + adapter: &run.project_adapter, + kind: CompletionKind::Sync { refresh }, + style: run.style, + }, )?; let settings: Bound<'py, PyAny> = capture_settings( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 14018b8ddf..adadca3f31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::result_output::documents::format_scenario_executi use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, - write_completion, + CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, + progress_relay, write_completion, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, ScenarioProject, boolean, cli_user_error, compile, discover_project, flush, int, @@ -213,12 +213,12 @@ fn run_scenarios( let relay = progress_relay( &run.stream, §ion, - ( - &run.project.project_dir, - &run.adapter, - CompletionKind::Run { target_dir }, - run.style, - ), + RelayContext { + project_dir: &run.project.project_dir, + adapter: &run.adapter, + kind: CompletionKind::Run { target_dir }, + style: run.style, + }, )?; let project_name: Bound<'_, PyAny> = native_field( &native_field(&run.project.discovered, "project_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 46e1d892fa..58e7eddeb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -33,21 +33,30 @@ type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); /// Consumer, declaration, usage kind, what it is used through, and the enum member. -pub(crate) type UsageRow = ( - QualifiedRow, - QualifiedRow, - String, - Option, - Option, -); +#[derive(Clone, Hash, PartialEq, Eq)] +pub(crate) struct UsageRow { + pub(crate) consumer: QualifiedRow, + pub(crate) declaration: QualifiedRow, + pub(crate) kind: String, + pub(crate) through: Option, + pub(crate) enum_member: Option, +} /// Path, scope, private owner, owning path and ownership root path of a named record. -type RecordRow = (String, String, Option, Option, String); +struct RecordRow { + path: String, + scope: String, + owner: Option, + owning_path: Option, + ownership_root: String, +} /// Name, the schema it extends, and each column's name and type. type SchemaRow = (String, Option, Vec<(String, Option)>); -/// Report key, message, path and help of one S006. -type DiagnosticRow = (Vec, String, String, String); /// Usages as `(consumer, declaration)` record positions, S006s, else a scope path error. -type SchemaUsageRows = (Vec<(usize, usize)>, Vec, Option); +type SchemaUsageRows = ( + Vec<(usize, usize)>, + Vec, + Option, +); /// `(own, position)`: an own dependency's position, else a distinct usage's. pub(crate) type SourceRow = (bool, usize); /// Distinct collected positions, merged sources per declaration, and final usage positions. @@ -58,7 +67,13 @@ fn invalid(what: &str) -> pyo3::PyErr { } fn named_record(handle: usize, row: RecordRow) -> PyResult { - let (path, scope, owner, owning_path, ownership_root) = row; + let RecordRow { + path, + scope, + owner, + owning_path, + ownership_root, + } = row; let owner: Option = match owner { None => None, Some((kind, name)) => Some(ResourceIdentity { @@ -78,15 +93,6 @@ fn named_record(handle: usize, row: RecordRow) -> PyResult { }) } -fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { - ( - diagnostic.key, - diagnostic.message, - diagnostic.path, - diagnostic.help, - ) -} - /// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] pub(crate) fn model_schema_scope_usages<'py>( @@ -163,10 +169,10 @@ pub(crate) fn model_schema_scope_usages<'py>( fn record_row(record: &Bound<'_, PyAny>) -> PyResult { let identity: Bound<'_, PyAny> = native_field(record, "identity")?; let owner: Bound<'_, PyAny> = native_field(&identity, "owner")?; - Ok(( - native_field(record, "path")?.extract()?, - native_field(&native_field(record, "scope")?, "value")?.extract()?, - if owner.is_none() { + Ok(RecordRow { + path: native_field(record, "path")?.extract()?, + scope: native_field(&native_field(record, "scope")?, "value")?.extract()?, + owner: if owner.is_none() { None } else { Some(( @@ -174,18 +180,25 @@ fn record_row(record: &Bound<'_, PyAny>) -> PyResult { native_field(&owner, "name")?.extract()?, )) }, - native_field(record, "owning_path")?.extract()?, - native_field(&native_field(record, "ownership_root")?, "path")?.extract()?, - )) + owning_path: native_field(record, "owning_path")?.extract()?, + ownership_root: native_field(&native_field(record, "ownership_root")?, "path")? + .extract()?, + }) } /// Report each S006 through the active compile diagnostic collector. -fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { +fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { if diagnostics.is_empty() { return Ok(()); } let path_class = py.import("pathlib")?.getattr("Path")?; - for (key, message, path, help) in diagnostics { + for InaccessibleDiagnostic { + key, + message, + path, + help, + } in diagnostics + { let fields: Bound<'_, PyDict> = PyDict::new(py); fields.set_item("phase", "compile")?; fields.set_item("severity", "error")?; @@ -230,11 +243,7 @@ fn scope_schema_usages( Ok(SchemaUsages { usages, diagnostics, - }) => ( - usages, - diagnostics.into_iter().map(diagnostic_row).collect(), - None, - ), + }) => (usages, diagnostics, None), Err(rejected) => ( Vec::new(), Vec::new(), @@ -265,7 +274,11 @@ pub(crate) fn merged_usage_rows( let mut consumed: Vec<(Option, QualifiedRow)> = Vec::with_capacity(distinct.len()); for position in &distinct { - let (consumer, declaration, _, _, _) = &collected[*position]; + let UsageRow { + consumer, + declaration, + .. + } = &collected[*position]; let by_declaration: Option = DeclarationKind::parse(&consumer.0) .is_some() .then(|| consumer.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs index 2644315669..8f9d194b3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs @@ -13,15 +13,15 @@ pub(crate) type QualifiedRow = (String, String, Option); /// An ownership root's path, kind and resource kind. pub(crate) type RootRow = (String, String, Option); /// Identity, path, line, column, scope, ownership root and owning path. -pub(crate) type DeclarationRow = ( - QualifiedRow, - String, - i64, - i64, - String, - RootRow, - Option, -); +pub(crate) struct DeclarationRow { + pub(crate) identity: QualifiedRow, + pub(crate) path: String, + pub(crate) line: i64, + pub(crate) column: i64, + pub(crate) scope: String, + pub(crate) root: RootRow, + pub(crate) owning_path: Option, +} fn invalid(what: &str) -> pyo3::PyErr { PyValueError::new_err(format!("invalid scope placement {what}")) @@ -73,7 +73,15 @@ pub(crate) fn scope_kind(value: &str) -> PyResult { /// A declaration row as its record, without macro dependencies. pub(crate) fn declaration_record(row: DeclarationRow) -> PyResult { - let (identity, path, line, column, scope, root, owning_path) = row; + let DeclarationRow { + identity, + path, + line, + column, + scope, + root, + owning_path, + } = row; Ok(DeclarationRecord { identity: declaration_identity(identity)?, path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index e0e672910c..564051531b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -24,8 +24,8 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; use crate::bindings::_helpers::scopes::scope_reports::{ - scope_browse, scope_explain, scope_list, scope_move_preview, scope_report, - serialize_scope_report, + ExplainArguments, ListArguments, ReportArguments, scope_browse, scope_explain, scope_list, + scope_move_preview, scope_report, serialize_scope_report, }; const TARGET_ERROR_CODE: &str = "C960"; @@ -249,12 +249,12 @@ fn query_result<'py>( return Ok(scope_list( py, lookup, - ( + ListArguments { folder, - Some(filters), - optional_text(py, &target), - prospective, - ), + filters: Some(filters), + target: optional_text(py, &target), + target_is_prospective: prospective, + }, )? .into_bound(py)); } @@ -264,12 +264,12 @@ fn query_result<'py>( let mut report: Bound<'_, PyAny> = scope_report( py, lookup, - ( - optional_text(py, &target), - optional_text(py, &at), + ReportArguments { + target: optional_text(py, &target), + at: optional_text(py, &at), directory, - Some(filters), - ), + filters: Some(filters), + }, )? .into_bound(py); let explain: Option = native_field(request, "explain")?.extract()?; @@ -277,12 +277,12 @@ fn query_result<'py>( let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_explain( py, lookup, - ( - PyString::new(py, &declaration).into_any(), - optional_text(py, &target), - optional_text(py, &at), + ExplainArguments { + declaration: PyString::new(py, &declaration).into_any(), + target: optional_text(py, &target), + at: optional_text(py, &at), directory, - ), + }, ) .map(|(explanation, diagnostics)| { (explanation.into_bound(py), diagnostics.into_bound(py)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index 0cc73f0663..3ee4cd952a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -29,7 +29,12 @@ use crate::bindings::types::CompilerDetach; const TYPES_MODULE: &str = "types"; /// `(declarations, canonical positions)` matched, else identity and path error messages. -pub(crate) type TargetRows = (bool, Vec, Option, Option); +pub(crate) struct TargetRows { + pub(crate) declarations: bool, + pub(crate) positions: Vec, + pub(crate) identity_error: Option, + pub(crate) path_error: Option, +} /// Visible `(position, reason, grant)`, inaccessible `(position, reason)`, else the path error. pub(crate) type ResolvedRows = ( Vec<(usize, &'static str, Option)>, @@ -119,27 +124,30 @@ impl NativeScopeLookup { None => ReportTarget::Text(text.to_owned()), }; Ok(match report_target(&self.records, &self.lookup, &target) { - Ok(TargetMatch::Resources(positions)) => ( - false, - canonical(&self.canonical.resources, positions), - None, - None, - ), - Ok(TargetMatch::Declarations(positions)) => ( - true, - canonical(&self.canonical.declarations, positions), - None, - None, - ), - Err(TargetFailure::Identity(failure)) => ( - false, - Vec::new(), - Some(identity_message(&failure, text)), - None, - ), - Err(TargetFailure::Path(rejected)) => { - (false, Vec::new(), None, Some(path_message(&rejected))) - } + Ok(TargetMatch::Resources(positions)) => TargetRows { + declarations: false, + positions: canonical(&self.canonical.resources, positions), + identity_error: None, + path_error: None, + }, + Ok(TargetMatch::Declarations(positions)) => TargetRows { + declarations: true, + positions: canonical(&self.canonical.declarations, positions), + identity_error: None, + path_error: None, + }, + Err(TargetFailure::Identity(failure)) => TargetRows { + declarations: false, + positions: Vec::new(), + identity_error: Some(identity_message(&failure, text)), + path_error: None, + }, + Err(TargetFailure::Path(rejected)) => TargetRows { + declarations: false, + positions: Vec::new(), + identity_error: None, + path_error: Some(path_message(&rejected)), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index ab0f6eb8d3..8f9205a9c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -21,8 +21,8 @@ use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::placement::{ - IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, ownership_root, - qualified_identity, resource_identity, + DeclarationRow, IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, + ownership_root, qualified_identity, resource_identity, }; use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; @@ -179,15 +179,15 @@ fn read_constant(facts: &Bound<'_, PyAny>) -> PyResult { /// One `DeclarationRecord` with its value metadata. pub(crate) fn read_declaration(record: &Bound<'_, PyAny>) -> PyResult { Ok(CachedDeclaration { - record: declaration_record(( - qualified_row(&native_field(record, "identity")?)?, - field(record, "path")?, - field(record, "line")?, - field(record, "column")?, - value_of(record, "scope")?, - root_row(&native_field(record, "ownership_root")?)?, - field(record, "owning_path")?, - ))?, + record: declaration_record(DeclarationRow { + identity: qualified_row(&native_field(record, "identity")?)?, + path: field(record, "path")?, + line: field(record, "line")?, + column: field(record, "column")?, + scope: value_of(record, "scope")?, + root: root_row(&native_field(record, "ownership_root")?)?, + owning_path: field(record, "owning_path")?, + })?, macro_facts: optional_item(record, "macro")? .map(|facts| read_macro(&facts)) .transpose()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 9583d93bff..bb699daa8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -40,26 +40,26 @@ use crate::bindings::_helpers::scopes::scope_report_objects::{ type PyObject = Py; /// Target, location, whether it is a directory, and the filters. -pub(crate) type ReportArguments<'py> = ( - Option>, - Option>, - bool, - Option>, -); +pub(crate) struct ReportArguments<'py> { + pub(crate) target: Option>, + pub(crate) at: Option>, + pub(crate) directory: bool, + pub(crate) filters: Option>, +} /// Declaration, target, location and whether it is a directory. -pub(crate) type ExplainArguments<'py> = ( - Bound<'py, PyAny>, - Option>, - Option>, - bool, -); +pub(crate) struct ExplainArguments<'py> { + pub(crate) declaration: Bound<'py, PyAny>, + pub(crate) target: Option>, + pub(crate) at: Option>, + pub(crate) directory: bool, +} /// Folder, filters, target and whether the target is prospective. -pub(crate) type ListArguments<'py> = ( - String, - Option>, - Option>, - bool, -); +pub(crate) struct ListArguments<'py> { + pub(crate) folder: String, + pub(crate) filters: Option>, + pub(crate) target: Option>, + pub(crate) target_is_prospective: bool, +} const CURRENT_FOLDER: &str = "."; @@ -148,7 +148,12 @@ pub(crate) fn scope_report( lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, ) -> PyResult { - let (target, at, directory, filters) = request; + let ReportArguments { + target, + at, + directory, + filters, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { Some(filters) => filters, @@ -178,7 +183,12 @@ pub(crate) fn scope_explain( lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, ) -> PyResult<(PyObject, PyObject)> { - let (declaration, target, at, directory) = request; + let ExplainArguments { + declaration, + target, + at, + directory, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let reports: LookupReports = lookup_reports(lookup)?; let request: ExplainRequest = ExplainRequest { @@ -240,7 +250,12 @@ pub(crate) fn scope_list( lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, ) -> PyResult { - let (folder, filters, target, target_is_prospective) = request; + let ListArguments { + folder, + filters, + target, + target_is_prospective, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let reports: LookupReports = lookup_reports(lookup)?; let request: FolderRequest = @@ -354,12 +369,12 @@ fn query_scope_report_binding<'py>( scope_report( py, &arguments.value("lookup")?, - ( - optional(&arguments, "target")?, - optional(&arguments, "at")?, - arguments.flag("directory")?, - optional(&arguments, "filters")?, - ), + ReportArguments { + target: optional(&arguments, "target")?, + at: optional(&arguments, "at")?, + directory: arguments.flag("directory")?, + filters: optional(&arguments, "filters")?, + }, ) } @@ -379,12 +394,12 @@ fn explain_scope_declaration_binding<'py>( scope_explain( py, &arguments.value("lookup")?, - ( - arguments.value("declaration")?, - optional(&arguments, "target")?, - optional(&arguments, "at")?, - arguments.flag("directory")?, - ), + ExplainArguments { + declaration: arguments.value("declaration")?, + target: optional(&arguments, "target")?, + at: optional(&arguments, "at")?, + directory: arguments.flag("directory")?, + }, ) } @@ -435,12 +450,12 @@ fn list_scope_declarations_binding<'py>( scope_list( py, &arguments.value("lookup")?, - ( - arguments.value("folder")?.extract()?, - optional(&arguments, "filters")?, - optional(&arguments, "target")?, - arguments.flag("target_is_prospective")?, - ), + ListArguments { + folder: arguments.value("folder")?.extract()?, + filters: optional(&arguments, "filters")?, + target: optional(&arguments, "target")?, + target_is_prospective: arguments.flag("target_is_prospective")?, + }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 8338ebd9bd..8ddc2ef87f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -57,15 +57,15 @@ fn tuple_from<'py>(py: Python<'py>, items: Vec>) -> PyResult) -> PyResult { - Ok(( - qualified_row(&native_field(usage, "consumer")?)?, - qualified_row(&native_field(usage, "declaration")?)?, - value_of(usage, "kind")?, - optional_item(usage, "through")? + Ok(UsageRow { + consumer: qualified_row(&native_field(usage, "consumer")?)?, + declaration: qualified_row(&native_field(usage, "declaration")?)?, + kind: value_of(usage, "kind")?, + through: optional_item(usage, "through")? .map(|through| qualified_row(&through)) .transpose()?, - field(usage, "enum_member")?, - )) + enum_member: field(usage, "enum_member")?, + }) } fn dependency_row( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 40d9e18fbf..46b48fc41a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -47,8 +47,12 @@ fn target_query<'py>( } else { None }; - let (declarations, positions, identity_error, path_error): TargetRows = - native_lookup(lookup)?.get().target(row, &raw)?; + let TargetRows { + declarations, + positions, + identity_error, + path_error, + }: TargetRows = native_lookup(lookup)?.get().target(row, &raw)?; if let Some(message) = identity_error { return Err(PyErr::from_value( error_class(py, ErrorId::InvalidQualifiedIdentity)?.call1((message,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 4be1bbc2fa..02e58040ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -16,7 +16,9 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::model_reads::binding_validated; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + ModelMetadataTexts, NativeCompiledModel, +}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_get_or; @@ -282,7 +284,12 @@ fn model_diagnostics<'py>( } else { &classes.warning }; - let row: FindingRow<'_> = (&audit.code, &audit.message, audit.line, audit.column); + let row: FindingRow<'_> = FindingRow { + code: &audit.code, + message: &audit.message, + line: audit.line, + column: audit.column, + }; diagnostics.push(model_error(classes, model, &row, severity)?); } for finding in findings.references { @@ -340,17 +347,32 @@ fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// `(code, message, line, column)`. -type FindingRow<'a> = (&'a str, &'a str, i64, i64); +/// The code, message, line and column of one finding. +struct FindingRow<'a> { + code: &'a str, + message: &'a str, + line: i64, + column: i64, +} fn finding_row(finding: &MetadataFinding) -> FindingRow<'_> { - (finding.code, &finding.message, finding.line, finding.column) + FindingRow { + code: finding.code, + message: &finding.message, + line: finding.line, + column: finding.column, + } } fn model_error<'py>( classes: &StageClasses<'py>, model: &Bound<'py, PyAny>, - &(code, message, line, column): &FindingRow<'_>, + &FindingRow { + code, + message, + line, + column, + }: &FindingRow<'_>, severity: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = classes.py(); @@ -446,8 +468,12 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes upstreams.push((upstream.str()?.extract()?, column_names(&columns)?)); } } - let (name, query_sql, authored_sql, calls_functions) = match model.cast::() - { + let ModelMetadataTexts { + name, + query_sql, + authored_sql, + calls_functions, + } = match model.cast::() { Ok(native) => native .get() .metadata_texts(model.py(), &FUNCTION_REFERENCE_KINDS)?, @@ -460,12 +486,12 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes break; } } - ( - native_field(model, "name")?.extract()?, - native_field(model, "query_sql")?.extract()?, - native_field(model, "authored_sql")?.extract()?, + ModelMetadataTexts { + name: native_field(model, "name")?.extract()?, + query_sql: native_field(model, "query_sql")?.extract()?, + authored_sql: native_field(model, "authored_sql")?.extract()?, calls_functions, - ) + } } }; Ok(MetadataModel { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 42644f97bf..cdf45c3cc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -9,8 +9,8 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ - AuditSqlInput, HookModelInput, HookSqlInput, ResourceSqlDiagnostic, ResourceSqlFailure, - ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, + AuditSqlInput, HookModelInput, HookSqlInput, ResourceSpan, ResourceSqlDiagnostic, + ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, }; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -272,7 +272,7 @@ fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { fn row_relations(rows: Vec) -> Vec { let mut names: Vec = rows .into_iter() - .filter_map(|(kind, name, _, _)| relation_name(&kind, name)) + .filter_map(|ReferenceRow { kind, name, .. }| relation_name(&kind, name)) .collect(); names.sort_unstable(); names.dedup(); @@ -318,7 +318,12 @@ fn python_diagnostic<'py>( classes: &DiagnosticClasses<'py>, diagnostic: ResourceSqlDiagnostic, ) -> PyResult> { - let (line, column, end_line, end_column) = diagnostic.span; + let ResourceSpan { + line, + column, + end_line, + end_column, + } = diagnostic.span; let location: Bound<'py, PyDict> = PyDict::new(py); location.set_item("path", classes.path.call1((&diagnostic.path,))?)?; location.set_item("line", line)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index e5dfd2cf62..04336f22b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, + BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, }; use crate::bindings::_helpers::analysis::model_reads::{ declared_rows, dynamic_columns_declared, inferred_rows, star_flags, unchecked_columns, @@ -74,7 +74,7 @@ fn model_relation_names(model: &Bound<'_, PyAny>) -> PyResult<(Vec, bool let referenced: bool = !rows.is_empty(); let names: Vec = relation_names( rows.into_iter() - .map(|(kind, name, _, _)| (kind, name)) + .map(|ReferenceRow { kind, name, .. }| (kind, name)) .collect(), ); return Ok((names, referenced)); @@ -155,12 +155,12 @@ fn binding_shape<'py>( if !catalog.is_truthy()? { let created: BindingCatalog = BindingCatalog::create( py, - ( - or_generic(&dialect)?.extract()?, - false, - native_field(profile, "semantic_known_functions")?.extract()?, - native_field(profile, "semantic_known_types")?.extract()?, - ), + CatalogKey { + dialect: or_generic(&dialect)?.extract()?, + quoted_ignore_case: false, + known_functions: native_field(profile, "semantic_known_functions")?.extract()?, + known_types: native_field(profile, "semantic_known_types")?.extract()?, + }, inputs.as_any(), )?; catalog = Bound::new(py, created)?.into_any(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 9dcd6a4f87..e35b5a5709 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -14,6 +14,7 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::fields::{ field as native_field, path_posix_text, path_stem, }; @@ -26,19 +27,11 @@ const INTERNAL_FAILURE: &str = "internal"; const DECIMAL_OVERFLOW: &str = "decimal_overflow"; const SQL_TEST_ASSEMBLY_CONTEXT: &str = "native SQL test assembly"; -/// `(line, column, end line, end column, message, help)` of one P013 diagnostic. -pub(crate) type DiagnosticRow = (usize, usize, usize, usize, String, String); -/// `(name, scope deps, target models, case fingerprint, diagnostic resource, diagnostics)`. -pub(crate) type FactsRow = ( - String, - Vec<(&'static str, String)>, - Vec, - Option, - String, - Vec, -); /// The test's facts, and the `(kind, message)` of the error its assembly raises, if any. -pub(crate) type AssemblyRow = (Option, Option<(&'static str, String)>); +pub(crate) type AssemblyRow = ( + Option, + Option<(&'static str, String)>, +); /// The compile inputs assembly reads: model inputs, SQL test inputs and lexical rules. pub(crate) struct AssemblyRequest<'py> { @@ -86,11 +79,18 @@ fn assembly_model(model: &Bound<'_, PyAny>) -> PyResult { references: record .references .iter() - .map(|(kind, name, package, _)| SqlTestAssemblyReference { - kind: kind.clone(), - name: name.clone(), - package: package.clone(), - }) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| SqlTestAssemblyReference { + kind: kind.clone(), + name: name.clone(), + package: package.clone(), + }, + ) .collect(), }) }); @@ -240,11 +240,10 @@ fn parameter_values(items: &Bound<'_, PyAny>) -> PyResult AssemblyRow { match outcome { - SqlTestAssemblyOutcome::Assembled(facts) => (Some(facts_row(facts)), None), - SqlTestAssemblyOutcome::FingerprintOverflow(facts) => ( - Some(facts_row(facts)), - Some((DECIMAL_OVERFLOW, String::new())), - ), + SqlTestAssemblyOutcome::Assembled(facts) => (Some(facts), None), + SqlTestAssemblyOutcome::FingerprintOverflow(facts) => { + (Some(facts), Some((DECIMAL_OVERFLOW, String::new()))) + } SqlTestAssemblyOutcome::Failed(SqlTestAssemblyFailure::Input(message)) => { (None, Some((INPUT_FAILURE, message))) } @@ -257,27 +256,3 @@ fn assembly_row(outcome: SqlTestAssemblyOutcome) -> AssemblyRow { ), } } - -fn facts_row(facts: AssembledSqlTestFacts) -> FactsRow { - ( - facts.name, - facts.scope_deps, - facts.target_model_names, - facts.case_fingerprint, - facts.diagnostic_resource_name, - facts - .diagnostics - .into_iter() - .map(|diagnostic| { - ( - diagnostic.line, - diagnostic.column, - diagnostic.end_line, - diagnostic.end_column, - diagnostic.message, - diagnostic.help, - ) - }) - .collect(), - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 029f6f42a9..4cec1db815 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiler::models::{AssembledSqlTestFacts, SqlTestHelperDiagnostic}; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::records::construct::{enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sql_test_glue::assembly::{ - AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, + AssemblyRequest, AssemblyRow, assemble_compiled_sql_tests, }; /// Fields a direct-logic compiled payload copies from its input payload, in order. @@ -159,7 +160,14 @@ fn compiled_payload<'py>( fn keyed_diagnostic<'py>( classes: &Classes<'py>, (test_input, resource_name): (&Bound<'py, PyAny>, &str), - (line, column, end_line, end_column, message, help): DiagnosticRow, + SqlTestHelperDiagnostic { + line, + column, + end_line, + end_column, + message, + help, + }: SqlTestHelperDiagnostic, ) -> PyResult> { let py: Python<'py> = classes.py(); let path: Bound<'py, PyAny> = @@ -203,11 +211,17 @@ fn keyed_diagnostic<'py>( fn compiled_test<'py>( classes: &Classes<'py>, test_input: &Bound<'py, PyAny>, - facts: FactsRow, + facts: AssembledSqlTestFacts, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { let py: Python<'py> = classes.py(); - let (name, scope_rows, target_model_names, case_fingerprint, resource_name, diagnostics) = - facts; + let AssembledSqlTestFacts { + name, + scope_deps: scope_rows, + target_model_names, + case_fingerprint, + diagnostic_resource_name: resource_name, + diagnostics, + } = facts; let scope_deps: Vec> = scope_rows .iter() .map(|(resource_type, dep)| classes.object_key(resource_type, dep)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs index 51dac5c47a..45d8a0755b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -2,13 +2,24 @@ use crate::bindings::_helpers::records::construct::{enum_from_value, new_record_with}; use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::sql_test_glue::planning::{AssertionRow, ChainRow, PlanRow}; +use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_analysis::compiler::models::{ + SqlTestAssertionStep, SqlTestChainStep, SqlTestPlan, SqlTestPlanWarning, +}; -fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult> { - let (model, sql, expected_sql, lifted, body, expected_columns, expected_lifted) = row; +fn chain_step<'py>(py: Python<'py>, step: SqlTestChainStep) -> PyResult> { + let SqlTestChainStep { + model_name: model, + resolved_sql: sql, + expected_cte_sql: expected_sql, + expected_lifted_ctes: expected_lifted, + lifted_ctes: lifted, + comparison_body_sql: body, + expected_columns, + } = step; new_record_with( py, RecordId::ChainStep, @@ -33,8 +44,13 @@ fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult ) } -fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult> { - let (name, sql, lifted, body) = row; +fn assertion_step<'py>(py: Python<'py>, step: SqlTestAssertionStep) -> PyResult> { + let SqlTestAssertionStep { + name, + resolved_sql: sql, + lifted_ctes: lifted, + comparison_body_sql: body, + } = step; new_record_with( py, RecordId::SqlTestAssertionStep, @@ -48,7 +64,17 @@ fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult, row: PlanRow) -> PyResult> { - let (sql, chain_rows, assertion_rows, model_names, warning_rows, error_messages) = row; + let PlanRow { + plan: + SqlTestPlan { + sql, + chain: chain_rows, + assertions: assertion_rows, + model_names, + warnings: warning_rows, + }, + error_messages, + } = row; let mut chain: Vec> = Vec::new(); for value in chain_rows { chain.push(chain_step(py, value)?); @@ -58,7 +84,12 @@ pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { assertions.push(assertion_step(py, value)?); } let mut warnings: Vec> = Vec::new(); - for (model_name, level, message) in warning_rows { + for SqlTestPlanWarning { + model_name, + severity: level, + message, + } in warning_rows + { warnings.push(new_record_with( py, RecordId::PlanWarning, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index df61cf369a..fa0241d2d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -7,9 +7,8 @@ use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; use sqlbuild_analysis::compiler::models::{ - SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainModel, SqlTestChainStep, SqlTestCte, - SqlTestPlan, SqlTestPlanBatch, SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, - SqlTestPlanTest, + SqlTestChainBatch, SqlTestChainModel, SqlTestCte, SqlTestPlan, SqlTestPlanBatch, + SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, SqlTestPlanTest, }; use crate::bindings::_helpers::analysis::model_reads; @@ -21,37 +20,30 @@ use crate::bindings::types::CompilerDetach; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -pub(crate) type CtePair = (String, String); -/// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. -pub(crate) type ChainRow = ( - String, - String, - Option, - Vec, - Option, - Option>, - Vec, -); -/// `(name, resolved SQL, lifted CTEs, comparison body)`. -pub(crate) type AssertionRow = (String, String, Vec, Option); -/// `(model, severity, message)`. -pub(crate) type WarningRow = (Option, &'static str, String); -/// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. -pub(crate) type PlanRow = ( - Option, - Vec, - Vec, - Vec, - Vec, - Vec, -); +/// A native SQL-test plan with its distinct error messages. +pub(crate) struct PlanRow { + pub(crate) plan: SqlTestPlan, + pub(crate) error_messages: Vec, +} + +positional_record! { + extract; + /// An SQL function's name with its rendered UDF and table-function call affixes. + pub(crate) struct FunctionRow { + pub(crate) name: String, + pub(crate) udf_prefix: String, + pub(crate) udf_suffix: String, + pub(crate) table_function_prefix: String, + pub(crate) table_function_suffix: String, + } +} /// The planning request: compiled models and tests, with the adapter's facts. pub(crate) struct PlanningRequest<'py> { pub(crate) models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. pub(crate) rendered_model_sql: HashMap, - pub(crate) functions: Vec<(String, String, String, String, String)>, + pub(crate) functions: Vec, pub(crate) tests: Vec>, /// Cursor-rendered model query overrides by test index, replacing the test's own. pub(crate) rendered_test_overrides: HashMap>, @@ -171,13 +163,13 @@ fn model_dependencies(model: &Bound<'_, PyAny>) -> PyResult> { } fn plan_function( - (name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix): ( - String, - String, - String, - String, - String, - ), + FunctionRow { + name, + udf_prefix, + udf_suffix, + table_function_prefix, + table_function_suffix, + }: FunctionRow, ) -> SqlTestPlanFunction { SqlTestPlanFunction { name, @@ -242,36 +234,8 @@ fn cte(value: &Bound<'_, PyAny>) -> PyResult { fn plan_row(plan: SqlTestPlan) -> PlanRow { let error_messages: Vec = sql_test_plan_error_messages(&plan); - ( - plan.sql, - plan.chain.into_iter().map(chain_row).collect(), - plan.assertions.into_iter().map(assertion_row).collect(), - plan.model_names, - plan.warnings - .into_iter() - .map(|warning| (warning.model_name, warning.severity, warning.message)) - .collect(), + PlanRow { + plan, error_messages, - ) -} - -fn chain_row(step: SqlTestChainStep) -> ChainRow { - ( - step.model_name, - step.resolved_sql, - step.expected_cte_sql, - step.lifted_ctes, - step.comparison_body_sql, - step.expected_columns, - step.expected_lifted_ctes, - ) -} - -fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { - ( - step.name, - step.resolved_sql, - step.lifted_ctes, - step.comparison_body_sql, - ) + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index f4eed0434b..c1ca49eb60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -6,10 +6,17 @@ use sqlbuild_sqltext::compiler::models::{ DeclarationReferenceStop, }; -/// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant. -type ReferenceRow = (u8, String, Option, usize, usize); +/// One natively scanned reference: kind 0 is an enum member, 1 a constant. +#[derive(Clone)] +pub(crate) struct ScannedReference { + pub(crate) kind: u8, + pub(crate) name: String, + pub(crate) member: Option, + pub(crate) start: usize, + pub(crate) end: usize, +} /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. -type ScanRow = (Vec, Option); +type ScanRow = (Vec, Option); /// One SQL string's references and stop code, scanned without walking past malformed ones. pub(crate) fn declaration_scan(sql: &str) -> ScanRow { @@ -30,16 +37,16 @@ fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { ) } -fn reference_row(reference: DeclarationReference) -> ReferenceRow { +fn reference_row(reference: DeclarationReference) -> ScannedReference { let kind: u8 = match reference.kind { DeclarationReferenceKind::Enum => 0, DeclarationReferenceKind::Constant => 1, }; - ( + ScannedReference { kind, - reference.name, - reference.member, - reference.start, - reference.end, - ) + name: reference.name, + member: reference.member, + start: reference.start, + end: reference.end, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 8d2aab2a9f..881982dc82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -54,21 +54,29 @@ fn tokenize_model_header(header: &str) -> PyResult> { }) } -pub(crate) type ColumnLocation = (String, usize, usize, usize, usize); +positional_record! { + into; + /// A column name and its authored line, column, end line and end column. + pub(crate) struct ColumnLocation { + pub(crate) name: String, + pub(crate) line: usize, + pub(crate) column: usize, + pub(crate) end_line: usize, + pub(crate) end_column: usize, + } +} pub(crate) fn location_rows( locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, ) -> Vec { locations .into_iter() - .map(|(name, location)| { - ( - name, - location.line, - location.column, - location.end_line, - location.end_column, - ) + .map(|(name, location)| ColumnLocation { + name, + line: location.line, + column: location.column, + end_line: location.end_line, + end_column: location.end_column, }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index a365dde1bc..ffd2158bac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -23,10 +23,29 @@ use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; -/// `(kind, name, package, call_argument_count)` for one reference. -type ReferenceRow = (&'static str, String, Option, Option); -/// `(kind, call, start, message, help, corrected_call)` for one rejected call. -type InvalidCallRow = (&'static str, String, usize, String, String, String); +positional_record! { + into; + /// `(kind, name, package, call_argument_count)` for one reference. + struct ReferenceRow { + kind: &'static str, + name: String, + package: Option, + call_argument_count: Option, + } +} + +positional_record! { + into; + /// `(kind, call, start, message, help, corrected_call)` for one rejected call. + struct InvalidCallRow { + kind: &'static str, + call: String, + start: usize, + message: String, + help: String, + corrected_call: String, + } +} /// Python references and rejected calls of one scan. type ScannedObjects<'py> = (Vec>, Vec>); /// The references and rejected calls, or the scan error and its start. @@ -73,23 +92,23 @@ impl SqlReferenceScanner { } fn reference_row(reference: SqlReference) -> ReferenceRow { - ( - reference.kind, - reference.name, - reference.package, - reference.call_argument_count, - ) + ReferenceRow { + kind: reference.kind, + name: reference.name, + package: reference.package, + call_argument_count: reference.call_argument_count, + } } fn invalid_call_row(call: InvalidReferenceCall) -> InvalidCallRow { - ( - call.kind, - call.call, - call.start, - call.message, - call.help, - call.corrected_call, - ) + InvalidCallRow { + kind: call.kind, + call: call.call, + start: call.start, + message: call.message, + help: call.help, + corrected_call: call.corrected_call, + } } fn text<'py>(py: Python<'py>, value: impl IntoPyObject<'py>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 415807dfa7..687b20670c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -10,16 +10,19 @@ use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; + const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -/// `(sql or None when unchanged, spans, reads, error message or None)` for one SQL text. -pub(crate) type InterpolationRow = ( - Option, - Vec<(usize, usize, usize, usize)>, - Vec<(&'static str, String)>, - Option, -); +/// One SQL text interpolated: its SQL, substitution spans, reads and error message. +pub(crate) struct InterpolationRow { + /// The interpolated SQL, or `None` when unchanged. + pub(crate) sql: Option, + pub(crate) spans: Vec, + pub(crate) reads: Vec<(&'static str, String)>, + pub(crate) error: Option, +} /// Project variables, the environment, context values (or `None`) and the variable renderer. #[derive(FromPyObject)] @@ -154,13 +157,18 @@ pub(crate) fn interpolation_batch( let mut rows = Vec::with_capacity(sqls.len()); for (sql, file_path) in sqls { let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => (result.sql, result.spans, read_rows(result.reads), None), - Err(failure) => ( - None, - Vec::new(), - read_rows(failure.reads), - Some(failure.message), - ), + Ok(result) => InterpolationRow { + sql: result.sql, + spans: result.spans.into_iter().map(SpanRow::from).collect(), + reads: read_rows(result.reads), + error: None, + }, + Err(failure) => InterpolationRow { + sql: None, + spans: Vec::new(), + reads: read_rows(failure.reads), + error: Some(failure.message), + }, }; if let Some(error) = host.raised.borrow_mut().take() { return Err(error); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs index 6007f4e19d..74fe591603 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs @@ -10,14 +10,17 @@ use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(name, family, precision, scale, length)` of one normalized type. -type NormalizedRow = ( - String, - &'static str, - Option, - Option, - Option, -); +positional_record! { + into; + /// `(name, family, precision, scale, length)` of one normalized type. + struct NormalizedRow { + name: String, + family: &'static str, + precision: Option, + scale: Option, + length: Option, + } +} /// Normalize one type: `(normalized row, Polyglot parse error)`, or Python's exception. #[pyfunction] @@ -51,13 +54,13 @@ pub(crate) fn normalization_error(python: Python<'_>, error: &TypeNormalizationE } fn normalized_row(normalized: NormalizedType) -> NormalizedRow { - ( - normalized.normalized_name, - normalized.family.as_str(), - normalized.precision.map(|value| value.as_str().to_owned()), - normalized.scale.map(|value| value.as_str().to_owned()), - normalized.length.map(|value| value.as_str().to_owned()), - ) + NormalizedRow { + name: normalized.normalized_name, + family: normalized.family.as_str(), + precision: normalized.precision.map(|value| value.as_str().to_owned()), + scale: normalized.scale.map(|value| value.as_str().to_owned()), + length: normalized.length.map(|value| value.as_str().to_owned()), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 08e37dd711..cf4f54599c 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -1,8 +1,13 @@ //! Python-facing classes and mapping requests, converted to native models at the boundary. +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyErr, PyResult, Python}; +use pyo3::types::PyTuple; use pyo3::{FromPyObject, pyclass}; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::types::{Expansion, Relations}; +use sqlbuild_analysis::semantic_validation::models::Expansion; +use sqlbuild_analysis::semantic_validation::types::Relations; + +use crate::bindings::types::{FromPositional, IntoPositional}; use std::collections::HashMap; #[derive(FromPyObject, Debug)] @@ -32,7 +37,7 @@ pub(crate) struct PositionInput { pub(crate) query: String, pub(crate) expanded: String, pub(crate) cleaned: String, - pub(crate) passes: Vec>, + pub(crate) passes: Vec>>, } #[pyclass(module = "sqlbuild._native")] @@ -83,7 +88,11 @@ impl From for validation::PositionInput { query: input.query, expanded: input.expanded, cleaned: input.cleaned, - passes: input.passes, + passes: input + .passes + .into_iter() + .map(Positional::unwrap_all) + .collect(), } } } @@ -148,3 +157,35 @@ pub(crate) struct DeclarationTextRequest { pub(crate) audit_keys: Vec, pub(crate) hook_keys: Vec, } + +/// A record of another crate exchanged with Python as its positional tuple. +#[derive(Clone, Debug)] +pub(crate) struct Positional(pub(crate) T); + +impl Positional { + /// Each record of `items`, for Python. + pub(crate) fn wrap_all(items: Vec) -> Vec { + items.into_iter().map(Self).collect() + } + + /// Each record Python sent in `items`. + pub(crate) fn unwrap_all(items: Vec) -> Vec { + items.into_iter().map(|item| item.0).collect() + } +} + +impl<'py, T: FromPositional> FromPyObject<'py> for Positional { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + T::from_positional(value).map(Self) + } +} + +impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { + type Target = PyTuple; + type Output = Bound<'py, PyTuple>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.0.into_positional(py) + } +} diff --git a/crates/sqlbuild-python/src/bindings/types.rs b/crates/sqlbuild-python/src/bindings/types.rs index f0038d008c..367b8e9c5e 100644 --- a/crates/sqlbuild-python/src/bindings/types.rs +++ b/crates/sqlbuild-python/src/bindings/types.rs @@ -1,6 +1,19 @@ +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; + pub(crate) trait CompilerDetach { fn compiler_detach Result + Send>( self, operation: F, ) -> Result; } + +/// A named record read from the positional tuple Python sends. +pub(crate) trait FromPositional: Sized { + fn from_positional(value: &Bound<'_, PyAny>) -> PyResult; +} + +/// A named record converted into the positional tuple Python receives. +pub(crate) trait IntoPositional { + fn into_positional<'py>(self, py: Python<'py>) -> PyResult>; +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs index 9a3becdfd7..fd68155fc5 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs @@ -46,8 +46,16 @@ pub(crate) struct ModelBody { pub(crate) passes: Vec>, } -/// `RESOURCE_CALL_PATTERN` on a site's text: `(kind, name, name_start, name_end)`. -fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { +/// A resource call a site's text holds, with its name span within that text. +struct ResourceCall { + kind: String, + name: String, + name_start: usize, + name_end: usize, +} + +/// `RESOURCE_CALL_PATTERN` on a site's text. +fn resource_call(text: &[char]) -> Option { if !starts_with(text, 0, "__") { return None; } @@ -82,12 +90,12 @@ fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { if !(rest.is_empty() || rest == ['\n']) { return None; } - Some(( - kind.to_owned(), - slice(text, name_start, name_end), + Some(ResourceCall { + kind: kind.to_owned(), + name: slice(text, name_start, name_end), name_start, name_end, - )) + }) } fn skip_space(text: &[char], mut index: usize) -> usize { @@ -102,7 +110,12 @@ pub(crate) fn resource_sites(text: &[char], context: &ScanContext) -> Vec AnalysisSql let call = resource_call(&chars(&site.text)); let kind: char = call .as_ref() - .and_then(|(kind, ..)| kind.chars().next()) + .and_then(|call| call.kind.chars().next()) .unwrap_or(GENERIC_PLACEHOLDER_KIND); let placeholder = placeholder(kind, index, site.end - site.start); - if let Some((kind, name, ..)) = call { + if let Some(ResourceCall { kind, name, .. }) = call { let key = (kind, name); match tables.iter_mut().find(|(item, _)| *item == key) { Some((_, placeholders)) => { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index fc694bf75b..eda014fbd7 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -68,9 +68,14 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result schema_facts(&row, schema, inputs)?, - None => (Vec::new(), Vec::new(), false, 0), + None => SchemaFacts::default(), }; Ok(Model { sql_analysis_disabled, @@ -97,7 +102,14 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result, Vec, bool, u32); +/// A model's declared columns and dynamic families with their proofs and audit count. +#[derive(Default)] +struct SchemaFacts { + columns: Vec, + dynamic_columns: Vec, + dynamic_columns_proven: bool, + audit_count: u32, +} fn schema_facts( row: &ModelRow, @@ -141,7 +153,12 @@ fn schema_facts( .map(|column| column.audit_count) .sum::(); if schema.dynamic_columns.is_empty() { - return Ok((columns, Vec::new(), false, audit_count)); + return Ok(SchemaFacts { + columns, + dynamic_columns: Vec::new(), + dynamic_columns_proven: false, + audit_count, + }); } let proof_types: HashMap<&str, Option<&str>> = row .dynamic_proof @@ -172,7 +189,12 @@ fn schema_facts( .dynamic_proof .as_ref() .is_some_and(|proof| proof.output_proven); - Ok((columns, dynamic_columns, proven, audit_count)) + Ok(SchemaFacts { + columns, + dynamic_columns, + dynamic_columns_proven: proven, + audit_count, + }) } /// Python's `types_equal`, normalizing each type once; Python raises what normalization rejects. diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 92bc83d6e9..b39a5d3784 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -208,7 +208,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann prepared .1 .first() - .map(|site| site.5.clone()) + .map(|site| site.original_text.clone()) .unwrap_or_default() }); assert_eq!( @@ -277,7 +277,7 @@ fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quo for test_case in test_cases { let lint_sql = format!("SELECT @m({} x) AS y", test_case.fragment); let lint_site = prepare(&lint_sql, &lint_sql, &[], test_case.dialect)? - .and_then(|prepared| prepared.1.first().map(|site| site.5.clone())) + .and_then(|prepared| prepared.1.first().map(|site| site.original_text.clone())) .unwrap_or_default(); assert_eq!( lint_site, test_case.expected_lint_site, @@ -311,7 +311,13 @@ fn given_dollar_quoted_apostrophe_when_preparing_lint_sites_then_later_reference let sql = "SELECT $$Customer's order$$ AS note, $tag$it's$tag$ AS tag FROM __ref(\"orders\") o"; for test_case in test_cases { let sites: Vec = prepare(sql, sql, &[], test_case.dialect)? - .map(|prepared| prepared.1.into_iter().map(|site| site.5).collect()) + .map(|prepared| { + prepared + .1 + .into_iter() + .map(|site| site.original_text) + .collect() + }) .unwrap_or_default(); assert_eq!(sites, test_case.expected_sites, "{}", test_case.description); } diff --git a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs index 5e6fa94418..4eff333623 100644 --- a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs +++ b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs @@ -51,7 +51,7 @@ pub fn compare( .iter() .filter_map(|(identity, count)| { (*count > *diagnostic.before.get(identity).unwrap_or(&0)) - .then_some(identity.0.as_str()) + .then_some(identity.code.as_str()) }) .collect(); if !codes.is_empty() { diff --git a/crates/sqlbuild-rules/src/fix_verification/models.rs b/crates/sqlbuild-rules/src/fix_verification/models.rs index 3d5cc6dc99..8f0d439363 100644 --- a/crates/sqlbuild-rules/src/fix_verification/models.rs +++ b/crates/sqlbuild-rules/src/fix_verification/models.rs @@ -29,13 +29,22 @@ pub struct ModelComparison { pub lineage_equal: bool, } +/// A diagnostic's location-insensitive identity. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct DiagnosticIdentity { + pub code: String, + pub resource_type: String, + pub resource_name: String, + pub message: String, +} + /// Location-insensitive diagnostic multiplicities for one owner. #[derive(Debug)] pub struct DiagnosticComparison { pub owner: usize, pub edited: bool, - pub before: BTreeMap<(String, String, String, String), i64>, - pub after: BTreeMap<(String, String, String, String), i64>, + pub before: BTreeMap, + pub after: BTreeMap, } /// Failures attributed to edited owners and changes outside those owners. diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 9599fb1e2b..43cb230326 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -217,11 +217,20 @@ fn keepends_lengths(text: &str) -> Vec { .collect() } +/// The fields of one native diagnostic, its span in lint-text characters. +struct DiagnosticFields { + code: String, + message: String, + remediation: String, + start: i64, + end: i64, +} + /// The fields of one diagnostic: code, message, remediation, start and end. fn diagnostic_fields( diagnostic: &Map, body_length: i64, -) -> Result<(String, String, String, i64, i64), String> { +) -> Result { let text = |key: &str| { diagnostic .get(key) @@ -242,7 +251,13 @@ fn diagnostic_fields( if start < 0 || end < start || end > body_length { return Err(INVALID_DIAGNOSTIC.to_string()); } - Ok((code, message, remediation, start, end)) + Ok(DiagnosticFields { + code, + message, + remediation, + start, + end, + }) } /// Map one native diagnostic onto authored source, or drop it when it belongs to generated SQL. @@ -255,7 +270,13 @@ pub(crate) fn authored_diagnostic( return Err(NOT_AN_OBJECT.to_string()); }; let body_length: i64 = body.lint_text.chars().count() as i64; - let (code, message, remediation, start, end) = diagnostic_fields(diagnostic, body_length)?; + let DiagnosticFields { + code, + message, + remediation, + start, + end, + } = diagnostic_fields(diagnostic, body_length)?; let reason: Option = match diagnostic.get("fix_unavailable_reason") { None | Some(Value::Null) => None, Some(Value::String(reason)) => Some(reason.clone()), diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs index 36bb34d459..aa80d6d1ab 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -54,6 +54,14 @@ fn skippable(line: &str) -> bool { stripped.is_empty() || stripped.starts_with("--") } +/// Where a retained violation sits, its code and whether a native rule reported it. +struct SuppressedTarget<'a> { + file: usize, + line: usize, + code: &'a str, + native: bool, +} + /// Whether a retained violation is the native finding one directive suppresses. pub(crate) fn matches_directive( item: &RetainedViolation, @@ -61,23 +69,28 @@ pub(crate) fn matches_directive( file: usize, directive: &SuppressionDirective, ) -> bool { - let (item_file, line, code, native): (usize, usize, &str, bool) = match item { + let target: SuppressedTarget<'_> = match item { RetainedViolation::Existing(index) => { let violation: &SuppressibleViolation = &violations[*index]; - ( - violation.file, - violation.line, - &violation.code, - violation.native, - ) + SuppressedTarget { + file: violation.file, + line: violation.line, + code: &violation.code, + native: violation.native, + } } - RetainedViolation::Added(finding) => (finding.file, finding.line, SUPPRESSION_CODE, true), + RetainedViolation::Added(finding) => SuppressedTarget { + file: finding.file, + line: finding.line, + code: SUPPRESSION_CODE, + native: true, + }, }; - item_file == file - && line == directive.target - && code == directive.code - && native - && code.starts_with(NATIVE_RULE_PREFIX) + target.file == file + && target.line == directive.target + && target.code == directive.code + && target.native + && target.code.starts_with(NATIVE_RULE_PREFIX) } /// Findings for every directive line when the project forbids suppressions. diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs index f514682f67..4ef2ad23dc 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs @@ -7,7 +7,7 @@ use regex::Regex; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::models::Neutralized; -use crate::sql_lint::types::{InterpolationSite, PreparedSql}; +use crate::sql_lint::types::PreparedSql; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; static CTES: LazyLock> = LazyLock::new(|| { @@ -68,21 +68,7 @@ pub(crate) fn prepare( } let mut referenced: Vec = referenced.into_iter().collect(); referenced.sort(); - let sites: Vec = neutralized - .sites - .into_iter() - .map(|site| { - ( - site.sentinel, - site.neutralized_start, - site.neutralized_end, - site.original_start, - site.original_end, - site.original_text, - ) - }) - .collect(); - Ok(Some((neutralized.text, sites, referenced))) + Ok(Some((neutralized.text, neutralized.sites, referenced))) } /// The byte offset of character position `position`, or `None` past the end. diff --git a/crates/sqlbuild-rules/src/sql_lint/types.rs b/crates/sqlbuild-rules/src/sql_lint/types.rs index 1cb260e2c3..ef7601e9a1 100644 --- a/crates/sqlbuild-rules/src/sql_lint/types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/types.rs @@ -1,2 +1,4 @@ -pub(crate) type InterpolationSite = (String, usize, usize, usize, usize, String); +use crate::lint_text::models::InterpolationSite; + +/// Lint text with sentinels, its interpolation sites and the CTEs interpolation references. pub type PreparedSql = (String, Vec, Vec); diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs index cde7244b9a..3f09e406fa 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs @@ -26,12 +26,14 @@ pub(crate) fn unique_grants(index: &ScopeIndex, grants: Vec) -> Vec .collect() } -type GrantKey<'a> = ( - &'a ResourceIdentity, - &'a DeclarationIdentity, - GrantThroughKey<'a>, - GrantKind, -); +/// What makes two grants the same: resource, declaration, what grants it and its kind. +#[derive(PartialEq, Eq, Hash)] +struct GrantKey<'a> { + resource: &'a ResourceIdentity, + declaration: &'a DeclarationIdentity, + through: GrantThroughKey<'a>, + kind: GrantKind, +} #[derive(PartialEq, Eq, Hash)] enum GrantThroughKey<'a> { @@ -40,17 +42,17 @@ enum GrantThroughKey<'a> { } fn grant_key<'a>(index: &'a ScopeIndex, grant: &'a GrantRecord) -> GrantKey<'a> { - ( - &grant.resource, - &index.declarations[grant.declaration].identity, - match &grant.through { + GrantKey { + resource: &grant.resource, + declaration: &index.declarations[grant.declaration].identity, + through: match &grant.through { GrantThrough::Model(name) => GrantThroughKey::Model(name), GrantThrough::Declaration(declaration) => { GrantThroughKey::Declaration(&index.declarations[*declaration].identity) } }, - grant.kind, - ) + kind: grant.kind, + } } /// Visibility and identity lookups over one index, as the Python grant functions use them. diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs index 0a9859764e..fb3f786713 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs @@ -11,14 +11,23 @@ pub(crate) fn resource_key(record: &ResourceRecord) -> (String, String) { (record.path.clone(), resource_text(&record.identity)) } +/// A declaration sort key: identity text, path, then line and column as `P` compares them. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct DeclarationKey

{ + identity: String, + path: String, + line: P, + column: P, +} + /// The builder's `_declaration_key`, which compares line and column as integers. -pub(crate) fn declaration_key(record: &DeclarationRecord) -> (String, String, i64, i64) { - ( - declaration_text(&record.identity), - record.path.clone(), - record.line, - record.column, - ) +pub(crate) fn declaration_key(record: &DeclarationRecord) -> DeclarationKey { + DeclarationKey { + identity: declaration_text(&record.identity), + path: record.path.clone(), + line: record.line, + column: record.column, + } } /// The lookup's `resource_sort_key`. @@ -34,13 +43,13 @@ pub(crate) fn canonical_resource_key( /// The lookup's `declaration_sort_key`, which compares line and column as text. pub(crate) fn canonical_declaration_key( record: &DeclarationRecord, -) -> Result<(String, String, String, String), ScopeDeferral> { - Ok(( - declaration_text(&record.identity), - normalize_path(&record.path)?, - record.line.to_string(), - record.column.to_string(), - )) +) -> Result, ScopeDeferral> { + Ok(DeclarationKey { + identity: declaration_text(&record.identity), + path: normalize_path(&record.path)?, + line: record.line.to_string(), + column: record.column.to_string(), + }) } /// Stable sort of `members` by a fallible key, as Python's `sorted(..., key=...)`. diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs index 50af50a8f9..9a6e8a92eb 100644 --- a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs @@ -14,7 +14,12 @@ use crate::scope_report::models::{ }; /// `_declaration_container`: role, visibility label, role root and bucket path. -pub(crate) type Container = (Option, &'static str, Option, Option); +pub(crate) struct Container { + pub(crate) role: Option, + pub(crate) visibility: &'static str, + pub(crate) role_root: Option, + pub(crate) bucket: Option, +} pub(crate) fn declaration_text(record: &DeclarationRecord) -> String { format_identity(&QualifiedIdentity::Declaration(record.identity.clone())) @@ -50,7 +55,12 @@ fn visibility_label(scope: ScopeKind) -> &'static str { pub(crate) fn container(record: &DeclarationRecord) -> Container { let visibility: &'static str = visibility_label(record.scope); if record.scope == ScopeKind::Private { - return (None, visibility, None, None); + return Container { + role: None, + visibility, + role_root: None, + bucket: None, + }; } let role_parts: &[&str] = record.identity.kind.role_parts(); let safe: String = safe_scope_path(&record.path); @@ -65,12 +75,12 @@ pub(crate) fn container(record: &DeclarationRecord) -> Container { let role_end: usize = (role_index + role_parts.len()).min(parts.len()); let bucket_end: usize = parts.len().saturating_sub(1).max(role_end); let bucket: String = parts[role_end..bucket_end].join("/"); - ( - Some(role_parts.join("/")), + Container { + role: Some(role_parts.join("/")), visibility, - Some(parts[..role_end].join("/")), - (!bucket.is_empty()).then_some(bucket), - ) + role_root: Some(parts[..role_end].join("/")), + bucket: (!bucket.is_empty()).then_some(bucket), + } } /// The report metadata of one declaration's values and container. @@ -112,7 +122,12 @@ fn metadata(values: Option<&DeclarationValues>, container: Container) -> Vec<(St push("render_as", optional_text(constant.render_as.as_deref())); } } - let (role, visibility, role_root, bucket) = container; + let Container { + role, + visibility, + role_root, + bucket, + } = container; push("role", optional_text(role.as_deref())); push("visibility_label", text(visibility)); push("role_root", optional_text(role_root.as_deref())); diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs index 07e79ead32..04f4774014 100644 --- a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs @@ -6,7 +6,9 @@ use crate::scope_index::main::format_identity::format_identity; use crate::scope_index::models::{ DeclarationIdentity, DeclarationRecord, QualifiedIdentity, ResourceIdentity, }; -use crate::scope_report::_helpers::declarations::{container, integer, optional_text, text, texts}; +use crate::scope_report::_helpers::declarations::{ + Container, container, integer, optional_text, text, texts, +}; use crate::scope_report::_helpers::json::{diagnostic_json, object}; use crate::scope_report::_helpers::paths::safe_scope_path; use crate::scope_report::constants::METADATA_SCHEMA_VERSION; @@ -91,7 +93,12 @@ fn declaration_projection( record: &DeclarationRecord, values: Option<&DeclarationValues>, ) -> JsonValue { - let (role, visibility, role_root, bucket) = container(record); + let Container { + role, + visibility, + role_root, + bucket, + } = container(record); object(vec![ ("identity", text(&declaration_key(&record.identity))), ("kind", text(record.identity.kind.as_str())), diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs index ea40a71473..ba1f16bef2 100644 --- a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs @@ -80,28 +80,47 @@ fn chain_lines( lines } +/// One availability section: its title, declarations, report section name and whether optional. +struct AvailabilityGroup<'a> { + title: &'a str, + declarations: &'a [DeclarationReport], + name: &'a str, + optional: bool, +} + fn availability_lines( (style, request): (CliStyle, &ScopeTextRequest), report: &ScopeReport, used: &HashSet, ) -> Vec { - let groups: [(&str, &[DeclarationReport], &str, bool); 3] = [ - ("Available", &report.available, "available", false), - ( - "Relationship grants", - &report.relationship_scope, - "relationship_scope", - true, - ), - ( - "Nearby unavailable", - &report.nearby_unavailable, - "nearby_unavailable", - true, - ), + let groups: [AvailabilityGroup<'_>; 3] = [ + AvailabilityGroup { + title: "Available", + declarations: &report.available, + name: "available", + optional: false, + }, + AvailabilityGroup { + title: "Relationship grants", + declarations: &report.relationship_scope, + name: "relationship_scope", + optional: true, + }, + AvailabilityGroup { + title: "Nearby unavailable", + declarations: &report.nearby_unavailable, + name: "nearby_unavailable", + optional: true, + }, ]; let mut lines: Vec = Vec::new(); - for (title, declarations, name, optional) in groups { + for AvailabilityGroup { + title, + declarations, + name, + optional, + } in groups + { let section: Option<&ScopeSection> = report_section(report, name); let collapsed_count: i64 = section.map_or(0, |section| section.collapsed_count); if optional && !request.verbose && declarations.is_empty() && collapsed_count == 0 { diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index da4557cda7..6d583b0502 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -1,7 +1,8 @@ //! `@@name`, `@@ENV:NAME` and `@@CTX:name` interpolation of authored SQL. +use crate::compiler::models::CharSpan; use crate::compiler::models::{InterpolatedSql, InterpolationFailure, InterpolationRead}; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::types::InterpolationHost; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; use sqlbuild_core::text::main::is_letter::is_letter; @@ -11,8 +12,13 @@ const TOKEN: &str = "@@"; const ENVIRONMENT_PREFIX: &str = "ENV:"; const CONTEXT_PREFIX: &str = "CTX:"; -/// One substitution in byte offsets: `(source start, source end, output start, output end)`. -type ByteSpan = (usize, usize, usize, usize); +/// One substitution in bytes: source start and end, output start and end. +struct ByteSpan { + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, +} struct Interpolation<'run, H: InterpolationHost> { host: &'run H, @@ -112,8 +118,12 @@ impl Interpolation<'_, H> { let text = &sql[..limit]; let output_start = self.output.len(); let end = self.rendered_token(text, start)?; - self.spans - .push((start, end, output_start, self.output.len())); + self.spans.push(ByteSpan { + source_start: start, + source_end: end, + output_start, + output_end: self.output.len(), + }); Ok(end) } @@ -256,13 +266,11 @@ fn character_spans(source: &str, output: &str, spans: &[ByteSpan]) -> Vec, - pub spans: Vec, + pub spans: Vec, pub reads: Vec, } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index a8225ae81c..0326016fed 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -4,12 +4,12 @@ use crate::compiler::_helpers::model_headers::tokenization::{ use crate::compiler::main::declaration_references::scan_declaration_references; use crate::compiler::main::model_header_single_parsing::parse_one; use crate::compiler::main::sql_interpolation::interpolate_sql; +use crate::compiler::models::CharSpan; use crate::compiler::models::{ AuthoredValue, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, InterpolationRead, }; use crate::compiler::tests::test_types::{HeaderNestingTestCase, MapHost}; -use crate::compiler::types::CharSpan; use std::collections::BTreeMap; use std::thread; @@ -251,13 +251,22 @@ pub(crate) fn interpolated(host: &MapHost, sql: &str) -> Result pub(crate) fn interpolation_facts( host: &MapHost, sql: &str, -) -> (Vec, Vec) { +) -> (Vec<[usize; 4]>, Vec) { interpolate_sql(host, sql, "models/orders.sql").map_or_else( |failure| (Vec::new(), failure.reads), - |result| (result.spans, result.reads), + |result| (result.spans.iter().map(span_parts).collect(), result.reads), ) } +fn span_parts(span: &CharSpan) -> [usize; 4] { + [ + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ] +} + /// Named locations from `[line, column, end_line, end_column]` rows. pub(crate) fn locations( columns: Vec<(&str, [usize; 4])>, diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs index 7a2eafc102..308d2501e4 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs @@ -136,10 +136,10 @@ fn given_tokens_when_interpolating_then_spans_count_code_points_and_reads_keep_o description: "variables, environment, context and a bare @@ after non-ASCII text", sql: "SELECT 'ü', @@größe, @@ENV:REGION, @@CTX:this @@", expected_spans: vec![ - (12, 19, 12, 17), - (21, 33, 19, 21), - (35, 45, 23, 29), - (46, 48, 30, 32), + [12, 19, 12, 17], + [21, 33, 19, 21], + [35, 45, 23, 29], + [46, 48, 30, 32], ], expected_reads: vec![ InterpolationRead::Environment("REGION".to_owned()), diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index 319db3016a..d116ba54d6 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::compiler::models::{InterpolationRead, NestingFailure}; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::types::InterpolationHost; use std::collections::BTreeMap; pub(crate) struct ModelHeaderTokenizationTestCase { @@ -51,7 +51,7 @@ pub(crate) struct InterpolationTestCase { pub(crate) struct InterpolationFactsTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, - pub(crate) expected_spans: Vec, + pub(crate) expected_spans: Vec<[usize; 4]>, pub(crate) expected_reads: Vec, } diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs index a16c3efdff..95b3a0bf9d 100644 --- a/crates/sqlbuild-sqltext/src/compiler/types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -1,6 +1,3 @@ -/// One substitution in code points: `(source start, source end, output start, output end)`. -pub type CharSpan = (usize, usize, usize, usize); - /// The project variables, process environment and context values one interpolation reads. pub trait InterpolationHost { /// The text of project variable `name`, `Err(message)` when it cannot be text, or `None`. diff --git a/fensu.toml b/fensu.toml index 9394b3890e..e762cf3943 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1151,6 +1151,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=15", @@ -1177,11 +1178,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=20", @@ -1198,11 +1199,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=1", @@ -1227,6 +1229,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=1", @@ -1259,9 +1262,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", @@ -1275,6 +1279,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=2", @@ -1288,7 +1293,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs=66", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", @@ -1304,10 +1309,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=2", @@ -1320,7 +1327,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=33", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=6", @@ -1342,12 +1349,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=4", From 7c7bf79c39c44ad3e1f09b7a3e55025f4c0f8ca8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 11/39] chore: enable fensu dead-code enforcement and remove dead code --- fensu.toml | 1137 ++++++++++++++++- .../_helpers/random_dag_project.py | 153 --- scripts/cold_compile_performance/models.py | 14 - .../_helpers/analysis_corpus/failure_cases.py | 3 - .../_helpers/comparing/capture_text.py | 43 + .../_helpers/coverage/analysis.py | 2 +- .../_helpers/coverage/capture_values.py | 2 +- .../_helpers/coverage/discovery.py | 2 +- .../_helpers/coverage/render.py | 2 +- .../classes/capture_file.py | 92 -- scripts/fensu_policy/constants.py | 14 - .../_helpers/host_roots.py | 54 + scripts/native_module_paths/constants.py | 3 + .../_helpers/capturing/canonical.py | 2 +- src/sqlbuild/adapter/contract/constants.py | 16 - src/sqlbuild/adapters/bigquery/constants.py | 3 - src/sqlbuild/adapters/clickhouse/constants.py | 1 - .../adapters/duckdb/_helpers/__init__.py | 1 - src/sqlbuild/adapters/snowflake/constants.py | 8 - src/sqlbuild/adapters/sqlite/constants.py | 1 - src/sqlbuild/compiler/frontier/__init__.py | 0 src/sqlbuild/compiler/lineage/__init__.py | 0 src/sqlbuild/compiler/migrations/__init__.py | 0 src/sqlbuild/compiler/planner/__init__.py | 0 src/sqlbuild/executor/clone/types.py | 4 - src/sqlbuild/executor/python_nodes/models.py | 10 - .../dagster/_helpers/translation.py | 12 - .../integrations/dagster/constants.py | 15 - .../_helpers/_test_types.py | 7 + .../native_module_paths/_helpers/helpers.py | 3 + .../_helpers/test_native_host.py | 25 +- .../compiler/frontier/test_stage_capture.py | 2 +- 32 files changed, 1249 insertions(+), 382 deletions(-) delete mode 100644 scripts/cold_compile_performance/_helpers/random_dag_project.py create mode 100644 scripts/compiler_differential/_helpers/comparing/capture_text.py delete mode 100644 scripts/compiler_differential/classes/capture_file.py create mode 100644 scripts/native_module_paths/_helpers/host_roots.py delete mode 100644 src/sqlbuild/adapters/duckdb/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/frontier/__init__.py delete mode 100644 src/sqlbuild/compiler/lineage/__init__.py delete mode 100644 src/sqlbuild/compiler/migrations/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/__init__.py diff --git a/fensu.toml b/fensu.toml index e762cf3943..4aa8804da8 100644 --- a/fensu.toml +++ b/fensu.toml @@ -21,6 +21,1097 @@ require_cacheable = true [targets.python.evaluation] exclude = ["tests/e2e/fixtures/**"] +[targets.python.dead_code] +enabled = true + +[[targets.python.dead_code.roots]] +modules = ["scripts.*"] +symbols = ["main"] +reason = "Repository tooling runs these modules with `python -m scripts.` from the Makefile, CI and pre-commit." + +[[targets.python.dead_code.roots]] +modules = ["scripts.fensu_policy.rules.*"] +symbols = ["*"] +reason = "Fensu loads these custom rule modules from rule_modules in this file." + +[[targets.python.dead_code.roots]] +modules = [ + "sqlbuild.rule_engine._helpers.host.custom_host", + "sqlbuild.rule_engine._helpers.host.custom_host_launch", +] +symbols = ["main"] +reason = "The custom-rule host processes run these modules with `python -m`; native code starts the launcher, which starts the host." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild._native_host"] +symbols = ["*"] +reason = "Native code imports this module by path and reads its attributes by name." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.classes.base_adapter"] +symbols = ["BaseAdapter"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.exceptions"] +symbols = ["AdapterUserError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.models"] +symbols = ["ColumnInfo", "ExpressionInferenceProfile", "RelationInfo", "RelationLookup", "RetentionRequest", "RowDiffSampling", "RowDiffTolerance", "RowDiffTolerances", "TableFreshnessMetadata", "TableFreshnessRequest"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.types"] +symbols = ["BuiltinAdapter", "FrameworkType", "LifeCycleEventKind", "MigrationTransfer", "RetentionScope", "TableFreshnessStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main._session_warehouse_support"] +symbols = ["adapter_supports_session_warehouse"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.builtins"] +symbols = ["builtin_adapter_classes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.project_adapters"] +symbols = ["discover_project_adapters"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.resolve_adapter"] +symbols = ["resolve_adapter"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main._get_columns_for_inspection"] +symbols = ["get_columns_for_inspection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main._list_relations_for_inspection"] +symbols = ["list_relations_for_inspection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.active_inspection_catalog"] +symbols = ["active_inspection_catalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.inspection_query_sink"] +symbols = ["inspection_query_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.open_inspection_catalog"] +symbols = ["open_inspection_catalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.record_inspection_query"] +symbols = ["record_inspection_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.relation_lookup"] +symbols = ["build_relation_lookup"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"] +symbols = ["resolve_relation_location_qualified_name"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.run_bounded_inspections"] +symbols = ["run_bounded_inspections"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.run_recorded_inspection_query"] +symbols = ["run_recorded_inspection_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.models"] +symbols = ["InspectionQueryRecord", "StatementMetadataEffect"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.conditional_result_nullability"] +symbols = ["conditional_result_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.first_arg_nullability"] +symbols = ["first_arg_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.normalize_relation_type"] +symbols = ["normalize_relation_type"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.normalize_type"] +symbols = ["normalize_type"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.safe_cast_nullability"] +symbols = ["safe_cast_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"] +symbols = ["dispatch_with_compute_logs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse._helpers.provider_settings"] +symbols = ["provider_settings_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse._helpers.startup_inputs"] +symbols = ["startup_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse.models"] +symbols = ["CompileReuseRequest"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.auditing.types"] +symbols = ["AuditEvaluationMode", "AuditOutcome", "AuditSeverity", "ThresholdOperator"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.diagnostics.python_resources"] +symbols = ["python_resource_rows"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.macro_bridge.call_store"] +symbols = ["target_context_with_macro_call_store"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.render.user_macros"] +symbols = ["inventory_project_macros", "load_project_macros", "macro_invocation_context"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile.types"] +symbols = ["AttachedAuditTargetKind", "FunctionLanguage", "SqlTestMode"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"] +symbols = ["discover_command_output_sink_functions"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.validation.discovery"] +symbols = ["validate_discovered_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery.main.runtime_extensions"] +symbols = ["discover_runtime_extensions"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.classes.macro_bridge"] +symbols = ["run_with_macro_bridge"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"] +symbols = ["macro_store_module_paths"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.main.active_macro_bridge"] +symbols = ["active_macro_bridge"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.pipeline._helpers.materializations"] +symbols = ["load_custom_materializations"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes._helpers.user_inventory"] +symbols = ["python_node_dependency_key", "python_node_identity"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes.main._hook_identities"] +symbols = ["build_hook_identities"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes.types"] +symbols = ["PythonNodeKind", "PythonNodeStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.references.models"] +symbols = ["ProjectRelation", "ProjectRelationIndex", "RelationName"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cost.models"] +symbols = ["CostResourceContext", "RunCostSummary"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cost.types"] +symbols = ["CostCapability", "CostStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cursor_algebra.exceptions"] +symbols = ["CursorAlgebraError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cursor_algebra.types"] +symbols = ["BoundSentinel"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics._helpers.process_resources"] +symbols = ["read_process_resources"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.constants"] +symbols = ["SQL_PRIVATE_RECORD_MARKER"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.diagnostics_context"] +symbols = ["diagnostics_context"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.log_debug_event"] +symbols = ["log_debug_event"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.log_sql"] +symbols = ["log_sql"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.process_resource_reporting"] +symbols = ["process_resource_reporting"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.models"] +symbols = ["ProcessResourceUsage"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.errors.contracts.exceptions"] +symbols = ["ExecutorInputError", "SharedInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.main._current_audit_result_projection"] +symbols = ["current_audit_result_projection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.models"] +symbols = ["AuditExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.build.models"] +symbols = ["BuildCallbacks", "BuildCustomizations", "BuildExecutionResult", "BuildInitialState", "BuildRuntimeParams", "FunctionExecutionResult", "SeedExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.build.types"] +symbols = ["BuildStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._build_destination_transient_models"] +symbols = ["build_destination_transient_models"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._build_retention_requests"] +symbols = ["build_destination_retention_requests"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._fingerprinting"] +symbols = ["copy_clone_fingerprints"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._run_prephase_clone_stream"] +symbols = ["run_prephase_clone_stream"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main.execute"] +symbols = ["execute_clone"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.models"] +symbols = ["CloneExecutionInput", "CloneItemResult", "CloneSourceEntries"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.types"] +symbols = ["CloneStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.classes.query_artifact_lifecycle"] +symbols = ["QueryDiffArtifactLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.exceptions"] +symbols = ["FullDiffSizeGuardError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.main._execute_query"] +symbols = ["execute_query_diff"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.main.execute"] +symbols = ["execute_diff"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.models"] +symbols = ["DiffExecutionOptions", "DiffExecutionResult", "FullDiffSizeLimits", "RowDiffSamplingOverride"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.janitor.main.execute"] +symbols = ["execute_janitor_plan"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.janitor.models"] +symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.load.main.run"] +symbols = ["run_load_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.load.models"] +symbols = ["LoadCallbacks", "LoadExecutionResult", "LoadRuntimeParams"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.migrations._helpers.relation_for_location"] +symbols = ["migration_relation_for_location"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.old_name_views._helpers.janitor"] +symbols = ["is_compatibility_view"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.auditing"] +symbols = ["run_audit_pipeline_with_callbacks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.scenario"] +symbols = ["run_scenario_capture_pipeline", "run_scenario_local_test_pipeline", "run_scenario_test_pipeline", "select_scenario_snapshot_capture_candidates"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.seeding"] +symbols = ["run_seed_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.testing"] +symbols = ["run_test_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.exceptions"] +symbols = ["AuditExecutionError", "TemporaryTestFunctionError", "WarehouseConnectionError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure"] +symbols = ["raise_warehouse_connection_failure"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.main.run"] +symbols = ["run_build_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.models"] +symbols = ["AuditPipelineCallbacks", "TestPipelineCallbacks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes._helpers.ingress_execution"] +symbols = ["execute_ingress_python_loader_nodes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.classes.run_state"] +symbols = ["PythonNodeRunState"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main._checks"] +symbols = ["execute_python_checks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main._ingress"] +symbols = ["run_ingress_python_loader_nodes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main.read_side"] +symbols = ["create_read_side_python_execution_tracker"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.models"] +symbols = ["IngressCallbacks", "PythonCheckCallbacks", "PythonCheckExecutionResult", "PythonNodeExecutionResult", "PythonNodeRuntime"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.run.models"] +symbols = ["ModelExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.scenario.models"] +symbols = ["ScenarioCaptureSettings", "ScenarioLocalReplaySource", "ScenarioSnapshotCaptureLimits", "ScenarioTestSettings"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.scheduling.types"] +symbols = ["ExecutionStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.testing.models"] +symbols = ["SqlTestExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.constants"] +symbols = ["ASSET_SELECTION_COMMANDS", "CHECK_METADATA_EXCLUDED_KEYS", "CLONE_COMMAND", "COMPLETED_EXECUTION_STATUSES", "DAGSTER_ASSET_ENABLED_FIELD", "DAGSTER_ASSET_NODE_KINDS", "DAGSTER_CHECK_ENABLED_FIELD", "DAGSTER_CHECK_NAME_FIELD", "DAGSTER_DIRECT_KIND_NODE_KINDS", "DEFAULT_SELECTABLE_NODE_KINDS", "EXPLICIT_SELECTION_FLAGS", "INSUFFICIENT_CHECK_STATUS", "LIVE_EVENT_COMMANDS", "LOADER_NODE_KIND", "LOAD_SELECTABLE_NODE_KINDS", "MATERIALIZABLE_NODE_KINDS", "MODEL_NODE_KIND", "SCENARIO_CHECK_KIND", "SOURCE_NODE_KIND", "STANDALONE_CHECK_COMMANDS", "SUCCESS_EXECUTION_STATUS", "VIEW_MATERIALIZATION_TYPE", "WARNING_CHECK_SEVERITY"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.exceptions"] +symbols = ["DagsterDagInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt._helpers.profile.load"] +symbols = ["load_dbt_yaml"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt._helpers.profile.render"] +symbols = ["render_selected_dbt_profile_output"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt.exceptions"] +symbols = ["DbtProfileError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.constants"] +symbols = ["DLT_INTEGRATION_KIND"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.exceptions"] +symbols = ["DltIntegrationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.main._loaders"] +symbols = ["dlt_loader_function"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.models"] +symbols = ["DltResourceConfig", "DltSourceConfig"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.constants"] +symbols = ["INGESTR_INTEGRATION_KIND"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.exceptions"] +symbols = ["IngestrIntegrationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.main._loaders"] +symbols = ["ingestr_loader_function"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.models"] +symbols = ["IngestrSourceConfig"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.rivers.constants"] +symbols = ["RIVERS_ASSET_NODE_KINDS"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.classes.event_codec"] +symbols = ["MicrobatchEventCodec"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.constants"] +symbols = ["MICROBATCH_COLUMNS", "MICROBATCH_COLUMN_TYPES", "MICROBATCH_DIRECT_INDEXES", "MICROBATCH_GENERATION_WILDCARD", "MICROBATCH_TABLE_NAME"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.exceptions"] +symbols = ["MicrobatchStateError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.models"] +symbols = ["MicrobatchCoverageProjection", "MicrobatchInterval", "ProjectedMicrobatchInterval", "ReplayRequirementProjection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.types"] +symbols = ["ReplayRequirementState"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.provider.exceptions"] +symbols = ["ProviderInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.provider.main.session"] +symbols = ["build_provider_session"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.constants"] +symbols = ["RELATION_RENDERER"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_asset_definition"] +symbols = ["read_asset_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_audit_factory_definition"] +symbols = ["read_audit_factory_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_check_definition"] +symbols = ["read_check_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_factory_definition"] +symbols = ["read_factory_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_hook_definition"] +symbols = ["read_hook_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_loader_definition"] +symbols = ["read_loader_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_task_definition"] +symbols = ["read_task_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.models"] +symbols = ["AuditCase", "SqlResourceRef"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.types"] +symbols = ["PythonCheckSeverity", "SqlResourceRefKind"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.engine._custom_catalogue"] +symbols = ["load_custom_rules"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.engine.custom_payloads"] +symbols = ["custom_rule_payloads"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.classes.text_tee"] +symbols = ["TextComputeLogTee"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.exceptions"] +symbols = ["CaptureAlreadyExistsError", "CaptureNotFoundError", "CaptureStateError", "ComputeLogMetadataError", "ComputeLogPathError", "InvalidCaptureIdError", "InvalidComputeLogCursorError", "InvalidComputeLogLimitError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.models"] +symbols = ["CaptureByteCounts", "CaptureInventory", "CaptureInventoryItem", "CaptureMetadata", "ComputeLogReadChunk", "FinalCaptureMetadata", "PruneResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.types"] +symbols = ["ComputeLogStream"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.contracts.models"] +symbols = ["ConnectionHooks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.contracts.types"] +symbols = ["ExecutionResourceKind"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.exceptions"] +symbols = ["EventExporterInputError", "EventExporterStateError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"] +symbols = ["invoke_lifecycle_event_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._run_command_output_sinks"] +symbols = ["run_command_output_sinks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition"] +symbols = ["get_lifecycle_event_sink_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.exceptions"] +symbols = ["ExecutionHistoryStorageError", "IntegrityConflictError", "InvalidCursorError", "InvalidEventError", "InvalidFilterError", "InvalidLimitError", "ProjectionConsistencyError", "UnsupportedSchemaVersionError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.models"] +symbols = ["EventPage", "RunPage", "RunRecord", "StoredEvent"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.types"] +symbols = ["RunStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.exceptions"] +symbols = ["ExecutionDurationLimitError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main.execution_deadline_scope"] +symbols = ["execution_deadline_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.models"] +symbols = ["RunItemTimeouts"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.event_dispatcher"] +symbols = ["EventDispatcher"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.operation_lifecycle"] +symbols = ["OperationLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.exceptions"] +symbols = ["ObservabilityValidationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.create_lifecycle_event"] +symbols = ["create_lifecycle_event"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.current_execution_identity"] +symbols = ["current_execution_identity"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.diagnostic_log_to_json"] +symbols = ["diagnostic_log_to_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.dispatcher_scope"] +symbols = ["dispatcher_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.invocation_external_context_scope"] +symbols = ["invocation_external_context_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.invocation_scope"] +symbols = ["invocation_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.lifecycle_event_from_json"] +symbols = ["lifecycle_event_from_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.lifecycle_event_to_json"] +symbols = ["lifecycle_event_to_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.run_scope"] +symbols = ["run_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.models"] +symbols = ["DiagnosticLog", "LifecycleEvent", "OperationAttributes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.exceptions"] +symbols = ["CommandOutputValidationError", "OutputCaptureInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.main.configured_output_capture_scope"] +symbols = ["configured_output_capture_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.models"] +symbols = ["BoundCommandOutputSink", "CommandOutputCaptureSummary", "CommandOutputRecord", "OutputCaptureContext"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.types"] +symbols = ["CommandOutputStream", "OutputRecordPriority"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.sql_values.exceptions"] +symbols = ["SqlValueRenderingError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.bigquery.classes.bigquery_adapter"] +symbols = ["BigQueryAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.clickhouse.classes.clickhouse_adapter"] +symbols = ["ClickHouseAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.databricks.classes.databricks_adapter"] +symbols = ["DatabricksAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.duckdb.classes.duckdb_adapter"] +symbols = ["DuckDbAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.motherduck.classes.motherduck_adapter"] +symbols = ["MotherDuckAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.postgres.classes.postgres_adapter"] +symbols = ["PostgresAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.snowflake.classes.snowflake_adapter"] +symbols = ["SnowflakeAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.sqlite.classes.sqlite_adapter"] +symbols = ["SqliteAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.sqlserver.classes.sqlserver_adapter"] +symbols = ["SqlServerAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.dense_project"] +symbols = ["dense_model_path", "dense_test_path"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.diagnostic_measurement"] +symbols = ["measure_diagnostic_compile", "measure_function_name_scan", "measure_position_mapping"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.diagnostic_project"] +symbols = ["write_cascade_project", "write_diagnostic_project", "write_function_name_project", "write_single_model_diagnostic_project", "write_undescribed_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_benchmark"] +symbols = ["run_varied_cache_benchmark"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_project"] +symbols = ["write_varied_compile_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_statistics"] +symbols = ["varied_project_statistics"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit"] +symbols = ["assert_required_cgroup_memory_limit"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.read_compile_measurement"] +symbols = ["read_compile_measurement"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.semantic_compile_fingerprint"] +symbols = ["semantic_compile_fingerprint"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.models"] +symbols = ["VariedProjectStatistics"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.corpus.emitted_codes"] +symbols = ["analysis_error_codes", "case_emitted_codes", "discovery_error_codes", "emitted_failure_codes", "inline_compile_codes", "render_error_codes", "unaccounted_inline_codes"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.analysis"] +symbols = ["analysis_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.discovery"] +symbols = ["discovery_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.render"] +symbols = ["render_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.constants"] +symbols = ["ANALYSIS_UNREACHABLE_CODES", "DISCOVERY_UNREACHABLE_CODES", "RENDER_UNREACHABLE_CODES"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.main.failure_cases"] +symbols = ["failure_cases"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.main.generate_project"] +symbols = ["generate_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.models"] +symbols = ["EmittedCodes"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series._helpers.outcomes"] +symbols = ["full_compile_difference"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.constants"] +symbols = ["DELTA_COMPILE_LINE", "DOUBLE_CAST", "DOUBLE_COLUMN", "FLOAT_CAST", "FLOAT_COLUMN", "PROJECT_CONFIG"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.run_delta_compile_sequence"] +symbols = ["run_delta_compile_sequence"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.run_delta_compile_series"] +symbols = ["run_delta_compile_series"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.write_delta_compile_project"] +symbols = ["write_delta_compile_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.models"] +symbols = ["DeltaCompileOutcome", "DeltaCompileSequence", "DeltaCompileSeries"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.types"] +symbols = ["SequenceEdit"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.docs_skill._helpers.navigation"] +symbols = ["list_ordered_page_paths"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.format_performance._helpers.model_project"] +symbols = ["write_model_format_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.crates"] +symbols = ["native_module_path_errors", "scan_native_crates"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.host_roots"] +symbols = ["host_dead_code_roots", "host_targets"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.scan"] +symbols = ["scan_native_source"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths.models"] +symbols = ["NativeModulePathScan"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.setting_help.main.find_settings_without_snippet"] +symbols = ["find_settings_without_snippet"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.setting_help.main.parse_setting_snippet"] +symbols = ["parse_setting_snippet"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.constants", "sqlbuild.integrations.rivers.constants"] +symbols = ["*_EXCLUSIONS", "*_MEMBERS"] +reason = "Exhaustiveness guards: tests prove every enum member is either a member or an exclusion of each derived set." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile.types"] +symbols = ["AuditInput", "TypedSqlValueRenderer"] +reason = "src/sqlbuild/_native.pyi annotates native bindings with these protocols; Fensu does not read stubs." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract._helpers.raw_statement_audit"] +symbols = ["audit_raw_statement_calls"] +reason = "A unit test runs this repository audit over the adapter sources." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._prephase_row_from_clone_item"] +symbols = ["prephase_row_from_clone_item"] +reason = "The clone prephase-row tests use this entry point; production calls the helper directly." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.auditing"] +symbols = ["run_audit_pipeline"] +reason = "The audit pipeline tests drive this callback-free form; production calls run_audit_pipeline_with_callbacks." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.scenario"] +symbols = ["_ScenarioPipelineResult"] +reason = "PEP 695 type-parameter bounds in this module use the protocol; Fensu does not count bounds as references." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.host.module_state"] +symbols = ["module_state_token"] +reason = "The custom-rule host tests compute expected module-state tokens with it." + [targets.python.thresholds] max_positional_args = 1 @@ -50,11 +1141,6 @@ rule = "FFR309" path = "src/sqlbuild/cost/models.py" reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models and types they use." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/lint/__init__.py" -reason = "Native bindings own lint entry points, expansion preparation and compiler-verified fix planning; this package retains boundary models, custom-rule and macro callbacks." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" @@ -67,33 +1153,38 @@ reason = "Native bindings own microbatch coverage and replay projection, event I [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/cursor_algebra/__init__.py" -reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." +path = "src/sqlbuild/lint/__init__.py" +reason = "Empty package kept so the tests of the native lint, expansion and fix planning keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/spec/contracts/__init__.py" -reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." +rule = "FFL108" +path = "src/sqlbuild/lint/__init__.py" +reason = "Empty package kept so the tests of the native lint, expansion and fix planning keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/integrations/dbt/__init__.py" -reason = "Native bindings own the dbt interop commands, the dbt runner, its JSON event stream and project initialisation; this package keeps the dbt models, types and errors and the PyYAML and Jinja profile glue they call." +path = "src/sqlbuild/presentation/__init__.py" +reason = "Empty package kept so the tests of the native CLI presentation keep their mirrored location; it declares nothing." + +[[targets.python.rule_exceptions]] +rule = "FFL108" +path = "src/sqlbuild/presentation/__init__.py" +reason = "Empty package kept so the tests of the native CLI presentation keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/frontier/__init__.py" -reason = "Native bindings own the compile frontier and the installed code identity; this package only declares the frontier constants, types and errors they read and raise." +path = "src/sqlbuild/cursor_algebra/__init__.py" +reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/planner/__init__.py" -reason = "Native bindings own planning, plan entries, plan output, command plans and the planner value records; this package only declares the plan models, types, constants and errors the executor, integrations and pipeline import." +path = "src/sqlbuild/spec/contracts/__init__.py" +reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/migrations/__init__.py" -reason = "Native bindings own migration planning and state rows; this package only declares the migration models, decisions and table names the executor imports." +path = "src/sqlbuild/integrations/dbt/__init__.py" +reason = "Native bindings own the dbt interop commands, the dbt runner, its JSON event stream and project initialisation; this package keeps the dbt models, types and errors and the PyYAML and Jinja profile glue they call." [[targets.python.rule_exceptions]] rule = "FFR309" @@ -115,11 +1206,6 @@ rule = "FFR309" path = "src/sqlbuild/errors/contracts/__init__.py" reason = "Native bindings read expected-error codes, messages and help; this package only declares the shared and executor input errors callers raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/presentation/__init__.py" -reason = "Native bindings own CLI styling, terminal width and the transient line coordinator; this package only declares the display options and presentation types callers pass." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" @@ -190,11 +1276,6 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/exceptions.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types and the rendering error adapters raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/lineage/__init__.py" -reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." - [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/scripts/cold_compile_performance/_helpers/random_dag_project.py b/scripts/cold_compile_performance/_helpers/random_dag_project.py deleted file mode 100644 index 7ecb629896..0000000000 --- a/scripts/cold_compile_performance/_helpers/random_dag_project.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Seeded DuckDB projects whose models bind against shapes inferred from their producers.""" - -from __future__ import annotations - -import random -from collections.abc import Callable -from pathlib import Path - -from scripts.cold_compile_performance.models import RandomDagProject - -_RANDOM_DAG_ROOTS: int = 3 -_RANDOM_DAG_ROOT_SHARE: float = 0.1 -_RANDOM_DAG_FEATURE_SHARE: float = 0.1 -_RANDOM_DAG_WINDOW: int = 12 -_RANDOM_DAG_SOURCES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("raw_orders", ("id", "quantity", "status")), - ("raw_customers", ("id", "region")), -) - - -def write_random_dag_project(*, project_dir: Path, project: RandomDagProject) -> tuple[str, ...]: - """Write seeded models over inferred shapes; return model names in topological order.""" - - rng: random.Random = random.Random(project.seed) - project_dir.mkdir(parents=True, exist_ok=True) - rules: str = ", ".join(f'"{code}"' for code in project.rules) - (project_dir / "sqlbuild_project.toml").write_text( - f'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = [{rules}]\n', encoding="utf-8" - ) - (project_dir / "sources").mkdir(exist_ok=True) - (project_dir / "sources" / "raw.yml").write_text( - "sources:\n" - " - name: raw_orders\n description: Test source raw_orders.\n" - " expression: SELECT 1 AS id, 2 AS quantity, 'open' AS status\n" - " - name: raw_customers\n description: Test source raw_customers.\n" - " expression: SELECT 1 AS id, 'north' AS region\n", - encoding="utf-8", - ) - models: Path = project_dir / "models" - models.mkdir(exist_ok=True) - columns: dict[str, tuple[str, ...]] = {} - names: list[str] = [] - for index in range(project.model_count): - name: str = f"orders_{index:03d}" - options: str = "" - body: str - output: tuple[str, ...] - if index < _RANDOM_DAG_ROOTS or rng.random() < _RANDOM_DAG_ROOT_SHARE: - source, output = rng.choice(_RANDOM_DAG_SOURCES) - body = f'SELECT * FROM __source("{source}")' - else: - body, output = _random_dag_query( - rng=rng, index=index, names=names, columns=columns, errors=project.errors - ) - if project.run_ids and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - options += ', tags ["${CTX:run.id}"]' - if project.analysis_opt_outs and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - options += ", sql_analysis false" - (models / f"{name}.sql").write_text( - f'MODEL (description "Test model {name}.", materialized view{options});\n{body}\n', - encoding="utf-8", - ) - columns[name] = output - names.append(name) - if project.missing_reference: - (models / "orders_missing.sql").write_text( - 'MODEL (description "Test model orders_missing.", materialized view);\n' - 'SELECT id FROM __ref("orders_absent")\n', - encoding="utf-8", - ) - if project.cycle: - for name, upstream in ( - ("orders_loop_a", "orders_loop_b"), - ("orders_loop_b", "orders_loop_a"), - ): - (models / f"{name}.sql").write_text( - f'MODEL (description "Test model {name}.", materialized view);\n' - f'SELECT id FROM __ref("{upstream}")\n', - encoding="utf-8", - ) - return tuple(names) - - -def _random_dag_query( - *, - rng: random.Random, - index: int, - names: list[str], - columns: dict[str, tuple[str, ...]], - errors: bool, -) -> tuple[str, tuple[str, ...]]: - upstream: str = rng.choice(names[max(0, index - _RANDOM_DAG_WINDOW) :]) - other: str = rng.choice(names) - if errors and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - return ( - f'SELECT id, missing_{index} FROM __ref("{upstream}")', - ("id", f"missing_{index}"), - ) - return rng.choice(_query_builders())( - index=index, upstream=upstream, other=other, columns=columns - ) - - -def _star_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del index, other - return f'SELECT * FROM __ref("{upstream}")', columns[upstream] - - -def _projection_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del other - return ( - f"SELECT id, {columns[upstream][-1]} AS kept_{index}, " - f'CAST(id AS BIGINT) * {index} AS value_{index} FROM __ref("{upstream}")', - ("id", f"kept_{index}", f"value_{index}"), - ) - - -def _join_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - return ( - f"SELECT a.*, b.{columns[other][-1]} AS joined_{index} " - f'FROM __ref("{upstream}") AS a LEFT JOIN __ref("{other}") AS b ON a.id = b.id', - (*columns[upstream], f"joined_{index}"), - ) - - -def _union_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del index, columns - return ( - f'SELECT id FROM __ref("{upstream}") UNION ALL SELECT id FROM __ref("{other}")', - ("id",), - ) - - -def _aggregate_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del other, columns - return ( - f'SELECT id, COUNT(*) AS row_count_{index} FROM __ref("{upstream}") GROUP BY id', - ("id", f"row_count_{index}"), - ) - - -def _query_builders() -> tuple[Callable[..., tuple[str, tuple[str, ...]]], ...]: - return (_star_query, _projection_query, _join_query, _union_query, _aggregate_query) diff --git a/scripts/cold_compile_performance/models.py b/scripts/cold_compile_performance/models.py index 566a6ca7d8..9118eaf908 100644 --- a/scripts/cold_compile_performance/models.py +++ b/scripts/cold_compile_performance/models.py @@ -13,17 +13,3 @@ class VariedProjectStatistics: leaf_models: int array_models: int window_models: int - - -@dataclass(frozen=True) -class RandomDagProject: - """One seeded DuckDB project whose models bind against inferred upstream shapes.""" - - seed: int - model_count: int - errors: bool = False - run_ids: bool = False - analysis_opt_outs: bool = False - missing_reference: bool = False - cycle: bool = False - rules: tuple[str, ...] = () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 63172895f0..8f95cc214a 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -54,9 +54,6 @@ _EXPRESSION_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' _EXPRESSION_AUDIT_PATH: str = "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql" _TEST_PATH: str = "tests/unit/test_stg_orders.sql" -_ORDER_ROW: str = ( - " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status\n" -) def _mart(body: str) -> dict[str, str]: diff --git a/scripts/compiler_differential/_helpers/comparing/capture_text.py b/scripts/compiler_differential/_helpers/comparing/capture_text.py new file mode 100644 index 0000000000..a783261382 --- /dev/null +++ b/scripts/compiler_differential/_helpers/comparing/capture_text.py @@ -0,0 +1,43 @@ +"""Expand the shared nodes of a stage capture document.""" + +from __future__ import annotations + +import json + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest +from scripts.compiler_differential.constants import ( + STAGE_CAPTURE_ROOT_KEY, + STAGE_CAPTURE_SHARED_NODES_KEY, +) + +_SHARED_DOCUMENT_KEYS: frozenset[str] = frozenset( + {STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_ROOT_KEY} +) + + +def expand_capture_text(text: str) -> object: + """Parse a whole capture document, replacing every shared reference by its node.""" + + document: object = json.loads(text) + mapping: dict[str, object] | None = as_json_object(document) + nodes: dict[str, object] | None = as_json_object( + None if mapping is None else mapping.get(STAGE_CAPTURE_SHARED_NODES_KEY) + ) + if mapping is None or nodes is None or set(mapping) != _SHARED_DOCUMENT_KEYS: + return document + expanded: dict[str, object] = {} + for digest, node in nodes.items(): + expanded[digest] = _expanded(value=node, nodes=expanded) + return _expanded(value=mapping[STAGE_CAPTURE_ROOT_KEY], nodes=expanded) + + +def _expanded(*, value: object, nodes: dict[str, object]) -> object: + digest: str | None = shared_digest(value) + if digest is not None: + return nodes[digest] + items: dict[str, object] | None = as_json_object(value) + if items is not None: + return {key: _expanded(value=item, nodes=nodes) for key, item in items.items()} + if isinstance(value, list): + return [_expanded(value=item, nodes=nodes) for item in value] + return value diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index 5116c308c5..d7a532970f 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -7,6 +7,7 @@ from collections.abc import Iterator, Sequence from pathlib import Path +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -15,7 +16,6 @@ has_content, records, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( ANALYSIS_CALLABLE_CAPTURE_FIELDS, ANALYSIS_DETAIL_KINDS, diff --git a/scripts/compiler_differential/_helpers/coverage/capture_values.py b/scripts/compiler_differential/_helpers/coverage/capture_values.py index 16d29d8956..12744bf938 100644 --- a/scripts/compiler_differential/_helpers/coverage/capture_values.py +++ b/scripts/compiler_differential/_helpers/coverage/capture_values.py @@ -5,8 +5,8 @@ import re from collections.abc import Iterator +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( CALLABLE_MARKER, COLLECTION_CAPTURE_MARKERS, diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index da0e2810aa..b6823157d5 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from pathlib import Path +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -17,7 +18,6 @@ named_values, records, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( CALLABLE_CAPTURE_FIELDS, CRLF_BYTES, diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 8f314e0bce..ee553ba958 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from pathlib import Path, PurePosixPath +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -20,7 +21,6 @@ macro_function_source, takes_typed_reference, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( BUILT_IN_AUDIT_NAMES, COMPILE_INPUTS_CAPTURE_TYPE, diff --git a/scripts/compiler_differential/classes/capture_file.py b/scripts/compiler_differential/classes/capture_file.py deleted file mode 100644 index 3017d571fd..0000000000 --- a/scripts/compiler_differential/classes/capture_file.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Read one stage capture from disk node by node instead of parsing it whole.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest -from scripts.compiler_differential._helpers.comparing.normalize import normalize_artifact_text -from scripts.compiler_differential.constants import ( - STAGE_CAPTURE_ROOT_KEY, - STAGE_CAPTURE_SHARED_NODES_KEY, -) - -_SHARED_HEADER: bytes = f"{{{json.dumps(STAGE_CAPTURE_SHARED_NODES_KEY)}: {{\n".encode() -_ROOT_PREFIX: bytes = f"{json.dumps(STAGE_CAPTURE_ROOT_KEY)}: ".encode() -_SHARED_DOCUMENT_KEYS: frozenset[str] = frozenset( - {STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_ROOT_KEY} -) -_DIGEST_CHARACTERS: int = 64 -_NODE_PREFIX_BYTES: int = _DIGEST_CHARACTERS + len(b'"": ') - - -class CaptureFile: - """A normalized stage capture whose shared nodes are parsed only when resolved.""" - - def __init__(self, path: Path) -> None: - self._path: Path = path - self._offsets: dict[str, int] = {} - self.root: object = self._index() - - def resolve(self, value: object) -> object: - """Return the shared node a reference names, or the value itself.""" - - digest: str | None = shared_digest(value) - return value if digest is None else self._node(digest) - - def _index(self) -> object: - with self._path.open("rb") as stream: - if stream.readline() != _SHARED_HEADER: - return _parse(self._path.read_bytes()) - root: object = None - offset: int = stream.tell() - for line in iter(stream.readline, b""): - if line.startswith(_ROOT_PREFIX): - root = _parse(line[len(_ROOT_PREFIX) :]) - elif line.startswith(b'"') and len(line) > _NODE_PREFIX_BYTES: - self._offsets[line[1 : _DIGEST_CHARACTERS + 1].decode("ascii")] = offset - offset += len(line) - return root - - def _node(self, digest: str) -> object: - offset: int | None = self._offsets.get(digest) - if offset is None: - raise json.JSONDecodeError(f"missing shared node {digest}", str(self._path), 0) - with self._path.open("rb") as stream: - _ = stream.seek(offset) - line: bytes = stream.readline() - return _parse(line[_NODE_PREFIX_BYTES:]) - - -def expand_capture_text(text: str) -> object: - """Parse a whole capture document, replacing every shared reference by its node.""" - - document: object = json.loads(text) - mapping: dict[str, object] | None = as_json_object(document) - nodes: dict[str, object] | None = as_json_object( - None if mapping is None else mapping.get(STAGE_CAPTURE_SHARED_NODES_KEY) - ) - if mapping is None or nodes is None or set(mapping) != _SHARED_DOCUMENT_KEYS: - return document - expanded: dict[str, object] = {} - for digest, node in nodes.items(): - expanded[digest] = _expanded(value=node, nodes=expanded) - return _expanded(value=mapping[STAGE_CAPTURE_ROOT_KEY], nodes=expanded) - - -def _expanded(*, value: object, nodes: dict[str, object]) -> object: - digest: str | None = shared_digest(value) - if digest is not None: - return nodes[digest] - items: dict[str, object] | None = as_json_object(value) - if items is not None: - return {key: _expanded(value=item, nodes=nodes) for key, item in items.items()} - if isinstance(value, list): - return [_expanded(value=item, nodes=nodes) for item in value] - return value - - -def _parse(data: bytes) -> object: - text: str = data.decode("utf-8").rstrip("\n") - return json.loads(normalize_artifact_text(text.removesuffix(","))) diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index f7d6084c08..d199a76cf2 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -125,15 +125,6 @@ ADAPTER_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild", "adapter") ADAPTERS_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild", "adapters") ADAPTER_PACKAGE_IMPORT_PREFIX: str = "sqlbuild.adapters." -ADAPTER_CONTRACT_CLASS_IMPORT_PREFIX: tuple[str, ...] = ( - "sqlbuild", - "adapter", - "contract", - "classes", -) -ADAPTER_CONTRACT_BASE_CLASS_NAMES: frozenset[str] = frozenset( - {"BaseAdapter", "DuckDbBackedAdapter", "StrictAdapter"} -) ADAPTER_CONTRACT_BASE_SYMBOLS: frozenset[tuple[str, ...]] = frozenset( { ( @@ -185,7 +176,6 @@ PLANNER_PATH_PREFIX: str = "src/sqlbuild/compiler/planner/" POLICY_EVALUATION_SCOPES: frozenset[str] = frozenset({"root", "tooling"}) POLICY_IMPLEMENTATION_PATH_PREFIX: str = "scripts/fensu_policy/" -PROVIDER_CLASS_NAME: str = "Provider" PROVIDER_MODULE_PARTS: tuple[str, ...] = ("src", "sqlbuild", "providers.py") PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) ROOT_SCOPE_NAME: str = "root" @@ -280,10 +270,6 @@ EVENT_EXPORTER_DECORATOR_NAMES: frozenset[str] = frozenset( {"lifecycle_event_sink", "command_output_sink"} ) -OBSERVABILITY_MODULE_PREFIXES: tuple[tuple[str, ...], ...] = ( - ("sqlbuild", "observability"), - ("sqlbuild", "runtime", "observability"), -) OBSERVABILITY_LIFECYCLE_EVENT_MODULES: tuple[tuple[str, ...], ...] = ( ("sqlbuild", "observability"), ("sqlbuild", "runtime", "observability", "models"), diff --git a/scripts/native_module_paths/_helpers/host_roots.py b/scripts/native_module_paths/_helpers/host_roots.py new file mode 100644 index 0000000000..7c425af671 --- /dev/null +++ b/scripts/native_module_paths/_helpers/host_roots.py @@ -0,0 +1,54 @@ +"""Fensu dead-code roots that keep the native host table's Python targets alive.""" + +import ast +import importlib.util +import tomllib +from importlib.machinery import ModuleSpec +from pathlib import Path +from typing import Any + +import sqlbuild._native_host as native_host +from scripts.native_module_paths.constants import ( + HOST_ROOT_REASON, + MODULE_FILE_ATTRIBUTE, + PYTHON_SUFFIX, +) + + +def defining_target(*, target: tuple[str, str]) -> tuple[str, str]: + """Follow `from module import name as name` re-exports to the module that defines a target.""" + module, attribute = target + spec: ModuleSpec | None = importlib.util.find_spec(module) + origin: str | None = None if spec is None else spec.origin + if origin is None or not origin.endswith(PYTHON_SUFFIX): + return target + for node in ast.parse(Path(origin).read_text(encoding="utf-8")).body: + if not isinstance(node, ast.ImportFrom) or node.level != 0 or node.module is None: + continue + for alias in node.names: + if (alias.asname or alias.name) == attribute: + return defining_target(target=(node.module, alias.name)) + return target + + +def host_targets() -> set[tuple[str, str]]: + """The defining module and symbol of every host table entry that names a symbol.""" + targets: set[tuple[str, str]] = set() + for target in native_host.host_targets().values(): + if target[1] != MODULE_FILE_ATTRIBUTE: + targets.add(defining_target(target=target)) + return targets + + +def host_dead_code_roots(*, config: Path) -> set[tuple[str, str]]: + """Every module and symbol the Fensu dead-code roots in `config` keep for the host table.""" + parsed: dict[str, Any] = tomllib.loads(config.read_text(encoding="utf-8")) + roots: list[dict[str, Any]] = parsed["targets"]["python"]["dead_code"]["roots"] + found: set[tuple[str, str]] = set() + for root in roots: + if root["reason"] != HOST_ROOT_REASON: + continue + for module in root["modules"]: + for symbol in root["symbols"]: + found.add((module, symbol)) + return found diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 192e7b5ecf..407caf0bd7 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -34,3 +34,6 @@ ) HOST_MODULE: str = "sqlbuild._native_host" HOST_ENTRY: re.Pattern[str] = re.compile(r"\bHostEntry\s*\{\s*name:\s*" + ATTRIBUTE_NAME) +HOST_ROOT_REASON: str = "Native code resolves these through the sqlbuild._native_host name table." +MODULE_FILE_ATTRIBUTE: str = "__file__" +PYTHON_SUFFIX: str = ".py" diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py index 927fb20359..051a6d8013 100644 --- a/scripts/planner_differential/_helpers/capturing/canonical.py +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -6,7 +6,7 @@ import re from datetime import date, timedelta -from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.planner_differential.constants import ( COMPACT_TIMESTAMP_PATTERN, DATE_PATTERN, diff --git a/src/sqlbuild/adapter/contract/constants.py b/src/sqlbuild/adapter/contract/constants.py index 2f323717dd..0a895325e8 100644 --- a/src/sqlbuild/adapter/contract/constants.py +++ b/src/sqlbuild/adapter/contract/constants.py @@ -1,21 +1,5 @@ """Adapter constants.""" -INTEGER_TYPE_NAMES: frozenset[str] = frozenset( - {"TINYINT", "SMALLINT", "INT", "INTEGER", "BIGINT", "LONG", "INT64"} -) -DECIMAL_TYPE_NAMES: frozenset[str] = frozenset( - {"DECIMAL", "BIGDECIMAL", "NUMERIC", "NUMBER", "BIGNUMERIC"} -) -FLOAT_TYPE_NAMES: frozenset[str] = frozenset({"DOUBLE", "FLOAT", "FLOAT64", "REAL"}) -STRING_TYPE_NAMES: frozenset[str] = frozenset({"VARCHAR", "CHAR", "CHARACTER", "TEXT", "STRING"}) -BOOLEAN_TYPE_NAMES: frozenset[str] = frozenset({"BOOLEAN", "BOOL"}) -TIMESTAMP_TYPE_NAMES: frozenset[str] = frozenset( - {"TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMPNTZ", "TIMESTAMPLTZ"} -) -DATE_TYPE_NAME: str = "DATE" -DATETIME_TYPE_NAME: str = "DATETIME" -POLYGLOT_CUSTOM_TYPE_NAME: str = "CUSTOM" -TIMESTAMP_TYPE_TOKEN: str = "TIMESTAMP" INTEGER_TYPE_TOKEN: str = "INT" TYPED_OBJECT_ENTRY_PART_COUNT: int = 2 TABLE_RELATION_TYPE_NAMES: frozenset[str] = frozenset( diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index d3024011fb..498d176b89 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -2,15 +2,12 @@ BOOLEAN_METADATA_TYPE_NAME: str = "BOOLEAN" BOOLEAN_WIRE_TYPE_NAME: str = "BOOL" -BIGNUMERIC_TYPE_NAME: str = "BIGNUMERIC" -CUSTOM_NORMALIZATION_TYPE_NAMES: frozenset[str] = frozenset({"INT64", "FLOAT64", "BIGNUMERIC"}) DATE_TYPE_NAME: str = "DATE" DATETIME_TYPE_NAME: str = "DATETIME" DECIMAL_METADATA_TYPE_NAMES: frozenset[str] = frozenset({"NUMERIC", "BIGNUMERIC"}) FLOAT_METADATA_TYPE_NAME: str = "FLOAT" FLOAT_WIRE_TYPE_NAME: str = "FLOAT64" INTEGER_METADATA_TYPE_NAME: str = "INTEGER" -INTEGER_PARSE_TYPE_NAMES: frozenset[str] = frozenset({"INT", "BIGINT"}) INTEGER_TYPE_TOKEN: str = "INT" INTEGER_WIRE_TYPE_NAME: str = "INT64" NOT_FOUND_ERROR_CLASS_NAME: str = "NotFound" diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 7dce598a92..7b9047e237 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -40,7 +40,6 @@ ) VIEW_ENGINES: frozenset[str] = frozenset({"View", "MaterializedView", "LiveView", "WindowView"}) RELATION_NOT_FOUND_CODES: frozenset[int] = frozenset({60, 81}) -ATOMIC_DATABASE_ENGINES: frozenset[str] = frozenset({"Atomic", "Replicated"}) NULLABLE_WRAPPER: str = "Nullable" TYPE_WRAPPERS: tuple[str, ...] = ("Nullable", "LowCardinality") TYPE_ALIASES: dict[str, str] = { diff --git a/src/sqlbuild/adapters/duckdb/_helpers/__init__.py b/src/sqlbuild/adapters/duckdb/_helpers/__init__.py deleted file mode 100644 index d39eda62bb..0000000000 --- a/src/sqlbuild/adapters/duckdb/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""DuckDB adapter helpers.""" diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 3816e406b6..c816a0575e 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -10,17 +10,9 @@ SECONDARY_ROLES_ALL: str = "ALL" SECONDARY_ROLES_NONE: str = "NONE" NUMBER_TYPE_NAME: str = "NUMBER" -DEFAULT_TEXT_LENGTH: int = 16_777_216 -INTEGER_PRECISION: int = 38 -INTEGER_SCALE: int = 0 -NORMALIZED_LTZ_INPUT_TYPE_NAME: str = "TIMESTAMPLTZ" -NORMALIZED_NTZ_INPUT_TYPE_NAMES: frozenset[str] = frozenset({"TIMESTAMP", "TIMESTAMPNTZ"}) -NORMALIZED_TZ_INPUT_TYPE_NAME: str = "TIMESTAMPTZ" STATUS_COLUMN_NAME: str = "status" SUCCESS_STATUS_TOKENS: tuple[str, ...] = ("success", "created", "dropped") TEXT_TYPE_NAMES: frozenset[str] = frozenset({"TEXT", "VARCHAR", "CHAR", "CHARACTER"}) -TEXT_TYPE_NAME: str = "TEXT" -UNBOUNDED_TEXT_TYPE_NAMES: frozenset[str] = frozenset({"STRING", "TEXT", "VARCHAR"}) TRUE_METADATA_VALUE: str = "YES" SHOW_LIKE_ESCAPE_CHARACTER: str = "\\" CURRENT_DATABASE_ATTRIBUTE: str = "_sqlbuild_current_database" diff --git a/src/sqlbuild/adapters/sqlite/constants.py b/src/sqlbuild/adapters/sqlite/constants.py index dcebc49096..8b35836965 100644 --- a/src/sqlbuild/adapters/sqlite/constants.py +++ b/src/sqlbuild/adapters/sqlite/constants.py @@ -15,7 +15,6 @@ LOCKED_ERROR_FRAGMENTS: tuple[str, ...] = ("database is locked", "database table is locked") MISSING_RELATION_ERROR_FRAGMENT: str = "no such table" READ_STATEMENT_KEYWORDS: frozenset[str] = frozenset({"SELECT", "PRAGMA", "EXPLAIN", "VALUES"}) -TRANSACTION_END_KEYWORDS: frozenset[str] = frozenset({"COMMIT", "END", "ROLLBACK"}) WRITE_GATE_POLL_SECONDS: float = 0.1 TYPEOF_TYPE_NAMES: dict[str, str] = { "integer": "INTEGER", diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/lineage/__init__.py b/src/sqlbuild/compiler/lineage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/__init__.py b/src/sqlbuild/compiler/planner/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/executor/clone/types.py b/src/sqlbuild/executor/clone/types.py index 9f30a97f63..2a9606126d 100644 --- a/src/sqlbuild/executor/clone/types.py +++ b/src/sqlbuild/executor/clone/types.py @@ -19,10 +19,6 @@ def __call__( ) -> None: ... -class CloneFingerprintProgressReporter(CloneFingerprintProgressCallback, Protocol): - def write_interrupted(self) -> None: ... - - class CloneStatus(StrEnum): SUCCESS = "success" WARNING = "warning" diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 1ff25a8a67..4237e12f99 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -112,16 +112,6 @@ class PythonNodeFanInDecision: skip_mode: SkipMode | None = None -@dataclass(frozen=True) -class CursorWindow: - """Resolved cursor bounds for one Python execution run.""" - - start_cursor_ts: datetime | None = None - end_cursor_ts: datetime | None = None - start_cursor_int: int | None = None - end_cursor_int: int | None = None - - from sqlbuild.executor.python_nodes.classes.run_state import PythonNodeRunState # noqa: E402 diff --git a/src/sqlbuild/integrations/dagster/_helpers/translation.py b/src/sqlbuild/integrations/dagster/_helpers/translation.py index 4a1997d92e..a8dff56731 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/translation.py +++ b/src/sqlbuild/integrations/dagster/_helpers/translation.py @@ -41,15 +41,3 @@ def translate_sqlbuild_dag( ) return {**dag, "nodes": translated_nodes, "checks": translated_checks} - - -def is_dagster_asset_enabled(node: Mapping[str, Any]) -> bool: - """Return whether a translated DAG node is owned by the SQLBuild asset definition.""" - - return bool(node.get(DAGSTER_ASSET_ENABLED_FIELD, True)) - - -def is_dagster_check_enabled(check: Mapping[str, Any]) -> bool: - """Return whether a translated DAG check is owned by the SQLBuild asset definition.""" - - return bool(check.get(DAGSTER_CHECK_ENABLED_FIELD, True)) diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index d02a1fee47..8afa16ddce 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -49,23 +49,9 @@ CLONE_COMMAND: str = "clone" CHECK_COMMAND: str = "check" STANDALONE_CHECK_COMMANDS: frozenset[str] = frozenset({"audit", CHECK_COMMAND, "test"}) -EVENT_OUTPUT_FLAG: str = "--event-output" LIVE_EVENT_COMMANDS: frozenset[str] = ASSET_SELECTION_COMMANDS EXPLICIT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "-s", "--select-file"}) -JSON_OUTPUT_FLAGS: frozenset[str] = frozenset({"--json", "--json-output"}) -JSON_OUTPUT_FLAG: str = "--json-output" -SELECT_FILE_FLAG: str = "--select-file" -SCENARIO_TEST_COMMAND: tuple[str, str] = ("scenario", "test") SCENARIO_CHECK_KIND: str = NodeKind.SCENARIO.value -SCENARIO_VALUE_FLAGS: frozenset[str] = frozenset( - { - "--max-snapshot-rows", - "--max-snapshot-total-rows", - "--max-snapshot-bytes", - "--max-snapshot-total-bytes", - } -) -LOAD_COMMAND: str = "load" LOAD_SELECTABLE_NODE_KIND_MEMBERS: frozenset[NodeKind] = frozenset( {NodeKind.SOURCE, NodeKind.LOADER} ) @@ -146,7 +132,6 @@ ) SUCCESS_EXECUTION_STATUS: str = ExecutionStatus.SUCCESS.value FAILED_EXECUTION_STATUS: str = ExecutionStatus.FAILED.value -CHECK_NAME_SEPARATOR_CHARACTER: str = "_" CHECK_METADATA_EXCLUDED_KEYS: frozenset[str] = frozenset( {"passed", "steps", "expected_results", "assertion_results"} ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py index 7040d2fcc8..89d2e334f2 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py +++ b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py @@ -40,3 +40,10 @@ class HostUnknownNameCase: description: str name: str expected_error: str + + +@dataclass(frozen=True) +class HostDeadCodeRootCase: + description: str + expected_missing_targets: tuple[tuple[str, str], ...] + expected_extra_targets: tuple[tuple[str, str], ...] diff --git a/tests/unit/scripts/native_module_paths/_helpers/helpers.py b/tests/unit/scripts/native_module_paths/_helpers/helpers.py index facfc3700e..5f2fe828c8 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/helpers.py +++ b/tests/unit/scripts/native_module_paths/_helpers/helpers.py @@ -1,7 +1,10 @@ +from pathlib import Path + from scripts.native_module_paths._helpers.crates import native_module_path_errors from scripts.native_module_paths._helpers.scan import scan_native_source SOURCE_LABEL: str = "lib.rs" +FENSU_CONFIG: Path = Path(__file__).parents[5] / "fensu.toml" def native_source_errors(source: str) -> tuple[str, ...]: diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py b/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py index 5ff77bc5ee..aa49f0ad8f 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py @@ -10,13 +10,15 @@ import sqlbuild._native as native import sqlbuild._native_host as native_host +from scripts.native_module_paths._helpers.host_roots import host_dead_code_roots, host_targets from tests.unit.scripts.native_module_paths._helpers._test_types import ( + HostDeadCodeRootCase, HostImportCase, HostKindCase, HostTableCase, HostUnknownNameCase, ) -from tests.unit.scripts.native_module_paths._helpers.helpers import resolved_kind +from tests.unit.scripts.native_module_paths._helpers.helpers import FENSU_CONFIG, resolved_kind @pytest.mark.parametrize( @@ -115,5 +117,26 @@ def test_given_unknown_name_when_reading_native_host_then_raises_attribute_error assert str(raised.value) == test_case.expected_error +@pytest.mark.parametrize( + "test_case", + [ + HostDeadCodeRootCase( + description="every host target is a dead-code root and every host root a target", + expected_missing_targets=(), + expected_extra_targets=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_native_host_table_when_reading_fensu_roots_then_roots_match_targets( + test_case: HostDeadCodeRootCase, +) -> None: + targets: set[tuple[str, str]] = host_targets() + roots: set[tuple[str, str]] = host_dead_code_roots(config=FENSU_CONFIG) + + assert tuple(sorted(targets - roots)) == test_case.expected_missing_targets + assert tuple(sorted(roots - targets)) == test_case.expected_extra_targets + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 86d7bd65ee..616cc76a25 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -8,7 +8,7 @@ import pytest -from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from sqlbuild._native import ( STAGE_CAPTURE_DIR_ENV_VAR, CompilerStage, From 738e9bfc17145dc9b096d43cd29a4845b2b0fcd0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:53:38 +0100 Subject: [PATCH 12/39] refactor: name large tuple types introduced on main --- .../scenario_command/capture_command.rs | 2 +- .../src/store/main/read_output_transcript.rs | 25 ++++++------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs index b3ad6ee36e..470ab6f736 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs @@ -105,7 +105,7 @@ pub(crate) fn run_scenario_capture_command( project_dir: &project.project_dir, adapter: &adapter, kind: CompletionKind::Capture, - style: style, + style, }, )?; let results: Vec> = call_with( diff --git a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs index c21864bb78..44be378a54 100644 --- a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs +++ b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs @@ -52,7 +52,7 @@ pub fn read_output_transcript( .optional(), "read Recall invocation", )?; - let completeness: Option<(i64, i64, i64, bool, bool, Option)> = checked( + checked( store .connection .query_row( @@ -60,26 +60,17 @@ pub fn read_output_transcript( gap_reason FROM output_invocations WHERE invocation_id = ?1", [invocation_id], |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - )) + transcript.missing_sequences = row.get(0)?; + transcript.reported_drops = row.get(1)?; + transcript.filtered = row.get(2)?; + transcript.flush_complete = row.get(3)?; + transcript.complete = row.get(4)?; + transcript.gap_reason = row.get(5)?; + Ok(()) }, ) .optional(), "read Recall output completeness", )?; - if let Some((missing, drops, filtered, flushed, complete, reason)) = completeness { - transcript.missing_sequences = missing; - transcript.reported_drops = drops; - transcript.filtered = filtered; - transcript.flush_complete = flushed; - transcript.complete = complete; - transcript.gap_reason = reason; - } Ok(transcript) } From 2655063bb600220fccef743cbff24a5326aab1f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:20:57 +0100 Subject: [PATCH 13/39] chore: satisfy fensu 0.27 in code added on main --- .../src/clickhouse/constants.rs | 52 ++++++++++++++----- fensu.toml | 10 ++++ 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index 46bf41da3d..d4a540e63a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -6,24 +6,50 @@ use crate::schema::models::Table; pub const BATCH_ROWS: usize = 5_000; /// Longest wait for a batch to fill, so each table gets at most about one part per second per writer. pub const BATCH_WINDOW_MS: u64 = 1_000; +/// ClickHouse `READONLY`. +pub const READONLY_CODE: u64 = 164; +/// ClickHouse `DATABASE_ACCESS_DENIED`. +pub const DATABASE_ACCESS_DENIED_CODE: u64 = 291; +/// ClickHouse `ACCESS_DENIED`. +pub const ACCESS_DENIED_CODE: u64 = 497; +/// ClickHouse `AUTHENTICATION_FAILED`. +pub const AUTHENTICATION_FAILED_CODE: u64 = 516; +/// ClickHouse `TIMEOUT_EXCEEDED`. +pub const TIMEOUT_EXCEEDED_CODE: u64 = 159; +/// ClickHouse `TOO_MANY_SIMULTANEOUS_QUERIES`. +pub const TOO_MANY_SIMULTANEOUS_QUERIES_CODE: u64 = 202; +/// ClickHouse `SOCKET_TIMEOUT`. +pub const SOCKET_TIMEOUT_CODE: u64 = 209; +/// ClickHouse `NETWORK_ERROR`. +pub const NETWORK_ERROR_CODE: u64 = 210; +/// ClickHouse `MEMORY_LIMIT_EXCEEDED`. +pub const MEMORY_LIMIT_EXCEEDED_CODE: u64 = 241; +/// ClickHouse `TABLE_IS_READ_ONLY`. +pub const TABLE_IS_READ_ONLY_CODE: u64 = 242; +/// ClickHouse `TOO_MANY_PARTS`. +pub const TOO_MANY_PARTS_CODE: u64 = 252; +/// ClickHouse `UNKNOWN_STATUS_OF_INSERT`. +pub const UNKNOWN_STATUS_OF_INSERT_CODE: u64 = 319; +/// ClickHouse `SYSTEM_ERROR`. +pub const SYSTEM_ERROR_CODE: u64 = 425; /// Server error codes meaning the user may not create or write the store. pub const PERMISSION_CODES: [u64; 4] = [ - 164, // READONLY - 291, // DATABASE_ACCESS_DENIED - 497, // ACCESS_DENIED - 516, // AUTHENTICATION_FAILED + READONLY_CODE, + DATABASE_ACCESS_DENIED_CODE, + ACCESS_DENIED_CODE, + AUTHENTICATION_FAILED_CODE, ]; /// Server error codes worth retrying with the same rows and deduplication tokens. pub const TRANSIENT_CODES: [u64; 9] = [ - 159, // TIMEOUT_EXCEEDED - 202, // TOO_MANY_SIMULTANEOUS_QUERIES - 209, // SOCKET_TIMEOUT - 210, // NETWORK_ERROR - 241, // MEMORY_LIMIT_EXCEEDED - 242, // TABLE_IS_READ_ONLY - 252, // TOO_MANY_PARTS - 319, // UNKNOWN_STATUS_OF_INSERT - 425, // SYSTEM_ERROR + TIMEOUT_EXCEEDED_CODE, + TOO_MANY_SIMULTANEOUS_QUERIES_CODE, + SOCKET_TIMEOUT_CODE, + NETWORK_ERROR_CODE, + MEMORY_LIMIT_EXCEEDED_CODE, + TABLE_IS_READ_ONLY_CODE, + TOO_MANY_PARTS_CODE, + UNKNOWN_STATUS_OF_INSERT_CODE, + SYSTEM_ERROR_CODE, ]; /// UNKNOWN_DATABASE: Recall never creates databases. pub const UNKNOWN_DATABASE_CODE: u64 = 81; diff --git a/fensu.toml b/fensu.toml index 4aa8804da8..a58a5bdf7f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -167,6 +167,11 @@ modules = ["sqlbuild.adapter.type_system.main.safe_cast_nullability"] symbols = ["safe_cast_nullability"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.clickhouse._helpers.recall_store"] +symbols = ["close_recall_store", "connect_recall_store", "insert_recall_rows", "run_recall_command", "run_recall_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"] symbols = ["dispatch_with_compute_logs"] @@ -737,6 +742,11 @@ modules = ["sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"] symbols = ["invoke_lifecycle_event_sink"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._invoke_recall_sink"] +symbols = ["invoke_recall_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.runtime.event_exporting.main._run_command_output_sinks"] symbols = ["run_command_output_sinks"] From 2dd43629a59117122465d726799dd22b8871af9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:55:50 +0100 Subject: [PATCH 14/39] fix(clickhouse): accept ClickHouse Cloud SharedMergeTree engines Shared* MergeTree engines now pass model validation like their MergeTree counterparts. Replicated* engines stay rejected. Support is best effort and untested on ClickHouse Cloud. --- .../_helpers/materialization.rs | 2 +- .../src/model_validation/constants.rs | 4 +- .../tests/test_physical_storage.rs | 38 ++++++++++++++++++- .../docs/concepts/adapters/clickhouse.md | 8 ++-- .../adapters/clickhouse/test_table_layout.py | 12 ++++++ .../docs/concepts/adapters/clickhouse.mdx | 8 ++-- 6 files changed, 61 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index 0500b3c29b..46bc71a1ad 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -434,7 +434,7 @@ fn check_engine(config: &ConfigView<'_, N>) -> Check { _ => Err(ValidationStop::Error( config .config_error(format!( - "{ENGINE_KEY} '{}' must be a single-server MergeTree-family table engine", + "{ENGINE_KEY} '{}' must be a non-replicated MergeTree-family table engine", value.python_str() )) .with_help(model_header_help( diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index fdef85e14e..74d56179a8 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -300,5 +300,5 @@ pub const PHYSICAL_STORAGE_KEYS: [&str; 5] = [ pub const PHYSICAL_STORAGE_MATERIALIZATIONS: [&str; 2] = ["table", "incremental"]; /// The name suffix every supported table engine shares. pub const ENGINE_FAMILY_SUFFIX: &str = "MergeTree"; -/// Engine name prefixes of replicated and shared engines, which need a cluster. -pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 2] = ["Replicated", "Shared"]; +/// Engine name prefixes of replicated engines, which need a cluster. +pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 1] = ["Replicated"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs index 06366bc6d4..c367ec9999 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs @@ -50,6 +50,40 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a query_sql: "select 1", expected_outcome: "model 'orders_daily': order_by is only valid for table and incremental models", }, + ValidateTestCase { + description: "a shared engine", + config: table(vec![("engine", Value::Str("SharedMergeTree"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a shared replacing engine with arguments", + config: table(vec![( + "engine", + Value::Str("SharedReplacingMergeTree(updated_at)"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a replicated replacing engine", + config: table(vec![( + "engine", + Value::Str("ReplicatedReplacingMergeTree(updated_at)"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'ReplicatedReplacingMergeTree(updated_at)' must be a non-replicated MergeTree-family table engine", + }, + ValidateTestCase { + description: "a shared engine outside the MergeTree family", + config: table(vec![("engine", Value::Str("SharedSet"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'SharedSet' must be a non-replicated MergeTree-family table engine", + }, ValidateTestCase { description: "a replicated engine", config: table(vec![( @@ -58,14 +92,14 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a )]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a single-server MergeTree-family table engine", + expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a non-replicated MergeTree-family table engine", }, ValidateTestCase { description: "an engine outside the MergeTree family", config: table(vec![("engine", Value::Str("Log"))]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': engine 'Log' must be a single-server MergeTree-family table engine", + expected_outcome: "model 'orders_daily': engine 'Log' must be a non-replicated MergeTree-family table engine", }, ValidateTestCase { description: "a blank sorting expression", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index b8bdd8f42d..83c01eecc8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -113,13 +113,15 @@ MODEL ( | Field | Value | Default | |-------|-------|---------| -| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `engine` | A non-replicated MergeTree-family engine, with its arguments | `MergeTree` | | `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | | `partition_by` | A partitioning expression | No partitioning | | `primary_key` | A prefix of `order_by` | The sorting key | | `table_settings` | Setting names mapped to strings, integers or booleans | None | -Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. +Quote expressions that are not plain column names. Compile rejects malformed values, `Replicated*` engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +ClickHouse Cloud's `Shared*` engines (`SharedMergeTree`, `SharedReplacingMergeTree` and the other `SharedMergeTree` variants) are accepted and handled like their `MergeTree` counterparts. Support is best effort: SQLBuild is not tested on ClickHouse Cloud. ClickHouse Cloud already creates a `MergeTree` table as `SharedMergeTree`, so naming the plain engine works there too. The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. @@ -181,7 +183,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## Limits -- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. ClickHouse Cloud and its `Shared*` engines are best effort and untested. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). - Snapshots always use an unsorted `MergeTree` table. diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py index ff381b1932..3305b4054f 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py @@ -39,6 +39,18 @@ layout={"order_by": "order_id"}, expected_clauses="ENGINE = MergeTree ORDER BY (order_id)", ), + TableLayoutTestCase( + description="a shared engine", + layout={"engine": "SharedMergeTree", "partition_by": "toYYYYMM(ordered_at)"}, + expected_clauses=( + "ENGINE = SharedMergeTree ORDER BY tuple() PARTITION BY toYYYYMM(ordered_at)" + ), + ), + TableLayoutTestCase( + description="a shared replacing engine with its version column", + layout={"engine": "SharedReplacingMergeTree(version)", "order_by": ["customer_id"]}, + expected_clauses="ENGINE = SharedReplacingMergeTree(version) ORDER BY (customer_id)", + ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 5aa6becff3..3876b673df 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -100,13 +100,15 @@ MODEL ( | Field | Value | Default | |-------|-------|---------| -| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `engine` | A non-replicated MergeTree-family engine, with its arguments | `MergeTree` | | `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | | `partition_by` | A partitioning expression | No partitioning | | `primary_key` | A prefix of `order_by` | The sorting key | | `table_settings` | Setting names mapped to strings, integers or booleans | None | -Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. +Quote expressions that are not plain column names. Compile rejects malformed values, `Replicated*` engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +ClickHouse Cloud's `Shared*` engines (`SharedMergeTree`, `SharedReplacingMergeTree` and the other `SharedMergeTree` variants) are accepted and handled like their `MergeTree` counterparts. Support is best effort: SQLBuild is not tested on ClickHouse Cloud. ClickHouse Cloud already creates a `MergeTree` table as `SharedMergeTree`, so naming the plain engine works there too. The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. @@ -168,7 +170,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## Limits -- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. ClickHouse Cloud and its `Shared*` engines are best effort and untested. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). - Snapshots always use an unsorted `MergeTree` table. From c438f702cf7d67d7c0d562895e1e8d7dc85e5e7d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:11:10 +0100 Subject: [PATCH 15/39] test: pin buffered stdout in broken-pipe parity tests and list the compute log logger --- tests/integration/src/sqlbuild/cli/entry/helpers.py | 5 ++++- .../native_module_paths/_helpers/test_module_paths.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 15825c8079..0db3751b7d 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -299,13 +299,16 @@ def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> La reader, writer = os.pipe() os.close(reader) + environment: dict[str, str] = {**os.environ, "NO_COLOR": "1"} + # The broken-pipe contract follows Python's default buffered stdout. + _ = environment.pop("PYTHONUNBUFFERED", None) try: completed: subprocess.CompletedProcess[bytes] = subprocess.run( [str(program), *args], cwd=cwd, stdout=writer, stderr=subprocess.PIPE, - env={**os.environ, "NO_COLOR": "1"}, + env=environment, check=False, timeout=600, ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index f8e98b8787..5fd81a5f4d 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -125,6 +125,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli", "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", + "sqlbuild.cli.compute_logs", "sqlbuild.cli.observability", "sqlbuild.compile", "sqlbuild.compile.reuse", From c4fbf351b56ea8eefb32e1533dfce9d403614b9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:31:46 +0100 Subject: [PATCH 16/39] perf: run sqb query from the project configuration without full discovery sqb query only needs the adapter and connection settings. Load the project and local configuration, validate target warehouses and connections, and connect, instead of parsing every declaration and importing project Python. Ad hoc SQL now also runs when an unrelated model or Python file is broken. --- .../compile_driver/config_validation.rs | 7 + .../_helpers/discovery/configuration.rs | 40 ++- .../bindings/_helpers/query_command/dag.rs | 6 +- .../_helpers/query_command/project.rs | 22 +- .../bindings/_helpers/query_command/query.rs | 10 +- .../cli/commands/main/query/_test_types.py | 30 +++ .../main/query/test_query_project_inputs.py | 243 ++++++++++++++++++ website/src/content/docs/docs/cli/query.mdx | 2 + 8 files changed, 344 insertions(+), 16 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs index d1e6f34625..9d29ac2895 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs @@ -193,6 +193,13 @@ pub(crate) fn reject_invalid_project_configuration( config: &ProjectConfiguration<'_, '_>, ) -> PyResult<()> { reject_local_sql_analysis_opt_out(config)?; + reject_invalid_connection_configuration(config) +} + +/// Validate the target warehouses and target connections a warehouse connection depends on. +pub(crate) fn reject_invalid_connection_configuration( + config: &ProjectConfiguration<'_, '_>, +) -> PyResult<()> { reject_unsupported_target_warehouses(config)?; reject_invalid_target_connections(config) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs index b7ff43f7eb..d0b17cbd19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/configuration.rs @@ -3,6 +3,9 @@ use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyString; +use crate::bindings::_helpers::compile_driver::config_validation::{ + ProjectConfiguration, reject_invalid_connection_configuration, +}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, within}; use crate::bindings::_helpers::contracts::project_config::{ load_local_config_file, load_project_config_file, @@ -16,6 +19,22 @@ use crate::bindings::_helpers::records::ids::RecordId; pub(crate) fn discover_project_configuration<'py>( py: Python<'py>, project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + configuration_inputs(py, project_dir, false) +} + +/// `discover_project_configuration` with the target warehouses and connections validated. +pub(crate) fn discover_connection_configuration<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, +) -> PyResult> { + configuration_inputs(py, project_dir, true) +} + +fn configuration_inputs<'py>( + py: Python<'py>, + project_dir: &Bound<'py, PyAny>, + validate_connections: bool, ) -> PyResult> { let lifecycle: Bound<'py, PyAny> = call_with( &host::entry(py, ExecutorHost::OperationLifecycle)?, @@ -28,19 +47,24 @@ pub(crate) fn discover_project_configuration<'py>( ], )?; within(&lifecycle, |_| { + let project_config: Bound<'py, PyAny> = + load_project_config_file(py, project_dir)?.into_bound(py); + let local_config: Bound<'py, PyAny> = + load_local_config_file(py, project_dir)?.into_bound(py); + if validate_connections { + reject_invalid_connection_configuration(&ProjectConfiguration { + project_dir, + project_config: &project_config, + local_config: &local_config, + })?; + } new_record_with( py, RecordId::DiscoveredProjectInputs, &[ ("project_dir", project_dir.clone()), - ( - "project_config", - load_project_config_file(py, project_dir)?.into_bound(py), - ), - ( - "local_config", - load_local_config_file(py, project_dir)?.into_bound(py), - ), + ("project_config", project_config), + ("local_config", local_config), ], ) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs index 183172518e..49dfea7d02 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs @@ -12,7 +12,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::dag_artifact::artifact::build_dag_json; use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; -use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; use crate::bindings::_helpers::records::fields::field as native_field; fn array_length(document: &serde_json::Value, key: &str) -> PyResult { @@ -47,7 +49,7 @@ pub(crate) fn run_dag_command<'py>( project_dir, discovered, adapter, - } = project_adapter(py, project_dir)?; + } = project_adapter(py, project_dir, ProjectInputs::Discovered)?; let resolver: Bound<'py, PyAny> = resolve_external_sql_reference_resolver(project_dir, discovered.clone())?; let graph: Bound<'py, PyAny> = build_project_graph( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs index eeaaa42ae5..9973b9619a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs @@ -7,6 +7,7 @@ use crate::bindings::_helpers::compile_driver::discovery::discover_project_input use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::contracts::spec_resolution::resolve_effective_adapter_name; +use crate::bindings::_helpers::discovery::configuration::discover_connection_configuration; use crate::bindings::_helpers::records::fields::field as native_field; /// The project directory, defaulting to the working directory, with its inputs and adapter. @@ -16,18 +17,31 @@ pub(crate) struct ProjectAdapter<'py> { pub(crate) adapter: Bound<'py, PyAny>, } +/// How much of the project a command reads before it resolves the adapter. +#[derive(Clone, Copy)] +pub(crate) enum ProjectInputs { + /// Every declaration file and the imported project Python. + Discovered, + /// Only the project and local configuration, with its connection settings validated. + Configuration, +} + pub(crate) fn project_adapter<'py>( py: Python<'py>, project_dir: Option>, + inputs: ProjectInputs, ) -> PyResult> { let project_dir: Bound<'py, PyAny> = match project_dir.filter(|path| !path.is_none()) { Some(path) => path, None => imported(py, "pathlib", "Path")?.call_method0("cwd")?, }; - let discovered: Bound<'py, PyAny> = discover_project_inputs( - py, - Some(keywords(py, &[("project_dir", project_dir.clone())])?), - )?; + let discovered: Bound<'py, PyAny> = match inputs { + ProjectInputs::Discovered => discover_project_inputs( + py, + Some(keywords(py, &[("project_dir", project_dir.clone())])?), + )?, + ProjectInputs::Configuration => discover_connection_configuration(py, &project_dir)?, + }; let adapter_name: Bound<'py, PyAny> = resolve_effective_adapter_name( &native_field(&discovered, "project_config")?, &native_field(&discovered, "local_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index 0adfe98ad0..4d3f947a2e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -19,7 +19,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::query_command::project::{ProjectAdapter, project_adapter}; +use crate::bindings::_helpers::query_command::project::{ + ProjectAdapter, ProjectInputs, project_adapter, +}; use crate::bindings::_helpers::query_command::query_values::{ csv_text, display_text, json_cell, python_json_rows, strict_width, }; @@ -323,7 +325,11 @@ pub(crate) fn run_query_command( Some("pass a non-negative integer or use --no-limit"), )); } - let project: ProjectAdapter<'_> = project_adapter(py, Some(arguments.value("project_dir")?))?; + let project: ProjectAdapter<'_> = project_adapter( + py, + Some(arguments.value("project_dir")?), + ProjectInputs::Configuration, + )?; let result: Bound<'_, PyAny> = run_sql( &project, Some(arguments.value("selected_target")?), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py index 42d64cf8f6..6bfcfc0b91 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/_test_types.py @@ -32,3 +32,33 @@ class QueryExecutionErrorCliTestCase: command: tuple[str, ...] expected_stderr_fragment: str expected_returncode: int = 2 + + +@dataclass(frozen=True) +class QueryUnrelatedProjectFaultTestCase: + description: str + repo_files: dict[str, str] + expected_rows: tuple[dict[str, int], ...] = ({"order_id": 7},) + + +@dataclass(frozen=True) +class QueryProjectDiscoveryTestCase: + description: str + command: tuple[str, ...] + expected_exit_code: int + expected_project_discoveries: int + + +@dataclass(frozen=True) +class QueryTargetConnectionTestCase: + description: str + command: tuple[str, ...] + expected_stdout_fragment: str + + +@dataclass(frozen=True) +class QueryConnectionConfigErrorTestCase: + description: str + project_toml: str + expected_stderr_fragment: str + expected_returncode: int = 1 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py new file mode 100644 index 0000000000..dcac52db7c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py @@ -0,0 +1,243 @@ +"""`sqb query` reads only the project configuration it needs to connect.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.query._test_types import ( + QueryConnectionConfigErrorTestCase, + QueryProjectDiscoveryTestCase, + QueryTargetConnectionTestCase, + QueryUnrelatedProjectFaultTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + execute_duckdb, + prepare_inline_project, + run_sqb, +) + +PROJECT_TOML: str = ( + 'name = "orders_project"\nadapter = "duckdb"\n\n[connection]\ndatabase = "orders.duckdb"\n' +) +VALID_MODEL_SQL: str = "MODEL (materialized table);\n\nSELECT 1 AS order_id\n" +TARGETS_PROJECT_TOML: str = dedent( + """ + name = "orders_targets" + adapter = "duckdb" + default_target = "dev" + + [connections.dev] + database = "dev.duckdb" + + [connections.prod] + database = "prod.duckdb" + + [targets.dev] + connection = "dev" + + [targets.prod] + connection = "prod" + """ +).lstrip() +STAGE_WORK_SCRIPT: str = """ +import contextlib, io, json, sys +import sqlbuild._native as native +from sqlbuild.cli.entry.main.entry import main + +_ = native.compile_stage_work_info(reset=True) +with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + code = main(json.loads(sys.argv[1])) +work = native.compile_stage_work_info() +print(json.dumps({"code": code, "discoveries": work.get("discover_project_inputs", 0)})) +""" + + +@pytest.mark.parametrize( + "test_case", + ( + QueryUnrelatedProjectFaultTestCase( + description="model without a MODEL header", + repo_files={"models/orders.sql": "SELECT FROM WHERE (\n"}, + ), + QueryUnrelatedProjectFaultTestCase( + description="project Python that fails to import", + repo_files={"python/broken.py": 'raise RuntimeError("broken project python")\n'}, + ), + QueryUnrelatedProjectFaultTestCase( + description="local SQL analysis opt-out the project forbids", + repo_files={ + "sqlbuild_project.toml": PROJECT_TOML + + "\n[settings]\nrequire_sql_analysis = true\n", + "sqlbuild_local.toml": "[settings]\nsql_analysis = false\n", + }, + ), + ), + ids=lambda case: case.description, +) +def test_given_unrelated_broken_project_input_when_running_ad_hoc_query_then_sql_runs( + tmp_path: Path, + test_case: QueryUnrelatedProjectFaultTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={"sqlbuild_project.toml": PROJECT_TOML, **test_case.repo_files}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("query", "SELECT 7 AS order_id", "--format", "json"), + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == list(test_case.expected_rows) + assert "Declaration parsing" not in result.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + QueryProjectDiscoveryTestCase( + description="ad hoc query discovers no project files", + command=("query", "SELECT 1 AS order_id"), + expected_exit_code=0, + expected_project_discoveries=0, + ), + QueryProjectDiscoveryTestCase( + description="dag still discovers the project", + command=("dag", "--json"), + expected_exit_code=0, + expected_project_discoveries=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_project_when_running_command_then_project_discovery_runs_only_when_needed( + tmp_path: Path, + test_case: QueryProjectDiscoveryTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_project", + repo_files={"sqlbuild_project.toml": PROJECT_TOML, "models/orders.sql": VALID_MODEL_SQL}, + ) + argv: list[str] = ["--project-dir", str(project_dir), *test_case.command] + + result: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", STAGE_WORK_SCRIPT, json.dumps(argv)], + check=True, + capture_output=True, + text=True, + cwd=tmp_path, + ) + outcome: dict[str, int] = json.loads(result.stdout.strip().splitlines()[-1]) + + assert outcome == { + "code": test_case.expected_exit_code, + "discoveries": test_case.expected_project_discoveries, + } + + +@pytest.mark.parametrize( + "test_case", + ( + QueryTargetConnectionTestCase( + description="default target connection", + command=("query", "SELECT label FROM destination", "--format", "csv"), + expected_stdout_fragment="label\ndev\n", + ), + QueryTargetConnectionTestCase( + description="explicit target connection", + command=( + "query", + "SELECT label FROM destination", + "--format", + "csv", + "--target", + "prod", + ), + expected_stdout_fragment="label\nprod\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_target_connections_when_running_query_then_selected_target_database_is_read( + tmp_path: Path, + test_case: QueryTargetConnectionTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_targets", + repo_files={"sqlbuild_project.toml": TARGETS_PROJECT_TOML}, + ) + for label in ("dev", "prod"): + execute_duckdb( + db_path=project_dir / f"{label}.duckdb", + sql=f"CREATE TABLE destination AS SELECT '{label}' AS label", + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout == test_case.expected_stdout_fragment + + +@pytest.mark.parametrize( + "test_case", + ( + QueryConnectionConfigErrorTestCase( + description="target warehouse under an adapter without warehouses", + project_toml=TARGETS_PROJECT_TOML + '\n[targets.dev.warehouses]\nquery = "ORDERS_WH"\n', + expected_stderr_fragment=( + "[targets.dev.warehouses] selects a query warehouse, but the 'duckdb' adapter " + "has no warehouse to select" + ), + ), + QueryConnectionConfigErrorTestCase( + description="target selecting an unknown connection", + project_toml=TARGETS_PROJECT_TOML.replace( + 'connection = "dev"\n', 'connection = "missing"\n' + ), + expected_stderr_fragment="error[P001]: Unknown connection 'missing' selected by target.", + ), + QueryConnectionConfigErrorTestCase( + description="no connection configured", + project_toml='name = "orders_project"\nadapter = "duckdb"\n', + expected_stderr_fragment=( + "no target is selected and no top-level [connection] is configured" + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_invalid_connection_configuration_when_running_query_then_reports_config_error( + tmp_path: Path, + test_case: QueryConnectionConfigErrorTestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="orders_config_error", + repo_files={"sqlbuild_project.toml": test_case.project_toml}, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("query", "SELECT 1 AS order_id"), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_returncode + assert test_case.expected_stderr_fragment in result.stderr + assert "RECORD" not in result.stdout + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/website/src/content/docs/docs/cli/query.mdx b/website/src/content/docs/docs/cli/query.mdx index 388a675e77..94128a39b7 100644 --- a/website/src/content/docs/docs/cli/query.mdx +++ b/website/src/content/docs/docs/cli/query.mdx @@ -21,6 +21,8 @@ sqb --project-dir query --file my_query.sql File paths are resolved from the current working directory. +`sqb query` reads only `sqlbuild_project.toml` and `sqlbuild_local.toml` to connect. It does not parse models or import project Python, so it starts quickly in large projects and still runs when a model or Python file is broken. The SQL runs as written: SQLBuild does not resolve model names in it. + ## Flags | Flag | Description | From 36b563ddfa341219d3a3cbdc360e26ad3c8cffbd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:43:58 +0100 Subject: [PATCH 17/39] test: guard the startup import footprint of scope, lineage, dag and query --- .../sqlbuild/cli/commands/main/_test_types.py | 1 + .../main/test_startup_import_footprint.py | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 6dc1cc9791..a391b0993f 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1068,6 +1068,7 @@ class StartupImportFootprintTestCase: argv: tuple[str, ...] expected_exit_code: int forbidden_modules: tuple[str, ...] + local_config: str = "" @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 85d8a7db1d..aaf1acff51 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -62,6 +62,26 @@ "sqlbuild.adapter.contract.classes.base_adapter", "sqlbuild.compiler.pipeline", ) +INSPECTION_FORBIDDEN: tuple[str, ...] = ( + *WAREHOUSE_AND_INTEGRATION_MODULES, + "filelock", + "polyglot_sql", + "pydantic", + "yaml", + "sqlbuild.lint", + "sqlbuild.rule_engine", + *PRESENTATION_AND_PROVIDER_MODULES, +) +PROJECT_DISCOVERY_MODULES: tuple[str, ...] = ( + "sqlbuild.compiler.discovery._helpers.validation", + "sqlbuild.compiler.macro_bridge", + "sqlbuild.compiler.python_nodes", +) +QUERY_FORBIDDEN: tuple[str, ...] = ( + *(module for module in INSPECTION_FORBIDDEN if module != "duckdb"), + *PROJECT_DISCOVERY_MODULES, +) +MEMORY_CONNECTION: str = '[connection]\ndatabase = ":memory:"\n' REUSED_COMPILE_FORBIDDEN: tuple[str, ...] = ( *WAREHOUSE_AND_INTEGRATION_MODULES, *FULL_COMPILE_MODULES, @@ -118,6 +138,31 @@ expected_exit_code=0, forbidden_modules=COMPILE_FORBIDDEN, ), + StartupImportFootprintTestCase( + description="scope", + argv=("scope", "orders"), + expected_exit_code=0, + forbidden_modules=INSPECTION_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="lineage", + argv=("lineage", "orders", "--json"), + expected_exit_code=0, + forbidden_modules=INSPECTION_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="dag", + argv=("dag", "--json"), + expected_exit_code=0, + forbidden_modules=INSPECTION_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="ad hoc query", + argv=("query", "SELECT 1 AS order_id"), + expected_exit_code=0, + forbidden_modules=QUERY_FORBIDDEN, + local_config=MEMORY_CONNECTION, + ), ], ids=lambda case: case.description, ) @@ -125,6 +170,7 @@ def test_given_fresh_interpreter_when_running_cli_then_heavy_modules_stay_unload test_case: StartupImportFootprintTestCase, tmp_path: Path ) -> None: write_compile_startup_project(tmp_path) + _ = (tmp_path / "sqlbuild_local.toml").write_text(test_case.local_config, encoding="utf-8") argv: list[str] = ["--project-dir", str(tmp_path), *test_case.argv] result: subprocess.CompletedProcess[str] = subprocess.run( From 9e1577602d2ba3fec38c3ed03f5488e4a418998c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:09:32 +0100 Subject: [PATCH 18/39] docs: regenerate the bundled query reference --- .../.agents/skills/sqlbuild/references/docs/cli/query.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md index 6b3b907b0d..c2dbb5f2fa 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md @@ -30,6 +30,8 @@ sqb --project-dir query --file my_query.sql File paths are resolved from the current working directory. +`sqb query` reads only `sqlbuild_project.toml` and `sqlbuild_local.toml` to connect. It does not parse models or import project Python, so it starts quickly in large projects and still runs when a model or Python file is broken. The SQL runs as written: SQLBuild does not resolve model names in it. + ## Flags | Flag | Description | From 1261a1cf8c4b19fd5cb2bf72d959259e474fa503 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:20:44 +0100 Subject: [PATCH 19/39] fix(snowflake): collect run cost for large runs in bounded history slices Large runs always timed out collecting cost: one query-history read covered the whole run and embedded every run query ID in the SQL. Read history in slices of at most 15 minutes, select CREATE text by the run's query tag so the SQL no longer grows with the run, split a slice that reaches its statement timeout, and record cost as pending, naming the timeout, when the collection budget runs out. Other failures now name the Snowflake error. --- .../src/cost_allocation/_helpers/mod.rs | 1 + .../cost_allocation/_helpers/query_text.rs | 34 ++ .../src/cost_allocation/constants.rs | 4 + .../cost_allocation/main/query_history_sql.rs | 23 +- .../src/cost_allocation/models.rs | 4 +- .../_helpers/cost_telemetry/snowflake.rs | 245 ++++++++++++--- .../docs/concepts/adapters/snowflake.md | 8 + src/sqlbuild/_native.pyi | 3 +- .../adapters/snowflake/_helpers/cost.py | 16 + .../snowflake/classes/snowflake_adapter.py | 9 +- .../snowflake/test_cost_collection.py | 4 +- tests/unit/src/sqlbuild/cost/_test_types.py | 30 ++ tests/unit/src/sqlbuild/cost/helpers.py | 60 ++++ .../unit/src/sqlbuild/cost/test_snowflake.py | 23 +- .../src/sqlbuild/cost/test_snowflake_scale.py | 294 ++++++++++++++++++ .../docs/docs/concepts/adapters/snowflake.mdx | 8 + 16 files changed, 688 insertions(+), 78 deletions(-) create mode 100644 crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs create mode 100644 src/sqlbuild/adapters/snowflake/_helpers/cost.py create mode 100644 tests/unit/src/sqlbuild/cost/test_snowflake_scale.py diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs index 3b9c8a22a1..a9f8414daa 100644 --- a/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod allocation; pub(crate) mod decimal; +pub(crate) mod query_text; diff --git a/crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs b/crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs new file mode 100644 index 0000000000..53a4e23f66 --- /dev/null +++ b/crates/sqlbuild-core/src/cost_allocation/_helpers/query_text.rs @@ -0,0 +1,34 @@ +//! The `QUERY_TEXT` column of a query-history read, selected by the run's query tag. + +use crate::cost_allocation::constants::{ + HOOK_TAG_FRAGMENT, NULL_QUERY_TEXT, SQLBUILD_TAG_FRAGMENT, +}; + +/// One run's query-tag fragment as `json.dumps` writes it, else the SQLBuild app fragment. +fn run_tag_fragment(run_id: &str) -> String { + let plain: bool = run_id + .chars() + .all(|character| character.is_ascii_graphic() && character != '"' && character != '\\'); + if plain && !run_id.is_empty() { + format!("\"run_id\":\"{run_id}\"") + } else { + SQLBUILD_TAG_FRAGMENT.to_owned() + } +} + +fn like_contains(fragment: &str) -> String { + format!("'%{}%'", fragment.replace('\'', "''")) +} + +/// CREATE text of the run's non-hook statements, selected by query tag, or masked text. +pub(crate) fn query_text_expression(run_id: Option<&str>) -> String { + let Some(run_id) = run_id else { + return NULL_QUERY_TEXT.to_owned(); + }; + format!( + "CASE WHEN (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') \ +AND QUERY_TAG LIKE {} AND QUERY_TAG NOT LIKE {} THEN QUERY_TEXT END AS QUERY_TEXT", + like_contains(&run_tag_fragment(run_id)), + like_contains(HOOK_TAG_FRAGMENT), + ) +} diff --git a/crates/sqlbuild-core/src/cost_allocation/constants.rs b/crates/sqlbuild-core/src/cost_allocation/constants.rs index 09c13bd940..57789c24a3 100644 --- a/crates/sqlbuild-core/src/cost_allocation/constants.rs +++ b/crates/sqlbuild-core/src/cost_allocation/constants.rs @@ -25,6 +25,10 @@ pub(crate) const CREDITS_PER_HOUR: [(&str, u64); 10] = [ pub const QUERY_HISTORY_RESULT_LIMIT: usize = 10_000; pub(crate) const QUERY_HISTORY_FUNCTION: &str = "INFORMATION_SCHEMA.QUERY_HISTORY"; pub(crate) const NULL_QUERY_TEXT: &str = "NULL AS QUERY_TEXT"; +/// The compact, key-sorted query-tag fragment every SQLBuild-tagged statement carries. +pub(crate) const SQLBUILD_TAG_FRAGMENT: &str = "\"app\":\"sqlbuild\""; +/// The query-tag fragment of hook statements, whose text is never requested. +pub(crate) const HOOK_TAG_FRAGMENT: &str = "\"resource_type\":\"hook\""; /// The rounding digit at which `ROUND_HALF_EVEN` looks past the first dropped digit. pub(crate) const HALF_DIGIT: u8 = 5; /// `str(Decimal)` switches to exponent form at or below this many left digits. diff --git a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs index 40b6028249..54fa520146 100644 --- a/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs +++ b/crates/sqlbuild-core/src/cost_allocation/main/query_history_sql.rs @@ -1,6 +1,7 @@ //! `query_history_sql`: the Snowflake `QUERY_HISTORY` read of one end-time window. -use crate::cost_allocation::constants::{NULL_QUERY_TEXT, QUERY_HISTORY_FUNCTION}; +use crate::cost_allocation::_helpers::query_text::query_text_expression; +use crate::cost_allocation::constants::QUERY_HISTORY_FUNCTION; use crate::cost_allocation::models::QueryHistoryWindow; fn query_history_function(database: Option<&str>) -> String { @@ -13,24 +14,6 @@ fn query_history_function(database: Option<&str>) -> String { } } -fn query_text_expression(query_ids: &[String]) -> String { - if query_ids.is_empty() { - return NULL_QUERY_TEXT.to_owned(); - } - let mut sorted: Vec<&String> = query_ids.iter().collect(); - sorted.sort(); - sorted.dedup(); - let literals: Vec = sorted - .into_iter() - .map(|query_id| format!("'{}'", query_id.replace('\'', "''"))) - .collect(); - format!( - "CASE WHEN QUERY_ID IN ({}) AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') \ -THEN QUERY_TEXT END AS QUERY_TEXT", - literals.join(", ") - ) -} - /// Query-history SQL for queries ending after `start`, and before `end` when it is bounded. pub fn query_history_sql(window: &QueryHistoryWindow<'_>) -> String { let end_range: String = match window.end { @@ -58,7 +41,7 @@ FROM TABLE({}( RESULT_LIMIT => {} )) ORDER BY START_TIME, QUERY_ID", - query_text_expression(window.classification_query_ids), + query_text_expression(window.text_run_id), query_history_function(window.database), window.start, window.result_limit, diff --git a/crates/sqlbuild-core/src/cost_allocation/models.rs b/crates/sqlbuild-core/src/cost_allocation/models.rs index 349f862fbe..d23c4e9be0 100644 --- a/crates/sqlbuild-core/src/cost_allocation/models.rs +++ b/crates/sqlbuild-core/src/cost_allocation/models.rs @@ -63,12 +63,12 @@ pub struct RunAllocation { pub limitations: Vec, } -/// The time window and classified query IDs of one query-history read. +/// One query-history read's window and the run whose CREATE text it requests by query tag. #[derive(Clone, Debug, PartialEq, Eq)] pub struct QueryHistoryWindow<'a> { pub database: Option<&'a str>, pub start: &'a str, pub end: Option<&'a str>, - pub classification_query_ids: &'a [String], + pub text_run_id: Option<&'a str>, pub result_limit: usize, } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 405562e494..5cb6da8ff2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -3,8 +3,10 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use pyo3::exceptions::{PyTimeoutError, PyValueError}; -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; -use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyTuple}; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; @@ -25,8 +27,15 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::RecordId; -const DEADLINE_SECONDS: f64 = 10.0; -const QUERY_TIMEOUT_SECONDS: &str = "5"; +/// Overall budget of one collection; a run's history is read in slices well inside it. +const DEADLINE_SECONDS: f64 = 30.0; +/// Statement timeout of one query-history read. +const QUERY_TIMEOUT_SECONDS: &str = "10"; +/// Widest end-time slice of one read; long runs are read as consecutive slices of this width. +const SLICE_SECONDS: i64 = 15 * 60; +/// Narrowest slice still split in half after a statement timeout. +const MIN_TIMEOUT_SPLIT_SECONDS: i64 = 60; +const STATEMENT_TIMEOUT_CODE: &str = "000630"; const RUNNING_STATUS: &str = "RUNNING"; const HOOK_RESOURCE_TYPE: &str = "hook"; const ROW_WIDTH: usize = 14; @@ -43,11 +52,12 @@ const PARTIAL_STATUS: &str = "PARTIAL"; const COMPLETE_STATUS: &str = "COMPLETE"; const UNAVAILABLE_STATUS: &str = "UNAVAILABLE"; -/// The connection, window inputs and deadline every query-history read shares. +/// Connection, window inputs and deadline of every read; text is kept only for classified IDs. struct History<'py> { connection: Bound<'py, PyAny>, database: Option, - classification_query_ids: Vec, + text_run_id: Option, + classification_query_ids: HashSet, deadline: f64, result_limit: usize, } @@ -92,7 +102,7 @@ fn history_rows<'py>( database: history.database.as_deref(), start: &start_text, end: end_text.as_deref(), - classification_query_ids: &history.classification_query_ids, + text_run_id: history.text_run_id.as_deref(), result_limit: history.result_limit, }); let parameters: Bound<'py, PyDict> = PyDict::new(py); @@ -134,20 +144,46 @@ fn merged_rows<'py>( Ok(by_query_id.into_values().collect()) } -/// End-time-bounded rows, halving the window while a read reaches the result limit. +/// Whether a query-history read failed on Snowflake's statement timeout. +pub(crate) fn statement_timeout(py: Python<'_>, error: &PyErr) -> bool { + error + .value(py) + .str() + .is_ok_and(|text| text.to_string_lossy().contains(STATEMENT_TIMEOUT_CODE)) +} + +/// End-time-bounded rows, halving the window while a read reaches the result limit or times out. fn bounded_rows<'py>( history: &History<'py>, (start, end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult<(Vec>, bool)> { let py: Python<'py> = start.py(); - let rows: Vec> = history_rows(history, (start, Some(end)))?; + let width: Bound<'py, PyAny> = end.sub(start)?; + let rows: Vec> = match history_rows(history, (start, Some(end))) { + Ok(rows) => rows, + Err(error) + if statement_timeout(py, &error) + && width.gt(timedelta(py, ("seconds", MIN_TIMEOUT_SPLIT_SECONDS))?)? => + { + return split_rows(history, (start, end), &width); + } + Err(error) => return Err(error), + }; if rows.len() < history.result_limit { return Ok((rows, false)); } - let width: Bound<'py, PyAny> = end.sub(start)?; if width.le(timedelta(py, ("milliseconds", 1))?)? { return Ok((rows, true)); } + split_rows(history, (start, end), &width) +} + +/// Rows of both halves of a bounded window, merged by query ID. +fn split_rows<'py>( + history: &History<'py>, + (start, end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + width: &Bound<'py, PyAny>, +) -> PyResult<(Vec>, bool)> { let midpoint: Bound<'py, PyAny> = start.add(width.div(2)?)?; let (left, left_saturated) = bounded_rows(history, (start, &midpoint))?; let (right, right_saturated) = bounded_rows(history, (&midpoint, end))?; @@ -194,7 +230,7 @@ fn optional_str<'py>(value: &Bound<'py, PyAny>) -> PyResult> { /// A history row as an observation, or the reason it carries no attributable compute. fn row_observation<'py>( row: &Bound<'py, PyAny>, - require_complete: bool, + (require_complete, classify): (bool, bool), ) -> PyResult, &'static str>> { let py: Python<'py> = row.py(); let fields: Vec> = unpacked(row)?; @@ -224,7 +260,7 @@ fn row_observation<'py>( } else { String::new() }; - let query_text: Option = if query_text.is_instance_of::() { + let query_text: Option = if classify && query_text.is_instance_of::() { Some(query_text.extract()?) } else { None @@ -288,7 +324,81 @@ fn row_observation<'py>( Ok(Ok(observation)) } -/// Observations of the run window: open-ended history, bounded windows when saturated. +/// Equal slice starts of a window, each at most `SLICE_SECONDS`; one slice for a short run. +fn slice_starts<'py>( + (window_start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + run: &Bound<'py, PyAny>, +) -> PyResult>> { + let py: Python<'py> = run.py(); + let width: Bound<'py, PyAny> = window_end.sub(window_start)?; + let slice: Bound<'py, PyAny> = timedelta(py, ("seconds", SLICE_SECONDS))?; + let count: usize = if run.le(&slice)? { + 1 + } else { + width + .sub(timedelta(py, ("microseconds", 1))?)? + .floor_div(&slice)? + .extract::()? + + 1 + }; + (0..count) + .map(|index| window_start.add(width.mul(index)?.div(count)?)) + .collect() +} + +/// Open-ended rows from `start`, which include running queries, or bounded windows if saturated. +fn open_rows<'py>( + history: &History<'py>, + (start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), +) -> PyResult<(Vec>, bool)> { + let py: Python<'py> = start.py(); + let open_rows: Vec> = match history_rows(history, (start, None)) { + Ok(rows) => rows, + Err(error) if statement_timeout(py, &error) => { + return bounded_rows(history, (start, window_end)); + } + Err(error) => return Err(error), + }; + if open_rows.len() < history.result_limit { + return Ok((open_rows, false)); + } + let (bounded, bounded_saturated) = bounded_rows(history, (start, window_end))?; + let mut still_running: Vec> = Vec::new(); + for row in open_rows { + if running(&row.get_item(7)?, &row.get_item(10)?)? { + still_running.push(row); + } + } + Ok(( + merged_rows(still_running.into_iter().chain(bounded))?, + bounded_saturated, + )) +} + +/// Rows of the run window, read as bounded slices with an open-ended final slice. +fn sliced_rows<'py>( + history: &History<'py>, + (window_start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + run: &Bound<'py, PyAny>, +) -> PyResult<(Vec>, bool)> { + let starts: Vec> = slice_starts((window_start, window_end), run)?; + let mut rows: Vec> = Vec::new(); + let mut saturated: bool = false; + for (index, start) in starts.iter().enumerate() { + let (slice, slice_saturated) = match starts.get(index + 1) { + Some(end) => bounded_rows(history, (start, end))?, + None => open_rows(history, (start, window_end))?, + }; + rows.extend(slice); + saturated |= slice_saturated; + } + if starts.len() == 1 { + return Ok((rows, saturated)); + } + Ok((merged_rows(rows.into_iter())?, saturated)) +} + +/// Observations of the run window: sliced history, bounded windows when saturated. fn query_observations<'py>( history: &History<'py>, (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -297,29 +407,20 @@ fn query_observations<'py>( let py: Python<'py> = started_at.py(); let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; - let open_rows: Vec> = history_rows(history, (&window_start, None))?; - let mut saturated: bool = open_rows.len() >= history.result_limit; - let rows: Vec> = if saturated { - let (bounded, bounded_saturated) = - bounded_rows(history, (&window_start, &completed_at.add(&skew)?))?; - let mut still_running: Vec> = Vec::new(); - for row in open_rows { - if running(&row.get_item(7)?, &row.get_item(10)?)? { - still_running.push(row); - } - } - saturated = bounded_saturated; - merged_rows(still_running.into_iter().chain(bounded))? - } else { - open_rows - }; + let window_end: Bound<'py, PyAny> = completed_at.add(&skew)?; + let (rows, saturated) = sliced_rows( + history, + (&window_start, &window_end), + &completed_at.sub(started_at)?, + )?; let observations: Bound<'py, PyList> = PyList::empty(py); let mut observed: HashSet = HashSet::new(); let mut skipped: Vec<(String, &'static str)> = Vec::new(); for row in rows { let id: String = query_id(&row)?; observed.insert(id.clone()); - match row_observation(&row, expected.contains(&id))? { + let classify: bool = history.classification_query_ids.contains(&id); + match row_observation(&row, (expected.contains(&id), classify))? { Ok(observation) => observations.append(observation)?, Err(reason) => match skipped.iter_mut().find(|(skipped_id, _)| *skipped_id == id) { Some(entry) => entry.1 = reason, @@ -340,6 +441,43 @@ fn query_observations<'py>( }) } +/// The budget a query-history read exhausted, or `None` for any other failure. +fn budget_exhausted(py: Python<'_>, error: &PyErr) -> Option { + if error.is_instance_of::(py) + && error + .value(py) + .str() + .is_ok_and(|text| text.to_string_lossy() == DEADLINE_MESSAGE) + { + return Some(format!( + "{DEADLINE_SECONDS:.0}-second collection deadline exceeded" + )); + } + statement_timeout(py, error).then(|| { + format!("query-history read reached its {QUERY_TIMEOUT_SECONDS}-second statement timeout") + }) +} + +/// A pending summary for a run whose history did not return within the collection budget. +fn budget_pending( + (run_id, cause): (&str, &str), + statement_count: usize, + failure: Option<&str>, +) -> Summary { + let mut pending: Summary = with_ledger_failure( + partial(&format!( + "Snowflake query history did not return within the cost collection budget \ +({cause}). Run `sqb cost {run_id}` to retry." + )), + failure, + ); + if failure.is_none() { + pending.state.status = PENDING_STATUS; + } + pending.state.counts = [statement_count, 0, statement_count, 0]; + pending +} + /// A collected summary: the allocation it prices, if any, and its derived fields. struct Summary { allocation: Option, @@ -485,14 +623,15 @@ fn collect<'py>( } } let mut expected_ids: HashSet = HashSet::new(); - let mut classification: Vec = Vec::new(); + let mut classification: HashSet = HashSet::new(); for (id, entry) in pyo3::types::PyDictMethods::iter(&by_query_id) { let id: String = id.extract()?; if !native_field(&entry, "resource_type")?.eq(HOOK_RESOURCE_TYPE)? { - classification.push(id.clone()); + classification.insert(id.clone()); } expected_ids.insert(id); } + history.text_run_id = (!classification.is_empty()).then(|| run_id.clone()); history.classification_query_ids = classification; if !ledger.is_empty() && expected_ids.is_empty() { let mut summary: Summary = with_ledger_failure( @@ -514,11 +653,26 @@ fn collect<'py>( saturated: false, }; for attempt in 0..collection.attempts { - observed = query_observations( + observed = match query_observations( &history, (&collection.started_at, &collection.completed_at), &expected_ids, - )?; + ) { + Ok(observed) => observed, + Err(error) => { + let Some(cause) = budget_exhausted(py, &error) else { + return Err(error); + }; + if summary.is_some() { + break; + } + return Ok(budget_pending( + (&run_id, &cause), + ledger.len(), + failure.as_deref(), + )); + } + }; let attributed: Vec> = observed .observations .iter() @@ -644,7 +798,8 @@ fn collect_snowflake_cost<'py>( let history: History<'py> = History { connection: arguments.value("connection")?, database: arguments.value("database")?.extract()?, - classification_query_ids: Vec::new(), + text_run_id: None, + classification_query_ids: HashSet::new(), deadline: 0.0, result_limit: optional( "result_limit", @@ -684,14 +839,23 @@ fn snowflake_query_observations<'py>( "classification_query_ids", "deadline", ], - &["database", "result_limit"], + &["database", "result_limit", "run_id"], ), )?; let result_limit: Bound<'py, PyAny> = arguments.value("result_limit")?; + let classification_query_ids: HashSet = + strings(&arguments.value("classification_query_ids")?)? + .into_iter() + .collect(); let history: History<'py> = History { connection: arguments.value("connection")?, database: arguments.value("database")?.extract()?, - classification_query_ids: strings(&arguments.value("classification_query_ids")?)?, + text_run_id: if classification_query_ids.is_empty() { + None + } else { + arguments.value("run_id")?.extract()? + }, + classification_query_ids, deadline: arguments.value("deadline")?.extract()?, result_limit: if result_limit.is_none() { QUERY_HISTORY_RESULT_LIMIT @@ -737,8 +901,8 @@ fn render_snowflake_query_history_sql( "render_snowflake_query_history_sql", options, ( - &["started_at", "classification_query_ids"], - &["database", "completed_at", "result_limit"], + &["started_at"], + &["database", "completed_at", "result_limit", "run_id"], ), )?; let start: String = arguments @@ -753,11 +917,12 @@ fn render_snowflake_query_history_sql( }; let result_limit: Bound<'_, PyAny> = arguments.value("result_limit")?; let database: Option = arguments.value("database")?.extract()?; + let run_id: Option = arguments.value("run_id")?.extract()?; Ok(query_history_sql(&QueryHistoryWindow { database: database.as_deref(), start: &start, end: end.as_deref(), - classification_query_ids: &strings(&arguments.value("classification_query_ids")?)?, + text_run_id: run_id.as_deref(), result_limit: if result_limit.is_none() { QUERY_HISTORY_RESULT_LIMIT } else { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index ea93ec6ce1..0b104feb87 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -132,6 +132,14 @@ contract rate. Use `sqb cost`, `sqb cost latest`, `sqb cost `, or provide a versioned, decimal-safe output contract. Pending detail records are refreshed from Snowflake when inspected again. +SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` when the run finishes, in end-time slices of +at most 15 minutes, so a long run with tens of thousands of statements costs several short reads +rather than one slow one. Each read has a 10-second statement timeout; a slice that reaches it is +split in half and read again. If history still cannot be read within the 30-second collection +budget, the build is unaffected and its cost is recorded as `pending`, with a message naming the +timeout; `sqb cost ` collects it later. Any other failure is recorded as +`collection_failed` with the Snowflake error that caused it. + These values are attributed compute credits and estimated cost, not Snowflake-billed credits or invoice reconciliation. The estimate uses only query history visible to the executing role and does not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60-second minimum, diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 19bf31f222..cd77ef32eb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7592,14 +7592,15 @@ def snowflake_query_observations( deadline: float, database: str | None = None, result_limit: int | None = None, + run_id: str | None = None, ) -> tuple[tuple[QueryCostObservation, ...], frozenset[str], dict[str, str], bool]: ... def render_snowflake_query_history_sql( *, started_at: datetime, - classification_query_ids: Iterable[str], database: str | None = None, completed_at: datetime | None = None, result_limit: int = 10000, + run_id: str | None = None, ) -> str: ... def snowflake_statement_is_metadata_only(*, query_type: object, query_text: object) -> bool: ... def parse_sqlbuild_query_tag(value: object) -> dict[str, str | None] | None: ... diff --git a/src/sqlbuild/adapters/snowflake/_helpers/cost.py b/src/sqlbuild/adapters/snowflake/_helpers/cost.py new file mode 100644 index 0000000000..149fa8f000 --- /dev/null +++ b/src/sqlbuild/adapters/snowflake/_helpers/cost.py @@ -0,0 +1,16 @@ +"""Describe Snowflake cost-collection failures.""" + +from __future__ import annotations + +_FAILURE_DETAIL_LIMIT: int = 200 + + +def cost_failure_cause(error: Exception) -> str: + """Name a cost-collection failure by type and its message on one bounded line.""" + + detail: str = " ".join(str(error).split()) + if not detail: + return type(error).__name__ + if len(detail) > _FAILURE_DETAIL_LIMIT: + detail = f"{detail[: _FAILURE_DETAIL_LIMIT - 3]}..." + return f"{type(error).__name__}: {detail}" diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 1ea747e1c7..b9a9088eea 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -118,6 +118,7 @@ from sqlbuild.adapter.type_system.main.first_arg_nullability import first_arg_nullability from sqlbuild.adapter.type_system.main.normalize_numeric_family import normalize_numeric_family from sqlbuild.adapter.type_system.main.types_equal import types_equal +from sqlbuild.adapters.snowflake._helpers.cost import cost_failure_cause from sqlbuild.adapters.snowflake._helpers.grants import ( render_snowflake_view_grants, render_snowflake_view_revokes, @@ -168,6 +169,8 @@ _STATE_TABLE_RETENTION_ATTRIBUTE: str = "_state_table_retention_fallbacks_state" _METADATA_INSPECTION_CONCURRENCY: int = 8 _SHOW_RESULT_LIMIT: int = 10_000 +_COST_LOGIN_TIMEOUT_SECONDS: int = 5 +_COST_NETWORK_TIMEOUT_SECONDS: int = 15 _SHOW_COLUMN_KIND: str = "COLUMN" _SHOW_COLUMN_KINDS: frozenset[str] = frozenset({_SHOW_COLUMN_KIND, "VIRTUAL_COLUMN"}) _UNRESOLVED: object = object() @@ -357,8 +360,8 @@ def collect_run_cost( rate_source: str, ) -> RunCostSummary: telemetry_connection_config: dict[str, object] = dict(connection_config) - telemetry_connection_config["login_timeout"] = 5 - telemetry_connection_config["network_timeout"] = 5 + telemetry_connection_config["login_timeout"] = _COST_LOGIN_TIMEOUT_SECONDS + telemetry_connection_config["network_timeout"] = _COST_NETWORK_TIMEOUT_SECONDS try: connection: Any = self.connect(telemetry_connection_config) except Exception as error: @@ -384,7 +387,7 @@ def collect_run_cost( status=CostStatus.COLLECTION_FAILED, usd_per_credit=usd_per_credit, rate_source=rate_source, - message=f"Snowflake cost collection failed ({type(error).__name__}).", + message=f"Snowflake cost collection failed ({cost_failure_cause(error)}).", ) finally: try: diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py index 695632573f..afbc4b0c70 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py @@ -68,7 +68,7 @@ def __call__(self, **kwargs: Any) -> RunCostSummary: collection_error=True, close_error=False, expected_status=CostStatus.COLLECTION_FAILED, - expected_message_fragment="collection failed (PermissionError)", + expected_message_fragment="collection failed (PermissionError: query history denied)", ), SnowflakeCostCollectionTestCase( description="close failure downgrades successful collection to partial", @@ -113,6 +113,6 @@ def test_given_telemetry_stage_failure_when_collecting_then_safe_status_is_retur assert summary.status == test_case.expected_status assert connect.config["login_timeout"] == 5 - assert connect.config["network_timeout"] == 5 + assert connect.config["network_timeout"] == 15 assert (test_case.expected_message_fragment or "") in (summary.message or "") assert (test_case.expected_limitation_fragment or "") in " ".join(summary.limitations) diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index ac1e03b09f..b6246b99c4 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from datetime import timedelta from pathlib import Path from sqlbuild.cost.types import CostStatus @@ -101,3 +102,32 @@ class CostResourceScopeTestCase: expected_resource_name: str expected_phase: str expected_attempt: int + + +@dataclass(frozen=True) +class SnowflakeScaleTestCase: + description: str + statement_count: int + duration_minutes: int + timeout_window: timedelta | None + expected_status: CostStatus + expected_query_count: int + expected_message_fragments: tuple[str, ...] = () + expected_max_sql_length: int = 2_000 + expected_max_slice: timedelta = timedelta(minutes=15) + + +@dataclass(frozen=True) +class SnowflakeLargeRunCostTestCase: + description: str + timeout_window: timedelta | None + expected_status: CostStatus + expected_message_fragment: str + expected_max_sql_length: int = 2_000 + + +@dataclass(frozen=True) +class SnowflakeCostFailureMessageTestCase: + description: str + expected_status: CostStatus + expected_message: str diff --git a/tests/unit/src/sqlbuild/cost/helpers.py b/tests/unit/src/sqlbuild/cost/helpers.py index 6cb346cdb9..cab18bd7b3 100644 --- a/tests/unit/src/sqlbuild/cost/helpers.py +++ b/tests/unit/src/sqlbuild/cost/helpers.py @@ -1,5 +1,8 @@ +import json from datetime import UTC, datetime, timedelta from decimal import Decimal +from pathlib import Path +from typing import Any from sqlbuild._native import CostRunRecord, QueryCostObservation, ResourceCost from sqlbuild.cost.models import RunCostSummary @@ -67,3 +70,60 @@ def build_cost_run_record(*, run_id: str, completed_at: datetime) -> CostRunReco estimated_usd=resource.estimated_usd, ), ) + + +def history_row(*, query_id: str, ended_at: datetime) -> tuple[Any, ...]: + """One successful one-second warehouse query ending at `ended_at`.""" + + return ( + query_id, + "", + "TRANSFORM_WH", + "X-Small", + "STANDARD", + 1, + ended_at - timedelta(seconds=1), + ended_at, + 1000, + 100, + "SUCCESS", + "INSERT", + None, + ended_at, + ) + + +def run_history_rows( + *, origin: datetime, statement_count: int, duration: timedelta +) -> list[tuple[Any, ...]]: + """History rows of one run whose statements end evenly across `duration`.""" + + step: timedelta = duration / statement_count + return [ + history_row(query_id=f"q-{index:05d}", ended_at=origin + step * (index + 1)) + for index in range(statement_count) + ] + + +def write_ledger(*, path: Path, run_id: str, rows: list[tuple[Any, ...]]) -> None: + """Write one statement-ledger entry for each history row.""" + + with path.open("w", encoding="utf-8") as ledger: + for index, row in enumerate(rows): + ledger.write( + json.dumps( + { + "statement_id": f"statement-{index}", + "run_id": run_id, + "resource_type": "model", + "resource_name": f"orders_{index % 1400}", + "phase": "execute", + "attempt": 1, + "query_id": row[0], + "status": "success", + "started_at": row[6].isoformat(), + "completed_at": row[7].isoformat(), + } + ) + + "\n" + ) diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake.py b/tests/unit/src/sqlbuild/cost/test_snowflake.py index 0a558b5c77..fabafcbfca 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake.py @@ -75,8 +75,10 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: "RESULT_LIMIT => 10000", "EXECUTION_STATUS", "QUERY_TYPE", - "CASE WHEN QUERY_ID IN ('q-create') AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE " - "'CREATE_%') THEN QUERY_TEXT END AS QUERY_TEXT", + "CASE WHEN (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') " + 'AND QUERY_TAG LIKE \'%"run_id":"run-1"%\' ' + 'AND QUERY_TAG NOT LIKE \'%"resource_type":"hook"%\' ' + "THEN QUERY_TEXT END AS QUERY_TEXT", "CURRENT_TIMESTAMP()", ), expected_excluded_fragment="WHERE EXECUTION_STATUS = 'SUCCESS'", @@ -91,7 +93,7 @@ def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_t database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), completed_at=datetime(2026, 8, 23, 10, 5, tzinfo=UTC), - classification_query_ids=frozenset({"q-create"}), + run_id="run-1", ) for fragment in test_case.expected_fragments: @@ -110,8 +112,10 @@ def test_given_time_range_when_rendering_query_history_then_uses_valid_bounded_t "RESULT_LIMIT => 10000", "EXECUTION_STATUS", "QUERY_TYPE", - "CASE WHEN QUERY_ID IN ('q-create') AND (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE " - "'CREATE_%') THEN QUERY_TEXT END AS QUERY_TEXT", + "CASE WHEN (QUERY_TYPE = 'CREATE' OR QUERY_TYPE LIKE 'CREATE_%') " + 'AND QUERY_TAG LIKE \'%"run_id":"run-1"%\' ' + 'AND QUERY_TAG NOT LIKE \'%"resource_type":"hook"%\' ' + "THEN QUERY_TEXT END AS QUERY_TEXT", "CURRENT_TIMESTAMP()", ), expected_excluded_fragment="END_TIME_RANGE_END", @@ -125,7 +129,7 @@ def test_given_start_time_when_rendering_open_history_then_running_queries_are_r sql: str = render_snowflake_query_history_sql( database="analytics", started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), - classification_query_ids=frozenset({"q-create"}), + run_id="run-1", ) for fragment in test_case.expected_fragments: @@ -149,7 +153,6 @@ def test_given_no_classifiable_create_ids_when_rendering_history_then_query_text ) -> None: sql: str = render_snowflake_query_history_sql( started_at=datetime(2026, 8, 23, 10, 0, tzinfo=UTC), - classification_query_ids=frozenset(), ) for fragment in test_case.expected_fragments: @@ -161,9 +164,9 @@ def test_given_no_classifiable_create_ids_when_rendering_history_then_query_text "test_case", [ RenderQueryHistorySqlTestCase( - description="exact create hook is omitted from query text allowlist", + description="a run of only hooks requests no query text", expected_fragments=("NULL AS QUERY_TEXT",), - expected_excluded_fragment="q-hook", + expected_excluded_fragment="THEN QUERY_TEXT", ) ], ids=lambda case: case.description, @@ -865,7 +868,7 @@ def test_given_retry_delay_exceeds_remaining_deadline_when_collecting_then_parti None, origin + timedelta(seconds=1), ) - monotonic_values: Iterator[float] = iter((0.0, 0.0, 0.0, 0.0, 9.5)) + monotonic_values: Iterator[float] = iter((0.0, 0.0, 0.0, 0.0, 29.5)) monkeypatch.setattr( "time.monotonic", lambda: next(monotonic_values), diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py new file mode 100644 index 0000000000..4e68dd5f6c --- /dev/null +++ b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py @@ -0,0 +1,294 @@ +"""Snowflake cost collection over runs with tens of thousands of statements.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from pathlib import Path +from typing import Any + +import pytest + +from sqlbuild._native import collect_snowflake_cost +from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter +from sqlbuild.cost.models import RunCostSummary +from sqlbuild.cost.types import CostStatus +from tests.unit.src.sqlbuild.cost._test_types import ( + SnowflakeCostFailureMessageTestCase, + SnowflakeLargeRunCostTestCase, + SnowflakeScaleTestCase, +) +from tests.unit.src.sqlbuild.cost.helpers import history_row, run_history_rows, write_ledger + +_ORIGIN: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) +_RUN_ID: str = "run-nightly" + + +_START_PATTERN: re.Pattern[str] = re.compile( + r"END_TIME_RANGE_START => TO_TIMESTAMP_LTZ\('([^']+)'\)" +) +_END_PATTERN: re.Pattern[str] = re.compile(r"END_TIME_RANGE_END => TO_TIMESTAMP_LTZ\('([^']+)'\)") +_LIMIT_PATTERN: re.Pattern[str] = re.compile(r"RESULT_LIMIT => (\d+)") +STATEMENT_TIMEOUT_ERRNO: int = 630 + + +class StatementTimeoutError(Exception): + """The error a Snowflake read raises when it reaches its statement timeout.""" + + def __init__(self, *, seconds: str) -> None: + super().__init__( + f"000630 (57014): Statement reached its statement or warehouse timeout of " + f"{seconds} second(s) and was canceled." + ) + self.errno: int = STATEMENT_TIMEOUT_ERRNO + + +@dataclass +class HistoryRead: + """One query-history read: its window, SQL length and statement timeout.""" + + start: datetime + end: datetime | None + sql_length: int + timeout_seconds: str + + +class _Cursor: + def __init__(self, rows: list[tuple[Any, ...]]) -> None: + self.rows: list[tuple[Any, ...]] = rows + + def fetchall(self) -> list[tuple[Any, ...]]: + return self.rows + + +@dataclass +class _FakeQueryHistory: + """Answers `INFORMATION_SCHEMA.QUERY_HISTORY` reads from fixed rows, like Snowflake does. + + Rows are filtered by end time, the open-ended read also returns running queries, and at most + `RESULT_LIMIT` rows are returned. A read wider than `timeout_window` raises Snowflake's + statement-timeout error, standing in for a read too slow for its statement timeout. + """ + + rows: list[tuple[Any, ...]] + timeout_window: timedelta | None = None + reads: list[HistoryRead] = field(default_factory=list) + + @property + def widest_bounded_read(self) -> timedelta: + """The widest end-time window of any bounded read.""" + + return max( + (read.end - read.start for read in self.reads if read.end is not None), + default=timedelta(0), + ) + + @property + def longest_sql(self) -> int: + """The length of the longest SQL statement read.""" + + return max(read.sql_length for read in self.reads) + + def execute(self, sql: str, **kwargs: Any) -> _Cursor: + assert "INFORMATION_SCHEMA.QUERY_HISTORY" in sql + start_match: re.Match[str] | None = _START_PATTERN.search(sql) + limit_match: re.Match[str] | None = _LIMIT_PATTERN.search(sql) + assert start_match is not None and limit_match is not None + end_match: re.Match[str] | None = _END_PATTERN.search(sql) + start: datetime = datetime.fromisoformat(start_match.group(1)) + end: datetime | None = ( + datetime.fromisoformat(end_match.group(1)) if end_match is not None else None + ) + timeout_seconds: str = kwargs["statement_params"]["STATEMENT_TIMEOUT_IN_SECONDS"] + self.reads.append( + HistoryRead(start=start, end=end, sql_length=len(sql), timeout_seconds=timeout_seconds) + ) + latest_end: datetime = end if end is not None else max(row[7] or start for row in self.rows) + if self.timeout_window is not None and latest_end - start > self.timeout_window: + raise StatementTimeoutError(seconds=timeout_seconds) + matched: list[tuple[Any, ...]] = [ + row + for row in self.rows + if (row[7] is None and end is None) + or (row[7] is not None and row[7] >= start and (end is None or row[7] < end)) + ] + return _Cursor(matched[: int(limit_match.group(1))]) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeScaleTestCase( + description="a 78-minute run of 20,000 statements is read in bounded slices", + statement_count=20_000, + duration_minutes=78, + timeout_window=None, + expected_status=CostStatus.COMPLETE, + expected_query_count=20_000, + ), + SnowflakeScaleTestCase( + description="slices that reach the statement timeout are split and retried", + statement_count=6_500, + duration_minutes=42, + timeout_window=timedelta(minutes=8), + expected_status=CostStatus.COMPLETE, + expected_query_count=6_500, + ), + SnowflakeScaleTestCase( + description="history that times out at every width is reported pending with its cause", + statement_count=6_500, + duration_minutes=42, + timeout_window=timedelta(0), + expected_status=CostStatus.PENDING, + expected_query_count=0, + expected_message_fragments=( + "10-second statement timeout", + f"Run `sqb cost {_RUN_ID}` to retry.", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_large_run_when_collecting_snowflake_cost_then_sql_stays_bounded( + test_case: SnowflakeScaleTestCase, + tmp_path: Path, +) -> None: + duration: timedelta = timedelta(minutes=test_case.duration_minutes) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=test_case.statement_count, duration=duration + ) + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id=_RUN_ID, rows=rows) + history: _FakeQueryHistory = _FakeQueryHistory( + rows=rows, timeout_window=test_case.timeout_window + ) + + summary: RunCostSummary = collect_snowflake_cost( + connection=history, + run_id=_RUN_ID, + started_at=_ORIGIN, + completed_at=_ORIGIN + duration, + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + attempts=1, + ) + + assert summary.status == test_case.expected_status + assert summary.query_count == test_case.expected_query_count + assert summary.expected_statement_count == test_case.statement_count + for fragment in test_case.expected_message_fragments: + assert fragment in (summary.message or "") + assert history.longest_sql < test_case.expected_max_sql_length + assert {read.timeout_seconds for read in history.reads} == {"10"} + assert history.widest_bounded_read <= test_case.expected_max_slice + + +class _ClosingHistory(_FakeQueryHistory): + def close(self) -> None: + return None + + +class ProgrammingError(Exception): + """Stands in for the Snowflake connector's error of the same name.""" + + +class _FailingHistory(_ClosingHistory): + def execute(self, sql: str, **kwargs: Any) -> Any: + del sql, kwargs + raise ProgrammingError( + "002003 (02000): SQL compilation error:\nDatabase 'ANALYTICS' does not exist." + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeLargeRunCostTestCase( + description="a large run's history is collected completely", + timeout_window=None, + expected_status=CostStatus.COMPLETE, + expected_message_fragment="", + ), + SnowflakeLargeRunCostTestCase( + description="history too slow to read leaves the run pending, naming the timeout", + timeout_window=timedelta(0), + expected_status=CostStatus.PENDING, + expected_message_fragment="statement timeout", + ), + ], + ids=lambda case: case.description, +) +def test_given_large_run_when_adapter_collects_cost_then_failures_stay_non_fatal( + test_case: SnowflakeLargeRunCostTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + origin: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) + rows: list[tuple[Any, ...]] = [ + history_row(query_id=f"q-{index:05d}", ended_at=origin + timedelta(seconds=index // 3)) + for index in range(12_000) + ] + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id="run-large", rows=rows) + history: _ClosingHistory = _ClosingHistory(rows=rows, timeout_window=test_case.timeout_window) + monkeypatch.setattr(adapter, "connect", lambda config: history) + + summary: RunCostSummary = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id="run-large", + started_at=origin, + completed_at=origin + timedelta(seconds=4_000), + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert summary.status == test_case.expected_status + assert test_case.expected_message_fragment in (summary.message or "") + assert history.longest_sql < test_case.expected_max_sql_length + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostFailureMessageTestCase( + description="a multi-line Snowflake error is named on one line", + expected_status=CostStatus.COLLECTION_FAILED, + expected_message=( + "Snowflake cost collection failed (ProgrammingError: 002003 (02000): " + "SQL compilation error: Database 'ANALYTICS' does not exist.)." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_query_history_error_when_adapter_collects_cost_then_message_names_cause( + test_case: SnowflakeCostFailureMessageTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + origin: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) + rows: list[tuple[Any, ...]] = [history_row(query_id="q-1", ended_at=origin)] + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id="run-failed", rows=rows) + monkeypatch.setattr(adapter, "connect", lambda config: _FailingHistory(rows=rows)) + + summary: RunCostSummary = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id="run-failed", + started_at=origin, + completed_at=origin + timedelta(seconds=1), + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert summary.status == test_case.expected_status + assert summary.message == test_case.expected_message diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 76d70017ad..3daf486fe1 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -118,6 +118,14 @@ contract rate. Use `sqb cost`, `sqb cost latest`, `sqb cost `, or provide a versioned, decimal-safe output contract. Pending detail records are refreshed from Snowflake when inspected again. +SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` when the run finishes, in end-time slices of +at most 15 minutes, so a long run with tens of thousands of statements costs several short reads +rather than one slow one. Each read has a 10-second statement timeout; a slice that reaches it is +split in half and read again. If history still cannot be read within the 30-second collection +budget, the build is unaffected and its cost is recorded as `pending`, with a message naming the +timeout; `sqb cost ` collects it later. Any other failure is recorded as +`collection_failed` with the Snowflake error that caused it. + These values are attributed compute credits and estimated cost, not Snowflake-billed credits or invoice reconciliation. The estimate uses only query history visible to the executing role and does not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60-second minimum, From d752ab3f10f4ed845be1c4d31c6ea58ffff2a8be Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:20:50 +0100 Subject: [PATCH 20/39] fix(dagster): report seed and end-of-run audits as asset checks The integration-result contract rejected the seed attachment kind, so every seed audit degraded its record and printed a warning. Accept it, and map singular audits, whose results name where they ran, to the DAG check that lists every resource they read, so end-of-run audits reach Dagster instead of being dropped. --- .../src/integration_results/constants.rs | 3 +- .../src/integration_results/tests/helpers.rs | 37 +++++ .../tests/test_integration_results.rs | 49 +++++- .../integration_results/tests/test_types.rs | 6 + .../_helpers/dagster_invocation/results.rs | 18 +++ .../references/docs/integrations/dagster.md | 2 +- .../integrations/dagster/_test_types.py | 7 + .../sqlbuild/integrations/dagster/helpers.py | 92 ++++++++++++ .../dagster/test_audit_attachment_checks.py | 139 ++++++++++++++++++ .../src/sqlbuild/cli/output/_test_types.py | 15 +- .../test_measurement_integration_envelope.py | 80 +++++++++- .../docs/docs/integrations/dagster.mdx | 2 +- 12 files changed, 441 insertions(+), 9 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py diff --git a/crates/sqlbuild-core/src/integration_results/constants.rs b/crates/sqlbuild-core/src/integration_results/constants.rs index 512d74e0aa..6139ea82ef 100644 --- a/crates/sqlbuild-core/src/integration_results/constants.rs +++ b/crates/sqlbuild-core/src/integration_results/constants.rs @@ -48,7 +48,8 @@ pub(crate) const CHECK_STATUSES: [&str; 5] = ["error", "fail", "insufficient", " pub(crate) const CHECK_NON_FAIL_STATUSES: [&str; 2] = ["insufficient", "pass"]; /// `AuditSeverity` values, in declaration order. pub(crate) const AUDIT_SEVERITIES: [&str; 2] = ["warn", "error"]; -pub(crate) const CHECK_ATTACHMENT_KINDS: [&str; 3] = ["end", "model", "source"]; +/// Every audit attachment kind: model, source and seed attachments, and end-of-run audits. +pub(crate) const CHECK_ATTACHMENT_KINDS: [&str; 4] = ["end", "model", "seed", "source"]; pub(crate) const CHECK_RUN_SCOPE_PHASES: [&str; 2] = ["delta_and_final", "final"]; pub(crate) const EVALUATION_MODES: [&str; 2] = ["violations", "measurement"]; pub(crate) const BOUND_OPERATORS: [&str; 2] = ["below", "above"]; diff --git a/crates/sqlbuild-core/src/integration_results/tests/helpers.rs b/crates/sqlbuild-core/src/integration_results/tests/helpers.rs index 53f50afe2f..d1a0702766 100644 --- a/crates/sqlbuild-core/src/integration_results/tests/helpers.rs +++ b/crates/sqlbuild-core/src/integration_results/tests/helpers.rs @@ -50,3 +50,40 @@ pub(super) fn orders_asset(status: &str) -> JsonValue { ("microbatch", object(vec![("run_type", text("normal"))])), ]) } + +/// A completed audit record whose single check carries the given attachment kind. +pub(super) fn audit_check_record(attachment_kind: &str) -> JsonValue { + object(vec![ + ("schema_version", integer(1)), + ("record_kind", text("integration_result")), + ("event_id", text("e1")), + ("event_sequence", integer(0)), + ("event_type", text("resource_attempt_completed")), + ("occurred_at", text("2026-09-02T12:00:00Z")), + ("invocation_id", text("i1")), + ("run_id", text("r1")), + ("resource_id", text("audit:not_null:orders")), + ("resource_attempt_id", text("a1")), + ("operation_id", JsonValue::Null), + ("statement_id", JsonValue::Null), + ("resource_kind", text("audit")), + ("resource_name", text("not_null")), + ("attempt_number", integer(1)), + ("duration_ms", integer(3)), + ("output_kind", text("check")), + ("command", text("audit")), + ("asset", JsonValue::Null), + ( + "checks", + JsonValue::Array(vec![object(vec![ + ("kind", text("audit")), + ("name", text("not_null")), + ("check_id", text("audit:not_null:orders")), + ("passed", JsonValue::Bool(true)), + ("status", text("pass")), + ("attachment_kind", text(attachment_kind)), + ("audit_definition_name", text("not_null")), + ])]), + ), + ]) +} diff --git a/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs b/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs index 5dbf05e0a4..ba01a2bcd7 100644 --- a/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs +++ b/crates/sqlbuild-core/src/integration_results/tests/test_integration_results.rs @@ -1,8 +1,10 @@ use crate::integration_results::main::encode_integration_value::encode_integration_value; use crate::integration_results::main::normalize_integration_record::normalize_integration_record; use crate::integration_results::main::strip_optional_enrichment::strip_optional_enrichment; -use crate::integration_results::tests::helpers::{integer, object, orders_asset, record, text}; -use crate::integration_results::tests::test_types::NormalizeTestCase; +use crate::integration_results::tests::helpers::{ + audit_check_record, integer, object, orders_asset, record, text, +}; +use crate::integration_results::tests::test_types::{AttachmentKindTestCase, NormalizeTestCase}; use crate::json::models::JsonValue; #[test] @@ -99,3 +101,46 @@ fn given_newer_enrichment_when_stripping_then_record_keeps_core_fields() { ); } } + +#[test] +fn given_every_audit_attachment_kind_when_normalizing_check_then_it_is_supported() { + let test_cases = [ + AttachmentKindTestCase { + description: "a model audit check is supported", + attachment_kind: "model", + expected_error: None, + }, + AttachmentKindTestCase { + description: "a source audit check is supported", + attachment_kind: "source", + expected_error: None, + }, + AttachmentKindTestCase { + description: "a seed audit check is supported", + attachment_kind: "seed", + expected_error: None, + }, + AttachmentKindTestCase { + description: "an end-of-run audit check is supported", + attachment_kind: "end", + expected_error: None, + }, + AttachmentKindTestCase { + description: "an unknown attachment kind is rejected", + attachment_kind: "warehouse", + expected_error: Some("integration check attachment_kind is unsupported"), + }, + ]; + + for test_case in test_cases { + let normalized = + normalize_integration_record(audit_check_record(test_case.attachment_kind)); + let error: Option = normalized.as_ref().err().map(|error| error.message.clone()); + assert_eq!( + error.as_deref(), + test_case.expected_error, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/integration_results/tests/test_types.rs b/crates/sqlbuild-core/src/integration_results/tests/test_types.rs index bacd65819f..fee2995e50 100644 --- a/crates/sqlbuild-core/src/integration_results/tests/test_types.rs +++ b/crates/sqlbuild-core/src/integration_results/tests/test_types.rs @@ -6,3 +6,9 @@ pub(super) struct NormalizeTestCase { pub(super) expected_error: Option<&'static str>, pub(super) expected_encoded: Option<&'static str>, } + +pub(super) struct AttachmentKindTestCase { + pub(super) description: &'static str, + pub(super) attachment_kind: &'static str, + pub(super) expected_error: Option<&'static str>, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs index 2fb97dc797..3914bb5227 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dagster_invocation/results.rs @@ -27,6 +27,7 @@ const CHECK_ASSET_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_f const MATERIALIZE: &str = "materialize"; const MATERIALIZATION: &str = "materialization"; const CHECK: &str = "check"; +const AUDIT_CHECK_KIND: &str = "audit"; /// DAG nodes upstream first, or in artifact order when edges form a cycle. pub(crate) fn sorted_nodes<'py>(dag: &Bound<'py, PyAny>) -> PyResult>> { @@ -241,6 +242,23 @@ fn dag_check_for<'py>( return Ok(Some(dag_check)); } } + singular_audit_check(dag, check) +} + +/// The DAG check `audit:` of a singular audit whose result names where it ran. +fn singular_audit_check<'py>( + dag: &Bound<'py, PyAny>, + check: &Bound<'py, PyAny>, +) -> PyResult>> { + if text_of(check, "kind")? != AUDIT_CHECK_KIND { + return Ok(None); + } + let wanted: String = format!("{AUDIT_CHECK_KIND}:{}", text_of(check, "name")?); + for dag_check in items(dag, "checks")? { + if text_of(&dag_check, "id")? == wanted && get(&dag_check, "attachment_kind")?.is_none() { + return Ok(Some(dag_check)); + } + } Ok(None) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md index 361ffc5e31..c1503e8ff4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md @@ -102,7 +102,7 @@ precedence; SQLBuild does not silently combine it with Dagster's implicit asset SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster asset checks: - **Unit tests** become checks attached to the models they test -- **Audits** become checks attached to the model or source they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN` +- **Audits** become checks attached to the model, source, or seed they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN`. A singular audit that runs at the end of the build becomes a check on each model it reads - **Scenarios** become checks attached to the models they exercise - **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py b/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py index dbef4d15bb..a9ac76f605 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/_test_types.py @@ -145,3 +145,10 @@ class DagsterEmptyCursorInputsCheckE2ETestCase: model_sql: str asset_key: tuple[str, ...] expected_check_names: tuple[str, ...] + + +@dataclass(frozen=True) +class DagsterAuditAttachmentChecksE2ETestCase: + description: str + command: tuple[str, ...] + expected_checked_assets: tuple[tuple[str, ...], ...] diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index f6eacfa60a..36917cf737 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -5,6 +5,7 @@ import time from collections import defaultdict from collections.abc import Callable +from dataclasses import dataclass from pathlib import Path from threading import Event from types import MappingProxyType @@ -391,3 +392,94 @@ def sqlbuild_assets(context): defs = dg.Definitions(assets=[sqlbuild_assets], jobs=[dg.define_asset_job("stdout_capture_job")]) """ + + +@dataclass(frozen=True) +class AuditCheckOutcome: + """The asset checks one Dagster run declared and evaluated.""" + + success: bool + declared: frozenset[tuple[AssetKey, str]] + declared_assets: frozenset[AssetKey] + evaluated: frozenset[tuple[AssetKey, str]] + all_passed: bool + + +ATTACHMENT_CHECK_PROJECT_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "attachment_checks"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "attachment_checks.duckdb"\n' + ), + "seeds/products.csv": "product_id,product_name\n1,widget\n2,gadget\n", + "seeds/products.yml": ( + "seeds:\n" + " - name: products\n" + " description: Product catalogue.\n" + " columns:\n" + " - name: product_id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - name: product_name\n" + " type: VARCHAR\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n" + " description: Raw orders.\n" + " expression: |\n" + " SELECT 1 AS order_id, 1 AS product_id\n" + " UNION ALL SELECT 2 AS order_id, 2 AS product_id\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + " audits:\n" + " - not_null\n" + " - name: product_id\n" + " type: INTEGER\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.',\n" + " columns (order_id (), product_id ()),\n" + " audits [not_null (column order_id)],\n" + ");\n\n" + 'SELECT order_id, product_id FROM __source("raw_orders")\n' + ), + "models/customers.sql": ( + "MODEL (description 'Customers.');\n\nSELECT 1 AS customer_id, 1 AS order_id\n" + ), + "audits/singular/orders_have_customers.sql": ( + 'AUDIT (name "orders_have_customers", severity error);\n\n' + "SELECT o.order_id\n" + 'FROM __ref("orders") o\n' + 'LEFT JOIN __ref("customers") c ON o.order_id = c.order_id\n' + "WHERE c.order_id IS NULL AND o.order_id = 1\n" + ), + "audits/singular/orders_match_raw_orders.sql": ( + 'AUDIT (name "orders_match_raw_orders", severity error);\n\n' + "SELECT o.order_id\n" + 'FROM __ref("orders") o\n' + 'LEFT JOIN __source("raw_orders") r ON o.order_id = r.order_id\n' + "WHERE r.order_id IS NULL\n" + ), +} + + +def audit_check_outcome(*, result: ExecuteInProcessResult, assets: Any) -> AuditCheckOutcome: + """The asset checks a Dagster run of `assets` declared and evaluated.""" + + declared: frozenset[tuple[AssetKey, str]] = frozenset( + (spec.asset_key, spec.name) for spec in assets.check_specs + ) + evaluations: list[Any] = list(result.get_asset_check_evaluations()) + return AuditCheckOutcome( + success=result.success, + declared=declared, + declared_assets=frozenset(asset_key for asset_key, _ in declared), + evaluated=frozenset( + (evaluation.asset_key, evaluation.check_name) for evaluation in evaluations + ), + all_passed=all(evaluation.passed for evaluation in evaluations), + ) diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py new file mode 100644 index 0000000000..4b3cc97628 --- /dev/null +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_audit_attachment_checks.py @@ -0,0 +1,139 @@ +"""E2E coverage of Dagster asset checks for every audit attachment kind. + +The project has seed, source and model audits, a singular audit inferred to attach to a model, +and a singular audit that runs at the end of the build. +""" + +import subprocess +from collections.abc import Iterator +from pathlib import Path + +import pytest +from dagster import AssetExecutionContext, AssetKey, ExecuteInProcessResult, materialize + +from sqlbuild.integrations.dagster import SqlBuildCliResource, SqlBuildProject, sqlbuild_assets +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + REPO_ROOT, + prepare_inline_project, + run_sqb, +) +from tests.e2e.src.sqlbuild.integrations.dagster._test_types import ( + DagsterAuditAttachmentChecksE2ETestCase, +) +from tests.e2e.src.sqlbuild.integrations.dagster.helpers import ( + ATTACHMENT_CHECK_PROJECT_FILES, + AuditCheckOutcome, + audit_check_outcome, +) + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterAuditAttachmentChecksE2ETestCase( + description="build reports every audit attachment kind as checks", + command=("build",), + expected_checked_assets=( + ("main", "products"), + ("raw_orders",), + ("main", "orders"), + ("main", "customers"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_every_audit_attachment_kind_when_building_through_dagster_then_checks_are_reported( + test_case: DagsterAuditAttachmentChecksE2ETestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="attachment_checks", + repo_files=ATTACHMENT_CHECK_PROJECT_FILES, + ) + + sqlbuild_project: SqlBuildProject = SqlBuildProject( + project_dir=project_dir, sqb_command=(str(REPO_ROOT / ".venv" / "bin" / "sqb"),) + ) + sqlbuild_project.prepare() + + @sqlbuild_assets(project=sqlbuild_project, required_resource_keys={"sqb"}) + def sqlbuild_attachment_checks(context: AssetExecutionContext) -> Iterator[object]: + yield from context.resources.sqb.cli(list(test_case.command), context=context).stream() + + result: ExecuteInProcessResult = materialize( + [sqlbuild_attachment_checks], + resources={"sqb": SqlBuildCliResource(project_dir=sqlbuild_project)}, + ) + + checks: AuditCheckOutcome = audit_check_outcome( + result=result, assets=sqlbuild_attachment_checks + ) + + assert checks.success is True + assert { + AssetKey(list(path)) for path in test_case.expected_checked_assets + } <= checks.declared_assets + assert checks.declared <= checks.evaluated + assert checks.all_passed is True + assert "integration output degraded" not in capsys.readouterr().err + + +@pytest.mark.parametrize( + "test_case", + [ + DagsterAuditAttachmentChecksE2ETestCase( + description="audit reports every audit attachment kind as checks", + command=("audit",), + expected_checked_assets=( + ("main", "products"), + ("raw_orders",), + ("main", "orders"), + ("main", "customers"), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_every_audit_attachment_kind_when_auditing_through_dagster_then_checks_are_reported( + test_case: DagsterAuditAttachmentChecksE2ETestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="attachment_checks", + repo_files=ATTACHMENT_CHECK_PROJECT_FILES, + ) + prebuild: subprocess.CompletedProcess[str] = run_sqb( + command=("build",), project_dir=project_dir + ) + + sqlbuild_project: SqlBuildProject = SqlBuildProject( + project_dir=project_dir, sqb_command=(str(REPO_ROOT / ".venv" / "bin" / "sqb"),) + ) + sqlbuild_project.prepare() + + @sqlbuild_assets(project=sqlbuild_project, required_resource_keys={"sqb"}) + def sqlbuild_attachment_checks(context: AssetExecutionContext) -> Iterator[object]: + yield from context.resources.sqb.cli(list(test_case.command), context=context).stream() + + result: ExecuteInProcessResult = materialize( + [sqlbuild_attachment_checks], + resources={"sqb": SqlBuildCliResource(project_dir=sqlbuild_project)}, + ) + + checks: AuditCheckOutcome = audit_check_outcome( + result=result, assets=sqlbuild_attachment_checks + ) + + assert prebuild.returncode == 0, prebuild.stdout + prebuild.stderr + assert checks.success is True + assert { + AssetKey(list(path)) for path in test_case.expected_checked_assets + } <= checks.declared_assets + assert checks.declared <= checks.evaluated + assert checks.all_passed is True + assert "integration output degraded" not in capsys.readouterr().err diff --git a/tests/integration/src/sqlbuild/cli/output/_test_types.py b/tests/integration/src/sqlbuild/cli/output/_test_types.py index 5d3f28690c..806697685e 100644 --- a/tests/integration/src/sqlbuild/cli/output/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/output/_test_types.py @@ -2,7 +2,8 @@ from dataclasses import dataclass -from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.compiler.auditing.types import AuditAttachmentKind, AuditOutcome +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind @dataclass(frozen=True) @@ -11,3 +12,15 @@ class MeasurementEnvelopeTestCase: outcome: AuditOutcome expected_status: str expected_passed: bool + + +@dataclass(frozen=True) +class AttachmentKindEnvelopeTestCase: + description: str + attachment_kind: AuditAttachmentKind + attached_target_kind: AttachedAuditTargetKind | None + attached_target_name: str | None + expected_attachment_kind: str + expected_asset_name: str | None + expected_degraded: bool = False + expected_passed: bool = True diff --git a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py index aeb5db14ee..37a802f2e5 100644 --- a/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py +++ b/tests/integration/src/sqlbuild/cli/output/test_measurement_integration_envelope.py @@ -1,4 +1,4 @@ -"""Measurement audit integration-envelope round-trip coverage.""" +"""Audit integration-envelope round-trip coverage.""" from __future__ import annotations @@ -19,10 +19,17 @@ AuditSeverity, ThresholdOperator, ) +from sqlbuild.compiler.compile.types import AttachedAuditTargetKind from sqlbuild.executor.auditing.models import AuditExecutionResult from sqlbuild.runtime.observability.models import LifecycleEvent -from tests.integration.src.sqlbuild.cli.output._test_types import MeasurementEnvelopeTestCase -from tests.unit.src.sqlbuild.cli.output._helpers.helpers import written_integration_record +from tests.integration.src.sqlbuild.cli.output._test_types import ( + AttachmentKindEnvelopeTestCase, + MeasurementEnvelopeTestCase, +) +from tests.unit.src.sqlbuild.cli.output._helpers.helpers import ( + audit_terminal, + written_integration_record, +) from tests.unit.src.sqlbuild.runtime.observability.helpers import lifecycle_event @@ -86,3 +93,70 @@ def test_given_measurement_execution_result_when_building_envelope_then_decode_r assert check.thresholds == {"warn": {"operator": "below", "limit": 100.0}} assert check.evidence_count == 0 assert check.evidence_truncated is False + + +@pytest.mark.parametrize( + "test_case", + ( + AttachmentKindEnvelopeTestCase( + "model audit", + AuditAttachmentKind.MODEL, + AttachedAuditTargetKind.MODEL, + "orders", + expected_attachment_kind="model", + expected_asset_name="orders", + ), + AttachmentKindEnvelopeTestCase( + "source audit", + AuditAttachmentKind.SOURCE, + AttachedAuditTargetKind.SOURCE, + "raw_orders", + expected_attachment_kind="source", + expected_asset_name="raw_orders", + ), + AttachmentKindEnvelopeTestCase( + "seed audit", + AuditAttachmentKind.SEED, + AttachedAuditTargetKind.SEED, + "products", + expected_attachment_kind="seed", + expected_asset_name="products", + ), + AttachmentKindEnvelopeTestCase( + "end-of-run audit", + AuditAttachmentKind.END, + None, + None, + expected_attachment_kind="end", + expected_asset_name=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_every_attachment_kind_when_building_envelope_then_check_is_not_degraded( + test_case: AttachmentKindEnvelopeTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + result: AuditExecutionResult = AuditExecutionResult( + audit_name="not_null", + audit_definition_name="not_null", + attachment_kind=test_case.attachment_kind, + attached_target_kind=test_case.attached_target_kind, + attached_target_name=test_case.attached_target_name, + severity=AuditSeverity.ERROR, + outcome=AuditOutcome.PASS, + row_count=0, + executed_sql="SELECT 1 WHERE FALSE", + ) + + envelope: IntegrationResultEnvelope = written_integration_record( + result=result, terminal=audit_terminal(result), directory=tmp_path, command="audit" + ) + + check: IntegrationCheckResult = envelope.checks[0] + assert envelope.projection_degraded is test_case.expected_degraded + assert check.attachment_kind == test_case.expected_attachment_kind + assert check.asset_name == test_case.expected_asset_name + assert check.passed is test_case.expected_passed + assert "integration output degraded" not in capsys.readouterr().err diff --git a/website/src/content/docs/docs/integrations/dagster.mdx b/website/src/content/docs/docs/integrations/dagster.mdx index c8b0939558..34d319eced 100644 --- a/website/src/content/docs/docs/integrations/dagster.mdx +++ b/website/src/content/docs/docs/integrations/dagster.mdx @@ -86,7 +86,7 @@ precedence; SQLBuild does not silently combine it with Dagster's implicit asset SQLBuild tests, audits, scenarios, and Python checks are registered as Dagster asset checks: - **Unit tests** become checks attached to the models they test -- **Audits** become checks attached to the model or source they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN` +- **Audits** become checks attached to the model, source, or seed they audit, with severity mapped to `AssetCheckSeverity.ERROR` or `AssetCheckSeverity.WARN`. A singular audit that runs at the end of the build becomes a check on each model it reads - **Scenarios** become checks attached to the models they exercise - **Python checks** (`@check`) become checks attached to the tasks, assets, or loaders they validate From 600d40b2bb6c0ad3b10dcb320214421f632b4faf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:20:53 +0100 Subject: [PATCH 21/39] feat: allow concurrency per target A target can set concurrency in [targets.]. Commands use --concurrency, then the selected target's value, then the local and project [settings] value. Invalid values fail with D001, and sqb build --verbose shows the effective limit and where it came from. --- Makefile | 1 + .../src/effective/constants.rs | 7 + .../effective/main/effective_concurrency.rs | 28 ++++ .../sqlbuild-config/src/effective/main/mod.rs | 1 + .../sqlbuild-config/src/effective/models.rs | 40 +++++ .../src/effective/tests/test_effective.rs | 64 +++++++- .../src/effective/tests/test_types.rs | 10 ++ .../src/project/_helpers/targets.rs | 21 ++- .../sqlbuild-config/src/project/constants.rs | 3 +- .../build_command/main/render_run_context.rs | 10 +- .../sqlbuild-core/src/build_command/models.rs | 2 + .../build_command/tests/test_build_command.rs | 3 +- .../_helpers/audit_command/command.rs | 16 +- .../_helpers/build_command/execution.rs | 12 +- .../_helpers/build_command/invocation.rs | 14 +- .../_helpers/command_runtime/concurrency.rs | 45 ++++++ .../bindings/_helpers/command_runtime/mod.rs | 1 + .../_helpers/contract_records/constants.rs | 2 + .../bindings/_helpers/load/load_command.rs | 18 +-- .../_helpers/scenario_command/test_command.rs | 18 +-- .../bindings/_helpers/seed_command/command.rs | 17 +-- .../_helpers/sql_test_command/test_command.rs | 17 +-- fensu.toml | 4 +- .../docs/concepts/project-configuration.md | 30 +++- src/sqlbuild/_native.pyi | 4 + .../cli/commands/main/build/_test_types.py | 9 ++ .../cli/commands/main/build/helpers.py | 36 +++++ .../main/build/test_target_concurrency.py | 141 ++++++++++++++++++ .../discovery/_helpers/_test_types.py | 9 ++ .../_helpers/test_target_concurrency.py | 78 ++++++++++ .../docs/concepts/project-configuration.mdx | 30 +++- 31 files changed, 619 insertions(+), 72 deletions(-) create mode 100644 crates/sqlbuild-config/src/effective/main/effective_concurrency.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py create mode 100644 tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py diff --git a/Makefile b/Makefile index efba5d41b3..7baf460205 100644 --- a/Makefile +++ b/Makefile @@ -106,6 +106,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_scoped_declarations.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_selector_surface.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_table_function_dependency.py \ + tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py \ tests/e2e/src/sqlbuild/cli/commands/main/build/test_validation_failures.py diff --git a/crates/sqlbuild-config/src/effective/constants.rs b/crates/sqlbuild-config/src/effective/constants.rs index 840e99a048..b7a0f2b172 100644 --- a/crates/sqlbuild-config/src/effective/constants.rs +++ b/crates/sqlbuild-config/src/effective/constants.rs @@ -210,4 +210,11 @@ pub(crate) const TARGET_FIELDS: &[TargetField] = &[ decided_by: &["query"], }, }, + TargetField { + group: None, + name: "concurrency", + rule: MergeRule::Overlay { + decided_by: &["concurrency"], + }, + }, ]; diff --git a/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs new file mode 100644 index 0000000000..16a6aa60eb --- /dev/null +++ b/crates/sqlbuild-config/src/effective/main/effective_concurrency.rs @@ -0,0 +1,28 @@ +//! `effective_concurrency`: the worker limit of one command and the layer it came from. + +use crate::effective::models::{ConcurrencyLayers, ConcurrencySource, EffectiveConcurrency}; + +/// `--concurrency`, else the target's `concurrency`, else the local-then-project setting. +#[must_use] +pub fn effective_concurrency(layers: &ConcurrencyLayers<'_>) -> EffectiveConcurrency { + if let Some(limit) = layers.requested { + return EffectiveConcurrency { + limit, + source: ConcurrencySource::CommandLine, + }; + } + if let Some((target, limit)) = layers.target { + return EffectiveConcurrency { + limit, + source: ConcurrencySource::Target(target.to_owned()), + }; + } + EffectiveConcurrency { + limit: layers.settings, + source: if layers.local_override { + ConcurrencySource::LocalSettings + } else { + ConcurrencySource::ProjectSettings + }, + } +} diff --git a/crates/sqlbuild-config/src/effective/main/mod.rs b/crates/sqlbuild-config/src/effective/main/mod.rs index 80f728929d..1e419479b5 100644 --- a/crates/sqlbuild-config/src/effective/main/mod.rs +++ b/crates/sqlbuild-config/src/effective/main/mod.rs @@ -1,4 +1,5 @@ pub mod default_connection_name; +pub mod effective_concurrency; pub mod effective_connection; pub mod effective_settings; pub mod effective_target_name; diff --git a/crates/sqlbuild-config/src/effective/models.rs b/crates/sqlbuild-config/src/effective/models.rs index 34cb57057d..d26481562b 100644 --- a/crates/sqlbuild-config/src/effective/models.rs +++ b/crates/sqlbuild-config/src/effective/models.rs @@ -80,3 +80,43 @@ pub struct TargetChoice { pub default: Option, pub defined: Vec, } + +/// The concurrency each configuration layer sets, highest precedence first. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ConcurrencyLayers<'a> { + /// `--concurrency` on the command line. + pub requested: Option, + /// The selected target's name and its `concurrency`, when it sets one. + pub target: Option<(&'a str, i64)>, + /// `[settings] concurrency` after any `sqlbuild_local.toml` override. + pub settings: i64, + /// Whether `sqlbuild_local.toml` `[settings]` overrides `concurrency`. + pub local_override: bool, +} + +/// Where the effective concurrency came from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum ConcurrencySource { + CommandLine, + Target(String), + LocalSettings, + ProjectSettings, +} + +impl std::fmt::Display for ConcurrencySource { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::CommandLine => formatter.write_str("--concurrency"), + Self::Target(name) => write!(formatter, "targets.{name}.concurrency"), + Self::LocalSettings => formatter.write_str("sqlbuild_local.toml settings.concurrency"), + Self::ProjectSettings => formatter.write_str("settings.concurrency"), + } + } +} + +/// One command's worker limit and the layer that set it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EffectiveConcurrency { + pub limit: i64, + pub source: ConcurrencySource, +} diff --git a/crates/sqlbuild-config/src/effective/tests/test_effective.rs b/crates/sqlbuild-config/src/effective/tests/test_effective.rs index e4c38903f3..dd2ffc3b5f 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_effective.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_effective.rs @@ -1,17 +1,18 @@ use crate::effective::errors::EffectiveConfigError; use crate::effective::main::default_connection_name::default_connection_name; +use crate::effective::main::effective_concurrency::effective_concurrency; use crate::effective::main::effective_connection::effective_connection; use crate::effective::main::effective_settings::effective_settings; use crate::effective::main::effective_target_name::effective_target_name; use crate::effective::main::effective_vars::effective_vars; use crate::effective::main::merged_target::merged_target; use crate::effective::models::{ - ConnectionChoice, ConnectionLayers, FieldValue, TargetChoice, TargetLayer, + ConcurrencyLayers, ConnectionChoice, ConnectionLayers, FieldValue, TargetChoice, TargetLayer, }; use crate::effective::tests::helpers::{entries, field_value, local_layer, project_layer}; use crate::effective::tests::test_types::{ - ConnectionTestCase, DefaultConnectionTestCase, MergeTestCase, MessageTestCase, Pairs, - TargetNameTestCase, UnionTestCase, + ConcurrencyTestCase, ConnectionTestCase, DefaultConnectionTestCase, MergeTestCase, + MessageTestCase, Pairs, TargetNameTestCase, UnionTestCase, }; use crate::effective::types::Entries; @@ -295,3 +296,60 @@ fn given_ambiguous_connection_when_rendering_then_names_join_into_the_python_mes ); } } + +#[test] +fn given_concurrency_layers_when_resolving_then_cli_beats_target_beats_settings() { + let test_cases = [ + ConcurrencyTestCase { + description: "the command line beats the target", + requested: Some(2), + target: Some(("prod", 8)), + settings: 16, + local_override: true, + expected_limit: 2, + expected_source: "--concurrency", + }, + ConcurrencyTestCase { + description: "the target beats a local settings override", + requested: None, + target: Some(("prod", 8)), + settings: 4, + local_override: true, + expected_limit: 8, + expected_source: "targets.prod.concurrency", + }, + ConcurrencyTestCase { + description: "a local settings override applies without a target value", + requested: None, + target: None, + settings: 4, + local_override: true, + expected_limit: 4, + expected_source: "sqlbuild_local.toml settings.concurrency", + }, + ConcurrencyTestCase { + description: "the project setting is the default", + requested: None, + target: None, + settings: 16, + local_override: false, + expected_limit: 16, + expected_source: "settings.concurrency", + }, + ]; + + for test_case in test_cases { + let resolved = effective_concurrency(&ConcurrencyLayers { + requested: test_case.requested, + target: test_case.target, + settings: test_case.settings, + local_override: test_case.local_override, + }); + assert_eq!( + (resolved.limit, resolved.source.to_string().as_str()), + (test_case.expected_limit, test_case.expected_source), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-config/src/effective/tests/test_types.rs b/crates/sqlbuild-config/src/effective/tests/test_types.rs index 0199cdea3a..6653868d84 100644 --- a/crates/sqlbuild-config/src/effective/tests/test_types.rs +++ b/crates/sqlbuild-config/src/effective/tests/test_types.rs @@ -48,3 +48,13 @@ pub(super) struct MessageTestCase { pub(super) names: &'static [&'static str], pub(super) expected_message: &'static str, } + +pub(super) struct ConcurrencyTestCase { + pub(super) description: &'static str, + pub(super) requested: Option, + pub(super) target: Option<(&'static str, i64)>, + pub(super) settings: i64, + pub(super) local_override: bool, + pub(super) expected_limit: i64, + pub(super) expected_source: &'static str, +} diff --git a/crates/sqlbuild-config/src/project/_helpers/targets.rs b/crates/sqlbuild-config/src/project/_helpers/targets.rs index 2ffa133ab5..0ba2fcebd1 100644 --- a/crates/sqlbuild-config/src/project/_helpers/targets.rs +++ b/crates/sqlbuild-config/src/project/_helpers/targets.rs @@ -15,7 +15,7 @@ use crate::project::_helpers::keys::{ use crate::project::_helpers::sections::{fixed_duration_seconds, item_timeout}; use crate::project::_helpers::values::{int_below, is_int}; use crate::project::constants::{ - CLONE_KEYS, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, MAX_FULL_ROWS_KEY, + CLONE_KEYS, CONCURRENCY_KEY, CONFIRMATION_POLICIES, EXECUTION_LIMIT_KEYS, MAX_FULL_ROWS_KEY, RETENTION_MATERIALIZATIONS, TARGET_KEYS, UNLIMITED_DIFF_FULL_ROWS, WAREHOUSE_GROUPS, }; use crate::project::errors::ConfigFailure; @@ -127,6 +127,24 @@ pub(crate) fn seconds_value(seconds: &str) -> ConfigValue { ) } +/// A target's worker limit, which overrides `[settings] concurrency` for that target. +fn target_concurrency(mapping: &Entries, target: TargetPlace<'_>) -> Loaded { + let invalid = || { + ConfigFailure::at( + target.file, + format!( + "targets.{}.concurrency must be an integer >= 1", + target.name + ), + ) + }; + let value = optional_strict_int(mapping, CONCURRENCY_KEY).map_err(|_| invalid())?; + if value.as_ref().is_some_and(|number| int_below(number, 1)) { + return Err(invalid()); + } + Ok(value_or_none(value)) +} + fn target_connection(mapping: &Entries, target: TargetPlace<'_>) -> Loaded<(Built, Built)> { match get(mapping, "connection") { None => Ok((Built::Value(ConfigValue::Map(Vec::new())), none())), @@ -396,6 +414,7 @@ fn local_target(payload: &ConfigValue, target: TargetPlace<'_>) -> Loaded>()?; fields.push(("clone", object("LocalClonePolicy", clone_fields))); + fields.push(("concurrency", target_concurrency(mapping, target)?)); Ok(fields) } diff --git a/crates/sqlbuild-config/src/project/constants.rs b/crates/sqlbuild-config/src/project/constants.rs index faa5439e1b..4c745d97c2 100644 --- a/crates/sqlbuild-config/src/project/constants.rs +++ b/crates/sqlbuild-config/src/project/constants.rs @@ -171,7 +171,7 @@ pub(crate) const MODEL_HEADER_KEYS: [&str; 73] = [ "valid_from_column", "valid_to_column", ]; -pub(crate) const TARGET_KEYS: [&str; 19] = [ +pub(crate) const TARGET_KEYS: [&str; 20] = [ "connection", "vars", "database", @@ -191,6 +191,7 @@ pub(crate) const TARGET_KEYS: [&str; 19] = [ "diff", "warehouses", "recall", + "concurrency", ]; pub(crate) const CLONE_KEYS: [&str; 2] = ["allow_as_clone_origin", "allow_as_clone_destination"]; pub(crate) const EXECUTION_LIMIT_KEYS: [&str; 5] = [ diff --git a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs index 6c902db1f2..b84acd88e7 100644 --- a/crates/sqlbuild-core/src/build_command/main/render_run_context.rs +++ b/crates/sqlbuild-core/src/build_command/main/render_run_context.rs @@ -30,7 +30,15 @@ pub fn render_run_context(facts: &RunContextFacts, style: CliStyle) -> String { ("warehouse", warehouse), ( "concurrency", - format!("{} configured limit", facts.concurrency), + if facts.concurrency_source.is_empty() { + format!("{} configured limit", facts.concurrency) + } else { + format!( + "{} configured limit ({})", + facts.concurrency, + sanitize(&facts.concurrency_source) + ) + }, ), ("full_refresh", facts.full_refresh.to_string()), ( diff --git a/crates/sqlbuild-core/src/build_command/models.rs b/crates/sqlbuild-core/src/build_command/models.rs index 952e90d832..bf8b0d29e7 100644 --- a/crates/sqlbuild-core/src/build_command/models.rs +++ b/crates/sqlbuild-core/src/build_command/models.rs @@ -179,6 +179,8 @@ pub struct RunContextFacts { /// The warehouse and where it came from. pub warehouse: Option<(String, String)>, pub concurrency: i64, + /// The configuration layer the concurrency came from. + pub concurrency_source: String, pub full_refresh: bool, pub selected_count: usize, pub total_count: usize, diff --git a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs index a2611194dd..f9bb0b89ad 100644 --- a/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs +++ b/crates/sqlbuild-core/src/build_command/tests/test_build_command.rs @@ -121,12 +121,13 @@ fn given_run_context_when_rendering_then_sanitizes_and_groups_counts() { command: "sqb build".to_owned(), run_id: Some("run\u{1b}1".to_owned()), concurrency: 4, + concurrency_source: "targets.prod.concurrency".to_owned(), selected_count: 1200, total_count: 1_234_567, date_vars: (Some("2026-01-01".to_owned()), Some("2026-01-31".to_owned())), ..RunContextFacts::default() }, - expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", + expected_text: "Execution\n command sqb build\n run_id run\\x1b1\n target not set\n database not set\n schema not set\n warehouse not set\n concurrency 4 configured limit (targets.prod.concurrency)\n full_refresh false\n selected 1,200 of 1,234,567 build resources\n date vars 2026-01-01 to 2026-01-31\n\n", }, ContextTestCase { description: "a warehouse, one date and selector files", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index c6047e2bd5..6555702ee3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -15,6 +15,7 @@ use crate::bindings::_helpers::build_progress::audit_progress::AuditProgressRepo use crate::bindings::_helpers::cli::progress::current_native_progress_projector; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_text::write_execution_header; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, keywords, within}; @@ -184,15 +185,12 @@ fn prepare_run<'py>( let py: Python<'py> = request.py(); let entries: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?.getattr("audit_entries")?; - let requested: Bound<'py, PyAny> = native_field(request, "concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field(pipeline, "project")? - .getattr("settings")? - .getattr("concurrency")? - .extract::()? - } else { - requested.extract::()? - }; + let concurrency: i64 = resolve_concurrency( + &native_field(request, "concurrency")?, + &native_field(invocation, "discovered_inputs")?, + &native_field(request, "selected_target")?, + )? + .limit; let audit_count: usize = entries.len()?; let worker_count: i64 = concurrency.min(i64::try_from(audit_count).unwrap_or(i64::MAX)); let stream: Bound<'py, PyAny> = native_field(invocation, "progress_stream")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index ec7ba2f74d..d20adc24a2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -1,8 +1,10 @@ //! Prepare build execution, run the build pipeline and the post-build Python checks. +use pyo3::IntoPyObject; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_core::build_command::main::render_run_context::render_run_context; use sqlbuild_core::build_command::models::{RunContextFacts, SelectorFileFact}; use sqlbuild_core::cli_text::models::CliStyle; @@ -81,7 +83,7 @@ fn validated_date(py: Python<'_>, value: Option>) -> PyResult, pipeline: &Bound<'_, PyAny>, - concurrency: &Bound<'_, PyAny>, + concurrency: &EffectiveConcurrency, ) -> PyResult { let py: Python<'_> = context.py(); let project: Bound<'_, PyAny> = native_field(pipeline, "project")?; @@ -142,7 +144,8 @@ fn run_context_facts( database: display_text(&native_field(&project, "effective_target_database")?)?, schema: display_text(&native_field(&project, "effective_target_schema")?)?, warehouse, - concurrency: concurrency.extract()?, + concurrency: concurrency.limit, + concurrency_source: concurrency.source.to_string(), full_refresh: context.request_flag("full_refresh")?, selected_count, total_count, @@ -159,7 +162,7 @@ pub(crate) fn write_run_context( context: &BuildContext<'_>, pipeline: &Bound<'_, PyAny>, ) -> PyResult<()> { - let concurrency: Bound<'_, PyAny> = context.concurrency(&native_field(pipeline, "project")?)?; + let concurrency: EffectiveConcurrency = context.concurrency()?; let facts: RunContextFacts = run_context_facts(context, pipeline, &concurrency)?; emit( &context.stream, @@ -220,8 +223,7 @@ pub(crate) fn prepare_build_execution<'py>( ) .and_then(|value| pyo3::Bound::new(py, value)) .map(pyo3::Bound::into_any)?; - let concurrency: Bound<'py, PyAny> = - context.concurrency(&native_field(pipeline, "project")?)?; + let concurrency: Bound<'py, PyAny> = context.concurrency()?.limit.into_pyobject(py)?.into_any(); if context.verbose { write_run_context(context, pipeline)?; } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs index 1742421c42..c2fca8e628 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs @@ -3,6 +3,7 @@ use std::time::Instant; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_config::effective::models::EffectiveConcurrency; use sqlbuild_core::build_command::errors::BuildCommandError; use sqlbuild_core::build_command::main::duration_limit_error::duration_limit_error; use sqlbuild_core::build_command::models::DurationLimit; @@ -12,6 +13,7 @@ use crate::bindings::_helpers::build_command::support::{boolean, or_none, text}; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_reporters::PlanningProgressReporter; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -51,11 +53,13 @@ impl<'py> BuildContext<'py> { native_field(&self.invocation, name) } - /// The requested concurrency, or the project setting. - pub(crate) fn concurrency(&self, project: &Bound<'py, PyAny>) -> PyResult> { - or_none(self.request.getattr("concurrency")?, || { - native_field(project, "settings")?.getattr("concurrency") - }) + /// `--concurrency`, else the selected target's `concurrency`, else `[settings]`. + pub(crate) fn concurrency(&self) -> PyResult { + resolve_concurrency( + &self.request.getattr("concurrency")?, + &self.get("discovered_inputs")?, + &native_field(&self.request, "selected_target")?, + ) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs new file mode 100644 index 0000000000..a9fab0784a --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/concurrency.rs @@ -0,0 +1,45 @@ +//! The worker limit of one command: `--concurrency`, the target, then `[settings]`. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_config::effective::main::effective_concurrency::effective_concurrency; +use sqlbuild_config::effective::models::{ConcurrencyLayers, EffectiveConcurrency}; + +use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; +use crate::bindings::_helpers::contracts::spec_resolution::{ + resolve_target_config, resolve_target_name, +}; +use crate::bindings::_helpers::records::fields::field as native_field; + +const CONCURRENCY: &str = "concurrency"; + +/// The effective concurrency of a command on `selected_target`, and the layer that set it. +pub(crate) fn resolve_concurrency( + requested: &Bound<'_, PyAny>, + discovered: &Bound<'_, PyAny>, + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + let py = discovered.py(); + let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; + let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; + let target: Option<(String, i64)> = + match resolve_target_name(py, &project_config, &local_config, selected_target)? { + Some(name) => { + let config: Bound<'_, PyAny> = + resolve_target_config(py, &project_config, &local_config, name.as_any())?; + let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; + limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) + } + None => None, + } + .transpose()?; + let settings: i64 = + native_field(&build_effective_settings_config(discovered)?, CONCURRENCY)?.extract()?; + let local_override: bool = + native_field(&local_config, "setting_overrides")?.contains(CONCURRENCY)?; + Ok(effective_concurrency(&ConcurrencyLayers { + requested: requested.extract()?, + target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), + settings, + local_override, + })) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs index 2d06334d13..fa6ea5ff50 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod adapters; +pub(crate) mod concurrency; pub(crate) mod connection; pub(crate) mod invocation; pub(crate) mod preview_target; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs index 8df9e95178..925d7e16ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contract_records/constants.rs @@ -146,6 +146,7 @@ const R_TARGET_CONFIG: RecordSpec = RecordSpec { "warehouses", FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), + FieldSpec::new("concurrency", FieldDefault::None), ], }; const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { @@ -184,6 +185,7 @@ const R_LOCAL_TARGET_CONFIG: RecordSpec = RecordSpec { "warehouses", FieldDefault::NativeCall(RecordId::TargetWarehousesConfig), ), + FieldSpec::new("concurrency", FieldDefault::None), ], }; const R_SETTINGS_CONFIG: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs index e7b86460f5..8007be6dac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs @@ -19,9 +19,9 @@ use sqlbuild_core::load_command::models::{LoadCounts, LoadResultText}; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_text::write_execution_header; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_runtime_config; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_target_config; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; use crate::bindings::_helpers::compile_driver::python_calls::call_with; @@ -300,16 +300,12 @@ fn prepare<'py>( ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? .extract()?; - let requested: Bound<'py, PyAny> = native_field(request, "concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &build_effective_settings_config(&discovered.clone())?, - "concurrency", - )? - .extract()? - } else { - requested.extract()? - }; + let concurrency: i64 = resolve_concurrency( + &native_field(request, "concurrency")?, + discovered, + &native_field(request, "selected_target")?, + )? + .limit; let typed: Bound<'py, PyAny> = type_cursor_overrides(&native_field(request, "cursor_overrides")?, &[])?; let cursor_overrides: Bound<'py, PyAny> = if typed.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 14018b8ddf..147ac28380 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -13,7 +13,7 @@ use sqlbuild_core::scenario_command::models::{ ScenarioNamespace, ScenarioTestFlags, SqlAnalysisCommand, }; -use crate::bindings::_helpers::compile_attachment::effective_config::build_effective_settings_config; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::compile_driver::python_calls::call_with; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::native_host::executor::ExecutorHost; @@ -98,16 +98,12 @@ fn prepare<'py>(py: Python<'py>, request: &Bound<'py, PyAny>) -> PyResult = request.getattr("concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &build_effective_settings_config(&project.discovered.clone())?, - "concurrency", - )? - .extract()? - } else { - requested.extract()? - }; + let concurrency: i64 = resolve_concurrency( + &request.getattr("concurrency")?, + &project.discovered, + &py.None().into_bound(py), + )? + .limit; let sys: Bound<'py, PyModule> = py.import("sys")?; let stream: Bound<'py, PyAny> = sys.getattr(if flag(request, "json_output")? { "stderr" diff --git a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs index f43a61d2c9..80fc6e43fe 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs @@ -17,6 +17,7 @@ use sqlbuild_core::seed_command::models::SeedRow; use crate::bindings::_helpers::cli::progress::current_native_progress_projector; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; use crate::bindings::_helpers::cli::progress_text::write_execution_header; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::plan_result::connection_with_hooks; @@ -437,16 +438,12 @@ pub(crate) fn run_seed_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py let invocation: Bound<'_, PyAny> = resolve_command_invocation(request, &record_class(py, RecordId::SeedInvocation)?)?; let pipeline: Bound<'_, PyAny> = compile_seeds(request, &invocation)?; - let requested: Bound<'_, PyAny> = native_field(request, "concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &native_field(&native_field(&pipeline, "project")?, "settings")?, - "concurrency", - )? - .extract::()? - } else { - requested.extract::()? - } + let concurrency: i64 = resolve_concurrency( + &native_field(request, "concurrency")?, + &native_field(&invocation, "discovered_inputs")?, + &native_field(request, "selected_target")?, + )? + .limit .max(1); write_header(&invocation, &pipeline, concurrency)?; let started: Instant = Instant::now(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs index 67b23cf31d..f0d0586175 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_command/test_command.rs @@ -30,6 +30,7 @@ use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReport use crate::bindings::_helpers::cli::progress_text::{ format_expectation_detail, format_expectation_name, write_execution_header, }; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::command_runtime::invocation::resolve_reported_command_invocation; use crate::bindings::_helpers::compile_driver::pipeline_entries::build_static_pipeline_result; use crate::bindings::_helpers::compile_driver::python_calls::keywords; @@ -674,16 +675,12 @@ pub(crate) fn run_test_command(py: Python<'_>, request: &Bound<'_, PyAny>) -> Py if inspect_only { return inspect(&stream, &pipeline.getattr("plan_output")?); } - let requested: Bound<'_, PyAny> = request.getattr("concurrency")?; - let concurrency: i64 = if requested.is_none() { - native_field( - &native_field(&native_field(&pipeline, "project")?, "settings")?, - "concurrency", - )? - .extract()? - } else { - requested.extract()? - }; + let concurrency: i64 = resolve_concurrency( + &request.getattr("concurrency")?, + &native_field(&invocation, "discovered_inputs")?, + &request.getattr("selected_target")?, + )? + .limit; stream.call_method1("write", ("\n",))?; let header: Bound<'_, PyDict> = PyDict::new(py); header.set_item("stream", &stream)?; diff --git a/fensu.toml b/fensu.toml index 154893a523..9f8cd60850 100644 --- a/fensu.toml +++ b/fensu.toml @@ -524,7 +524,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=25", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -536,7 +536,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/build_command/completion.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/build_command/defer_clone.rs=20", "crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/build_command/invocation.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs=22", "crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/build_command/policies.rs=5", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b4a40fe677..f7195f1341 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -186,6 +186,7 @@ A target is a named build context - the database and schema you build into, plus | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | | `warehouses` | Default Snowflake warehouse per command group, `build` and `query` (see below) | +| `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | ```toml [connections.warehouse] @@ -253,6 +254,33 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Target concurrency + +Targets often run on very different warehouses. A target can set its own worker limit, so local +and CI runs use many workers while production stays within what its warehouse runs without +queuing: + +```toml +[settings] +concurrency = 16 + +[targets.prod] +concurrency = 8 +``` + +`sqb build`, `sqb test`, `sqb audit`, `sqb seed`, `sqb load`, and `sqb scenario test` use the first +value set in this order: + +1. `--concurrency` on the command line, for that invocation only +2. The selected target's `concurrency`, from `[targets.]` in either file +3. `[settings] concurrency` in `sqlbuild_local.toml` +4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) + +The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build +--verbose` shows the effective limit and the setting it came from, for example +`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and +uses the active target's value. + ### Command-group warehouses Builds and ad-hoc inspection usually need different compute. A Snowflake target can choose a @@ -661,7 +689,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](models/configuration.md#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](models/contracts.md). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; `--concurrency` overrides it | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](python-nodes/loaders.md). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index cd77ef32eb..c94a73c0be 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -13699,6 +13699,7 @@ class TargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig diff: TargetDiffConfig warehouses: TargetWarehousesConfig + concurrency: int | None def __init__( self, connection: dict[str, object] = ..., @@ -13721,6 +13722,7 @@ class TargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig = ..., diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., + concurrency: int | None = ..., ) -> None: ... class LocalTargetConfig(NativeRecord): @@ -13746,6 +13748,7 @@ class LocalTargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig diff: TargetDiffConfig warehouses: TargetWarehousesConfig + concurrency: int | None def __init__( self, connection: dict[str, object] = ..., @@ -13769,6 +13772,7 @@ class LocalTargetConfig(NativeRecord): execution_limits: ExecutionLimitsConfig = ..., diff: TargetDiffConfig = ..., warehouses: TargetWarehousesConfig = ..., + concurrency: int | None = ..., ) -> None: ... class SettingsConfig(NativeRecord): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 4067389ef9..ecb5927eca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1372,3 +1372,12 @@ class MaterializationTypeChangeTestCase: description: str materializations: tuple[str, ...] expected_table_types: tuple[str, ...] + + +@dataclass(frozen=True) +class TargetConcurrencyE2ETestCase: + description: str + target_concurrency: str + local_settings: str + command: tuple[str, ...] + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 6a99667565..24006988ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1151,3 +1151,39 @@ def order_totals_model_sql(*, materialized: str) -> str: f"MODEL (description 'Order totals.', materialized {materialized});\n\n" "SELECT 1 AS order_id, 250 AS total_cents\n" ) + + +def target_concurrency_project_files( + *, target_concurrency: str, local_settings: str +) -> dict[str, str]: + """A DuckDB project with `[settings] concurrency = 2` and a `ci` target with extra lines.""" + + return { + "sqlbuild_project.toml": ( + 'name = "target_concurrency"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[settings]\n" + "concurrency = 2\n\n" + "[targets.dev]\n" + 'schema = "main"\n\n' + "[targets.ci]\n" + 'schema = "main"\n' + f"{target_concurrency}" + ), + "sqlbuild_local.toml": local_settings, + "seeds/products.csv": "product_id\n1\n2\n", + "seeds/products.yml": ( + "seeds:\n" + " - name: products\n" + " description: Products.\n" + " columns:\n" + " - name: product_id\n" + " type: INTEGER\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', materialized table);\n\nSELECT 1 AS order_id\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py new file mode 100644 index 0000000000..602e8abdaa --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py @@ -0,0 +1,141 @@ +"""E2E coverage for per-target concurrency and its precedence.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + TargetConcurrencyE2ETestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + target_concurrency_project_files, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + run_sqb, + table_exists, +) + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyE2ETestCase( + description="the selected target's concurrency replaces the project setting", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="3 configured limit (targets.ci.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="--concurrency overrides the target", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "build", "--verbose", "--target", "ci", "--concurrency", "1"), + expected_fragment="1 configured limit (--concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="the target overrides a local settings override", + target_concurrency="concurrency = 3\n", + local_settings="[settings]\nconcurrency = 5\n", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="3 configured limit (targets.ci.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="a target without concurrency keeps the local settings override", + target_concurrency="", + local_settings="[settings]\nconcurrency = 5\n", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="5 configured limit (sqlbuild_local.toml settings.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="a local target concurrency overrides the project target", + target_concurrency="concurrency = 3\n", + local_settings="[targets.ci]\nconcurrency = 4\n", + command=("--no-color", "build", "--verbose", "--target", "ci"), + expected_fragment="4 configured limit (targets.ci.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="other targets keep the project setting", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "build", "--verbose"), + expected_fragment="2 configured limit (settings.concurrency)", + ), + TargetConcurrencyE2ETestCase( + description="seed uses the target's concurrency", + target_concurrency="concurrency = 3\n", + local_settings="", + command=("--no-color", "seed", "--target", "ci"), + expected_fragment="concurrency: 3", + ), + ], + ids=lambda case: case.description, +) +def test_given_target_concurrency_when_running_then_documented_precedence_applies( + tmp_path: Path, + test_case: TargetConcurrencyE2ETestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="target_concurrency", + repo_files=target_concurrency_project_files( + target_concurrency=test_case.target_concurrency, + local_settings=test_case.local_settings, + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert test_case.expected_fragment in result.stdout + result.stderr + assert table_exists(db_path=project_dir / "warehouse.duckdb", table_name="products") + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyE2ETestCase( + description="zero is rejected with a coded configuration error", + target_concurrency="concurrency = 0\n", + local_settings="", + command=("--no-color", "build", "--target", "ci"), + expected_fragment="targets.ci.concurrency must be an integer >= 1", + ), + TargetConcurrencyE2ETestCase( + description="a non-integer is rejected with a coded configuration error", + target_concurrency='concurrency = "8"\n', + local_settings="", + command=("--no-color", "build", "--target", "ci"), + expected_fragment="targets.ci.concurrency must be an integer >= 1", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_target_concurrency_when_running_then_config_error_is_coded( + tmp_path: Path, + test_case: TargetConcurrencyE2ETestCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="target_concurrency", + repo_files=target_concurrency_project_files( + target_concurrency=test_case.target_concurrency, + local_settings=test_case.local_settings, + ), + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + output: str = result.stdout + result.stderr + + assert result.returncode != 0 + assert "D001" in output + assert test_case.expected_fragment in output + assert not (project_dir / "warehouse.duckdb").exists() diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 4f4499d076..5a48424c59 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -1039,3 +1039,12 @@ class OversizedDurationTestCase: description: str section: str expected_error_fragment: str + + +@dataclass(frozen=True) +class TargetConcurrencyConfigTestCase: + description: str + target_lines: str + expected_concurrency: int | None = None + expected_error_fragment: str = "" + expected_error_code: str = "D001" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py new file mode 100644 index 0000000000..4a1e1523ad --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_target_concurrency.py @@ -0,0 +1,78 @@ +"""Loading and validation of per-target concurrency.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import ProjectConfig, ProjectConfigError +from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( + TargetConcurrencyConfigTestCase, +) +from tests.unit.src.sqlbuild.compiler.discovery._helpers.helpers import load_project_config + +_PROJECT: str = 'name = "shop"\nadapter = "duckdb"\n\n[targets.prod]\nschema = "main"\n' + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyConfigTestCase( + description="a positive integer is loaded", + target_lines="concurrency = 8\n", + expected_concurrency=8, + ), + TargetConcurrencyConfigTestCase( + description="an absent value leaves the target unset", + target_lines="", + expected_concurrency=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_valid_target_concurrency_when_loading_then_it_is_kept( + test_case: TargetConcurrencyConfigTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + _PROJECT + test_case.target_lines, encoding="utf-8" + ) + + config: ProjectConfig = load_project_config(project_dir=tmp_path) + + assert config.targets["prod"].concurrency == test_case.expected_concurrency + + +@pytest.mark.parametrize( + "test_case", + [ + TargetConcurrencyConfigTestCase( + description="zero is rejected", + target_lines="concurrency = 0\n", + expected_error_fragment="targets.prod.concurrency must be an integer >= 1", + ), + TargetConcurrencyConfigTestCase( + description="a boolean is rejected", + target_lines="concurrency = true\n", + expected_error_fragment="targets.prod.concurrency must be an integer >= 1", + ), + TargetConcurrencyConfigTestCase( + description="a string is rejected", + target_lines='concurrency = "8"\n', + expected_error_fragment="targets.prod.concurrency must be an integer >= 1", + ), + ], + ids=lambda case: case.description, +) +def test_given_invalid_target_concurrency_when_loading_then_coded_error_is_raised( + test_case: TargetConcurrencyConfigTestCase, tmp_path: Path +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + _PROJECT + test_case.target_lines, encoding="utf-8" + ) + + with pytest.raises(ProjectConfigError) as error: + load_project_config(project_dir=tmp_path) + + assert error.value.code == test_case.expected_error_code + assert test_case.expected_error_fragment in str(error.value) diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index d2badeca2e..0e4822f823 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -163,6 +163,7 @@ A target is a named build context - the database and schema you build into, plus | `clone` | Clone policy (see below) | | `missing_migration_origin` | What a build does when a declared `migrate_from` origin does not exist in this target: `allow` (default), `require_confirmation`, or `deny` (see below) | | `warehouses` | Default Snowflake warehouse per command group, `build` and `query` (see below) | +| `concurrency` | Worker limit for commands run against this target, replacing `[settings] concurrency` (see below) | ```toml [connections.warehouse] @@ -230,6 +231,33 @@ password = "${ENV:SNOWFLAKE_PASSWORD}" Commands then use `dev` automatically. An explicit command such as `sqb build --target prod` still takes precedence for that invocation. +### Target concurrency + +Targets often run on very different warehouses. A target can set its own worker limit, so local +and CI runs use many workers while production stays within what its warehouse runs without +queuing: + +```toml +[settings] +concurrency = 16 + +[targets.prod] +concurrency = 8 +``` + +`sqb build`, `sqb test`, `sqb audit`, `sqb seed`, `sqb load`, and `sqb scenario test` use the first +value set in this order: + +1. `--concurrency` on the command line, for that invocation only +2. The selected target's `concurrency`, from `[targets.]` in either file +3. `[settings] concurrency` in `sqlbuild_local.toml` +4. `[settings] concurrency` in `sqlbuild_project.toml` (default `1`) + +The value must be an integer of at least `1`; anything else fails with `D001`. `sqb build +--verbose` shows the effective limit and the setting it came from, for example +`8 configured limit (targets.prod.concurrency)`. `sqb scenario test` has no `--target` option and +uses the active target's value. + ### Command-group warehouses Builds and ad-hoc inspection usually need different compute. A Snowflake target can choose a @@ -638,7 +666,7 @@ delta_compile = true | `require_sql_analysis` | `false` | Allow `sql_analysis false` (in a MODEL header or a `[path_defaults]` entry) only on models whose SQL the parser cannot read, and reject `[settings] sql_analysis = false` in `sqlbuild_local.toml`. Project file only. See [SQL analysis](/docs/concepts/models/configuration#sql-analysis). | | `query_change_tracking` | `true` | Track query fingerprints for change detection. A fingerprint covers the SQL tokens of the compiled query and ignores whitespace, comments, and the letter case of reserved keywords and built-in function names, so formatting-only edits do not rebuild. Every other word, including non-reserved keywords, identifiers, aliases, JSON path keys and user-defined function names, counts exactly, as do string literals, quoted identifiers, numbers, and optimizer hints. | | `column_contract_mode` | `implicit` | Controls whether column declarations on models without a `contract` declaration activate static shape/nullability validation. `implicit` preserves that validation; `explicit` treats columns as metadata and audit attachment unless the model declares `contract enforced`. Model-level `contract enforced` and `contract none` override this setting. Explicit type enforcement remains independent. See [Contracts](/docs/concepts/models/contracts). | -| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; `--concurrency` overrides it | +| `concurrency` | `1` | Default number of worker connections for commands that run resources in parallel, such as `sqb build`, `sqb test`, `sqb audit`, and `sqb seed`; a target's `concurrency` and `--concurrency` override it (see [Target concurrency](#target-concurrency)) | | `microbatch_concurrency` | `false` | Explicitly permit models with `batch_concurrency > 1`; concurrent batches use immutable coordination facts | | `auto_load_sources` | `true` | Automatically run source loaders before building dependent models during `sqb build`. See [Loaders](/docs/concepts/python-nodes/loaders). | | `table_promotion_mode` | adapter default | `staged` (CTAS to staging, audit, then promote) or `immediate` (CTAS directly to target, then audit) | From 6bfaff3cb22b8d966230c57c3236f6626def05f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:45:12 +0100 Subject: [PATCH 22/39] fix: use the effective concurrency for the microbatch warning and dbt interop The planner's concurrent-microbatch warning and the SQLBuild work in sqb dbt read the project setting directly, ignoring a target's concurrency. Resolve both through the same precedence as execution: --concurrency, the target, then the local and project settings. --- .../_helpers/build_command/planning.rs | 1 + .../_helpers/command_runtime/concurrency.rs | 35 ++++++++--- .../_helpers/compile_driver/plan_result.rs | 1 + .../_helpers/compiler_records/constants.rs | 1 + .../_helpers/dbt_interop/pipeline_phases.rs | 14 ++++- .../_helpers/dbt_interop/sqlbuild_work.rs | 2 +- .../_helpers/planner_entries/model_entries.rs | 7 ++- .../_helpers/planner_entries/models.rs | 2 + .../planner_phases/execution_changes.rs | 30 +++++++++ .../_helpers/planner_records/constants.rs | 1 + .../bindings/_helpers/records/constants.rs | 1 + fensu.toml | 2 +- src/sqlbuild/_native.pyi | 6 ++ .../cli/commands/main/build/_test_types.py | 7 +++ .../cli/commands/main/build/helpers.py | 46 ++++++++++++++ .../main/build/test_target_concurrency.py | 61 +++++++++++++++++++ .../cli/commands/main/dbt/_test_types.py | 8 +++ .../cli/commands/main/dbt/test_execute.py | 30 +++++++++ 18 files changed, 241 insertions(+), 14 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs index 0e9d5993c4..dc05338074 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/planning.rs @@ -70,6 +70,7 @@ fn request_options<'py>(context: &BuildContext<'py>) -> PyResult, selected_target: &Bound<'_, PyAny>, ) -> PyResult { - let py = discovered.py(); - let project_config: Bound<'_, PyAny> = native_field(discovered, "project_config")?; - let local_config: Bound<'_, PyAny> = native_field(discovered, "local_config")?; + resolve_configured_concurrency( + requested, + ( + &native_field(discovered, "project_config")?, + &native_field(discovered, "local_config")?, + ), + selected_target, + ) +} + +/// `resolve_concurrency` from the project and local configs themselves. +pub(crate) fn resolve_configured_concurrency( + requested: &Bound<'_, PyAny>, + (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + selected_target: &Bound<'_, PyAny>, +) -> PyResult { + let py = project_config.py(); let target: Option<(String, i64)> = - match resolve_target_name(py, &project_config, &local_config, selected_target)? { + match resolve_target_name(py, project_config, local_config, selected_target)? { Some(name) => { let config: Bound<'_, PyAny> = - resolve_target_config(py, &project_config, &local_config, name.as_any())?; + resolve_target_config(py, project_config, local_config, name.as_any())?; let limit: Option = native_field(&config, CONCURRENCY)?.extract()?; limit.map(|limit| Ok::<_, pyo3::PyErr>((name.extract::()?, limit))) } None => None, } .transpose()?; - let settings: i64 = - native_field(&build_effective_settings_config(discovered)?, CONCURRENCY)?.extract()?; + let settings: i64 = native_field( + &build_effective_settings(project_config, local_config)?, + CONCURRENCY, + )? + .extract()?; let local_override: bool = - native_field(&local_config, "setting_overrides")?.contains(CONCURRENCY)?; + native_field(local_config, "setting_overrides")?.contains(CONCURRENCY)?; Ok(effective_concurrency(&ConcurrencyLayers { requested: requested.extract()?, target: target.as_ref().map(|(name, limit)| (name.as_str(), *limit)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs index 407cf70237..33d0a026eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/plan_result.rs @@ -99,6 +99,7 @@ pub(crate) fn pipeline_result<'py>( "full_refresh", "reload_sources", "max_microbatches", + "concurrency", ], )?, )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 6cef6bd314..2ab3931847 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -395,6 +395,7 @@ const R_COMPILE_PIPELINE_OPTIONS: RecordSpec = RecordSpec { FieldSpec::new("resolve_python_run_selectors", FieldDefault::False), FieldSpec::new("no_cache", FieldDefault::False), FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("concurrency", FieldDefault::None), FieldSpec::new("selection_diagnostics", FieldDefault::False), FieldSpec::new("plan_sql_tests", FieldDefault::True), FieldSpec::new("record_migration_fingerprints", FieldDefault::True), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs index 4450fc5cda..9a3746dd1c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline_phases.rs @@ -2,10 +2,11 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::PyDict; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::build_command::support::{boolean, emit, none, text, tuple}; use crate::bindings::_helpers::cli::progress_reporters::ConnectionProgressReporter; +use crate::bindings::_helpers::command_runtime::concurrency::resolve_concurrency; use crate::bindings::_helpers::compile_driver::python_calls::{KeywordArguments, keywords}; use crate::bindings::_helpers::contracts::spec_resolution::resolve_target_config; use crate::bindings::_helpers::dbt_interop::execution::render_dbt_execution_summary_footer; @@ -302,6 +303,17 @@ pub(crate) fn run_dbt_sqlbuild_work<'py>( native_field(&project, "effective_target_name")?, )?, ), + ( + "concurrency", + resolve_concurrency( + &py.None().into_bound(py), + &discovered, + &native_field(&project, "effective_target_name")?, + )? + .limit + .into_pyobject(py)? + .into_any(), + ), ], )?; execute_dbt_sqlbuild_work( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs index 9c6b9873ad..27ce82fdec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs @@ -138,7 +138,7 @@ pub(crate) fn execute_dbt_sqlbuild_work<'py>( native_field(&command, "value")?.extract::()? ) }; - let concurrency = native_field(&project, "settings")?.getattr("concurrency")?; + let concurrency = native_field(&context, "concurrency")?; let header = pyo3::IntoPyObjectExt::into_bound_py_any( format_execution_header( &text(py, &display).extract::()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs index 9d67a16ef4..bf5a563cdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/model_entries.rs @@ -304,6 +304,7 @@ fn check_microbatch_concurrency<'py>( project: &Bound<'py, PyAny>, adapter: &Bound<'py, PyAny>, entries: &[Bound<'py, PyAny>], + concurrency: i64, ) -> PyResult>> { let py = project.py(); let skip = planner_type(py, RecordId::PlanAction, "SKIP")?; @@ -353,7 +354,7 @@ fn check_microbatch_concurrency<'py>( concurrent.push(entry); } } - if concurrent.is_empty() || !native_field(&settings, "concurrency")?.gt(1)? { + if concurrent.is_empty() || concurrency <= 1 { return Ok(None); } let count = concurrent.len(); @@ -404,6 +405,7 @@ pub(crate) fn plan_entries<'py>( cursor_overrides, full_refresh, build_inputs: inputs, + concurrency, } = request; let py = project.py(); let inputs = if inputs.is_none() { @@ -532,7 +534,8 @@ pub(crate) fn plan_entries<'py>( warnings.push(warning?); } } - if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries)? { + if let Some(warning) = check_microbatch_concurrency(&project, &adapter, &entries, concurrency)? + { warnings.push(warning); } Ok(ModelEntryResults { entries, warnings }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs index 6692769335..52e10ea560 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/models.rs @@ -29,6 +29,8 @@ pub(crate) struct PlanEntriesRequest<'py> { pub(crate) cursor_overrides: Bound<'py, PyAny>, pub(crate) full_refresh: bool, pub(crate) build_inputs: Bound<'py, PyAny>, + /// The effective worker limit execution will use. + pub(crate) concurrency: i64, } /// What assembling the plan output's fields reads. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs index 8adedd54f4..dae677b9a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/execution_changes.rs @@ -1,5 +1,6 @@ //! Write-hash reconciliation and plan entry construction. +use crate::bindings::_helpers::command_runtime::concurrency::resolve_configured_concurrency; use crate::bindings::_helpers::compile_driver::python_calls::keywords; use crate::bindings::_helpers::planner_entries::contexts::default_microbatch_limit_action; use crate::bindings::_helpers::planner_entries::model_entries::plan_entries; @@ -65,6 +66,26 @@ fn config_or<'py>( Ok(value) } +/// The concurrency execution will use; without configs, the request or the compiled setting. +fn planned_concurrency( + requested: &Bound<'_, PyAny>, + (project_config, local_config): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + project: &Bound<'_, PyAny>, +) -> PyResult { + if project_config.is_none() || local_config.is_none() { + if !requested.is_none() { + return requested.extract(); + } + return native_field(&native_field(project, "settings")?, "concurrency")?.extract(); + } + Ok(resolve_configured_concurrency( + requested, + (project_config, local_config), + &native_field(project, "effective_target_name")?, + )? + .limit) +} + /// Model plan entries for the pruned execution scope, with the freshness and override inputs they need. pub(crate) fn build_planner_entry_results<'py>( (runtime, warehouse): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -133,6 +154,14 @@ pub(crate) fn build_planner_entry_results<'py>( }, )? .into_any(); + let concurrency: i64 = planned_concurrency( + &native_field(overrides, "concurrency")?, + ( + &config, + &runtime_field(runtime, |runtime| &runtime.local_config)?, + ), + &project, + )?; plan_entries(PlanEntriesRequest { project, adapter: runtime_field(runtime, |runtime| &runtime.adapter)?, @@ -143,5 +172,6 @@ pub(crate) fn build_planner_entry_results<'py>( cursor_overrides: native_field(overrides, "cursor_overrides")?, full_refresh: native_field(overrides, "full_refresh")?.is_truthy()?, build_inputs, + concurrency, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs index 523d1e4eb2..d201e7b24d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_records/constants.rs @@ -86,6 +86,7 @@ const R_PLANNER_OVERRIDES: RecordSpec = RecordSpec { FieldSpec::new("forced_stale_model_names", FieldDefault::EmptyTuple), FieldSpec::new("external_blocked_model_names", FieldDefault::EmptyTuple), FieldSpec::new("max_microbatches", FieldDefault::None), + FieldSpec::new("concurrency", FieldDefault::None), ], }; const R_DEFERRAL_INPUTS: RecordSpec = RecordSpec { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs index 39f8e07f68..da4f43df58 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/records/constants.rs @@ -352,6 +352,7 @@ const DBT_SQLBUILD_WORK_CONTEXT: RecordSpec = RecordSpec { FieldSpec::required("use_color"), FieldSpec::required("snapshots_config"), FieldSpec::required("execution_limits"), + FieldSpec::required("concurrency"), ], }; const PLAN_SAFETY_GATES: RecordSpec = RecordSpec { diff --git a/fensu.toml b/fensu.toml index 9f8cd60850..cbccaf8b0f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -719,7 +719,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_config.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_execution.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runtime_results.rs=8", - "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/sqlbuild_work.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=26", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/console_records.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=8", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index c94a73c0be..d065ba38ad 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7876,6 +7876,7 @@ class DbtSqlbuildWorkContext(NativeRecord): use_color: bool snapshots_config: SnapshotsConfig execution_limits: ExecutionLimitsConfig + concurrency: int def __init__( self, plan_output: PlanOutput, @@ -7886,6 +7887,7 @@ class DbtSqlbuildWorkContext(NativeRecord): use_color: bool, snapshots_config: SnapshotsConfig, execution_limits: ExecutionLimitsConfig, + concurrency: int, ) -> None: ... class PlanSafetyGates(NativeRecord): @@ -11900,6 +11902,7 @@ class CompilePipelineOptions(NativeRecord): resolve_python_run_selectors: bool no_cache: bool max_microbatches: int | None + concurrency: int | None selection_diagnostics: bool plan_sql_tests: bool record_migration_fingerprints: bool @@ -11924,6 +11927,7 @@ class CompilePipelineOptions(NativeRecord): resolve_python_run_selectors: bool = ..., no_cache: bool = ..., max_microbatches: int | None = ..., + concurrency: int | None = ..., selection_diagnostics: bool = ..., plan_sql_tests: bool = ..., record_migration_fingerprints: bool = ..., @@ -12189,6 +12193,7 @@ class PlannerOverrides(NativeRecord): forced_stale_model_names: tuple[str, ...] external_blocked_model_names: tuple[str, ...] max_microbatches: int | None + concurrency: int | None def __init__( self, cursor_overrides: CursorOverrides | None = ..., @@ -12197,6 +12202,7 @@ class PlannerOverrides(NativeRecord): forced_stale_model_names: tuple[str, ...] = ..., external_blocked_model_names: tuple[str, ...] = ..., max_microbatches: int | None = ..., + concurrency: int | None = ..., ) -> None: ... class DeferralInputs(NativeRecord): diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index ecb5927eca..19826f9db5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1381,3 +1381,10 @@ class TargetConcurrencyE2ETestCase: local_settings: str command: tuple[str, ...] expected_fragment: str + + +@dataclass(frozen=True) +class TargetMicrobatchWarningE2ETestCase: + description: str + command: tuple[str, ...] + expected_warning_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 24006988ff..9e6c1db9e3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1187,3 +1187,49 @@ def target_concurrency_project_files( "MODEL (description 'Orders.', materialized table);\n\nSELECT 1 AS order_id\n" ), } + + +def target_concurrency_microbatch_files() -> dict[str, str]: + """A project whose `ci` target raises concurrency above `[settings]` for a microbatch model.""" + + return { + "sqlbuild_project.toml": ( + 'name = "target_microbatch"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connection]\n" + 'database = "warehouse.duckdb"\n\n' + "[settings]\n" + "concurrency = 1\n" + "microbatch_concurrency = true\n\n" + "[targets.dev]\n" + 'schema = "main"\n\n' + "[targets.ci]\n" + 'schema = "main"\n' + "concurrency = 3\n" + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_events\n" + " description: Raw events.\n" + " schema: main\n" + " table: raw_events\n" + ), + "models/orders.sql": ( + 'MODEL (description "Orders.",\n' + " materialized incremental,\n" + " incremental_strategy delete_insert,\n" + " incremental_mode microbatch,\n" + " microbatch_strategy watermark,\n" + " cursor_watermark_mode all,\n" + " cursor event_time,\n" + " cursor_type timestamp,\n" + " cursor_grain hour,\n" + " cursor_inputs (raw_events (column event_time, roles [filter, watermark])),\n" + " batch_size 1h,\n" + " batch_concurrency 3,\n" + ");\n\n" + 'SELECT id, event_time FROM __source("raw_events")\n' + "WHERE event_time >= __cursor_start() AND event_time < __cursor_end()\n" + ), + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py index 602e8abdaa..7c09f6274b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py @@ -5,12 +5,15 @@ import subprocess from pathlib import Path +import duckdb import pytest from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( TargetConcurrencyE2ETestCase, + TargetMicrobatchWarningE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + target_concurrency_microbatch_files, target_concurrency_project_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -139,3 +142,61 @@ def test_given_invalid_target_concurrency_when_running_then_config_error_is_code assert "D001" in output assert test_case.expected_fragment in output assert not (project_dir / "warehouse.duckdb").exists() + + +@pytest.mark.parametrize( + "test_case", + [ + TargetMicrobatchWarningE2ETestCase( + description="the target's concurrency enables the concurrent microbatch warning", + command=("--no-color", "build", "--target", "ci"), + expected_warning_count=1, + ), + TargetMicrobatchWarningE2ETestCase( + description="--concurrency 1 overrides the target and suppresses the warning", + command=("--no-color", "build", "--target", "ci", "--concurrency", "1"), + expected_warning_count=0, + ), + TargetMicrobatchWarningE2ETestCase( + description="a target without concurrency keeps the serial project setting", + command=("--no-color", "build", "--target", "dev"), + expected_warning_count=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_target_concurrency_when_planning_microbatches_then_warning_uses_it( + test_case: TargetMicrobatchWarningE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="target_microbatch", + repo_files=target_concurrency_microbatch_files(), + ) + connection: duckdb.DuckDBPyConnection = duckdb.connect(str(project_dir / "warehouse.duckdb")) + connection.execute("CREATE TABLE raw_events (id INTEGER, event_time TIMESTAMP)") + connection.execute( + "INSERT INTO raw_events VALUES (1, '2026-01-01 00:30:00'), (2, '2026-01-01 01:30:00')" + ) + connection.close() + initial: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + connection = duckdb.connect(str(project_dir / "warehouse.duckdb")) + connection.execute( + "INSERT INTO raw_events VALUES (3, '2026-01-01 02:30:00'), (4, '2026-01-01 03:30:00')" + ) + connection.close() + + incremental: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = incremental.stdout + incremental.stderr + assert initial.returncode == 0, initial.stdout + initial.stderr + assert incremental.returncode == 0, output + assert ( + output.count("Concurrent microbatch execution is enabled") + == test_case.expected_warning_count + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py index dee4c8baf4..2ff3651c63 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/_test_types.py @@ -122,3 +122,11 @@ class DbtPlanSafetyE2ETestCase: command: tuple[str, ...] expected_fragments: tuple[str, ...] unexpected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class DbtTargetConcurrencyE2ETestCase: + description: str + target_settings: str + command: tuple[str, ...] + expected_fragment: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py index c364a9b16d..ed4273cb4f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/dbt/test_execute.py @@ -12,10 +12,12 @@ DbtExecutionFailureCliTestCase, DbtExecutionQueryAssertion, DbtMissingOriginE2ETestCase, + DbtTargetConcurrencyE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.dbt.helpers import ( break_dbt_interop_fact_orders_model, declare_missing_origin_requiring_confirmation, + declare_sqlbuild_main_target, load_json_stdout, prepare_dbt_interop_project, skip_unless_dbt_is_runnable, @@ -579,3 +581,31 @@ def test_given_missing_origin_requiring_confirmation_when_running_dbt_then_refus assert test_case.expected_output in result.stdout + result.stderr assert "Pass --allow-missing-migration-origin" not in result.stdout + result.stderr assert not table_exists(db_path=project_dir / "dbt_interop.duckdb", table_name="local_only") + + +@pytest.mark.parametrize( + "test_case", + [ + DbtTargetConcurrencyE2ETestCase( + description="SQLBuild work in dbt interop uses the target's concurrency", + target_settings="concurrency = 3\n", + command=("--no-color", "dbt", "run", "--select", "local_only"), + expected_fragment="concurrency: 3", + ), + ], + ids=lambda case: case.description, +) +def test_given_target_concurrency_when_running_dbt_interop_then_sqlbuild_work_uses_it( + test_case: DbtTargetConcurrencyE2ETestCase, tmp_path: Path +) -> None: + skip_unless_dbt_is_runnable() + project_dir: Path = prepare_dbt_interop_project(tmp_path=tmp_path) + declare_sqlbuild_main_target(project_dir=project_dir, target_settings=test_case.target_settings) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + assert test_case.expected_fragment in output From ea98d4c37d3aabe92f1a92f8520a44ead9b56bd3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:48:10 +0100 Subject: [PATCH 23/39] feat(rules)!: make configuration the only Rule exception mechanism SQBRSQL038 no longer reports a cross join when either side is provably at most one row. The proof is inferred from the query: a seed whose CSV has one data row, an aggregate without GROUP BY, a one-row VALUES, LIMIT 1, or QUALIFY ROW_NUMBER() = 1 without a partition, carried through CTEs and subqueries that only filter or project. Anything else is still reported. Findings now name the joined relation and its CTE. Rule exceptions now live only in sqlbuild_project.toml: - an inline `-- sqb: ignore` directive is a SQBRSQL000 error that suppresses nothing and prints the exact [[rules.rule_exceptions]] entry to add; - `[rules] allow_exceptions` is removed and is a configuration error explaining the removal; - [[rules.rule_exceptions]] accepts `relation` and an optional `cte` for SQBRSQL038, so one exception covers exactly one join; relation-level exceptions are stale-checked like file-wide ones. BREAKING CHANGE: inline `-- sqb: ignore` comments are errors and `[rules] allow_exceptions` is rejected. Move each reviewed deviation into a [[rules.rule_exceptions]] entry and delete the setting. --- .../project_validation/_helpers/csv_header.rs | 56 ++ .../src/project_validation/constants.rs | 4 + .../main/at_most_one_csv_data_row.rs | 30 + .../src/project_validation/main/mod.rs | 1 + .../src/project_validation/tests.rs | 2 + .../tests/test_at_most_one_csv_data_row.rs | 85 +++ .../project_validation/tests/test_types.rs | 7 + .../_helpers/compiler_records/constants.rs | 1 - .../_helpers/discovery/project_facts.rs | 11 +- .../_helpers/format_command/config.rs | 29 +- .../bindings/_helpers/format_command/run.rs | 11 +- .../_helpers/rule_records/constants.rs | 3 +- .../bindings/_helpers/rules/early_sql_lint.rs | 16 +- .../bindings/_helpers/rules/lint_pipeline.rs | 68 ++- .../_helpers/rules/lint_sql_bodies.rs | 14 +- .../src/bindings/_helpers/rules/lint_text.rs | 70 +-- .../bindings/_helpers/rules/relation_keys.rs | 49 +- .../src/bindings/_helpers/rules/rule_cache.rs | 1 - .../bindings/_helpers/rules/rules_config.rs | 25 +- .../src/bindings/_helpers/rules/sql_rules.rs | 5 +- .../src/configuration/_helpers/loading.rs | 56 +- .../src/configuration/_helpers/mod.rs | 1 + .../_helpers/removed_settings.rs | 26 + .../src/configuration/errors.rs | 9 + .../main/forbidden_exceptions.rs | 57 -- .../configuration/main/load_rules_config.rs | 3 +- .../src/configuration/main/mod.rs | 1 - crates/sqlbuild-rules/src/constants.rs | 4 + .../src/engine/_helpers/evaluation.rs | 84 ++- crates/sqlbuild-rules/src/engine/tests.rs | 2 + .../src/engine/tests/helpers.rs | 49 +- .../engine/tests/test_relation_exceptions.rs | 73 +++ .../src/engine/tests/test_types.rs | 12 + .../src/lint_text/_helpers/sql_requests.rs | 18 +- .../src/lint_text/_helpers/suppressions.rs | 206 ++++--- .../sqlbuild-rules/src/lint_text/constants.rs | 27 +- .../src/lint_text/main/apply_suppressions.rs | 40 -- .../sqlbuild-rules/src/lint_text/main/mod.rs | 2 +- .../main/reject_inline_suppressions.rs | 18 + crates/sqlbuild-rules/src/lint_text/models.rs | 30 +- crates/sqlbuild-rules/src/lint_text/tests.rs | 2 + .../src/lint_text/tests/helpers.rs | 25 + .../src/lint_text/tests/test_lint_text.rs | 110 ++-- .../src/lint_text/tests/test_types.rs | 6 +- crates/sqlbuild-rules/src/models.rs | 8 +- .../src/rule_cache/main/lint_key.rs | 3 +- .../sqlbuild-rules/src/rule_cache/models.rs | 1 - .../src/rule_catalogue/constants.rs | 6 +- .../rule_catalogue/main/render_skills_body.rs | 8 +- .../src/sql_lint/_helpers/engine.rs | 28 +- .../src/sql_lint/_helpers/join_predicates.rs | 524 +++++++++++++++++- .../sqlbuild-rules/src/sql_lint/constants.rs | 4 + .../src/sql_lint/main/cross_join_subject.rs | 15 + .../sqlbuild-rules/src/sql_lint/main/mod.rs | 1 + crates/sqlbuild-rules/src/sql_lint/models.rs | 19 +- crates/sqlbuild-rules/src/sql_lint/tests.rs | 2 + .../src/sql_lint/tests/helpers.rs | 37 +- .../src/sql_lint/tests/test_cross_joins.rs | 220 ++++++++ .../src/sql_lint/tests/test_types.rs | 14 + .../src/sql_quality/_helpers/cte_columns.rs | 2 +- .../src/sql_quality/_helpers/json_parses.rs | 2 +- .../src/sql_quality/_helpers/literals.rs | 2 +- .../src/sql_quality/_helpers/ranking.rs | 4 +- .../main/declared_relation_keys.rs | 5 +- .../sqlbuild-rules/src/sql_quality/models.rs | 2 + fensu.toml | 2 +- src/sqlbuild/_native.pyi | 12 +- .../cli/commands/main/rules/_test_types.py | 23 + .../cli/commands/main/rules/helpers.py | 49 ++ .../main/rules/test_cross_join_one_row.py | 191 +++++++ .../main/rules/test_rule_exception_config.py | 140 +++++ .../sqlbuild/cli/commands/main/_test_types.py | 4 +- .../main/test_rule_exception_policy.py | 87 ++- .../sqlbuild/cli/commands/main/test_rules.py | 9 +- .../test_sql_rule_cache_publication.py | 4 +- tests/unit/src/sqlbuild/lint/_test_types.py | 9 + tests/unit/src/sqlbuild/lint/test_run_lint.py | 108 +--- .../_helpers/engine/_test_types.py | 7 +- .../_helpers/engine/test_config.py | 79 +-- 79 files changed, 2291 insertions(+), 689 deletions(-) create mode 100644 crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs create mode 100644 crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs create mode 100644 crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs delete mode 100644 crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs create mode 100644 crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs delete mode 100644 crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs create mode 100644 crates/sqlbuild-rules/src/lint_text/tests/helpers.rs create mode 100644 crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs create mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py diff --git a/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs index 1859d7aa66..c4f5c1211c 100644 --- a/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs +++ b/crates/sqlbuild-discovery/src/project_validation/_helpers/csv_header.rs @@ -5,6 +5,8 @@ use crate::project_validation::errors::HeaderError; use crate::project_validation::models::CsvSettings; use sqlbuild_core::text::main::decode_text::decode_text; +/// The quote character seed loaders use when a seed declares none. +const DEFAULT_QUOTECHAR: char = '"'; /// The size of the chunks `TextIOWrapper` decodes. const DECODE_CHUNK: usize = 8192; const UNQUOTED_NEWLINE: &str = @@ -320,6 +322,60 @@ fn first_record( Ok(None) } +/// Non-blank records in `bytes`, at most `limit`; `None` for a prefix ending before `limit`. +pub(crate) fn csv_record_count( + bytes: &[u8], + settings: &CsvSettings, + complete: bool, + limit: usize, +) -> Result, HeaderError> { + let mut dialect: Dialect = Dialect::new(settings, usize::MAX); + if settings.quotechar.is_none() { + dialect.quotechar = Some(DEFAULT_QUOTECHAR); + } + let text: String = match std::str::from_utf8(bytes) { + Ok(text) => text.to_owned(), + Err(error) if error.error_len().is_none() && !complete => { + String::from_utf8_lossy(&bytes[..error.valid_up_to()]).into_owned() + } + Err(_) => decoded_prefix(bytes, bytes.len())?, + }; + let mut parser: Parser<'_> = Parser { + settings: &dialect, + state: State::StartRecord, + field: String::new(), + field_len: 0, + fields: Vec::new(), + }; + let mut count: usize = 0; + let mut lines = lines(&text).peekable(); + while let Some(line) = lines.next() { + let terminated: bool = line.ends_with(['\n', '\r']); + if lines.peek().is_none() && !complete && !terminated_line(line, terminated, &text) { + return Ok(None); + } + for character in line.chars() { + parser.process(Input::Char(character))?; + } + parser.process(Input::Eol)?; + if parser.state == State::StartRecord { + if !std::mem::take(&mut parser.fields).is_empty() { + count += 1; + } + if count >= limit { + return Ok(Some(count)); + } + } + } + if !complete { + return Ok(None); + } + if parser.field_len != 0 || parser.state == State::InQuotedField || !parser.fields.is_empty() { + count += 1; + } + Ok(Some(count.min(limit))) +} + /// Whether a final line is known to be whole: it ends with `\n`, or `\r` not followed later. fn terminated_line(line: &str, terminated: bool, text: &str) -> bool { terminated && (line.ends_with('\n') || !text.ends_with('\r')) diff --git a/crates/sqlbuild-discovery/src/project_validation/constants.rs b/crates/sqlbuild-discovery/src/project_validation/constants.rs index 6122e20503..e03d600d5a 100644 --- a/crates/sqlbuild-discovery/src/project_validation/constants.rs +++ b/crates/sqlbuild-discovery/src/project_validation/constants.rs @@ -8,3 +8,7 @@ pub(crate) const CURRENT_DIRECTORY: &str = "."; pub(crate) const KNOWN_FOLDER_LIMIT: usize = 5; /// Model names SQLBuild reserves for compiled output structure. pub(crate) const RESERVED_MODEL_NAMES: [&str; 1] = ["_chain_"]; +/// The first seed read, grown until the header and two data rows are seen or the file ends. +pub(crate) const INITIAL_SEED_READ_BYTES: u64 = 64 * 1024; +/// A header and two data rows: enough to know a seed holds more than one data row. +pub(crate) const RECORDS_FOR_MANY_ROWS: usize = 3; diff --git a/crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs b/crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs new file mode 100644 index 0000000000..168e69135f --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/main/at_most_one_csv_data_row.rs @@ -0,0 +1,30 @@ +//! Whether a seed CSV holds a header and at most one data row. + +use std::fs::File; +use std::io::Read; +use std::path::Path; + +use crate::project_validation::_helpers::csv_header::csv_record_count; +use crate::project_validation::constants::{INITIAL_SEED_READ_BYTES, RECORDS_FOR_MANY_ROWS}; +use crate::project_validation::models::CsvSettings; + +/// Whether the CSV at `path` has at most one data row; any read or parse failure answers `false`. +#[must_use] +pub fn at_most_one_csv_data_row(path: &Path, settings: &CsvSettings) -> bool { + let Ok(mut file) = File::open(path) else { + return false; + }; + let mut bytes: Vec = Vec::new(); + let mut want: u64 = INITIAL_SEED_READ_BYTES; + loop { + let Ok(read) = (&mut file).take(want).read_to_end(&mut bytes) else { + return false; + }; + let complete: bool = (read as u64) < want; + match csv_record_count(&bytes, settings, complete, RECORDS_FOR_MANY_ROWS) { + Ok(Some(count)) => return count < RECORDS_FOR_MANY_ROWS, + Ok(None) if !complete => want = want.saturating_mul(2), + Ok(None) | Err(_) => return false, + } + } +} diff --git a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs index 1fc67badd8..2c19ca312a 100644 --- a/crates/sqlbuild-discovery/src/project_validation/main/mod.rs +++ b/crates/sqlbuild-discovery/src/project_validation/main/mod.rs @@ -1 +1,2 @@ +pub mod at_most_one_csv_data_row; pub mod validate_project_inputs; diff --git a/crates/sqlbuild-discovery/src/project_validation/tests.rs b/crates/sqlbuild-discovery/src/project_validation/tests.rs index ff3236ffdb..d7aebec514 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_at_most_one_csv_data_row.rs"] +mod at_most_one_csv_data_row; #[path = "tests/test_csv_header.rs"] mod csv_header; #[path = "tests/helpers.rs"] diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs new file mode 100644 index 0000000000..dc07731a42 --- /dev/null +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_at_most_one_csv_data_row.rs @@ -0,0 +1,85 @@ +use std::path::PathBuf; + +use crate::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; +use crate::project_validation::models::CsvSettings; +use crate::project_validation::tests::test_types::OneDataRowTestCase; + +#[test] +fn given_seed_csv_text_when_counting_data_rows_then_only_one_row_files_qualify() { + let test_cases = [ + OneDataRowTestCase { + description: "header and one row", + text: "version,currency\n1,EUR\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "header only", + text: "version,currency\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "one row without a final newline", + text: "version,currency\n1,EUR".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "blank lines are not rows", + text: "version,currency\n\n1,EUR\n\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "a quoted newline stays inside one row", + text: "version,note\n1,\"first\nsecond\"\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: true, + }, + OneDataRowTestCase { + description: "two rows", + text: "version,currency\n1,EUR\n2,USD\n".to_owned(), + settings: CsvSettings::default(), + expected_one_row: false, + }, + OneDataRowTestCase { + description: "two rows with a custom delimiter", + text: "version;currency\n1;EUR\n2;USD\n".to_owned(), + settings: CsvSettings { + delimiter: Some(";".to_owned()), + ..CsvSettings::default() + }, + expected_one_row: false, + }, + OneDataRowTestCase { + description: "many rows past the first read", + text: format!("id\n{}", "1\n".repeat(70_000)), + settings: CsvSettings::default(), + expected_one_row: false, + }, + OneDataRowTestCase { + description: "one row longer than the first read", + text: format!("id\n\"{}\"\n", "x".repeat(200_000)), + settings: CsvSettings::default(), + expected_one_row: true, + }, + ]; + let directory = tempfile::TempDir::new().unwrap_or_else(|error| panic!("{error}")); + + for test_case in test_cases { + let path: PathBuf = directory.path().join("seed.csv"); + std::fs::write(&path, &test_case.text).unwrap_or_else(|error| panic!("{error}")); + + assert_eq!( + at_most_one_csv_data_row(&path, &test_case.settings), + test_case.expected_one_row, + "{}", + test_case.description + ); + } + assert!(!at_most_one_csv_data_row( + &directory.path().join("missing.csv"), + &CsvSettings::default() + )); +} diff --git a/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs b/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs index 6881934f58..08e6d0affd 100644 --- a/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs +++ b/crates/sqlbuild-discovery/src/project_validation/tests/test_types.rs @@ -42,3 +42,10 @@ pub(super) fn csv_error(message: &str) -> Result, String> { pub(super) fn failure(kind: FailureKind, message: &str) -> Result<(), String> { Err(format!("{kind:?}: {message}")) } + +pub(crate) struct OneDataRowTestCase { + pub(crate) description: &'static str, + pub(crate) text: String, + pub(crate) settings: CsvSettings, + pub(crate) expected_one_row: bool, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs index 6cef6bd314..031bb669e0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiler_records/constants.rs @@ -324,7 +324,6 @@ const R_LINT_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("header_rules_enabled", FieldDefault::True), FieldSpec::new("max_literal_length", FieldDefault::Int(100)), FieldSpec::new("max_ranking_order_by", FieldDefault::Int(5)), - FieldSpec::new("allow_suppressions", FieldDefault::True), FieldSpec { name: "relation_keys", default: FieldDefault::EmptyDict, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs index 4a8806e1bf..639f068c24 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/project_facts.rs @@ -156,16 +156,21 @@ pub(crate) fn column_name(column: &Bound<'_, PyAny>) -> PyResult { native_field(column, "name")?.extract() } -fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult { +/// The CSV reader settings a seed declaration authors, without the dialect check. +pub(crate) fn seed_csv_settings(entry: &Bound<'_, PyAny>) -> PyResult { let settings: Bound<'_, PyAny> = native_field(entry, "csv_settings")?; - let mut csv: CsvSettings = CsvSettings { + Ok(CsvSettings { delimiter: optional_text(&settings, "delimiter")?, quotechar: optional_text(&settings, "quotechar")?, escapechar: optional_text(&settings, "escapechar")?, doublequote: native_field(&settings, "doublequote")?.extract()?, skipinitialspace: native_field(&settings, "skipinitialspace")?.extract()?, dialect_error: None, - }; + }) +} + +fn seed_entry_fact(py: Python<'_>, entry: &Bound<'_, PyAny>) -> PyResult { + let mut csv: CsvSettings = seed_csv_settings(entry)?; csv.dialect_error = csv_dialect_error(py, &csv)?; Ok(SeedEntryFact { name: column_name(entry)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs index 64a98f3881..7b312e7408 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs @@ -13,13 +13,12 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::rules_config::load_rules_config; -/// Format settings and whether lint suppressions are allowed. +/// Format settings resolved for one run. pub(crate) struct FormatConfig { pub(crate) settings: FormatSettings, - pub(crate) allow_suppressions: bool, } -/// Resolve the settings, loading the rules configuration when the project file exists. +/// Resolve the settings, validating the rules configuration when the project file exists. pub(crate) fn format_config( py: Python<'_>, project_dir: &Bound<'_, PyAny>, @@ -27,23 +26,15 @@ pub(crate) fn format_config( let directory: std::path::PathBuf = project_dir.extract()?; let settings: FormatSettings = resolve_format_settings(Path::new(&directory)).map_err(PyValueError::new_err)?; - let allow_suppressions: bool = if settings.has_project_config { - load_rules_config( + if settings.has_project_config { + let _ = load_rules_config( py, project_dir .call_method0("resolve")? .extract::()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))? - .getattr("allow_exceptions")? - .extract()? - } else { - true - }; - Ok(FormatConfig { - settings, - allow_suppressions, - }) + )?; + } + Ok(FormatConfig { settings }) } /// The Python `LintConfig` of `config`, linting as `dialect`. @@ -74,12 +65,6 @@ pub(crate) fn lint_config<'py>( "max_ranking_order_by", settings.max_ranking_order_by.into_pyobject(py)?.into_any(), ), - ( - "allow_suppressions", - pyo3::types::PyBool::new(py, config.allow_suppressions) - .to_owned() - .into_any(), - ), ], ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs index 9826bf58d9..bd12368073 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs @@ -25,8 +25,8 @@ use crate::bindings::_helpers::rules::dsl_headers::safe_format_files; use crate::bindings::_helpers::rules::dsl_headers::{header_violations, rules_config}; use crate::bindings::_helpers::rules::fixture_formatting::format_redundant_fixture_nulls; use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; -use crate::bindings::_helpers::rules::lint_text::apply_lint_suppressions; use crate::bindings::_helpers::rules::lint_text::format_lint_sql_bodies; +use crate::bindings::_helpers::rules::lint_text::reject_lint_suppressions; use sqlbuild_rules::dsl_headers::main::scan_headers::scan_headers; use sqlbuild_rules::dsl_headers::models::{HeaderRulesConfig, HeaderViolation}; @@ -303,9 +303,12 @@ pub(crate) fn run_format<'py>(request: &FormatRequest<'_, 'py>) -> PyResult

> = drop_resolved_description_faults(request.project_dir, violations)?; let final_contents: Bound<'py, PyDict> = overlaid(&files, &updated)?; - let allow_suppressions: bool = native_field(request.config, "allow_suppressions")?.extract()?; - let violations: Vec> = - apply_lint_suppressions(py, violations, final_contents.clone(), allow_suppressions)?; + let violations: Vec> = reject_lint_suppressions( + py, + violations, + final_contents.clone(), + request.project_dir.extract()?, + )?; Ok(FormatOutcome { files_checked, violations, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs index 94767d13ea..212db5027c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rule_records/constants.rs @@ -19,6 +19,8 @@ const R_RULE_EXEMPTION: RecordSpec = RecordSpec { FieldSpec::required("rule"), FieldSpec::required("path"), FieldSpec::required("reason"), + FieldSpec::new("relation", FieldDefault::None), + FieldSpec::new("cte", FieldDefault::None), ], }; const R_GRAPH_EDGE_EXCLUSION: RecordSpec = RecordSpec { @@ -266,7 +268,6 @@ const R_RULES_CONFIG: RecordSpec = RecordSpec { FieldSpec::new("rule_exceptions", FieldDefault::EmptyTuple), FieldSpec::new("graph_edge_exceptions", FieldDefault::EmptyTuple), FieldSpec::new("rule_ignores", FieldDefault::EmptyTuple), - FieldSpec::new("allow_exceptions", FieldDefault::True), FieldSpec::new("select_star_allow", FieldDefault::EmptyTuple), FieldSpec::new("domains", FieldDefault::EmptyTuple), FieldSpec::new("approved_source_tokens", FieldDefault::EmptyTuple), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs index f1bf047ba8..61fad4cdb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/early_sql_lint.rs @@ -34,7 +34,7 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledSource, LintFile, LintOutcome, LintRequest, LintSettings, lint, lint_sources, }; use crate::bindings::_helpers::rules::relation_keys::{ - RelationKeyRow, RelationKeys, relation_keys, source_key_input, + RelationKeyRow, RelationKeys, one_row_seed_names, relation_keys, source_key_input, }; use crate::bindings::_helpers::rules::rule_cache::stored_entries; use crate::bindings::_helpers::rules::rules_config::rules_config_of; @@ -57,7 +57,7 @@ pub(crate) fn sql_lint_settings( dialect: &str, codes: &[String], threshold: impl Fn(&str) -> PyResult>, - (relation_keys, allow_suppressions): (RelationKeys, bool), + relation_keys: RelationKeys, ) -> PyResult { let defaults: LintSettings = LintSettings::default(); Ok(LintSettings { @@ -71,7 +71,6 @@ pub(crate) fn sql_lint_settings( .unwrap_or(DEFAULT_MAX_RANKING_ORDER_BY), ..defaults.sql }, - allow_suppressions, relation_keys, ..defaults }) @@ -116,9 +115,13 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult> = record_field(inputs, "seed_inputs")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&seed_inputs)?; let mut seeds: Vec> = Vec::new(); - for seed in record_field(inputs, "seed_inputs")?.try_iter()? { - seeds.push(record_field(&seed?, "schema_entry")?); + for seed in &seed_inputs { + seeds.push(record_field(seed, "schema_entry")?); } let mut sources: Vec> = Vec::new(); for source in record_field(inputs, "source_inputs")?.try_iter()? { @@ -131,12 +134,13 @@ fn early_lint_plan(inputs: &Bound<'_, PyAny>, dialect: &str) -> PyResult>)>, } @@ -89,7 +88,6 @@ impl Default for LintSettings { max_literal_length: 100, max_ranking_order_by: 5, }, - allow_suppressions: true, relation_keys: Vec::new(), } } @@ -119,7 +117,6 @@ impl LintSettings { max_literal_length: record_field(config, "max_literal_length")?.extract()?, max_ranking_order_by: record_field(config, "max_ranking_order_by")?.extract()?, }, - allow_suppressions: record_field(config, "allow_suppressions")?.is_truthy()?, relation_keys, }) } @@ -645,7 +642,7 @@ pub(crate) fn lint( check_stop(py, request.stop)?; } let violations: Vec = - suppressed(&files, rows, request.settings.allow_suppressions)?; + with_directive_findings(&files, request.project_dir, rows)?; Ok(LintOutcome { files, violations, @@ -1110,10 +1107,34 @@ fn lint_body( allows_dynamic_output_star: traits.dynamic_star, allows_empty_fixture_star: traits.ceremonial, relation_keys: Vec::new(), + relation_names: relation_names(&dependencies.relations), }; Ok((body, dependencies.relations)) } +/// `path` relative to `project_dir` with `/` separators, or as given outside the project. +pub(crate) fn project_relative_path(path: &Path, project_dir: &Path) -> String { + let relative: &Path = path.strip_prefix(project_dir).unwrap_or(path); + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") +} + +/// The relation name each dependency sentinel stands for, without its `kind:` prefix. +pub(crate) fn relation_names(relations: &[(String, String)]) -> Vec<(String, String)> { + relations + .iter() + .map(|(sentinel, identity)| { + let name: &str = identity + .split_once(':') + .map_or(identity.as_str(), |(_, name)| name); + (sentinel.clone(), name.to_owned()) + }) + .collect() +} + /// The declared keys of each dependency relation that declares any, by its sentinel. pub(crate) fn relation_keys( settings: &LintSettings, @@ -1145,33 +1166,26 @@ fn sentinel_spans(sites: &[InterpolationSite]) -> Vec { .collect() } -/// `rows` without suppressed violations, with suppression findings, sorted by path then place. -fn suppressed( +/// `rows` with a finding for every inline suppression directive, sorted by path then place. +fn with_directive_findings( files: &[LintFile], - rows: Vec, - allow_suppressions: bool, + project_dir: &Path, + mut violations: Vec, ) -> PyResult> { - let keys: Vec = rows + let paths: Vec = files .iter() - .map(|row| SuppressibleViolation { - file: row.file, - line: usize::try_from(row.line).unwrap_or(usize::MAX), - code: row.code.clone(), - native: row.engine == NATIVE_ENGINE, - }) + .map(|file| project_relative_path(&file.path, project_dir)) + .collect(); + let sources: Vec<(&str, &str)> = paths + .iter() + .zip(files) + .map(|(path, file)| (path.as_str(), file.contents.as_str())) .collect(); - let texts: Vec<&str> = files.iter().map(|file| file.contents.as_str()).collect(); - let retained: Vec = catch_compiler_panic(|| { - apply_suppressions(&texts, &keys, allow_suppressions).map_err(|error| error.to_string()) + let findings: Vec = catch_compiler_panic(|| { + reject_inline_suppressions(&sources).map_err(|error| error.to_string()) }) .map_err(value_error)?; - let mut violations: Vec = retained - .into_iter() - .map(|item| match item { - RetainedViolation::Existing(index) => rows[index].clone(), - RetainedViolation::Added(finding) => LintViolationRow::suppression(finding), - }) - .collect(); + violations.extend(findings.into_iter().map(LintViolationRow::suppression)); violations.sort_by(|left, right| { ( files[left.file].path.to_string_lossy(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs index 1ee6aae410..599e08ee1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_sql_bodies.rs @@ -11,7 +11,9 @@ use sqlbuild_rules::lint_text::models::{ use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; -use crate::bindings::_helpers::rules::lint_pipeline::{LintViolationRow, ViolationClasses}; +use crate::bindings::_helpers::rules::lint_pipeline::{ + LintViolationRow, ViolationClasses, relation_names as sentinel_relation_names, +}; use crate::bindings::types::CompilerDetach; fn spans(passes: &Bound<'_, PyAny>) -> PyResult>> { @@ -48,6 +50,15 @@ fn relation_keys( Ok(keys) } +/// The relation name each dependency sentinel of one body stands for. +fn relation_names(body: &Bound<'_, PyAny>) -> PyResult> { + let mut relations: Vec<(String, String)> = Vec::new(); + for relation in native_field(body, "dependency_relations")?.try_iter()? { + relations.push(relation?.extract()?); + } + Ok(sentinel_relation_names(&relations)) +} + fn lint_body( body: &Bound<'_, PyAny>, index_by_path: &Bound<'_, PyDict>, @@ -69,6 +80,7 @@ fn lint_body( allows_dynamic_output_star: native_field(body, "allows_dynamic_output_star")?.extract()?, allows_empty_fixture_star: native_field(body, "allows_empty_fixture_star")?.extract()?, relation_keys: relation_keys(body, declared)?, + relation_names: relation_names(body)?, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 7efa8a6682..4ecbbdcbb0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -1,4 +1,4 @@ -//! Interpolation sentinels, suppressions and fixture-null candidates over Python lint models. +//! Interpolation sentinels, inline directive findings and fixture-null candidates over lint models. use std::collections::BTreeSet; use std::path::PathBuf; @@ -7,18 +7,17 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyRes use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_rules::lint_text::errors::LintTextError; -use sqlbuild_rules::lint_text::main::apply_suppressions::apply_suppressions; use sqlbuild_rules::lint_text::main::format_sql_bodies::format_sql_bodies; use sqlbuild_rules::lint_text::main::interpolation_text_at::interpolation_text_at; use sqlbuild_rules::lint_text::main::named_declaration_role_kind::named_declaration_role_kind as role_kind; use sqlbuild_rules::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use sqlbuild_rules::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use sqlbuild_rules::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use sqlbuild_rules::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use sqlbuild_rules::lint_text::main::restore_interpolation::restore_interpolation; use sqlbuild_rules::lint_text::main::scan_fixture_null_candidates::scan_fixture_null_candidates; use sqlbuild_rules::lint_text::models::{ - FormatFile, FormattedBodies, InterpolationSite, Neutralized, RetainedViolation, - SuppressibleViolation, + FormatFile, FormattedBodies, InterpolationSite, Neutralized, SuppressionFinding, }; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; @@ -28,7 +27,7 @@ use crate::bindings::_helpers::records::construct::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::lint_pipeline::{ - LintViolationRow, NATIVE_ENGINE, ViolationClasses, + LintViolationRow, NATIVE_ENGINE, ViolationClasses, project_relative_path, }; use crate::bindings::types::CompilerDetach; @@ -140,58 +139,39 @@ fn named_declaration_role_kind<'py>( } } -/// Remove violations matched by suppressions and add the suppression findings, as lint models. +/// `violations` plus a rejection finding for every inline suppression directive, as lint models. #[pyfunction] -pub(crate) fn apply_lint_suppressions<'py>( +pub(crate) fn reject_lint_suppressions<'py>( py: Python<'py>, violations: Vec>, contents_by_path: Bound<'py, PyDict>, - allow_suppressions: bool, + project_dir: PathBuf, ) -> PyResult>> { - let index_by_path: Bound<'py, PyDict> = PyDict::new(py); let mut paths: Vec> = Vec::new(); + let mut relative: Vec = Vec::new(); let mut contents: Vec = Vec::new(); for (path, text) in contents_by_path.iter() { - index_by_path.set_item(&path, paths.len())?; + relative.push(project_relative_path( + &path.extract::()?, + &project_dir, + )); paths.push(path); contents.push(text.extract()?); } - let keys: Vec = violations + let sources: Vec<(&str, &str)> = relative .iter() - .map(|violation| suppressible(violation, &index_by_path)) - .collect::>()?; - let texts: Vec<&str> = contents.iter().map(String::as_str).collect(); - let retained: Vec = compiler_guard(|| { - apply_suppressions(&texts, &keys, allow_suppressions).map_err(value_error) - })?; + .zip(&contents) + .map(|(path, text)| (path.as_str(), text.as_str())) + .collect(); + let findings: Vec = + compiler_guard(|| reject_inline_suppressions(&sources).map_err(value_error))?; let classes: ViolationClasses<'py> = ViolationClasses::new(py)?; - retained - .into_iter() - .map(|item| match item { - RetainedViolation::Existing(index) => Ok(violations[index].clone()), - RetainedViolation::Added(finding) => { - let row: LintViolationRow = LintViolationRow::suppression(finding); - classes.violation(&paths[row.file], &row) - } - }) - .collect() -} - -/// The suppression identity of one Python lint violation. -fn suppressible( - violation: &Bound<'_, PyAny>, - index_by_path: &Bound<'_, PyDict>, -) -> PyResult { - let file: Option = match index_by_path.get_item(native_field(violation, "file_path")?)? { - Some(index) => Some(index.extract()?), - None => None, - }; - Ok(SuppressibleViolation { - file: file.unwrap_or(usize::MAX), - line: native_field(violation, "line")?.extract()?, - code: native_field(violation, "code")?.extract()?, - native: native_field(violation, "engine")?.extract::()? == NATIVE_ENGINE, - }) + let mut retained: Vec> = violations; + for finding in findings { + let row: LintViolationRow = LintViolationRow::suppression(finding); + retained.push(classes.violation(&paths[row.file], &row)?); + } + Ok(retained) } /// The parts of `path` below `project_dir`, or `None` outside the project. @@ -284,5 +264,5 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { )?)?; module.add_function(wrap_pyfunction!(format_lint_sql_bodies, module)?)?; module.add_function(wrap_pyfunction!(named_declaration_role_kind, module)?)?; - module.add_function(wrap_pyfunction!(apply_lint_suppressions, module)?) + module.add_function(wrap_pyfunction!(reject_lint_suppressions, module)?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 989f88060c..09f09f5d78 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -1,16 +1,20 @@ //! Declared unique keys of project relations, derived natively from their key facts. use std::collections::BTreeMap; +use std::path::PathBuf; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::compiled_project::models::SchemaColumnFact; +use sqlbuild_discovery::project_validation::main::at_most_one_csv_data_row::at_most_one_csv_data_row; +use sqlbuild_discovery::project_validation::models::CsvSettings; use sqlbuild_rules::sql_quality::main::declared_relation_keys::declared_relation_keys as derive_keys; use sqlbuild_rules::sql_quality::models::{DeclaredKeyColumn, DeclaredRelation}; use crate::bindings::_helpers::compiled_project::schema_facts::python_column_facts; use crate::bindings::_helpers::compiled_project::source_views::{SourceKeyFacts, read_source_view}; +use crate::bindings::_helpers::discovery::project_facts::seed_csv_settings; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::rules::lint_inputs::model_key_row; @@ -184,11 +188,36 @@ fn declared_specs<'py>(entry: &Bound<'py, PyAny>, name: &str) -> PyResult>)>; -/// Keys that prove one row per key, by `kind:name` identity, from model rows, seeds and sources. +/// Names of the seeds whose CSV holds at most one data row, from `seed_file` and `schema_entry`. +pub(crate) fn one_row_seed_names(seeds: &[Bound<'_, PyAny>]) -> PyResult> { + let mut candidates: Vec<(String, PathBuf, CsvSettings)> = Vec::with_capacity(seeds.len()); + for seed in seeds { + let entry: Bound<'_, PyAny> = native_field(seed, "schema_entry")?; + candidates.push(( + native_field(&entry, "name")?.extract()?, + native_field(&native_field(seed, "seed_file")?, "file_path")?.extract()?, + seed_csv_settings(&entry)?, + )); + } + let py: Python<'_> = match seeds.first() { + Some(seed) => seed.py(), + None => return Ok(Vec::new()), + }; + Ok(py.detach(|| { + candidates + .into_iter() + .filter(|(_, path, settings)| at_most_one_csv_data_row(path, settings)) + .map(|(name, _, _)| name) + .collect() + })) +} + +/// Keys proving one row per key by `kind:name`; seeds in `one_row_seeds` also get the empty key. pub(crate) fn relation_keys<'py>( py: Python<'py>, mut rows: Vec, (seeds, sources): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + one_row_seeds: &[String], ) -> PyResult { for (kind, entries) in [(RELATION_KIND_SEED, seeds), (RELATION_KIND_SOURCE, sources)] { for entry in entries.try_iter()? { @@ -206,7 +235,11 @@ pub(crate) fn relation_keys<'py>( )?); } } - let relations: Vec = rows.into_iter().map(declared_relation).collect(); + let mut relations: Vec = rows.into_iter().map(declared_relation).collect(); + for relation in &mut relations { + relation.at_most_one_row = + relation.kind == RELATION_KIND_SEED && one_row_seeds.contains(&relation.name); + } let keys: BTreeMap>> = py.detach(|| derive_keys(&relations)); Ok(keys.into_iter().collect()) } @@ -233,7 +266,7 @@ fn declared_relation_keys<'py>( seeds: &Bound<'py, PyAny>, sources: &Bound<'py, PyAny>, ) -> PyResult> { - keys_dict(py, relation_keys(py, model_rows, (seeds, sources))?) + keys_dict(py, relation_keys(py, model_rows, (seeds, sources), &[])?) } /// Declared keys of a compiled project's models, seeds and sources. @@ -243,9 +276,13 @@ pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult> = native_field(project, "seeds")? + .try_iter()? + .collect::>()?; + let one_row_seeds: Vec = one_row_seed_names(&compiled_seeds)?; let mut seeds: Vec> = Vec::new(); - for seed in native_field(project, "seeds")?.try_iter()? { - seeds.push(native_field(&seed?, "schema_entry")?); + for seed in &compiled_seeds { + seeds.push(native_field(seed, "schema_entry")?); } let mut sources: Vec> = Vec::new(); for source in native_field(project, "sources")?.try_iter()? { @@ -258,6 +295,7 @@ pub(crate) fn project_relation_keys(project: &Bound<'_, PyAny>) -> PyResult DeclaredRelation { }) .collect(), audits, + at_most_one_row: false, } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index 7334eb244e..5167647518 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -41,7 +41,6 @@ pub(crate) fn settings_lint_key(settings: &LintSettings) -> String { max_literal_length: settings.sql.max_literal_length, max_ranking_order_by: settings.sql.max_ranking_order_by, relation_keys: settings.relation_keys.clone(), - allow_suppressions: settings.allow_suppressions, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs index 83d3585991..65e094a21f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rules_config.rs @@ -8,7 +8,7 @@ use pyo3::prelude::{ use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, Py, pyfunction, wrap_pyfunction}; use serde_json::Value; -use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::errors::RulesConfigLoadError; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config as load_config; use sqlbuild_rules::models::RulesConfig; @@ -137,6 +137,8 @@ fn config_object<'py>(py: Python<'py>, config: &RulesConfig) -> PyResult(py: Python<'py>, config: &RulesConfig) -> PyResult, project_dir: PathBuf) -> PyResul pub(crate) enum InvalidRulesConfig { /// The configuration could not be read: a `RulesError`. Unreadable(String), - /// It allows exceptions it may not: a `RulesConfigError` with help. + /// It uses a setting SQLBuild rejects: a `RulesConfigError` with help. Forbidden { message: String, help: String }, } @@ -275,17 +273,16 @@ pub(crate) fn rules_config_of( let project_dir: PathBuf = project_dir .canonicalize() .or_else(|_| std::path::absolute(project_dir))?; - let config: RulesConfig = match compiler_guard(|| Ok(load_config(&project_dir)))? { - Ok(config) => config, - Err(message) => return Ok(Err(InvalidRulesConfig::Unreadable(message))), - }; - if let Some(error) = forbidden_exceptions(&config) { - return Ok(Err(InvalidRulesConfig::Forbidden { + match compiler_guard(|| Ok(load_config(&project_dir)))? { + Ok(config) => Ok(Ok(config)), + Err(RulesConfigLoadError::Invalid(message)) => { + Ok(Err(InvalidRulesConfig::Unreadable(message))) + } + Err(RulesConfigLoadError::Policy(error)) => Ok(Err(InvalidRulesConfig::Forbidden { message: error.message, help: error.help, - })); + })), } - Ok(Ok(config)) } /// The project's rules configuration, or `None` when it is invalid and the rules phase reports it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index ed839bc92a..5e70f8b301 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -126,10 +126,7 @@ pub(crate) fn run_planned_sql_rules<'py>( Some(value) => value.extract(), None => Ok(None), }, - ( - project_relation_keys(&field(&graph, "project")?)?, - field(&effective_config, "allow_exceptions")?.is_truthy()?, - ), + project_relation_keys(&field(&graph, "project")?)?, )?; let early_lint: Bound<'py, PyAny> = options.get("early_lint")?; let run = SqlRun { diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index ed6d6fa9af..b49214685a 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,37 +1,45 @@ -use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; +use crate::configuration::_helpers::removed_settings::removed_settings; +use crate::configuration::errors::RulesConfigLoadError; +use crate::constants::{MIN_CUSTOM_RULE_TEST_CASES_KEY, RELATION_EXCEPTION_RULES}; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use std::path::Path; -pub(crate) fn load(project_dir: &Path) -> Result { +pub(crate) fn load(project_dir: &Path) -> Result { + let invalid = RulesConfigLoadError::Invalid; let path = project_dir.join(PROJECT_CONFIG_FILENAME); let source = std::fs::read_to_string(&path).map_err(|error| { - format!( + invalid(format!( "could not load rules config from {}: {error}", path.display() - ) + )) })?; let root: toml::Table = toml::from_str(&source).map_err(|error| { - format!( + invalid(format!( "could not load rules config from {}: {error}", path.display() - ) + )) })?; if root.contains_key("policy") || root.contains_key("lint") || root.contains_key("kata") { - return Err("unsupported legacy rules configuration; use [rules]".to_owned()); + return Err(invalid( + "unsupported legacy rules configuration; use [rules]".to_owned(), + )); } let config = match root.get("rules") { None => RulesConfig::default(), Some(value) => { - validate_raw(value)?; + if let Some(error) = value.as_table().and_then(removed_settings) { + return Err(RulesConfigLoadError::Policy(error)); + } + validate_raw(value).map_err(invalid)?; value .clone() .try_into::() - .map_err(|error| format!("invalid rules config: {error}"))? + .map_err(|error| invalid(format!("invalid rules config: {error}")))? } }; - validate(&config)?; + validate(&config).map_err(invalid)?; Ok(config) } @@ -48,7 +56,6 @@ fn validate_raw(value: &toml::Value) -> Result<(), String> { "rule_exceptions", "graph_edge_exceptions", "rule_ignores", - "allow_exceptions", "select_star_allow", "domains", "approved_source_tokens", @@ -209,6 +216,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { if !grammar.rule_code_is_exact(&exception.rule) { return Err(format!("invalid rule code: {}", exception.rule)); } + validate_relation_scope(exception)?; } let mut graph_edges: std::collections::BTreeSet<(&str, &str)> = std::collections::BTreeSet::new(); @@ -259,6 +267,32 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { Ok(()) } +/// A relation-level exception names a non-empty relation, for a Rule that reports one. +fn validate_relation_scope(exception: &crate::models::RuleException) -> Result<(), String> { + let blank = |value: &Option| value.as_ref().is_some_and(|text| text.trim().is_empty()); + if blank(&exception.relation) || blank(&exception.cte) { + return Err("rules.rule_exceptions relation and cte must be non-empty strings".into()); + } + if exception.cte.is_some() && exception.relation.is_none() { + return Err(format!( + "rules.rule_exceptions entry for {} at {} sets cte without relation; cte narrows a \ + relation-level exception", + exception.rule, exception.path + )); + } + if exception.relation.is_some() && !RELATION_EXCEPTION_RULES.contains(&exception.rule.as_str()) + { + return Err(format!( + "rules.rule_exceptions entry for {} at {} sets relation, but only {} report a joined \ + relation; remove relation to cover the whole file", + exception.rule, + exception.path, + RELATION_EXCEPTION_RULES.join(", ") + )); + } + Ok(()) +} + fn validate_relative_path(value: &str, label: &str) -> Result<(), String> { let path = Path::new(value); if value.trim().is_empty() diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs b/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs index f33faa4b7b..c96e79b252 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/mod.rs @@ -1 +1,2 @@ pub mod loading; +pub mod removed_settings; diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs b/crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs new file mode 100644 index 0000000000..cc2c9406d1 --- /dev/null +++ b/crates/sqlbuild-rules/src/configuration/_helpers/removed_settings.rs @@ -0,0 +1,26 @@ +//! `[rules]` settings SQLBuild no longer supports, with how to migrate. + +use sqlbuild_analysis::contracts::main::join_helps::join_helps; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; + +use crate::configuration::errors::ConfigPolicyError; +use crate::constants::REMOVED_ALLOW_EXCEPTIONS; + +/// The error for a `[rules]` table that still sets a removed setting. +pub(crate) fn removed_settings(rules: &toml::Table) -> Option { + if !rules.contains_key(REMOVED_ALLOW_EXCEPTIONS) { + return None; + } + let message: String = format!( + "{PROJECT_CONFIG_FILE} sets [rules] {REMOVED_ALLOW_EXCEPTIONS}, which SQLBuild no longer \ + supports: Rule exceptions are declared only in {PROJECT_CONFIG_FILE}, where review sees \ + them, and inline -- sqb: ignore suppressions were removed" + ); + let help: String = join_helps(&[ + format!("remove the {REMOVED_ALLOW_EXCEPTIONS} line from [rules] in {PROJECT_CONFIG_FILE}"), + "keep each reviewed deviation as a [[rules.rule_exceptions]] entry with rule, path and \ + reason, or turn a Rule off for the whole project with [rules] ignore" + .to_owned(), + ]); + Some(ConfigPolicyError { message, help }) +} diff --git a/crates/sqlbuild-rules/src/configuration/errors.rs b/crates/sqlbuild-rules/src/configuration/errors.rs index cd92448b71..14cdec8370 100644 --- a/crates/sqlbuild-rules/src/configuration/errors.rs +++ b/crates/sqlbuild-rules/src/configuration/errors.rs @@ -6,3 +6,12 @@ pub struct ConfigPolicyError { pub message: String, pub help: String, } + +/// Why a project's rules configuration cannot be loaded. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RulesConfigLoadError { + /// The configuration is unreadable or malformed. + Invalid(String), + /// The configuration uses a setting SQLBuild rejects, with help showing the fix. + Policy(ConfigPolicyError), +} diff --git a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs b/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs deleted file mode 100644 index 690d165255..0000000000 --- a/crates/sqlbuild-rules/src/configuration/main/forbidden_exceptions.rs +++ /dev/null @@ -1,57 +0,0 @@ -//! `forbidden_exceptions`: exceptions or scoped ignores in a project that disallows them. - -use sqlbuild_analysis::contracts::main::join_helps::join_helps; -use sqlbuild_core::cli_text::main::format_count_noun::format_count_noun; - -use crate::configuration::errors::ConfigPolicyError; -use crate::configuration::main::setting_help::setting_help; -use crate::models::RulesConfig; -use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_CONFIG_FILE; - -/// The policy error when `allow_exceptions = false` and exception or ignore entries exist. -#[must_use] -pub fn forbidden_exceptions(config: &RulesConfig) -> Option { - if config.allow_exceptions { - return None; - } - let mut entries: Vec = Vec::new(); - let mut tables: Vec = Vec::new(); - for (key, count) in [ - ("rule_exceptions", config.rule_exceptions.len()), - ("rule_ignores", config.rule_ignores.len()), - ] { - if count == 0 { - continue; - } - let table: String = format!("[[rules.{key}]]"); - let count: i64 = i64::try_from(count).unwrap_or(i64::MAX); - entries.push(format_count_noun( - count, - &format!("{table} entry"), - Some(&format!("{table} entries")), - )); - tables.push(table); - } - if entries.is_empty() { - return None; - } - let message: String = format!( - "{PROJECT_CONFIG_FILE} has {}, which this project does not allow; \ - {PROJECT_CONFIG_FILE} sets [rules] allow_exceptions = false", - entries.join(" and ") - ); - let help: String = join_helps(&[ - format!( - "remove every {} entry and fix the findings it hides; to turn a Rule off for the \ - whole project, add its code to [rules] ignore", - tables.join(" and ") - ), - setting_help( - "to allow Rule exceptions, scoped ignores and inline suppressions", - "rules", - "allow_exceptions", - "true", - ), - ]); - Some(ConfigPolicyError { message, help }) -} diff --git a/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs index 2c3ba017d0..b65f6aa60e 100644 --- a/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs +++ b/crates/sqlbuild-rules/src/configuration/main/load_rules_config.rs @@ -2,9 +2,10 @@ use std::path::Path; +use crate::configuration::errors::RulesConfigLoadError; use crate::models::RulesConfig; /// The project's rules configuration, or the reason it cannot be loaded. -pub fn load_rules_config(project_dir: &Path) -> Result { +pub fn load_rules_config(project_dir: &Path) -> Result { crate::configuration::_helpers::loading::load(project_dir) } diff --git a/crates/sqlbuild-rules/src/configuration/main/mod.rs b/crates/sqlbuild-rules/src/configuration/main/mod.rs index 42488f2f4f..732d7c95f5 100644 --- a/crates/sqlbuild-rules/src/configuration/main/mod.rs +++ b/crates/sqlbuild-rules/src/configuration/main/mod.rs @@ -1,4 +1,3 @@ -pub mod forbidden_exceptions; pub mod load_rules_config; pub mod setting_help; pub mod toml_string_array; diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index 73e3aa809f..3a5ae371e3 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -33,3 +33,7 @@ pub(crate) const NATIVE_ROWS_MEMO_VERSION: &str = "native-rules-rows-response-v2 pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; /// Threshold key setting the fewest test cases a custom rule must ship. pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; +/// Rules whose findings name a joined relation, so an exception may cover just that relation. +pub(crate) const RELATION_EXCEPTION_RULES: [&str; 1] = ["SQBRSQL038"]; +/// The removed exception switch, which used to allow or forbid every escape hatch at once. +pub(crate) const REMOVED_ALLOW_EXCEPTIONS: &str = "allow_exceptions"; diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index f0f4e3c2ed..494165ea40 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -1,9 +1,9 @@ use crate::configuration::main::validate as config; -use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY}; +use crate::constants::{API_VERSION, NATIVE_BUILD_IDENTITY, RELATION_EXCEPTION_RULES}; use crate::engine::_helpers::cache::{Cache, PendingBucketWrite, RuleCacheEntry}; use crate::models::{ - EvaluateRequest, EvaluateResponse, Fault, Model, RuleMetadata, RuleScope, RulesCodeGrammar, - RulesConfig, + EvaluateRequest, EvaluateResponse, Fault, Model, RuleException, RuleMetadata, RuleScope, + RulesCodeGrammar, RulesConfig, }; use crate::rules::main::{ assemble_catalogue, evaluate as rules, evaluate_project, fingerprint, @@ -12,6 +12,7 @@ use crate::rules::main::{ use crate::rules::models::{ ModelEvaluationRequest, ProjectEvaluationRequest, QueryShapes, ResolvedThresholdOverride, }; +use crate::sql_lint::main::cross_join_subject::cross_join_subject; use fensu_policy::apply_suppressions; use fensu_policy::lifecycle::errors::LifecycleError; use fensu_policy::lifecycle::models::{ @@ -580,18 +581,17 @@ fn apply_fault_policy( ) }) .collect(); + let exceptions: &[RuleException] = &request.config.rule_exceptions; let findings = faults .into_iter() - .map(fault_to_finding) + .map(|fault| fault_to_finding(fault, exceptions)) .collect::, _>>()?; - let suppressions = request - .config - .rule_exceptions + let suppressions = exceptions .iter() .map(|entry| ExactSuppression { code: entry.rule.clone(), path: entry.path.clone(), - symbol: None, + symbol: exception_symbol(entry), reason: entry.reason.clone(), }) .collect::>(); @@ -629,13 +629,71 @@ fn apply_fault_policy( .collect()) } -fn fault_to_finding(fault: Fault) -> Result { +/// The symbol a relation-level exception and the findings it covers share. +fn relation_symbol(relation: &str, cte: Option<&str>) -> String { + let relation: String = relation.to_lowercase(); + match cte { + Some(cte) => format!("relation {relation} in CTE {}", cte.to_lowercase()), + None => format!("relation {relation}"), + } +} + +/// The symbol a relation-level exception covers, `None` for a file-wide exception. +fn exception_symbol(entry: &RuleException) -> Option { + let relation: &str = entry.relation.as_deref()?; + Some(relation_symbol(relation, entry.cte.as_deref())) +} + +/// The relation-level exception symbol that covers `fault`: the CTE-scoped one first. +fn covering_symbol(fault: &Fault, exceptions: &[RuleException]) -> Option { + if !RELATION_EXCEPTION_RULES.contains(&fault.code.as_str()) { + return None; + } + let scoped: Vec<&RuleException> = exceptions + .iter() + .filter(|entry| { + entry.rule == fault.code && entry.path == fault.path && entry.relation.is_some() + }) + .collect(); + if scoped.is_empty() { + return None; + } + let (relation, cte) = cross_join_subject(&fault.message)?; + let matches = |entry: &&RuleException, cte: Option<&str>| { + entry + .relation + .as_deref() + .is_some_and(|named| named.eq_ignore_ascii_case(&relation)) + && match (entry.cte.as_deref(), cte) { + (Some(named), Some(found)) => named.eq_ignore_ascii_case(found), + (None, _) => true, + (Some(_), None) => false, + } + }; + scoped + .iter() + .find(|entry| entry.cte.is_some() && matches(entry, cte.as_deref())) + .or_else(|| { + scoped + .iter() + .find(|entry| entry.cte.is_none() && matches(entry, None)) + }) + .map(|entry| { + relation_symbol( + entry.relation.as_deref().unwrap_or_default(), + entry.cte.as_deref(), + ) + }) +} + +fn fault_to_finding(fault: Fault, exceptions: &[RuleException]) -> Result { + let symbol: Option = covering_symbol(&fault, exceptions); Ok(Finding { code: fault.code, path: fault.path, line: Some(u32::try_from(fault.line).map_err(|_| "rule finding line exceeds u32")?), column: Some(u32::try_from(fault.column).map_err(|_| "rule finding column exceeds u32")?), - symbol: None, + symbol, message: fault.message, remediation: Some(fault.remediation), severity: FindingSeverity::Blocking, @@ -656,6 +714,12 @@ fn finding_to_fault(finding: Finding) -> Fault { fn lifecycle_error(error: LifecycleError) -> String { match error { + LifecycleError::StaleSuppression { + code, + path, + symbol: Some(symbol), + .. + } => format!("stale rule exception suppresses no finding: {code} at {path} ({symbol})"), LifecycleError::StaleSuppression { code, path, .. } => { format!("stale rule exception suppresses no finding: {code} at {path}") } diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index 5195d6dd8a..c217717c22 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -10,6 +10,8 @@ mod helpers; mod incremental_rules; #[path = "tests/test_query_facts.rs"] mod query_facts; +#[path = "tests/test_relation_exceptions.rs"] +mod relation_exceptions; #[path = "tests/test_retained_rows.rs"] mod retained_rows; #[path = "tests/test_scope_facts.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index d68e0cc521..47193872a6 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -12,12 +12,14 @@ use crate::engine::main::discard_speculative_rows::discard_speculative_rows; use crate::engine::main::retained_request::retained_request; use crate::models::{ Column, Declaration, DeclarationMember, DynamicColumnFamily, EvaluateRequest, Fault, Model, - Reference, RetainedRows, RowsEvaluation, RowsRequest, SpeculativeAnswer, SpeculativeRows, + Reference, RetainedRows, RowsEvaluation, RowsRequest, RulesConfig, SpeculativeAnswer, + SpeculativeRows, }; use crate::engine::_helpers::evaluation::evaluate_json; use crate::engine::_helpers::evaluation::evaluate_request as evaluate_parsed; use crate::engine::_helpers::evaluation::parse_parts; +use crate::engine::main::finalize_rows::finalize_rows; use crate::engine::tests::test_types; pub(crate) fn request(project_dir: &TempDir, config: &Value) -> String { @@ -735,3 +737,48 @@ pub(crate) fn without_sql(models: Vec) -> Value { .collect(); serde_json::to_value(stripped).unwrap_or_default() } + + +/// The model path the relation-exception cases use. +pub(crate) const CROSS_JOIN_MODEL_PATH: &str = "models/orders.sql"; +/// The Rule whose findings name a joined relation. +pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; + +fn cross_join_fault(message: &str, line: u64) -> Fault { + Fault { + unevaluated: false, + code: CROSS_JOIN_CODE.to_owned(), + path: CROSS_JOIN_MODEL_PATH.to_owned(), + line, + column: 1, + message: message.to_owned(), + remediation: String::new(), + } +} + +/// Finalize a top-level `order_policy` join and a `customers` join in CTE `checks`. +pub(crate) fn finalize_cross_joins(exceptions: Value) -> Result, String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + std::fs::create_dir_all(project_dir.path().join("models")) + .map_err(|error| error.to_string())?; + std::fs::write(project_dir.path().join(CROSS_JOIN_MODEL_PATH), "SELECT 1") + .map_err(|error| error.to_string())?; + let config: RulesConfig = serde_json::from_value(json!({"rule_exceptions": exceptions})) + .map_err(|error| error.to_string())?; + let findings: Vec = vec![ + cross_join_fault( + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + 3, + ), + cross_join_fault( + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + 7, + ), + ]; + finalize_rows( + project_dir.path().to_string_lossy().into_owned(), + config, + &[CROSS_JOIN_CODE.to_owned()], + findings, + ) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs b/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs new file mode 100644 index 0000000000..46ec32a515 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_relation_exceptions.rs @@ -0,0 +1,73 @@ +use serde_json::json; + +use crate::engine::tests::helpers::{CROSS_JOIN_CODE, CROSS_JOIN_MODEL_PATH, finalize_cross_joins}; +use crate::engine::tests::test_types::{RelationExceptionTestCase, StaleRelationExceptionTestCase}; + +#[test] +fn given_relation_level_exceptions_when_finalizing_then_exactly_the_named_joins_are_excepted() +-> Result<(), String> { + let test_cases = [ + RelationExceptionTestCase { + description: "a relation exception covers only that relation", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "order_policy", "reason": "Every order meets every policy."}]), + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + RelationExceptionTestCase { + description: "a CTE-scoped exception covers the join in that CTE only", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "cte": "checks", "reason": "Deliberate."}]), + expected_messages: &[ + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + ], + }, + RelationExceptionTestCase { + description: "relation names match case-insensitively", + exceptions: json!([ + {"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "Order_Policy", "reason": "Every order meets every policy."}, + {"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "reason": "Deliberate."} + ]), + expected_messages: &[], + }, + ]; + + for test_case in test_cases { + let kept = finalize_cross_joins(test_case.exceptions.clone())?; + + let messages: Vec<&str> = kept.iter().map(|fault| fault.message.as_str()).collect(); + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_relation_level_exceptions_matching_nothing_when_finalizing_then_they_are_stale() { + let test_cases = [ + StaleRelationExceptionTestCase { + description: "a relation exception that matches nothing is stale", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "shipments", "reason": "Gone."}]), + expected_error: "stale rule exception suppresses no finding: SQBRSQL038 at models/orders.sql (relation shipments)", + }, + StaleRelationExceptionTestCase { + description: "a CTE-scoped exception for another CTE is stale", + exceptions: json!([{"rule": CROSS_JOIN_CODE, "path": CROSS_JOIN_MODEL_PATH, "relation": "customers", "cte": "other", "reason": "Gone."}]), + expected_error: "stale rule exception suppresses no finding: SQBRSQL038 at models/orders.sql (relation customers in CTE other)", + }, + ]; + + for test_case in test_cases { + let error: String = finalize_cross_joins(test_case.exceptions.clone()) + .err() + .unwrap_or_default(); + + assert!( + error.contains(test_case.expected_error), + "{}: {error}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index c22ae94525..4d2516fd8d 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -298,3 +298,15 @@ pub(crate) struct RowCodecTestCase { /// How many truncations of the rows still read back. pub(crate) expected_truncations_read: usize, } + +pub(crate) struct RelationExceptionTestCase { + pub(crate) description: &'static str, + pub(crate) exceptions: Value, + pub(crate) expected_messages: &'static [&'static str], +} + +pub(crate) struct StaleRelationExceptionTestCase { + pub(crate) description: &'static str, + pub(crate) exceptions: Value, + pub(crate) expected_error: &'static str, +} diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index b5909ebcde..9f61cde2ae 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -6,7 +6,8 @@ use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; use crate::lint_text::constants::{ - INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, + CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, + OMITTED_CEREMONIAL_SELECT_SQL, }; use crate::lint_text::models::{LintSqlBody, LintSqlOptions}; use crate::sql_lint::main::batch_engine::lint_batch_json; @@ -26,6 +27,13 @@ pub(crate) fn omitted_select(body: &LintSqlBody) -> Option { } } +/// Whether `text` may hold a `CROSS JOIN`; other bodies omit relation names to share requests. +fn mentions_cross_join(text: &str) -> bool { + text.as_bytes() + .windows(CROSS_KEYWORD.len()) + .any(|window| window.eq_ignore_ascii_case(CROSS_KEYWORD)) +} + fn strings(values: &[String]) -> Value { Value::from(values.to_vec()) } @@ -95,6 +103,14 @@ pub(crate) fn lint_payload( .collect(); let _ = payload.insert("relation_keys".to_string(), Value::Object(keys)); } + if !body.relation_names.is_empty() && mentions_cross_join(&body.lint_text) { + let names: Map = body + .relation_names + .iter() + .map(|(sentinel, name)| (sentinel.clone(), Value::from(name.clone()))) + .collect(); + let _ = payload.insert("relation_names".to_string(), Value::Object(names)); + } Value::Object(payload) } diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs index 36bb34d459..72c3b00555 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -1,140 +1,130 @@ -//! Suppression directive parsing and matching. +//! Inline suppression directives, which SQLBuild rejects in favour of configured exceptions. use std::sync::LazyLock; use regex::Regex; -use sqlbuild_core::text::main::python_isspace::python_isspace; +use crate::constants::SETTING_SNIPPET_INDENT; use crate::lint_text::_helpers::lines::LineSpan; use crate::lint_text::_helpers::sites::compiled; use crate::lint_text::constants::{ - FORBIDDEN_FIX_REASON, FORBIDDEN_MESSAGE, FORBIDDEN_REMEDIATION, INVALID_MESSAGE, - INVALID_REMEDIATION, NATIVE_RULE_PREFIX, REASON_REQUIRED_FIX_REASON, SUPPRESSION_CODE, - SUPPRESSION_PATTERN, SUPPRESSION_PREFIX, UNUSED_REMEDIATION, + CROSS_JOIN_CODE, CROSS_JOIN_RELATION_PATTERN, DIRECTIVE_FIX_REASON, DIRECTIVE_MESSAGE, + DIRECTIVE_PATTERN, REASON_PLACEHOLDER, RULE_PLACEHOLDER, SUPPRESSION_PREFIX, }; use crate::lint_text::errors::LintTextError; -use crate::lint_text::models::{ - RetainedViolation, SuppressibleViolation, SuppressionDirective, SuppressionFinding, -}; +use crate::lint_text::models::SuppressionFinding; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::text::main::python_isspace::python_isspace; -static SUPPRESSION: LazyLock> = - LazyLock::new(|| Regex::new(SUPPRESSION_PATTERN)); +static DIRECTIVE: LazyLock> = + LazyLock::new(|| Regex::new(DIRECTIVE_PATTERN)); +static CROSS_JOIN_RELATION: LazyLock> = + LazyLock::new(|| Regex::new(CROSS_JOIN_RELATION_PATTERN)); -/// Valid directives, and the one-based lines of invalid ones. -pub(crate) fn parse_directives( +/// One inline directive: what it names, and what the exception entry for it holds. +struct Directive { + line: usize, + code: Option, + reason: Option, + relation: Option, +} + +/// The rejection finding for every directive line in one file. +pub(crate) fn directive_findings( + file: usize, + path: &str, lines: &[LineSpan<'_>], -) -> Result<(Vec, Vec), LintTextError> { - let pattern: &Regex = compiled(&SUPPRESSION)?; - let mut directives: Vec = Vec::new(); - let mut invalid: Vec = Vec::new(); +) -> Result, LintTextError> { + let pattern: &Regex = compiled(&DIRECTIVE)?; + let relation_pattern: &Regex = compiled(&CROSS_JOIN_RELATION)?; + let mut findings: Vec = Vec::new(); for (index, line) in lines.iter().enumerate() { if !line.content.to_lowercase().contains(SUPPRESSION_PREFIX) { continue; } - let Some(captures) = pattern.captures(line.content) else { - invalid.push(index + 1); - continue; + let captures = pattern.captures(line.content); + let text = |name: &str| -> Option { + captures + .as_ref() + .and_then(|found| found.name(name)) + .map(|found| found.as_str().trim().to_owned()) + .filter(|value| !value.is_empty()) }; - let mut target: usize = index + 1; - while target < lines.len() && skippable(lines[target].content) { - target += 1; - } - directives.push(SuppressionDirective { - code: captures["code"].to_string(), + let code: Option = text("code"); + let relation: Option = if code.as_deref() == Some(CROSS_JOIN_CODE) { + target_line(lines, index) + .and_then(|target| cross_joined_relation(relation_pattern, target)) + } else { + None + }; + let directive = Directive { line: index + 1, - target: target + 1, + code, + reason: text("reason"), + relation, + }; + findings.push(SuppressionFinding { + file, + line: directive.line, + message: DIRECTIVE_MESSAGE.to_owned(), + remediation: remediation(&directive, path), + fault: true, + fix: None, + fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), }); } - Ok((directives, invalid)) + Ok(findings) } -/// Blank lines and comment lines between a directive and the line it suppresses. -fn skippable(line: &str) -> bool { - let stripped: &str = line.trim_start_matches(python_isspace); - stripped.is_empty() || stripped.starts_with("--") +/// The next line that is neither blank nor a comment: the line a directive meant to cover. +fn target_line<'a>(lines: &[LineSpan<'a>], directive: usize) -> Option<&'a str> { + lines + .iter() + .skip(directive + 1) + .map(|line| line.content) + .find(|content| { + let stripped: &str = content.trim_start_matches(python_isspace); + !stripped.is_empty() && !stripped.starts_with("--") + }) } -/// Whether a retained violation is the native finding one directive suppresses. -pub(crate) fn matches_directive( - item: &RetainedViolation, - violations: &[SuppressibleViolation], - file: usize, - directive: &SuppressionDirective, -) -> bool { - let (item_file, line, code, native): (usize, usize, &str, bool) = match item { - RetainedViolation::Existing(index) => { - let violation: &SuppressibleViolation = &violations[*index]; - ( - violation.file, - violation.line, - &violation.code, - violation.native, - ) - } - RetainedViolation::Added(finding) => (finding.file, finding.line, SUPPRESSION_CODE, true), - }; - item_file == file - && line == directive.target - && code == directive.code - && native - && code.starts_with(NATIVE_RULE_PREFIX) +/// The relation a `CROSS JOIN` on `line` reads: a dependency's name or an authored identifier. +fn cross_joined_relation(pattern: &Regex, line: &str) -> Option { + let captures = pattern.captures(line)?; + captures + .name("dependency") + .or_else(|| captures.name("identifier")) + .map(|found| found.as_str().to_owned()) } -/// Findings for every directive line when the project forbids suppressions. -pub(crate) fn forbidden_findings( - file: usize, - directives: &[SuppressionDirective], - invalid: &[usize], -) -> Vec { - let mut lines: Vec = invalid +/// Remove-and-declare help, with the exact `[[rules.rule_exceptions]]` entry to add. +fn remediation(directive: &Directive, path: &str) -> String { + let mut entry: Vec = vec![ + "[[rules.rule_exceptions]]".to_owned(), + format!( + "rule = {}", + toml_string(directive.code.as_deref().unwrap_or(RULE_PLACEHOLDER)) + ), + format!("path = {}", toml_string(path)), + ]; + if let Some(relation) = &directive.relation { + entry.push(format!("relation = {}", toml_string(relation))); + } + entry.push(format!( + "reason = {}", + toml_string(directive.reason.as_deref().unwrap_or(REASON_PLACEHOLDER)) + )); + let rule: &str = directive.code.as_deref().unwrap_or("Rule"); + let snippet: String = entry .iter() - .copied() - .chain(directives.iter().map(|directive| directive.line)) + .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) .collect(); - lines.sort_unstable(); - lines.dedup(); - lines - .into_iter() - .map(|line| { - RetainedViolation::Added(SuppressionFinding { - file, - line, - message: FORBIDDEN_MESSAGE.to_string(), - remediation: FORBIDDEN_REMEDIATION, - fault: true, - fix: None, - fix_unavailable_reason: Some(FORBIDDEN_FIX_REASON), - }) - }) - .collect() + format!( + "Remove this comment, then fix the {rule} finding, or record it as a reviewed exception in \ + {PROJECT_CONFIG_FILENAME}:{snippet}" + ) } -/// The finding for an invalid directive line. -pub(crate) fn invalid_finding(file: usize, line: usize) -> RetainedViolation { - RetainedViolation::Added(SuppressionFinding { - file, - line, - message: INVALID_MESSAGE.to_string(), - remediation: INVALID_REMEDIATION, - fault: false, - fix: None, - fix_unavailable_reason: Some(REASON_REQUIRED_FIX_REASON), - }) -} - -/// The finding for a directive that suppressed nothing, with an edit removing its line. -pub(crate) fn unused_finding( - file: usize, - directive: &SuppressionDirective, - span: &LineSpan<'_>, -) -> RetainedViolation { - RetainedViolation::Added(SuppressionFinding { - file, - line: directive.line, - message: format!("Unused suppression for {}", directive.code), - remediation: UNUSED_REMEDIATION, - fault: false, - fix: Some((span.start, span.end)), - fix_unavailable_reason: None, - }) +fn toml_string(value: &str) -> String { + toml::Value::String(value.to_owned()).to_string() } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index fd41451e50..b76842b150 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -26,22 +26,19 @@ pub(crate) const LINT_SITE_PATTERN: &str = r"^(?:@@|\$\{|@|(?:__dbt_ref|__ref|__seed|__source|__table_fn|__udf)\()"; pub(crate) const FORMAT_SITE_PATTERN: &str = r"(?i)^(?:@@|\$\{|@|(?:__cursor_end|__cursor_start|__dbt_ref|__empty_fixture|__ref|__seed|__source|__table_fn|__udf)\s*\()"; -/// Diagnostic code of every suppression finding. +/// Diagnostic code of every inline suppression directive finding. pub const SUPPRESSION_CODE: &str = "SQBRSQL000"; pub(crate) const SUPPRESSION_PREFIX: &str = "sqb: ignore"; -pub(crate) const SUPPRESSION_PATTERN: &str = - r"^\s*--\s*sqb:\s*ignore\s+(?P[A-Za-z0-9_-]+)\s+because\s+(?P\S.*)\s*$"; -pub(crate) const NATIVE_RULE_PREFIX: &str = "SQBRSQL"; -pub(crate) const INVALID_MESSAGE: &str = "Suppression directive is invalid"; -pub(crate) const INVALID_REMEDIATION: &str = "Use '-- sqb: ignore CODE because '."; -pub(crate) const UNUSED_REMEDIATION: &str = "Remove the stale suppression directive."; -pub(crate) const FORBIDDEN_MESSAGE: &str = - "Inline Rule suppressions are not allowed in this project"; -pub(crate) const FORBIDDEN_REMEDIATION: &str = "Remove the directive and fix the finding it suppresses; sqlbuild_project.toml sets [rules] allow_exceptions = false. To allow inline suppressions, set this in sqlbuild_project.toml:\n [rules]\n allow_exceptions = true"; -pub(crate) const FORBIDDEN_FIX_REASON: &str = - "removing the directive exposes the finding it suppresses"; -pub(crate) const REASON_REQUIRED_FIX_REASON: &str = - "a valid suppression reason requires user intent"; +pub(crate) const DIRECTIVE_PATTERN: &str = + r"(?i)sqb:\s*ignore\b\s*(?P[A-Za-z0-9_-]*)(?:\s+because\s+(?P.*\S))?"; +pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; +pub(crate) const CROSS_JOIN_RELATION_PATTERN: &str = r#"(?i)\bcross\s+join\s+(?:__(?:ref|seed|source)\s*\(\s*['"](?P[^'"]+)['"]\s*\)|(?P[A-Za-z_][A-Za-z0-9_$.]*))"#; +pub(crate) const DIRECTIVE_MESSAGE: &str = + "Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml"; +pub(crate) const RULE_PLACEHOLDER: &str = ""; +pub(crate) const REASON_PLACEHOLDER: &str = ""; +pub(crate) const DIRECTIVE_FIX_REASON: &str = + "moving an exception into sqlbuild_project.toml requires review"; pub(crate) const FIXTURE_TYPED_NULL_PATTERN: &str = r"(?i)(?:CAST\s*\(\s*NULL\s+AS\b|NULL\s*::)"; pub(crate) const MODEL_FIXTURE_PATTERN: &str = @@ -162,3 +159,5 @@ pub(crate) const GROUPED_NAMED_ROLE_KINDS: [(&[&str], &str); 9] = [ (&["hooks", "python"], "python_hook"), (&["_hooks", "python"], "python_hook"), ]; +/// The keyword whose presence makes a lint request carry relation names. +pub(crate) const CROSS_KEYWORD: &[u8] = b"cross"; diff --git a/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs b/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs deleted file mode 100644 index 590a0f4259..0000000000 --- a/crates/sqlbuild-rules/src/lint_text/main/apply_suppressions.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! `apply_suppressions`: reason-required next-line suppressions applied to lint violations. - -use crate::lint_text::_helpers::lines::{LineSpan, line_spans}; -use crate::lint_text::_helpers::suppressions::{ - forbidden_findings, invalid_finding, matches_directive, parse_directives, unused_finding, -}; -use crate::lint_text::errors::LintTextError; -use crate::lint_text::models::{RetainedViolation, SuppressibleViolation}; - -/// Remove matching next-line diagnostics, or reject every directive when suppressions are off. -pub fn apply_suppressions( - files: &[&str], - violations: &[SuppressibleViolation], - allow_suppressions: bool, -) -> Result, LintTextError> { - let mut retained: Vec = (0..violations.len()) - .map(RetainedViolation::Existing) - .collect(); - for (file, contents) in files.iter().enumerate() { - let lines: Vec> = line_spans(contents); - let (directives, invalid) = parse_directives(&lines)?; - if !allow_suppressions { - retained.extend(forbidden_findings(file, &directives, &invalid)); - continue; - } - retained.extend(invalid.into_iter().map(|line| invalid_finding(file, line))); - for directive in directives { - match retained - .iter() - .position(|item| matches_directive(item, violations, file, &directive)) - { - Some(index) => { - let _ = retained.remove(index); - } - None => retained.push(unused_finding(file, &directive, &lines[directive.line - 1])), - } - } - } - Ok(retained) -} diff --git a/crates/sqlbuild-rules/src/lint_text/main/mod.rs b/crates/sqlbuild-rules/src/lint_text/main/mod.rs index d09920580a..2ff3fab536 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/mod.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/mod.rs @@ -1,4 +1,3 @@ -pub mod apply_suppressions; pub mod dependency_sites; pub mod first_header_only; pub mod format_fixture_blocks; @@ -9,6 +8,7 @@ pub mod named_declaration_role_kind; pub mod neutralize_audit_parameters; pub mod neutralize_context_interpolation; pub mod neutralize_interpolation; +pub mod reject_inline_suppressions; pub mod resolve_format_settings; pub mod restore_interpolation; pub mod scan_fixture_null_candidates; diff --git a/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs b/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs new file mode 100644 index 0000000000..75de3c5e87 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs @@ -0,0 +1,18 @@ +//! `reject_inline_suppressions`: an error for every inline `-- sqb: ignore` directive. + +use crate::lint_text::_helpers::lines::{LineSpan, line_spans}; +use crate::lint_text::_helpers::suppressions::directive_findings; +use crate::lint_text::errors::LintTextError; +use crate::lint_text::models::SuppressionFinding; + +/// A finding per directive line of each `(relative path, contents)`, printing its exception entry. +pub fn reject_inline_suppressions( + files: &[(&str, &str)], +) -> Result, LintTextError> { + let mut findings: Vec = Vec::new(); + for (file, (path, contents)) in files.iter().enumerate() { + let lines: Vec> = line_spans(contents); + findings.extend(directive_findings(file, path, &lines)?); + } + Ok(findings) +} diff --git a/crates/sqlbuild-rules/src/lint_text/models.rs b/crates/sqlbuild-rules/src/lint_text/models.rs index fc0b21277b..85a527e749 100644 --- a/crates/sqlbuild-rules/src/lint_text/models.rs +++ b/crates/sqlbuild-rules/src/lint_text/models.rs @@ -40,42 +40,18 @@ pub struct Neutralized { pub sites: Vec, } -/// The identity of one existing violation that a suppression can remove. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SuppressibleViolation { - pub file: usize, - pub line: usize, - pub code: String, - pub native: bool, -} - -/// One valid directive: its code, its line and the line it suppresses. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SuppressionDirective { - pub code: String, - pub line: usize, - pub target: usize, -} - -/// A suppression finding on one directive line; offsets are character positions. +/// The rejection finding on one inline directive line; offsets are character positions. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SuppressionFinding { pub file: usize, pub line: usize, pub message: String, - pub remediation: &'static str, + pub remediation: String, pub fault: bool, pub fix: Option<(usize, usize)>, pub fix_unavailable_reason: Option<&'static str>, } -/// One retained violation: an input violation by index, or a new suppression finding. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RetainedViolation { - Existing(usize), - Added(SuppressionFinding), -} - /// SQL-test files that may hold removable typed nulls, and the models their fixtures stand in for. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct FixtureNullCandidates { @@ -139,6 +115,8 @@ pub struct LintSqlBody { pub allows_dynamic_output_star: bool, pub allows_empty_fixture_star: bool, pub relation_keys: Vec<(String, Vec>)>, + /// The relation name each dependency sentinel stands for. + pub relation_names: Vec<(String, String)>, } /// The rule selection and thresholds of one SQL lint run. diff --git a/crates/sqlbuild-rules/src/lint_text/tests.rs b/crates/sqlbuild-rules/src/lint_text/tests.rs index 57c38b8979..5fb19d6a2d 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests.rs @@ -2,6 +2,8 @@ mod fixture_formatting; #[path = "tests/test_format_settings.rs"] mod format_settings; +#[path = "tests/helpers.rs"] +mod helpers; #[path = "tests/test_lint_text.rs"] mod lint_text; #[path = "tests/test_types.rs"] diff --git a/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs new file mode 100644 index 0000000000..96a8aef7f0 --- /dev/null +++ b/crates/sqlbuild-rules/src/lint_text/tests/helpers.rs @@ -0,0 +1,25 @@ +use crate::lint_text::constants::{DIRECTIVE_FIX_REASON, DIRECTIVE_MESSAGE}; +use crate::lint_text::models::SuppressionFinding; + +const INDENT: &str = "\n "; + +/// The rejection finding for a directive on `line`, printing `rest` after the entry header. +pub(crate) fn directive_finding(line: usize, rule: &str, rest: &[&str]) -> SuppressionFinding { + let mut remediation: String = format!( + "Remove this comment, then fix the {rule} finding, or record it as a reviewed exception \ + in sqlbuild_project.toml:{INDENT}[[rules.rule_exceptions]]" + ); + for line in rest { + remediation.push_str(INDENT); + remediation.push_str(line); + } + SuppressionFinding { + file: 0, + line, + message: DIRECTIVE_MESSAGE.to_owned(), + remediation, + fault: true, + fix: None, + fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), + } +} diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index d5466360bc..db434b6463 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -1,27 +1,24 @@ use crate::lint_text::_helpers::sites::site_pattern; -use crate::lint_text::constants::{ - FORBIDDEN_FIX_REASON, FORBIDDEN_MESSAGE, FORBIDDEN_REMEDIATION, INVALID_MESSAGE, - INVALID_REMEDIATION, REASON_REQUIRED_FIX_REASON, SITE_START_BYTES, UNUSED_REMEDIATION, -}; +use crate::lint_text::constants::SITE_START_BYTES; use crate::lint_text::errors::LintTextError; -use crate::lint_text::main::apply_suppressions::apply_suppressions; use crate::lint_text::main::format_sql_bodies::format_sql_bodies; use crate::lint_text::main::interpolation_text_at::interpolation_text_at; use crate::lint_text::main::lint_sql_bodies::lint_sql_bodies; use crate::lint_text::main::neutralize_audit_parameters::neutralize_audit_parameters; use crate::lint_text::main::neutralize_context_interpolation::neutralize_context_interpolation; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; +use crate::lint_text::main::reject_inline_suppressions::reject_inline_suppressions; use crate::lint_text::main::restore_interpolation::restore_interpolation; -use crate::lint_text::models::{ - ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions, RetainedViolation, - SuppressibleViolation, SuppressionFinding, -}; +use crate::lint_text::models::{ExpansionSpan, FormatFile, LintSqlBody, LintSqlOptions}; +use crate::lint_text::tests::helpers::directive_finding; use crate::lint_text::tests::test_types::{ Finding, FormatBodiesTestCase, LintBodiesTestCase, NeutralizeTestCase, ParameterTestCase, RestoreTestCase, SitePatternTestCase, SuppressionTestCase, TokenAtTestCase, }; -const SUPPRESSED_FILE: &str = "-- sqb: ignore SQBRSQL005 because kept\n\n-- note\nselect 1\n-- sqb: ignore X because y\n-- sqb: ignore nope\nselect 2\n"; +const SUPPRESSED_FILE: &str = + "-- sqb: ignore SQBRSQL005 because kept\n\n-- note\nselect 1\n-- sqb: ignore nope\nselect 2\n"; +const CROSS_JOIN_FILE: &str = "select o.order_id\nfrom orders as o\n-- sqb: ignore SQBRSQL038 because every order meets every policy\ncross join __seed(\"order_policy\") as p\n"; #[test] fn given_site_patterns_when_text_opens_with_another_byte_then_no_site_starts_there() @@ -227,72 +224,59 @@ fn given_character_offset_when_reading_token_then_only_site_starts_have_tokens() } #[test] -fn given_suppression_directives_when_applying_then_matching_findings_are_removed() +fn given_inline_directives_when_rejecting_then_each_prints_its_configured_exception() -> Result<(), LintTextError> { let test_cases = [ SuppressionTestCase { - description: "a used, an unused and an invalid directive", + description: "a reasoned directive and one without a reason", contents: SUPPRESSED_FILE, - violations: vec![(4, "SQBRSQL005"), (7, "SQBRSQL034")], - allow: true, - expected_retained: vec![ - RetainedViolation::Existing(1), - RetainedViolation::Added(SuppressionFinding { - file: 0, - line: 6, - message: INVALID_MESSAGE.to_string(), - remediation: INVALID_REMEDIATION, - fault: false, - fix: None, - fix_unavailable_reason: Some(REASON_REQUIRED_FIX_REASON), - }), - RetainedViolation::Added(SuppressionFinding { - file: 0, - line: 5, - message: "Unused suppression for X".to_string(), - remediation: UNUSED_REMEDIATION, - fault: false, - fix: Some((57, 84)), - fix_unavailable_reason: None, - }), + expected_findings: vec![ + directive_finding( + 1, + "SQBRSQL005", + &[ + "rule = \"SQBRSQL005\"", + "path = \"models/orders.sql\"", + "reason = \"kept\"", + ], + ), + directive_finding( + 5, + "nope", + &[ + "rule = \"nope\"", + "path = \"models/orders.sql\"", + "reason = \"\"", + ], + ), ], }, SuppressionTestCase { - description: "suppressions forbidden by the project", - contents: "select 1\n-- sqb: ignore nope\n", - violations: vec![(1, "SQBRSQL005")], - allow: false, - expected_retained: vec![ - RetainedViolation::Existing(0), - RetainedViolation::Added(SuppressionFinding { - file: 0, - line: 2, - message: FORBIDDEN_MESSAGE.to_string(), - remediation: FORBIDDEN_REMEDIATION, - fault: true, - fix: None, - fix_unavailable_reason: Some(FORBIDDEN_FIX_REASON), - }), - ], + description: "a cross join directive names the joined dependency", + contents: CROSS_JOIN_FILE, + expected_findings: vec![directive_finding( + 3, + "SQBRSQL038", + &[ + "rule = \"SQBRSQL038\"", + "path = \"models/orders.sql\"", + "relation = \"order_policy\"", + "reason = \"every order meets every policy\"", + ], + )], + }, + SuppressionTestCase { + description: "a file without directives", + contents: "select 1\n-- a plain comment\n", + expected_findings: vec![], }, ]; for test_case in test_cases { - let violations: Vec = test_case - .violations - .iter() - .map(|(line, code)| SuppressibleViolation { - file: 0, - line: *line, - code: (*code).to_string(), - native: true, - }) - .collect(); - - let retained = apply_suppressions(&[test_case.contents], &violations, test_case.allow)?; + let findings = reject_inline_suppressions(&[("models/orders.sql", test_case.contents)])?; assert_eq!( - retained, test_case.expected_retained, + findings, test_case.expected_findings, "{}", test_case.description ); diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs index 3df6eae669..52aeb3db6f 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::lint_text::models::{Neutralized, RetainedViolation}; +use crate::lint_text::models::{Neutralized, SuppressionFinding}; pub(crate) struct NeutralizeTestCase { pub(crate) description: &'static str, @@ -32,9 +32,7 @@ pub(crate) struct TokenAtTestCase { pub(crate) struct SuppressionTestCase { pub(crate) description: &'static str, pub(crate) contents: &'static str, - pub(crate) violations: Vec<(usize, &'static str)>, - pub(crate) allow: bool, - pub(crate) expected_retained: Vec, + pub(crate) expected_findings: Vec, } pub(crate) struct FormatBodiesTestCase { diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index 744677e574..d8a6b21248 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -65,6 +65,12 @@ pub struct RuleException { pub rule: String, pub path: String, pub reason: String, + /// The joined relation a relation-level exception covers, for Rules that report one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relation: Option, + /// The top-level CTE that contains the covered relation, when the exception is that narrow. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cte: Option, } #[derive(Clone, Debug, Deserialize, Serialize, Eq, Ord, PartialEq, PartialOrd)] @@ -147,7 +153,6 @@ pub struct RulesConfig { pub rule_exceptions: Vec, pub graph_edge_exceptions: Vec, pub rule_ignores: Vec, - pub allow_exceptions: bool, pub select_star_allow: Vec, pub domains: Vec, pub approved_source_tokens: Vec, @@ -168,7 +173,6 @@ impl Default for RulesConfig { rule_exceptions: vec![], graph_edge_exceptions: vec![], rule_ignores: vec![], - allow_exceptions: true, select_star_allow: vec![], domains: vec![], approved_source_tokens: vec![], diff --git a/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs index 8f5b8bdc4d..2884da48cc 100644 --- a/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs +++ b/crates/sqlbuild-rules/src/rule_cache/main/lint_key.rs @@ -12,7 +12,7 @@ fn key_sets(keys: &[Vec]) -> String { sets.join("\u{1d}") } -/// A digest of the dialect, literal and ranking limits, sorted relation keys and suppressions. +/// A digest of the dialect, literal and ranking limits and sorted relation keys. #[must_use] pub fn lint_key(input: &LintKeyInput) -> String { let mut relation_keys: Vec<(String, Vec>)> = input.relation_keys.clone(); @@ -29,7 +29,6 @@ pub fn lint_key(input: &LintKeyInput) -> String { &input.max_literal_length.to_string(), &input.max_ranking_order_by.to_string(), &relations, - if input.allow_suppressions { "1" } else { "0" }, ], ) } diff --git a/crates/sqlbuild-rules/src/rule_cache/models.rs b/crates/sqlbuild-rules/src/rule_cache/models.rs index accfe26913..8a99a27b88 100644 --- a/crates/sqlbuild-rules/src/rule_cache/models.rs +++ b/crates/sqlbuild-rules/src/rule_cache/models.rs @@ -8,7 +8,6 @@ pub struct LintKeyInput { pub max_ranking_order_by: i64, /// Each relation and its key column sets, in any order. pub relation_keys: Vec<(String, Vec>)>, - pub allow_suppressions: bool, } /// One model's SQL rule inputs. diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 6f6a815944..60c0c28f60 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -33,7 +33,7 @@ pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARA pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; -pub(crate) const RULE_EXAMPLES: [(&str, &str); 25] = [ +pub(crate) const RULE_EXAMPLES: [(&str, &str); 26] = [ ( "SQBRSQL040", "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", @@ -66,6 +66,10 @@ pub(crate) const RULE_EXAMPLES: [(&str, &str); 25] = [ "SQBRPROJECT101", "domain__int_clean__entity or domain__mart_v__entity.", ), + ( + "SQBRSQL038", + "JOIN customers AS c ON o.customer_id = c.customer_id, or CROSS JOIN a provably one-row relation such as a one-row seed or an aggregate without GROUP BY.", + ), ( "SQBRPROJECT105", "A domain__int_enriched__entity model lives under intermediate/enriched.", diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs index 2a44c33eb3..9882a1a4d0 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_skills_body.rs @@ -16,6 +16,7 @@ pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { "", "Rules enforce SQL model architecture and never modifies SQL files.", "Run `sqb rules run SQBR` before handing off model changes.", + "Fix findings in SQL. A deliberate deviation is a reviewed [[rules.rule_exceptions]] entry with a reason in sqlbuild_project.toml; inline `-- sqb: ignore` comments are errors.", "", "## Active Rules", ] @@ -48,8 +49,13 @@ pub fn render_skills_body(rules: &[RuleText], config: &RulesConfig) -> String { { body.extend([String::new(), "## Scoped Deviations".into(), String::new()]); for entry in &config.rule_exceptions { + let scope: String = match (&entry.relation, &entry.cte) { + (Some(relation), Some(cte)) => format!(" (relation `{relation}` in CTE `{cte}`)"), + (Some(relation), None) => format!(" (relation `{relation}`)"), + _ => String::new(), + }; body.push(format!( - "- Exception `{}` at `{}`: {}", + "- Exception `{}` at `{}`{scope}: {}", entry.rule, entry.path, entry.reason )); } diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs index 8e851b00bf..567376e688 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs @@ -11,12 +11,13 @@ use polyglot_sql::{ use crate::sql_lint::constants::LINT_API_VERSION; use crate::sql_lint::models::{ - AdditionalFactOptions, LintDiagnostic, LintEdit, LintRequest, LintResponse, LintRuleMetadata, - QueryFacts, + AdditionalFactOptions, CrossJoinRequest, LintDiagnostic, LintEdit, LintRequest, LintResponse, + LintRuleMetadata, QueryFacts, }; use crate::sql_lint::_helpers::additional::collect_additional_facts; use crate::sql_lint::_helpers::inline_relations::collect_inline_query_relation_spans; +use crate::sql_lint::_helpers::join_predicates::cross_join_diagnostics; use crate::sql_lint::_helpers::terminal_shape::collect_terminal_shape_facts; const NULL_COMPARISON: LintRuleMetadata = LintRuleMetadata { @@ -192,7 +193,7 @@ const NESTED_ELSE_CASE: LintRuleMetadata = LintRuleMetadata { const UNUSED_JOINED_RELATION: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL032", message: "Joined relation contributes no referenced values", - remediation: "Use the joined relation, remove the join, or suppress this intentional cardinality filter.", + remediation: "Use the joined relation or remove the join. When the join intentionally filters rows, record it as a reviewed [[rules.rule_exceptions]] entry with a reason in sqlbuild_project.toml.", }; const COMMENT_ATTACHMENT: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL033", @@ -222,7 +223,7 @@ const RECURSIVE_CTE: LintRuleMetadata = LintRuleMetadata { const CROSS_JOIN: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL038", message: "CROSS JOIN creates an explicit cartesian product", - remediation: "Use a keyed JOIN, or suppress this rule with a reason when the cartesian product is intentional.", + remediation: "Use a keyed JOIN, or make one side provably at most one row (LIMIT 1, an aggregate without GROUP BY, or a one-row seed). When the cartesian product is intentional, record it as a reviewed [[rules.rule_exceptions]] entry with rule, path, relation and reason in sqlbuild_project.toml.", }; const INLINE_QUERY_RELATION: LintRuleMetadata = LintRuleMetadata { code: "SQBRSQL039", @@ -420,6 +421,16 @@ pub(crate) fn lint(request: LintRequest) -> Result { enabled: &enabled, }; let mut diagnostics = diagnostics(&context); + if enabled.contains(CROSS_JOIN.code) { + let cross_joins = CrossJoinRequest { + statements: &statements, + tokens: &tokens, + cross_spans: &facts.cross_joins, + relation_keys: &request.relation_keys, + relation_names: &request.relation_names, + }; + diagnostics.extend(cross_join_diagnostics(&cross_joins, &CROSS_JOIN)); + } let depths = token_depths(&tokens); if enabled.contains(JOIN_EXPRESSION.code) { diagnostics.extend(diagnostics_for_spans( @@ -1713,13 +1724,6 @@ fn rule_migration_diagnostics(context: &DiagnosticContext<'_>) -> Vec bool { _ => false, } } + +/// A `rule` diagnostic for every cross join that can multiply rows, naming its relation and CTE. +pub(super) fn cross_join_diagnostics( + request: &CrossJoinRequest<'_>, + rule: &LintRuleMetadata, +) -> Vec { + let one_row_relations: HashSet = request + .relation_keys + .iter() + .filter(|(_, keys)| keys.iter().any(Vec::is_empty)) + .map(|(name, _)| name.to_ascii_lowercase()) + .collect(); + let relation_names: HashMap = request + .relation_names + .iter() + .map(|(sentinel, name)| (sentinel.to_ascii_lowercase(), name.clone())) + .collect(); + let inputs = OneRowInputs { + one_row_relations: &one_row_relations, + relation_names: &relation_names, + }; + multiplying_cross_joins( + request.statements, + request.tokens, + request.cross_spans, + &inputs, + ) + .into_iter() + .map(|finding| LintDiagnostic { + code: rule.code, + message: finding + .relation + .as_deref() + .map_or(Cow::Borrowed(rule.message), |relation| { + Cow::Owned(cross_join_message(relation, finding.cte.as_deref())) + }), + remediation: Cow::Borrowed(rule.remediation), + start: finding.span.start, + end: finding.span.end, + fix_unavailable_reason: Some(CROSS_JOIN_FIX_REASON), + fix: None, + }) + .collect() +} + +/// The finding message naming the joined relation and, inside a CTE, that CTE. +fn cross_join_message(relation: &str, cte: Option<&str>) -> String { + match cte { + Some(cte) => { + format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_CTE_MARKER}{cte}{CROSS_JOIN_SUFFIX}") + } + None => format!("{CROSS_JOIN_PREFIX}{relation}{CROSS_JOIN_SUFFIX}"), + } +} + +/// One explicit cross join found in the query tree. +struct CrossJoinTarget { + /// Character offsets where the joined relation may start in the token stream. + anchors: Vec, + /// Whether both joined sides may hold more than one row. + multiplies: bool, + /// The joined relation's name as authored, with dependency sentinels resolved. + relation: Option, + /// The top-level CTE whose query contains the join. + cte: Option, +} + +/// The cross join reported at one `CROSS` keyword. +struct CrossJoinFinding { + span: Span, + relation: Option, + cte: Option, +} + +/// What the analysis knows about relations named in the query. +struct OneRowInputs<'a> { + /// Lowercased dependency sentinels whose relation holds at most one row. + one_row_relations: &'a HashSet, + /// Lowercased dependency sentinels mapped to the relation names they stand for. + relation_names: &'a HashMap, +} + +struct Analysis<'a> { + inputs: &'a OneRowInputs<'a>, + ctes: HashMap, + targets: Vec, +} + +/// Cross joins at `cross_spans` that can multiply rows, each with its relation and CTE. +fn multiplying_cross_joins( + statements: &[Expression], + tokens: &[Token], + cross_spans: &[Span], + inputs: &OneRowInputs<'_>, +) -> Vec { + if cross_spans.is_empty() { + return Vec::new(); + } + let mut analysis = Analysis { + inputs, + ctes: HashMap::new(), + targets: Vec::new(), + }; + for statement in statements { + analysis.record_ctes(statement); + } + for statement in statements { + analysis.visit(statement, None); + } + let significant: Vec = (0..tokens.len()) + .filter(|&index| { + !matches!( + tokens[index].token_type, + TokenType::Space + | TokenType::Break + | TokenType::LineComment + | TokenType::BlockComment + ) + }) + .collect(); + let mut findings: Vec = Vec::new(); + for span in cross_spans { + let candidates: Vec = relation_anchor_candidates(tokens, &significant, *span); + let target: Option<&CrossJoinTarget> = analysis + .targets + .iter() + .find(|target| anchored_at(target, &candidates)); + match target { + Some(target) if !target.multiplies => {} + Some(target) => findings.push(CrossJoinFinding { + span: *span, + relation: target.relation.clone(), + cte: target.cte.clone(), + }), + None => findings.push(CrossJoinFinding { + span: *span, + relation: None, + cte: None, + }), + } + } + findings +} + +impl Analysis<'_> { + /// Record whether every CTE holds at most one row, in definition order. + fn record_ctes(&mut self, expression: &Expression) { + for node in expression.dfs() { + let Expression::Select(select) = node else { + continue; + }; + let Some(with) = &select.with else { + continue; + }; + for cte in &with.ctes { + let one_row: bool = !with.recursive && self.at_most_one_row(&cte.this); + let name: String = cte.alias.name.to_ascii_lowercase(); + let known: bool = self.ctes.get(&name).copied().unwrap_or(one_row); + self.ctes.insert(name, known && one_row); + } + } + } + + fn visit(&mut self, expression: &Expression, cte: Option<&str>) { + let Expression::Select(select) = expression else { + for child in expression.children() { + self.visit(child, cte); + } + return; + }; + let mut bodies: Vec<&Expression> = Vec::new(); + if let Some(with) = &select.with { + for definition in &with.ctes { + bodies.push(&definition.this); + self.visit(&definition.this, Some(&definition.alias.name)); + } + } + self.record_joins(select, cte); + for child in expression.children() { + if bodies.iter().any(|body| std::ptr::eq(*body, child)) { + continue; + } + self.visit(child, cte); + } + } + + fn record_joins(&mut self, select: &Select, cte: Option<&str>) { + let single_from: Option<&Expression> = select + .from + .as_ref() + .filter(|from| from.expressions.len() == 1) + .and_then(|from| from.expressions.first()); + for (index, join) in select.joins.iter().enumerate() { + if join.kind != JoinKind::Cross { + continue; + } + let left_one_row: bool = + index == 0 && single_from.is_some_and(|from| self.relation_one_row(from)); + let multiplies: bool = !(left_one_row || self.relation_one_row(&join.this)); + self.targets.push(CrossJoinTarget { + anchors: relation_anchors(&join.this), + multiplies, + relation: self.relation_name(&join.this), + cte: cte.map(str::to_owned), + }); + } + } + + fn relation_name(&self, relation: &Expression) -> Option { + match relation { + Expression::Table(table) => Some(self.table_name(table)), + Expression::Subquery(subquery) => { + subquery.alias.as_ref().map(|alias| alias.name.clone()) + } + Expression::Alias(alias) => Some(alias.alias.name.clone()), + _ => None, + } + } + + fn table_name(&self, table: &TableRef) -> String { + if table.schema.is_none() && table.catalog.is_none() { + let lowered: String = table.name.name.to_ascii_lowercase(); + if let Some(name) = self.inputs.relation_names.get(&lowered) { + return name.clone(); + } + } + [&table.catalog, &table.schema] + .into_iter() + .flatten() + .map(|part| part.name.as_str()) + .chain([table.name.name.as_str()]) + .collect::>() + .join(".") + } + + /// Whether a query expression returns at most one row. + fn at_most_one_row(&self, expression: &Expression) -> bool { + match expression { + Expression::Select(select) => self.select_one_row(select), + Expression::Subquery(subquery) => self.subquery_one_row(subquery), + Expression::Paren(paren) => self.at_most_one_row(&paren.this), + Expression::Values(values) => values.expressions.len() <= 1, + _ => false, + } + } + + fn subquery_one_row(&self, subquery: &Subquery) -> bool { + !subquery.lateral + && (subquery + .limit + .as_ref() + .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)) + || self.at_most_one_row(&subquery.this)) + } + + fn select_one_row(&self, select: &Select) -> bool { + if select.limit_by.is_some() || select.connect.is_some() || !select.lateral_views.is_empty() + { + return false; + } + if limited_to_one_row(select) { + return true; + } + if select.expressions.iter().any(generates_rows) { + return false; + } + if select.group_by.is_none() && select.expressions.iter().any(aggregates) { + return true; + } + if select + .qualify + .as_ref() + .is_some_and(|qualify| first_unpartitioned_row(&qualify.this, select)) + { + return true; + } + if select.group_by.is_some() || !select.joins.is_empty() { + return false; + } + match &select.from { + None => true, + Some(from) => match from.expressions.as_slice() { + [relation] => self.relation_one_row(relation), + _ => false, + }, + } + } + + /// Whether a relation in FROM or JOIN holds at most one row. + fn relation_one_row(&self, relation: &Expression) -> bool { + match relation { + Expression::Table(table) => { + if table.schema.is_some() + || table.catalog.is_some() + || table.identifier_func.is_some() + || table.changes.is_some() + { + return false; + } + let name: String = table.name.name.to_ascii_lowercase(); + match self.ctes.get(&name) { + Some(one_row) => *one_row, + None => self.inputs.one_row_relations.contains(&name), + } + } + Expression::Subquery(subquery) => self.subquery_one_row(subquery), + Expression::Paren(paren) => self.relation_one_row(&paren.this), + Expression::Values(values) => values.expressions.len() <= 1, + _ => false, + } + } +} + +fn limited_to_one_row(select: &Select) -> bool { + let limited: bool = select + .limit + .as_ref() + .is_some_and(|limit| !limit.percent && at_most_one(&limit.this)); + let fetched: bool = select.fetch.as_ref().is_some_and(|fetch| { + !fetch.percent && !fetch.with_ties && fetch.count.as_ref().is_some_and(at_most_one) + }); + let top: bool = select + .top + .as_ref() + .is_some_and(|top| !top.percent && !top.with_ties && at_most_one(&top.this)); + limited || fetched || top +} + +fn at_most_one(expression: &Expression) -> bool { + match expression { + Expression::Literal(literal) => match literal.as_ref() { + Literal::Number(text) => text.parse::().is_ok_and(|count| count <= 1), + _ => false, + }, + Expression::Paren(paren) => at_most_one(&paren.this), + _ => false, + } +} + +/// Whether a projected expression can return several rows per input row. +fn generates_rows(expression: &Expression) -> bool { + expression.dfs().any(|node| match node { + Expression::Unnest(_) + | Expression::Explode(_) + | Expression::ExplodeOuter(_) + | Expression::GenerateSeries(_) => true, + Expression::Function(function) => is_row_generator(&function.name), + _ => false, + }) +} + +fn is_row_generator(name: &str) -> bool { + let name: &str = name.rsplit('.').next().unwrap_or_default(); + ROW_GENERATOR_FUNCTIONS + .iter() + .any(|generator| generator.eq_ignore_ascii_case(name)) +} + +fn anchored_at(target: &CrossJoinTarget, candidates: &[usize]) -> bool { + target + .anchors + .iter() + .any(|anchor| candidates.contains(anchor)) +} + +/// Whether an expression aggregates outside any window function or subquery. +fn aggregates(expression: &Expression) -> bool { + if polyglot_sql::traversal::is_aggregate(expression) { + return true; + } + match expression { + Expression::WindowFunction(_) | Expression::Subquery(_) | Expression::Select(_) => false, + _ => expression.children().into_iter().any(aggregates), + } +} + +/// `QUALIFY ROW_NUMBER() OVER (ORDER BY ...) = 1` without a partition, directly or by alias. +fn first_unpartitioned_row(condition: &Expression, select: &Select) -> bool { + match condition { + Expression::And(operation) => { + first_unpartitioned_row(&operation.left, select) + || first_unpartitioned_row(&operation.right, select) + } + Expression::Paren(paren) => first_unpartitioned_row(&paren.this, select), + Expression::Eq(operation) => { + let compared: &Expression = match (&operation.left, &operation.right) { + (other, Expression::Literal(literal)) | (Expression::Literal(literal), other) + if matches!(literal.as_ref(), Literal::Number(text) if text == FIRST_ROW) => + { + other + } + _ => return false, + }; + match compared { + Expression::WindowFunction(_) => unpartitioned_row_number(compared), + Expression::Column(column) if column.table.is_none() => select + .expressions + .iter() + .any(|projection| match projection { + Expression::Alias(alias) => { + alias.alias.name.eq_ignore_ascii_case(&column.name.name) + && unpartitioned_row_number(&alias.this) + } + _ => false, + }), + _ => false, + } + } + _ => false, + } +} + +fn unpartitioned_row_number(expression: &Expression) -> bool { + matches!( + expression, + Expression::WindowFunction(window) + if matches!(window.this, Expression::RowNumber(_)) + && window.over.partition_by.is_empty() + ) +} + +/// Where a joined relation can start: its first name part, or a derived table's alias. +fn relation_anchors(relation: &Expression) -> Vec { + match relation { + Expression::Table(table) => { + let mut anchors: Vec = Vec::new(); + for identifier in [&table.catalog, &table.schema].into_iter().flatten() { + anchors.extend(identifier.span.map(|span| span.start)); + } + anchors.extend(table.name.span.map(|span| span.start)); + anchors.extend(table.span.map(|span| span.start)); + anchors + } + Expression::Subquery(subquery) => subquery + .alias + .as_ref() + .and_then(|alias| alias.span) + .map(|span| span.start) + .into_iter() + .collect(), + Expression::Alias(alias) => alias + .alias + .span + .map(|span| span.start) + .into_iter() + .collect(), + _ => Vec::new(), + } +} + +/// Where the relation after `CROSS JOIN` at `cross` starts, and a derived table's alias. +fn relation_anchor_candidates(tokens: &[Token], significant: &[usize], cross: Span) -> Vec { + let Some(position) = significant + .iter() + .position(|&index| tokens[index].span.start == cross.start) + else { + return Vec::new(); + }; + let Some(&first) = significant.get(position + 2) else { + return Vec::new(); + }; + let mut candidates: Vec = vec![tokens[first].span.start]; + if tokens[first].token_type == TokenType::LParen { + let mut depth: usize = 0; + let mut cursor: usize = position + 2; + while let Some(&index) = significant.get(cursor) { + match tokens[index].token_type { + TokenType::LParen => depth += 1, + TokenType::RParen => { + depth = depth.saturating_sub(1); + if depth == 0 { + break; + } + } + _ => {} + } + cursor += 1; + } + let mut alias: usize = cursor + 1; + if significant + .get(alias) + .is_some_and(|&index| tokens[index].token_type == TokenType::As) + { + alias += 1; + } + if let Some(&index) = significant.get(alias) { + candidates.push(tokens[index].span.start); + } + } + candidates +} diff --git a/crates/sqlbuild-rules/src/sql_lint/constants.rs b/crates/sqlbuild-rules/src/sql_lint/constants.rs index ebb457fe57..f28e9feab8 100644 --- a/crates/sqlbuild-rules/src/sql_lint/constants.rs +++ b/crates/sqlbuild-rules/src/sql_lint/constants.rs @@ -50,3 +50,7 @@ pub(crate) const RELATION_MODIFIER_KEYWORDS: [&str; 43] = [ "WHERE", "WINDOW", ]; +/// The parts of a `SQBRSQL038` message that names its joined relation and CTE. +pub(crate) const CROSS_JOIN_PREFIX: &str = "CROSS JOIN with `"; +pub(crate) const CROSS_JOIN_CTE_MARKER: &str = "` in CTE `"; +pub(crate) const CROSS_JOIN_SUFFIX: &str = "` creates an explicit cartesian product"; diff --git a/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs b/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs new file mode 100644 index 0000000000..c9a03220dd --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/cross_join_subject.rs @@ -0,0 +1,15 @@ +//! The relation and CTE a `SQBRSQL038` finding names, for relation-level Rule exceptions. + +use crate::sql_lint::constants::{CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PREFIX, CROSS_JOIN_SUFFIX}; + +/// The `(relation, cte)` a cross join finding message names, `None` for the generic message. +#[must_use] +pub fn cross_join_subject(message: &str) -> Option<(String, Option)> { + let named: &str = message + .strip_prefix(CROSS_JOIN_PREFIX)? + .strip_suffix(CROSS_JOIN_SUFFIX)?; + match named.split_once(CROSS_JOIN_CTE_MARKER) { + Some((relation, cte)) => Some((relation.to_owned(), Some(cte.to_owned()))), + None => Some((named.to_owned(), None)), + } +} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs index f14a92656f..59bc99e94c 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs @@ -2,6 +2,7 @@ pub mod backtick_identifiers; pub mod batch_engine; pub mod batch_formatter; pub mod catalogue; +pub mod cross_join_subject; pub mod engine; pub mod format_requests; pub mod formatter; diff --git a/crates/sqlbuild-rules/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs index 9371ecd79c..ea4fbc58b2 100644 --- a/crates/sqlbuild-rules/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules/src/sql_lint/models.rs @@ -1,7 +1,8 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; -use polyglot_sql::tokens::Span; +use polyglot_sql::Expression; +use polyglot_sql::tokens::{Span, Token}; use serde::{Deserialize, Serialize}; #[derive(Debug, Default)] @@ -80,6 +81,8 @@ pub(crate) struct LintRequest { pub max_ranking_order_by: Option, #[serde(default)] pub relation_keys: HashMap>>, + #[serde(default)] + pub relation_names: HashMap, } #[derive(Debug)] @@ -100,7 +103,7 @@ pub(crate) struct LintResponse { #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub(crate) struct LintDiagnostic { pub code: &'static str, - pub message: &'static str, + pub message: Cow<'static, str>, pub remediation: Cow<'static, str>, pub start: usize, pub end: usize, @@ -147,3 +150,15 @@ pub(crate) struct FormatResponse { #[serde(skip_serializing_if = "Option::is_none")] pub reason: Option, } + +/// One lint request's explicit cross joins, with what is known about the relations they read. +#[derive(Debug)] +pub(crate) struct CrossJoinRequest<'a> { + pub(crate) statements: &'a [Expression], + pub(crate) tokens: &'a [Token], + pub(crate) cross_spans: &'a [Span], + /// Declared keys by dependency sentinel; the empty key proves at most one row. + pub(crate) relation_keys: &'a HashMap>>, + /// The relation name each dependency sentinel stands for. + pub(crate) relation_names: &'a HashMap, +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests.rs b/crates/sqlbuild-rules/src/sql_lint/tests.rs index c46d727733..5e5d936598 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests.rs @@ -4,6 +4,8 @@ mod authored_tokens; mod batch_lint; #[path = "tests/test_comment_formatter.rs"] mod comment_formatter; +#[path = "tests/test_cross_joins.rs"] +mod cross_joins; #[path = "tests/test_empty_fixture_lint.rs"] mod empty_fixture_lint; #[path = "tests/test_format_corpus.rs"] diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs index 57aabf1ed2..614ae828c8 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs @@ -1,16 +1,20 @@ use std::str::FromStr; use polyglot_sql::{Dialect, DialectType}; -use serde_json::{Value, json}; +use serde_json::{Map, Value, json}; use crate::sql_lint::main::batch_formatter::format_batch_json; use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::main::formatter::format_json; +use crate::sql_lint::tests::test_types::CrossJoinTestCase; use sqlbuild_analysis::sql_tokens::constants::CALL_SYNTAX_FUNCTIONS; use sqlbuild_analysis::sql_tokens::main::builtin_function_names::builtin_function_names; use sqlbuild_analysis::sql_tokens::main::canonical_tokens::canonical_tokens; use sqlbuild_analysis::sql_tokens::main::query_fingerprint::query_fingerprint; +/// The dependency sentinel the cross join cases name `order_policy`. +pub(crate) const POLICY_SENTINEL: &str = "__sqb_lint_7__"; + pub(crate) fn nested_function_sql(depth: usize) -> String { format!("SELECT {}1{}", "F(".repeat(depth), ")".repeat(depth)) } @@ -253,3 +257,34 @@ fn is_tight_call(name: &str, sql: &str, arguments: &str) -> bool { .to_ascii_uppercase() .contains(&format!("{} (", name.to_ascii_uppercase())) } + +/// The `SQBRSQL038` messages for one cross join case. +pub(crate) fn cross_join_messages(test_case: &CrossJoinTestCase) -> Result, String> { + let keys: Map = test_case + .one_row_relations + .iter() + .map(|name| ((*name).to_owned(), json!([[]]))) + .collect(); + let request = json!({ + "version": 1, + "sql": test_case.sql, + "dialect": test_case.dialect, + "enabled_rules": ["SQBRSQL038"], + "relation_keys": keys, + "relation_names": { POLICY_SENTINEL: "order_policy" }, + }); + let response: Value = serde_json::from_str(&lint_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(response["diagnostics"] + .as_array() + .cloned() + .unwrap_or_default() + .iter() + .map(|diagnostic| { + diagnostic["message"] + .as_str() + .unwrap_or_default() + .to_owned() + }) + .collect()) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs new file mode 100644 index 0000000000..84be96dad0 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_cross_joins.rs @@ -0,0 +1,220 @@ +use crate::sql_lint::main::cross_join_subject::cross_join_subject; +use crate::sql_lint::tests::helpers::{POLICY_SENTINEL, cross_join_messages}; +use crate::sql_lint::tests::test_types::{CrossJoinSubjectTestCase, CrossJoinTestCase}; + +const GENERIC_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product"; + +#[test] +fn given_cross_joins_when_one_side_is_provably_one_row_then_only_multiplying_joins_are_reported() +-> Result<(), String> { + let test_cases = [ + CrossJoinTestCase { + description: "two unknown relations multiply rows", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a one-row seed on the right", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqb_lint_7__ AS p", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "a seed with several rows names its relation", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN __sqb_lint_7__ AS p", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `order_policy` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a one-row seed on the left of the first join", + sql: "SELECT o.order_id FROM __sqb_lint_7__ AS p CROSS JOIN orders AS o", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "an aggregate without GROUP BY carried through a filtering CTE", + sql: "WITH totals AS (SELECT COUNT(*) AS order_count FROM orders), \ + kept AS (SELECT order_count FROM totals WHERE order_count > 0) \ + SELECT o.order_id, k.order_count FROM orders AS o CROSS JOIN kept AS k", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "an aggregate with GROUP BY can return many rows", + sql: "WITH totals AS (SELECT customer_id, COUNT(*) AS order_count FROM orders GROUP BY customer_id) \ + SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a windowed count is not an aggregate", + sql: "WITH totals AS (SELECT COUNT(*) OVER () AS order_count FROM orders) \ + SELECT o.order_id FROM orders AS o CROSS JOIN totals AS t", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `totals` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "LIMIT 1", + sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "LIMIT 2 can return two rows", + sql: "WITH latest AS (SELECT order_id FROM orders ORDER BY order_id DESC LIMIT 2) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN latest AS l", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `latest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "one-row VALUES", + sql: "WITH settings AS (SELECT * FROM (VALUES (1, 'EUR')) AS v(version, currency)) \ + SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "two-row VALUES", + sql: "WITH settings AS (SELECT * FROM (VALUES (1), (2)) AS v(version)) \ + SELECT o.order_id FROM orders AS o CROSS JOIN settings AS s", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `settings` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "QUALIFY ROW_NUMBER() = 1 without a partition", + sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (ORDER BY order_id DESC) = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "snowflake", + one_row_relations: &[], + expected_messages: &[], + }, + CrossJoinTestCase { + description: "QUALIFY ROW_NUMBER() = 1 with a partition keeps one row per partition", + sql: "WITH newest AS (SELECT order_id FROM orders QUALIFY ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY order_id DESC) = 1) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN newest AS n", + dialect: "snowflake", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `newest` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "a join inside the carrying CTE stops the proof", + sql: "WITH totals AS (SELECT COUNT(*) AS n FROM orders), \ + joined AS (SELECT t.n FROM totals AS t INNER JOIN customers AS c ON c.n = t.n) \ + SELECT o.order_id FROM orders AS o CROSS JOIN joined AS j", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &["CROSS JOIN with `joined` creates an explicit cartesian product"], + }, + CrossJoinTestCase { + description: "UNNEST in a carrying CTE stops the proof", + sql: "WITH totals AS (SELECT LIST(order_id) AS ids FROM orders), \ + expanded AS (SELECT UNNEST(ids) AS order_id FROM totals) \ + SELECT c.customer_id FROM customers AS c CROSS JOIN expanded AS e", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `expanded` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a set operation stops the proof", + sql: "WITH both_totals AS (SELECT COUNT(*) AS n FROM orders UNION ALL SELECT COUNT(*) AS n FROM customers) \ + SELECT o.order_id FROM orders AS o CROSS JOIN both_totals AS b", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `both_totals` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "the finding names the CTE that contains the join", + sql: "WITH checks AS (SELECT o.order_id FROM orders AS o CROSS JOIN customers AS c) \ + SELECT order_id FROM checks", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "CROSS JOIN with `customers` in CTE `checks` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "a later join still multiplies the rows before it", + sql: "SELECT o.order_id FROM __sqb_lint_7__ AS p CROSS JOIN orders AS o CROSS JOIN customers AS c", + dialect: "duckdb", + one_row_relations: &[POLICY_SENTINEL], + expected_messages: &[ + "CROSS JOIN with `customers` creates an explicit cartesian product", + ], + }, + CrossJoinTestCase { + description: "an unrecognised relation keeps the generic finding", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[GENERIC_MESSAGE], + }, + ]; + + for test_case in test_cases { + let messages: Vec = cross_join_messages(&test_case)?; + + assert_eq!( + messages, test_case.expected_messages, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_cross_join_messages_when_reading_their_subject_then_relation_and_cte_are_named() { + let test_cases = [ + CrossJoinSubjectTestCase { + description: "a top-level join", + message: "CROSS JOIN with `order_policy` creates an explicit cartesian product", + expected_subject: Some(("order_policy", None)), + }, + CrossJoinSubjectTestCase { + description: "a join inside a CTE", + message: "CROSS JOIN with `order_policy` in CTE `order_checks` creates an explicit cartesian product", + expected_subject: Some(("order_policy", Some("order_checks"))), + }, + CrossJoinSubjectTestCase { + description: "the generic message names nothing", + message: GENERIC_MESSAGE, + expected_subject: None, + }, + ]; + + for test_case in test_cases { + let subject = cross_join_subject(test_case.message); + + assert_eq!( + subject + .as_ref() + .map(|(relation, cte)| (relation.as_str(), cte.as_deref())), + test_case.expected_subject, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs index 0235c87b20..d036ebdecc 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs @@ -150,3 +150,17 @@ pub(crate) struct FunctionDepthFailureTestCase { pub expected_code: &'static str, pub expected_limit: &'static str, } + +pub(crate) struct CrossJoinTestCase { + pub description: &'static str, + pub sql: &'static str, + pub dialect: &'static str, + pub one_row_relations: &'static [&'static str], + pub expected_messages: &'static [&'static str], +} + +pub(crate) struct CrossJoinSubjectTestCase { + pub description: &'static str, + pub message: &'static str, + pub expected_subject: Option<(&'static str, Option<&'static str>)>, +} diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs index c60f04a74a..c029f0779c 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/cte_columns.rs @@ -421,7 +421,7 @@ fn unused_outputs( }); found.push(LintDiagnostic { code: UNUSED_CTE_OUTPUT.code, - message: UNUSED_CTE_OUTPUT.message, + message: Cow::Borrowed(UNUSED_CTE_OUTPUT.message), remediation: Cow::Borrowed(if select.distinct { DISTINCT_UNUSED_REMEDIATION } else if select diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs index 38228dad45..6e95858613 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/json_parses.rs @@ -95,7 +95,7 @@ fn scope_diagnostics( let (start, end) = function_span(tokens, second); found.push(LintDiagnostic { code: REPEATED_JSON_PARSE.code, - message: REPEATED_JSON_PARSE.message, + message: Cow::Borrowed(REPEATED_JSON_PARSE.message), remediation: Cow::Owned(remediation(second, group.len(), quote)), start, end, diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs index 667db34102..d052eb7085 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/literals.rs @@ -23,7 +23,7 @@ pub(crate) fn diagnostics(request: &QualityRequest<'_>) -> Vec { let length: usize = token.text.chars().count(); (length > request.max_literal_length).then(|| LintDiagnostic { code: LONG_LITERAL.code, - message: LONG_LITERAL.message, + message: Cow::Borrowed(LONG_LITERAL.message), remediation: Cow::Owned(remediation(length, request.max_literal_length)), start: token.span.start, end: token.span.end, diff --git a/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs b/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs index eceee81cfa..fb4e463d6a 100644 --- a/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs +++ b/crates/sqlbuild-rules/src/sql_quality/_helpers/ranking.rs @@ -84,7 +84,7 @@ pub(crate) fn diagnostics( if exceeds { found.push(LintDiagnostic { code: RANKING_SORT_CAP.code, - message: RANKING_SORT_CAP.message, + message: Cow::Borrowed(RANKING_SORT_CAP.message), remediation: Cow::Owned(sort_cap_remediation( order.len(), request.max_ranking_order_by, @@ -98,7 +98,7 @@ pub(crate) fn diagnostics( if unproven { found.push(LintDiagnostic { code: UNSTABLE_ROW_NUMBER_CODE, - message: UNPROVEN_RANKING_MESSAGE, + message: Cow::Borrowed(UNPROVEN_RANKING_MESSAGE), remediation: Cow::Borrowed(UNPROVEN_RANKING_REMEDIATION), start, end, diff --git a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs b/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs index 8527f39658..f99921268b 100644 --- a/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs +++ b/crates/sqlbuild-rules/src/sql_quality/main/declared_relation_keys.rs @@ -4,7 +4,7 @@ use crate::sql_quality::constants::{RELATION_KIND_REF, RELATION_KIND_SEED}; use crate::sql_quality::declared_keys::{non_null_unique_columns, trusted_unique_key}; use crate::sql_quality::models::DeclaredRelation; -/// Keys that prove one row per key, by `kind:name` identity; seeds are also keyed as refs. +/// Keys proving one row per key by `kind:name`; seeds are also refs, one-row relations key `[]`. pub fn declared_relation_keys( relations: &[DeclaredRelation], ) -> BTreeMap>> { @@ -14,6 +14,9 @@ pub fn declared_relation_keys( for column in non_null_unique_columns(relation) { found.push(vec![column]); } + if relation.at_most_one_row { + found.push(Vec::new()); + } if found.is_empty() { continue; } diff --git a/crates/sqlbuild-rules/src/sql_quality/models.rs b/crates/sqlbuild-rules/src/sql_quality/models.rs index d57f1aee24..7b91480b64 100644 --- a/crates/sqlbuild-rules/src/sql_quality/models.rs +++ b/crates/sqlbuild-rules/src/sql_quality/models.rs @@ -32,6 +32,8 @@ pub struct DeclaredRelation { pub columns: Vec, /// Table-level audits as `(definition name, its string column argument)`. pub audits: Vec<(String, Option)>, + /// Whether the relation is known to hold at most one row, which the empty key proves. + pub at_most_one_row: bool, } /// One declared column: its name, its audit definition names and whether it is declared non-null. diff --git a/fensu.toml b/fensu.toml index 154893a523..4d53087cd0 100644 --- a/fensu.toml +++ b/fensu.toml @@ -771,7 +771,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_sql.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", - "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/format_command/selection.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/freshness/freshness_command.rs=17", diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 19bf31f222..d1a3b740eb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3966,10 +3966,10 @@ def format_lint_sql_bodies( def lint_native_sql_bodies( bodies: list[LintBody], contents_by_path: dict[Path, str], config: LintConfig ) -> list[LintViolation]: ... -def apply_lint_suppressions( +def reject_lint_suppressions( violations: list[LintViolation], contents_by_path: dict[Path, str], - allow_suppressions: bool, + project_dir: Path, ) -> list[LintViolation]: ... def load_rules_config(*, project_dir: Path) -> RulesConfig: ... def valid_rules_config(*, project_dir: Path) -> RulesConfig | None: ... @@ -10456,7 +10456,6 @@ class LintConfig(NativeRecord): header_rules_enabled: bool max_literal_length: int max_ranking_order_by: int - allow_suppressions: bool relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] def __init__( self, @@ -10469,7 +10468,6 @@ class LintConfig(NativeRecord): header_rules_enabled: bool = ..., max_literal_length: int = ..., max_ranking_order_by: int = ..., - allow_suppressions: bool = ..., relation_keys: Mapping[str, tuple[tuple[str, ...], ...]] = ..., ) -> None: ... @@ -12572,11 +12570,15 @@ class RuleExemption(NativeRecord): rule: str path: str reason: str + relation: str | None + cte: str | None def __init__( self, rule: str, path: str, reason: str, + relation: str | None = ..., + cte: str | None = ..., ) -> None: ... class GraphEdgeExclusion(NativeRecord): @@ -17878,7 +17880,6 @@ class RulesConfig(NativeRecord): rule_exceptions: tuple[RuleExemption, ...] graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] rule_ignores: tuple[RuleIgnore, ...] - allow_exceptions: bool select_star_allow: tuple[SelectStarAllow, ...] domains: tuple[str, ...] approved_source_tokens: tuple[str, ...] @@ -17896,7 +17897,6 @@ class RulesConfig(NativeRecord): rule_exceptions: tuple[RuleExemption, ...] = ..., graph_edge_exceptions: tuple[GraphEdgeExclusion, ...] = ..., rule_ignores: tuple[RuleIgnore, ...] = ..., - allow_exceptions: bool = ..., select_star_allow: tuple[SelectStarAllow, ...] = ..., domains: tuple[str, ...] = ..., approved_source_tokens: tuple[str, ...] = ..., diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 40e2f72c12..69609a0e34 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -245,3 +245,26 @@ class OneModelCustomRuleEditCase: expected_warm_counts: tuple[int, int] expected_edit_counts: tuple[int, int] expected_edit_exit_code: int + + +@dataclass(frozen=True) +class OneRowCrossJoinCase: + """One model whose CROSS JOIN reads a relation that may hold at most one row.""" + + description: str + seed_rows: tuple[str, ...] + model_sql: str + expected_messages: tuple[str, ...] + expected_exit: int + + +@dataclass(frozen=True) +class RuleExceptionConfigCase: + """One `sqlbuild_project.toml` exception setup compiled against two cross joins.""" + + description: str + rules_toml: str + model_sql: str + expected_exit: int + expected_messages: tuple[str, ...] = () + expected_fragments: tuple[str, ...] = () diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index 8c7a47a4be..5e480c12ff 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -626,3 +626,52 @@ def write_bulk_rules_cache(*, root: Path, file_name: str, text: str) -> None: _ = rules_bulk_cache_path(project_dir=root, file_name=file_name).write_text( text, encoding="utf-8" ) + + +def compile_diagnostics(result: subprocess.CompletedProcess[str]) -> list[dict[str, Any]]: + """Return every diagnostic of one `sqb compile --json` result.""" + + return list(json.loads(result.stdout)["diagnostics"]) + + +def diagnostic_messages(result: subprocess.CompletedProcess[str], code: str) -> tuple[str, ...]: + """Return the messages of one code's diagnostics, in report order.""" + + return tuple( + str(item["message"]) + for item in filter(lambda item: item["code"] == code, compile_diagnostics(result)) + ) + + +def diagnostic_text(result: subprocess.CompletedProcess[str]) -> str: + """Return every diagnostic message and help, followed by stderr.""" + + lines: list[str] = [ + f"{item['message']}\n{item.get('help', '')}" for item in compile_diagnostics(result) + ] + return "\n".join((*lines, result.stderr)) + + +def write_one_row_cross_join_project( + *, project_dir: Path, toml: str, seed_rows: tuple[str, ...], files: tuple[tuple[str, str], ...] +) -> Path: + """Write a project with the `order_policy` seed and model files; return the seed CSV.""" + + (project_dir / "sqlbuild_project.toml").write_text(toml, encoding="utf-8") + for relative, text in files: + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + seed: Path = project_dir / "seeds" / "order_policy.csv" + seed.write_text("max_amount\n" + "".join(f"{row}\n" for row in seed_rows), encoding="utf-8") + return seed + + +def write_model_files(*, project_dir: Path, toml: str, files: tuple[tuple[str, str], ...]) -> None: + """Write `sqlbuild_project.toml` and each `(relative path, text)` file.""" + + (project_dir / "sqlbuild_project.toml").write_text(toml, encoding="utf-8") + for relative, text in files: + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py new file mode 100644 index 0000000000..7b9d8ca95f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_cross_join_one_row.py @@ -0,0 +1,191 @@ +"""E2E coverage of SQBRSQL038 inferring that one side of a CROSS JOIN holds at most one row.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import OneRowCrossJoinCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + diagnostic_messages, + run_compile_cli, + write_one_row_cross_join_project, +) + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL038"]\n' +_SEED_YAML: str = ( + "seeds:\n" + " - name: order_policy\n" + " description: Order limits applied to every order.\n" + " columns:\n" + " - name: max_amount\n" + " type: INTEGER\n" +) +_RAW_ORDERS_SQL: str = "MODEL (description 'Raw orders.');\nSELECT 1 AS order_id, 3 AS amount\n" +_HEADER: str = "MODEL (description 'Orders checked against limits.');\n" +_DIRECT_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "final AS (\n" + " SELECT o.order_id, p.max_amount\n" + " FROM orders AS o\n" + ' CROSS JOIN __seed("order_policy") AS p\n' + ")\n" + "SELECT f.order_id, f.max_amount FROM final AS f\n" +) +_CARRIED_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "policy AS (\n" + ' SELECT p.max_amount FROM __seed("order_policy") AS p WHERE p.max_amount > 0\n' + "),\n" + "final AS (\n" + " SELECT o.order_id, p.max_amount FROM orders AS o CROSS JOIN policy AS p\n" + ")\n" + "SELECT f.order_id, f.max_amount FROM final AS f\n" +) +_AGGREGATE_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "totals AS (\n" + " SELECT SUM(o.amount) AS total_amount FROM orders AS o\n" + "),\n" + "final AS (\n" + " SELECT o.order_id, t.total_amount FROM orders AS o CROSS JOIN totals AS t\n" + ")\n" + "SELECT f.order_id, f.total_amount FROM final AS f\n" +) +_LIMIT_SQL: str = ( + _HEADER + "WITH orders AS (\n" + ' SELECT o.order_id, o.amount FROM __ref("raw_orders") AS o\n' + "),\n" + "largest AS (\n" + " SELECT o.amount AS largest_amount FROM orders AS o\n" + " ORDER BY o.amount DESC, o.order_id ASC\n" + " LIMIT 1\n" + "),\n" + "final AS (\n" + " SELECT o.order_id, l.largest_amount FROM orders AS o CROSS JOIN largest AS l\n" + ")\n" + "SELECT f.order_id, f.largest_amount FROM final AS f\n" +) +_MULTIPLYING: str = ( + "CROSS JOIN with `order_policy` in CTE `final` creates an explicit cartesian product" +) + + +_SEED_YAML_FILE: tuple[str, str] = ("seeds/order_policy.yml", _SEED_YAML) +_RAW_ORDERS_FILE: tuple[str, str] = ("models/raw_orders.sql", _RAW_ORDERS_SQL) + + +@pytest.mark.parametrize( + "test_case", + [ + OneRowCrossJoinCase( + description="one-row seed joined directly", + seed_rows=("100",), + model_sql=_DIRECT_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="one-row seed carried through a filtering CTE", + seed_rows=("100",), + model_sql=_CARRIED_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="aggregate without GROUP BY", + seed_rows=("100", "200"), + model_sql=_AGGREGATE_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="LIMIT 1", + seed_rows=("100", "200"), + model_sql=_LIMIT_SQL, + expected_messages=(), + expected_exit=0, + ), + OneRowCrossJoinCase( + description="two-row seed multiplies rows", + seed_rows=("100", "200"), + model_sql=_DIRECT_SQL, + expected_messages=(_MULTIPLYING,), + expected_exit=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_cross_join_when_compiling_then_only_multiplying_joins_are_reported( + test_case: OneRowCrossJoinCase, + tmp_path: Path, +) -> None: + _ = write_one_row_cross_join_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + seed_rows=test_case.seed_rows, + files=( + _SEED_YAML_FILE, + _RAW_ORDERS_FILE, + ("models/checked_orders.sql", test_case.model_sql), + ), + ) + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + assert diagnostic_messages(result, "SQBRSQL038") == test_case.expected_messages, ( + result.stdout + result.stderr + ) + assert result.returncode == test_case.expected_exit, result.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + OneRowCrossJoinCase( + description="adding a seed row brings the finding back", + seed_rows=("100",), + model_sql=_DIRECT_SQL, + expected_messages=(_MULTIPLYING,), + expected_exit=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_one_row_seed_when_a_row_is_added_then_the_cached_compile_reports_the_join( + test_case: OneRowCrossJoinCase, + tmp_path: Path, +) -> None: + seed: Path = write_one_row_cross_join_project( + project_dir=tmp_path, + toml=_PROJECT_TOML, + seed_rows=test_case.seed_rows, + files=( + _SEED_YAML_FILE, + _RAW_ORDERS_FILE, + ("models/checked_orders.sql", test_case.model_sql), + ), + ) + first: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + seed.write_text(seed.read_text(encoding="utf-8") + "200\n", encoding="utf-8") + + second: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + assert first.returncode == 0, first.stdout + first.stderr + assert diagnostic_messages(first, "SQBRSQL038") == () + assert diagnostic_messages(second, "SQBRSQL038") == test_case.expected_messages, ( + second.stdout + second.stderr + ) + assert second.returncode == test_case.expected_exit + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py new file mode 100644 index 0000000000..731ab315d2 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py @@ -0,0 +1,140 @@ +"""E2E coverage of configuration-only Rule exceptions, including relation-level exceptions.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import RuleExceptionConfigCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + diagnostic_messages, + diagnostic_text, + run_compile_cli, + write_model_files, +) + +_PROJECT_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL038"]\n' +_MODEL_PATH: str = "models/order_checks.sql" +_CUSTOMERS: str = ( + "CROSS JOIN with `customers` in CTE `order_checks` creates an explicit cartesian product" +) +_REGIONS: str = "CROSS JOIN with `regions` in CTE `final` creates an explicit cartesian product" +_EXCEPTION_HEAD: str = f'\n[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "{_MODEL_PATH}"\n' +_REASON: str = 'reason = "Every order is checked for every customer and region."\n' +_CUSTOMERS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "customers"\n{_REASON}' +_REGIONS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "regions"\n{_REASON}' +_SHIPMENTS_EXCEPTION: str = f'{_EXCEPTION_HEAD}relation = "shipments"\n{_REASON}' +_CTE_EXCEPTIONS: str = ( + f'{_EXCEPTION_HEAD}relation = "customers"\ncte = "order_checks"\n{_REASON}' + f'{_EXCEPTION_HEAD}relation = "regions"\ncte = "final"\n{_REASON}' +) +_MODEL_SQL: str = ( + "MODEL (description 'Order checks.');\n" + "WITH order_checks AS (\n" + ' SELECT o.order_id, c.customer_id FROM __ref("orders") AS o\n' + "{directive}" + ' CROSS JOIN __ref("customers") AS c\n' + "),\n" + "final AS (\n" + " SELECT k.order_id, k.customer_id, r.region_id FROM order_checks AS k\n" + ' CROSS JOIN __ref("regions") AS r\n' + ")\n" + "SELECT f.order_id, f.customer_id, f.region_id FROM final AS f\n" +) +_PLAIN_SQL: str = _MODEL_SQL.format(directive="") +_DIRECTIVE_SQL: str = _MODEL_SQL.format( + directive=" -- sqb: ignore SQBRSQL038 because every order meets every customer\n" +) +_UPSTREAM_FILES: tuple[tuple[str, str], ...] = ( + ("models/orders.sql", "MODEL (description 'Sample orders.');\nSELECT 1 AS order_id\n"), + ( + "models/customers.sql", + "MODEL (description 'Sample customers.');\nSELECT 1 AS customer_id\n", + ), + ("models/regions.sql", "MODEL (description 'Sample regions.');\nSELECT 1 AS region_id\n"), +) + + +@pytest.mark.parametrize( + "test_case", + [ + RuleExceptionConfigCase( + description="no exception reports both joins", + rules_toml="", + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_messages=(_CUSTOMERS, _REGIONS), + ), + RuleExceptionConfigCase( + description="relation-level exception covers exactly one join", + rules_toml=_CUSTOMERS_EXCEPTION, + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_messages=(_REGIONS,), + ), + RuleExceptionConfigCase( + description="CTE-scoped exceptions cover both joins", + rules_toml=_CTE_EXCEPTIONS, + model_sql=_PLAIN_SQL, + expected_exit=0, + ), + RuleExceptionConfigCase( + description="stale relation-level exception fails", + rules_toml=_CUSTOMERS_EXCEPTION + _REGIONS_EXCEPTION + _SHIPMENTS_EXCEPTION, + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_fragments=( + "stale rule exception suppresses no finding: SQBRSQL038 at " + "models/order_checks.sql (relation shipments)", + ), + ), + RuleExceptionConfigCase( + description="inline directive is an error that prints its exception", + rules_toml="", + model_sql=_DIRECTIVE_SQL, + expected_exit=1, + expected_messages=(_CUSTOMERS, _REGIONS), + expected_fragments=( + "Inline Rule suppressions are not supported", + '[[rules.rule_exceptions]]\n rule = "SQBRSQL038"\n' + ' path = "models/order_checks.sql"\n' + ' relation = "customers"\n' + ' reason = "every order meets every customer"', + ), + ), + RuleExceptionConfigCase( + description="removed allow_exceptions setting fails with migration help", + rules_toml="allow_exceptions = false\n", + model_sql=_PLAIN_SQL, + expected_exit=1, + expected_fragments=( + "sets [rules] allow_exceptions, which SQLBuild no longer supports", + "remove the allow_exceptions line from [rules]", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_exception_config_when_compiling_then_only_named_findings_are_excepted( + test_case: RuleExceptionConfigCase, + tmp_path: Path, +) -> None: + write_model_files( + project_dir=tmp_path, + toml=_PROJECT_TOML + test_case.rules_toml, + files=(*_UPSTREAM_FILES, (_MODEL_PATH, test_case.model_sql)), + ) + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + text: str = diagnostic_text(result) + assert result.returncode == test_case.expected_exit, text + assert diagnostic_messages(result, "SQBRSQL038") == test_case.expected_messages, text + for fragment in test_case.expected_fragments: + assert fragment in text + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 6dc1cc9791..80a1b181fd 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -983,7 +983,7 @@ class WarmTargetProjectConfigEditTestCase: @dataclass(frozen=True) class RuleExceptionPolicyTestCase: - """One `[rules] allow_exceptions` value with one escape hatch, compiled through the CLI.""" + """One Rule exception mechanism, compiled through the CLI.""" description: str rules_toml: str @@ -994,7 +994,7 @@ class RuleExceptionPolicyTestCase: @dataclass(frozen=True) class FormatExceptionPolicyTestCase: - """One `[rules] allow_exceptions` value read by `sqb format`.""" + """One exception-policy case read by `sqb format`.""" description: str setting: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py index e0f76b9caf..29dbf2e504 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rule_exception_policy.py @@ -1,4 +1,4 @@ -"""Real-CLI coverage for forbidding Rule exceptions, scoped ignores and inline suppressions.""" +"""Real-CLI coverage for configuration-only Rule exceptions and rejected inline suppressions.""" from __future__ import annotations @@ -22,70 +22,66 @@ 'reason = "Samples intentionally keep one row."\n' ) _DIRECTIVE: str = "-- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row\n" -_SETTING_HELP: str = "[rules]\n allow_exceptions = true" +_PRINTED_ENTRY: str = ( + "[[rules.rule_exceptions]]\n" + ' rule = "SQBRSQL004"\n' + ' path = "models/customers.sql"\n' + ' reason = "the sample intentionally keeps one row"' +) +_REMOVED: str = "sets [rules] allow_exceptions, which SQLBuild no longer supports" @pytest.mark.parametrize( "test_case", [ RuleExceptionPolicyTestCase( - description="exception forbidden", - rules_toml=f"allow_exceptions = false\n\n{_EXCEPTION}", - directive="", + description="inline directive is an error printing its exception", + rules_toml="", + directive=_DIRECTIVE, expected_exit_code=1, expected_fragments=( - "sqlbuild_project.toml has 1 [[rules.rule_exceptions]] entry", - "sets [rules] allow_exceptions = false", - _SETTING_HELP, + "SQBRSQL000", + "Inline Rule suppressions are not supported", + _PRINTED_ENTRY, + "SQBRSQL004", ), ), RuleExceptionPolicyTestCase( - description="scoped ignore forbidden", - rules_toml=f"allow_exceptions = false\n\n{_IGNORE}", + description="allow_exceptions false is a removed setting", + rules_toml=f"allow_exceptions = false\n\n{_EXCEPTION}", directive="", expected_exit_code=1, - expected_fragments=("1 [[rules.rule_ignores]] entry", _SETTING_HELP), + expected_fragments=(_REMOVED, "remove the allow_exceptions line from [rules]"), ), RuleExceptionPolicyTestCase( - description="inline suppression forbidden", - rules_toml="allow_exceptions = false\n", - directive=_DIRECTIVE, + description="allow_exceptions true is a removed setting", + rules_toml="allow_exceptions = true\n", + directive="", expected_exit_code=1, - expected_fragments=( - "SQBRSQL000", - "Inline Rule suppressions are not allowed in this project", - "SQBRSQL004", - "allow_exceptions = true", - ), + expected_fragments=(_REMOVED,), ), RuleExceptionPolicyTestCase( - description="project-wide ignore stays allowed", - rules_toml='allow_exceptions = false\nignore = ["SQBRSQL004"]\n', + description="configured exception", + rules_toml=_EXCEPTION, directive="", expected_exit_code=0, ), RuleExceptionPolicyTestCase( - description="exception allowed explicitly", - rules_toml=f"allow_exceptions = true\n\n{_EXCEPTION}", + description="scoped ignore", + rules_toml=_IGNORE, directive="", expected_exit_code=0, ), RuleExceptionPolicyTestCase( - description="exception allowed by default", - rules_toml=_EXCEPTION, + description="project-wide ignore", + rules_toml='ignore = ["SQBRSQL004"]\n', directive="", expected_exit_code=0, ), - RuleExceptionPolicyTestCase( - description="inline suppression allowed by default", - rules_toml="", - directive=_DIRECTIVE, - expected_exit_code=0, - ), ], ids=lambda case: case.description, ) -def test_given_allow_exceptions_setting_when_compiling_then_escape_hatches_follow_it( +def test_given_rule_exception_mechanism_when_compiling_then_only_configuration_applies( test_case: RuleExceptionPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -116,30 +112,23 @@ def test_given_allow_exceptions_setting_when_compiling_then_escape_hatches_follo "test_case", [ FormatExceptionPolicyTestCase( - description="string false is rejected like compile", - setting='allow_exceptions = "false"', - expected_exit_code=1, - expected_fragment="invalid rules config", - expected_compile_fragment="invalid rules config", - ), - FormatExceptionPolicyTestCase( - description="boolean false rejects the inline suppression", + description="removed setting is rejected like compile", setting="allow_exceptions = false", expected_exit_code=1, - expected_fragment="Inline Rule suppressions are not allowed in this project", - expected_compile_fragment="Inline Rule suppressions are not allowed in this project", + expected_fragment=_REMOVED, + expected_compile_fragment=_REMOVED, ), FormatExceptionPolicyTestCase( - description="boolean true keeps the inline suppression", - setting="allow_exceptions = true", - expected_exit_code=0, - expected_fragment="FAULT=0", - expected_compile_fragment="", + description="inline directive is rejected like compile", + setting="", + expected_exit_code=1, + expected_fragment="Inline Rule suppressions are not supported", + expected_compile_fragment="Inline Rule suppressions are not supported", ), ], ids=lambda case: case.description, ) -def test_given_allow_exceptions_setting_when_formatting_then_it_is_validated_like_compile( +def test_given_exception_policy_when_formatting_then_it_is_validated_like_compile( test_case: FormatExceptionPolicyTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index e8b679ba72..e123c6fd33 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -53,8 +53,7 @@ _CACHE_RULES_TOML: str = 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRSQL004"]\n' _CACHE_THRESHOLD_TOML: str = _CACHE_RULES_TOML + "thresholds = { max_literal_length = 7 }\n" -_CACHE_POLICY_TOML: str = _CACHE_THRESHOLD_TOML + "allow_exceptions = false\n" -_CACHE_SELECTION_TOML: str = _CACHE_POLICY_TOML.replace( +_CACHE_SELECTION_TOML: str = _CACHE_THRESHOLD_TOML.replace( '"SQBRSQL004"', '"SQBRSQL004", "SQBRSQL001"' ) _CACHE_MODEL_SQL: str = 'MODEL (description "Orders");\nSELECT order_id FROM orders LIMIT 1\n' @@ -2879,10 +2878,10 @@ def test_given_relative_project_path_when_compiling_twice_then_sql_finding_remai @pytest.mark.parametrize( "test_case", - [RulesIntegrationTestCase("unused inline suppression is diagnosed", 1, "SQBRSQL000")], + [RulesIntegrationTestCase("inline directive without a finding fails", 1, "SQBRSQL000")], ids=lambda case: case.description, ) -def test_given_unused_inline_suppression_when_compiling_then_stale_directive_is_reported( +def test_given_inline_directive_without_finding_when_compiling_then_directive_is_reported( test_case: RulesIntegrationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -3034,7 +3033,6 @@ def test_given_empty_rule_selection_when_compiling_then_only_configuration_is_va (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), (_CACHE_RULES_TOML, _CACHE_EDITED_SQL), (_CACHE_THRESHOLD_TOML, _CACHE_EDITED_SQL), - (_CACHE_POLICY_TOML, _CACHE_EDITED_SQL), (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), (_CACHE_SELECTION_TOML, _CACHE_EDITED_SQL), ), @@ -3045,7 +3043,6 @@ def test_given_empty_rule_selection_when_compiling_then_only_configuration_is_va (1, 0), (0, 1), (0, 1), - (0, 1), (1, 0), ), ) diff --git a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py index 033d1dc48d..ed977f959d 100644 --- a/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py +++ b/tests/integration/src/sqlbuild/rule_engine/native_rows/test_sql_rule_cache_publication.py @@ -64,9 +64,7 @@ def test_given_increasing_findings_when_publishing_cache_then_findings_are_store project=project, model_paths=frozenset(paths), selected=(SimpleNamespace(code="SQBRSQL004", custom=False),), - effective_config=SimpleNamespace( - thresholds={}, allow_exceptions=True, cache=SimpleNamespace(enabled=True) - ), + effective_config=SimpleNamespace(thresholds={}, cache=SimpleNamespace(enabled=True)), ) result: SqlRulesEvaluation = native.run_planned_sql_rules( plan=cast(Any, plan), diff --git a/tests/unit/src/sqlbuild/lint/_test_types.py b/tests/unit/src/sqlbuild/lint/_test_types.py index 67f78c8c4b..be67c4acfe 100644 --- a/tests/unit/src/sqlbuild/lint/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_test_types.py @@ -7,6 +7,15 @@ from sqlbuild._native import SqlLexicalSyntax +@dataclass(frozen=True) +class InlineDirectiveTestCase: + """One model with an inline suppression directive above a query.""" + + description: str + query: str + expected_violations: tuple[tuple[str, int, str], ...] + + @dataclass(frozen=True) class LintBehaviorTestCase: """Identity for one non-tabular lint behavior test.""" diff --git a/tests/unit/src/sqlbuild/lint/test_run_lint.py b/tests/unit/src/sqlbuild/lint/test_run_lint.py index a9e9193d7a..e93b623929 100644 --- a/tests/unit/src/sqlbuild/lint/test_run_lint.py +++ b/tests/unit/src/sqlbuild/lint/test_run_lint.py @@ -11,7 +11,7 @@ LintRunResult, LintSeverity, LintViolation, - apply_lint_suppressions, + reject_lint_suppressions, run_lint, ) from tests.unit.src.sqlbuild.lint._test_types import ( @@ -19,6 +19,7 @@ FormatDescriptionResolutionTestCase, FormatNewlineTestCase, FormatProjectTestCase, + InlineDirectiveTestCase, LintBehaviorTestCase, LintProjectTestCase, ) @@ -407,46 +408,24 @@ def test_given_fixture_ceremony_variants_when_formatting_and_linting_then_forms_ assert linted.violations == () -@pytest.mark.parametrize( - "test_case", - [LintBehaviorTestCase(description="reasoned matching local suppression", expected_value=())], - ids=lambda case: case.description, -) -def test_given_reasoned_local_suppression_when_linting_then_matching_warning_is_removed( - test_case: LintBehaviorTestCase, - tmp_path: Path, -) -> None: - _ = test_case - _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - target: Path = tmp_path / "models" / "sample.sql" - target.parent.mkdir() - _ = target.write_text( - 'MODEL (description "ok");\n' - "-- sqb: ignore SQBRSQL004 because this fixture intentionally samples one row\n" - "SELECT value FROM items LIMIT 1\n", - encoding="utf-8", - ) - - result: LintRunResult = run_lint( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) - - assert result.violations == test_case.expected_value - - @pytest.mark.parametrize( "test_case", [ - LintBehaviorTestCase( - description="forbidden suppression keeps the finding and fails", - expected_value=(("SQBRSQL000", 2, "fault"), ("SQBRSQL004", 3, "warning")), - ) + InlineDirectiveTestCase( + description="matching directive keeps the finding and fails", + query="SELECT value FROM items LIMIT 1\n", + expected_violations=(("SQBRSQL000", 2, "fault"), ("SQBRSQL004", 3, "warning")), + ), + InlineDirectiveTestCase( + description="directive without a finding still fails", + query="SELECT value FROM items ORDER BY value LIMIT 1\n", + expected_violations=(("SQBRSQL000", 2, "fault"),), + ), ], ids=lambda case: case.description, ) -def test_given_suppressions_forbidden_when_linting_then_directive_is_a_fault_and_suppresses_nothing( - test_case: LintBehaviorTestCase, +def test_given_inline_directive_when_linting_then_it_is_a_fault_printing_the_exception_entry( + test_case: InlineDirectiveTestCase, tmp_path: Path, ) -> None: _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") @@ -455,73 +434,42 @@ def test_given_suppressions_forbidden_when_linting_then_directive_is_a_fault_and _ = target.write_text( 'MODEL (description "ok");\n' "-- sqb: ignore SQBRSQL004 because this fixture intentionally samples one row\n" - "SELECT value FROM items LIMIT 1\n", + f"{test_case.query}", encoding="utf-8", ) result: LintRunResult = run_lint( project_dir=tmp_path, - config=LintConfig(dialect="duckdb", allow_suppressions=False), + config=LintConfig(dialect="duckdb"), ) assert ( tuple((item.code, item.line, str(item.severity)) for item in result.violations) - == test_case.expected_value + == test_case.expected_violations ) remediation: str = result.violations[0].remediation or "" - assert "sqlbuild_project.toml sets [rules] allow_exceptions = false" in remediation - assert "[rules]\n allow_exceptions = true" in remediation - - -@pytest.mark.parametrize( - "test_case", - [ - LintBehaviorTestCase( - description="unused local suppression", - expected_value="Unused suppression for SQBRSQL004", - ) - ], - ids=lambda case: case.description, -) -def test_given_unused_local_suppression_when_linting_then_reports_suppression_warning( - test_case: LintBehaviorTestCase, - tmp_path: Path, -) -> None: - _ = test_case - _ = (tmp_path / "sqlbuild_project.toml").write_text(PROJECT_TOML, encoding="utf-8") - target: Path = tmp_path / "models" / "ordered.sql" - target.parent.mkdir() - _ = target.write_text( - 'MODEL (description "ok");\n' - "-- sqb: ignore SQBRSQL004 because this query used to sample one row\n" - "SELECT value FROM items ORDER BY value LIMIT 1\n", - encoding="utf-8", - ) - - result: LintRunResult = run_lint( - project_dir=tmp_path, - config=LintConfig(dialect="duckdb"), - ) - - assert tuple(item.code for item in result.violations) == ("SQBRSQL000",) - assert result.violations[0].message == test_case.expected_value + assert ( + "[[rules.rule_exceptions]]\n" + ' rule = "SQBRSQL004"\n' + ' path = "models/sample.sql"\n' + ' reason = "this fixture intentionally samples one row"' + ) in remediation @pytest.mark.parametrize( "test_case", [ LintBehaviorTestCase( - description="mandatory fault suppression attempt", + description="mandatory fault directive", expected_value={"SQBRSQL000", "description-present"}, ) ], ids=lambda case: case.description, ) -def test_given_header_fault_suppression_when_linting_then_mandatory_fault_remains( +def test_given_header_fault_directive_when_rejecting_then_mandatory_fault_remains( test_case: LintBehaviorTestCase, tmp_path: Path, ) -> None: - _ = test_case target: Path = tmp_path / "models" / "missing_description.sql" contents: str = ( "-- sqb: ignore description-present because this must not bypass compiler policy\n" @@ -537,6 +485,10 @@ def test_given_header_fault_suppression_when_linting_then_mandatory_fault_remain engine=LINT_ENGINE_SQLBUILD, ) - result: list[LintViolation] = apply_lint_suppressions([mandatory], {target: contents}, True) + result: list[LintViolation] = reject_lint_suppressions( + [mandatory], {target: contents}, tmp_path + ) + + assert {item.code for item in result} == test_case.expected_value assert {item.code for item in result} == test_case.expected_value diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py index b125b43287..ef45160c21 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/_test_types.py @@ -13,12 +13,13 @@ class RulesConfigErrorTestCase: @dataclass(frozen=True) -class AllowExceptionsConfigTestCase: - """One `[rules] allow_exceptions` configuration that loads.""" +class RelationExceptionConfigTestCase: + """One `[[rules.rule_exceptions]]` configuration that loads, with its relation scope.""" description: str source: str - expected_allow_exceptions: bool + expected_relation: str | None + expected_cte: str | None @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index adc5dd4395..66b29d3fe3 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -13,8 +13,8 @@ load_rules_config, ) from tests.unit.src.sqlbuild.rule_engine._helpers.engine._test_types import ( - AllowExceptionsConfigTestCase, PolicyLayoutConfigTestCase, + RelationExceptionConfigTestCase, RuleIgnoreConfigTestCase, RulesConfigErrorTestCase, SqlTestRulesConfigTestCase, @@ -166,9 +166,20 @@ expected_error_pattern="must be a normalized path relative to tests/unit", ), RulesConfigErrorTestCase( - description="non-boolean allow_exceptions", - source='[rules]\nallow_exceptions = "no"\n', - expected_error_pattern="invalid rules config", + description="relation for a Rule that reports no relation", + source=( + '[[rules.rule_exceptions]]\nrule = "SQBRSQL004"\npath = "models/orders.sql"\n' + 'relation = "customers"\nreason = "A reviewed sample"\n' + ), + expected_error_pattern="sets relation, but only SQBRSQL038 report a joined relation", + ), + RulesConfigErrorTestCase( + description="cte without relation", + source=( + '[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "models/orders.sql"\n' + 'cte = "checks"\nreason = "Deliberate"\n' + ), + expected_error_pattern="sets cte without relation", ), RulesConfigErrorTestCase( description="unknown SQL test rules key", @@ -192,31 +203,21 @@ def test_given_invalid_rules_config_when_loading_then_raises_clear_error( "test_case", ( RulesConfigErrorTestCase( - description="exception entry", + description="false", source=f"[rules]\nallow_exceptions = false\n{_EXCEPTION_ENTRY}", expected_error_pattern=( - r"has 1 \[\[rules\.rule_exceptions\]\] entry, which this project does not " - r"allow; sqlbuild_project\.toml sets \[rules\] allow_exceptions = false" + r"sets \[rules\] allow_exceptions, which SQLBuild no longer supports" ), ), RulesConfigErrorTestCase( - description="scoped ignore entry", - source=f"[rules]\nallow_exceptions = false\n{_IGNORE_ENTRY}", - expected_error_pattern=r"has 1 \[\[rules\.rule_ignores\]\] entry,", - ), - RulesConfigErrorTestCase( - description="exceptions and scoped ignores", - source=f"[rules]\nallow_exceptions = false\n{_EXCEPTION_ENTRY}{_EXCEPTION_ENTRY}" - f"{_IGNORE_ENTRY}", - expected_error_pattern=( - r"has 2 \[\[rules\.rule_exceptions\]\] entries and " - r"1 \[\[rules\.rule_ignores\]\] entry," - ), + description="true", + source="[rules]\nallow_exceptions = true\n", + expected_error_pattern=r"inline -- sqb: ignore suppressions were removed", ), ), ids=lambda case: case.description, ) -def test_given_exceptions_forbidden_when_loading_then_raises_error_with_exact_setting_help( +def test_given_removed_allow_exceptions_when_loading_then_raises_error_explaining_the_removal( tmp_path: Path, test_case: RulesConfigErrorTestCase, ) -> None: @@ -225,42 +226,42 @@ def test_given_exceptions_forbidden_when_loading_then_raises_error_with_exact_se with pytest.raises(RulesConfigError, match=test_case.expected_error_pattern) as raised: load_rules_config(project_dir=tmp_path) - assert raised.value.help.endswith( - "set this in sqlbuild_project.toml:\n [rules]\n" - " allow_exceptions = true" - ) + assert "remove the allow_exceptions line from [rules]" in raised.value.help + assert "[[rules.rule_exceptions]]" in raised.value.help @pytest.mark.parametrize( "test_case", ( - AllowExceptionsConfigTestCase( - description="absent setting allows exceptions", + RelationExceptionConfigTestCase( + description="file-wide exception and scoped ignore", source=_EXCEPTION_ENTRY + _IGNORE_ENTRY, - expected_allow_exceptions=True, + expected_relation=None, + expected_cte=None, ), - AllowExceptionsConfigTestCase( - description="true allows exceptions", - source=f"[rules]\nallow_exceptions = true\n{_EXCEPTION_ENTRY}{_IGNORE_ENTRY}", - expected_allow_exceptions=True, - ), - AllowExceptionsConfigTestCase( - description="false with only the project-wide ignore list", - source='[rules]\nallow_exceptions = false\nignore = ["SQBRSQL004"]\n', - expected_allow_exceptions=False, + RelationExceptionConfigTestCase( + description="relation-level exception narrowed to a CTE", + source=( + '[[rules.rule_exceptions]]\nrule = "SQBRSQL038"\npath = "models/orders.sql"\n' + 'relation = "order_policy"\ncte = "order_checks"\n' + 'reason = "Every order is checked against every policy"\n' + ), + expected_relation="order_policy", + expected_cte="order_checks", ), ), ids=lambda case: case.description, ) -def test_given_allow_exceptions_setting_when_loading_then_config_records_it( +def test_given_rule_exceptions_when_loading_then_config_records_their_scope( tmp_path: Path, - test_case: AllowExceptionsConfigTestCase, + test_case: RelationExceptionConfigTestCase, ) -> None: (tmp_path / "sqlbuild_project.toml").write_text(test_case.source, encoding="utf-8") config: RulesConfig = load_rules_config(project_dir=tmp_path) - assert config.allow_exceptions is test_case.expected_allow_exceptions + assert config.rule_exceptions[0].relation == test_case.expected_relation + assert config.rule_exceptions[0].cte == test_case.expected_cte @pytest.mark.parametrize( From a04dd730518297bf7878d03ef34c124a3efdb790 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:48:10 +0100 Subject: [PATCH 24/39] feat(rules): add published-layer domain isolation and name shape rules SQBRGRAPH103 rejects a published model reading a published model of its own domain, except a __mart_v__ view reading __mart__. SQBRPROJECT107 requires one name shape per domain's published layer, with literal allowed_qualifiers that each carry a reason. Both are ordinary built-in errors that skip names outside the ____ grammar, take configurable published_layers (default mart, mart_v), and print the sqb mv or sqb rename command where one applies. --- .../_helpers/rules/builtin_catalogue.rs | 20 +- .../src/configuration/_helpers/loading.rs | 82 ++++- crates/sqlbuild-rules/src/constants.rs | 20 + crates/sqlbuild-rules/src/engine/tests.rs | 2 + .../src/engine/tests/helpers.rs | 50 +++ .../engine/tests/test_incremental_rules.rs | 2 + .../src/engine/tests/test_published_layers.rs | 170 +++++++++ .../src/engine/tests/test_types.rs | 8 + .../src/rule_catalogue/constants.rs | 10 +- .../src/rules/_helpers/catalogue.rs | 18 + .../src/rules/_helpers/evaluation.rs | 29 +- .../src/rules/_helpers/model_layers.rs | 342 +++++++++++++++++- .../src/rules/main/builtin_options.rs | 23 +- crates/sqlbuild-rules/src/rules/models.rs | 6 +- .../cli/commands/main/rules/_test_types.py | 11 + .../main/rules/test_published_layer_rules.py | 114 ++++++ .../_helpers/engine/test_catalogue.py | 4 + .../_helpers/engine/test_config.py | 13 + 18 files changed, 899 insertions(+), 25 deletions(-) create mode 100644 crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs index 2b27e8f645..bca5bbdfd0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/builtin_catalogue.rs @@ -33,7 +33,25 @@ fn option<'py>( let py: Python<'py> = option_class.py(); let (factory, default): (&str, Bound<'py, PyAny>) = match declared.kind { BuiltinOptionKind::Boolean => ("boolean", PyBool::new(py, false).to_owned().into_any()), - BuiltinOptionKind::StringList => ("string_list", PyTuple::empty(py).into_any()), + BuiltinOptionKind::StringList(defaults) => { + ("string_list", PyTuple::new(py, defaults)?.into_any()) + } + BuiltinOptionKind::TableList => { + let arguments = keywords( + py, + vec![ + ("name", PyString::new(py, declared.name).into_any()), + ("value_type", py.get_type::().into_any()), + ("default", PyTuple::empty(py).into_any()), + ( + "description", + PyString::new(py, declared.description).into_any(), + ), + ("item_type", py.get_type::().into_any()), + ], + )?; + return new_record(py, RecordId::RuleOption, (), Some(&arguments)); + } }; let arguments = keywords( py, diff --git a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs index b49214685a..f580959741 100644 --- a/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs +++ b/crates/sqlbuild-rules/src/configuration/_helpers/loading.rs @@ -1,6 +1,9 @@ use crate::configuration::_helpers::removed_settings::removed_settings; use crate::configuration::errors::RulesConfigLoadError; -use crate::constants::{MIN_CUSTOM_RULE_TEST_CASES_KEY, RELATION_EXCEPTION_RULES}; +use crate::constants::{ + ALLOWED_QUALIFIERS_OPTION, MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, + PUBLISHED_LAYERS_OPTION, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, RELATION_EXCEPTION_RULES, +}; use crate::models::RulesConfig; use fensu_policy::policy::types::RuleCodeGrammar; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; @@ -206,6 +209,7 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { return Err(format!("invalid rule code: {code}")); } } + validate_published_layer_options(config)?; for exception in &config.rule_exceptions { if exception.rule.trim().is_empty() || exception.path.trim().is_empty() @@ -267,6 +271,82 @@ pub(crate) fn validate(config: &RulesConfig) -> Result<(), String> { Ok(()) } +/// Published layers name model layers; allowed qualifiers are literal and carry a reason. +fn validate_published_layer_options(config: &RulesConfig) -> Result<(), String> { + for code in [PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE] { + let Some(layers) = config + .rule_options + .get(code) + .and_then(|options| options.get(PUBLISHED_LAYERS_OPTION)) + else { + continue; + }; + let label: String = format!("rules.rule_options.{code}.{PUBLISHED_LAYERS_OPTION}"); + let Some(layers) = layers.as_array() else { + return Err(format!("{label} must be a list of model name layers")); + }; + for layer in layers { + match layer.as_str() { + Some(name) if MODEL_NAME_LAYERS.contains(&name) => {} + _ => { + return Err(format!( + "{label} entries must be model name layers: {}", + MODEL_NAME_LAYERS.join(", ") + )); + } + } + } + } + let Some(entries) = config + .rule_options + .get(PUBLISHED_SHAPE_CODE) + .and_then(|options| options.get(ALLOWED_QUALIFIERS_OPTION)) + else { + return Ok(()); + }; + let label: String = + format!("rules.rule_options.{PUBLISHED_SHAPE_CODE}.{ALLOWED_QUALIFIERS_OPTION}"); + let shape: String = format!( + "{label} entries must be {{ qualifier = \"...\", reason = \"...\" }} tables with an \ + optional domain" + ); + let Some(entries) = entries.as_array() else { + return Err(shape); + }; + let mut seen: std::collections::BTreeSet<(Option<&str>, &str)> = + std::collections::BTreeSet::new(); + for entry in entries { + let Some(table) = entry.as_object() else { + return Err(shape); + }; + if table + .keys() + .any(|key| !["domain", "qualifier", "reason"].contains(&key.as_str())) + { + return Err(shape); + } + let text = |key: &str| table.get(key).map(|value| value.as_str().map(str::trim)); + let (Some(Some(qualifier)), Some(Some(reason))) = (text("qualifier"), text("reason")) + else { + return Err(shape); + }; + let domain: Option<&str> = match text("domain") { + None => None, + Some(Some(domain)) if !domain.is_empty() => Some(domain), + Some(_) => return Err(shape), + }; + if qualifier.is_empty() || reason.is_empty() { + return Err(format!( + "{label} entries need a non-empty qualifier and reason" + )); + } + if !seen.insert((domain, qualifier)) { + return Err(format!("duplicate {label} entry: {qualifier}")); + } + } + Ok(()) +} + /// A relation-level exception names a non-empty relation, for a Rule that reports one. fn validate_relation_scope(exception: &crate::models::RuleException) -> Result<(), String> { let blank = |value: &Option| value.as_ref().is_some_and(|text| text.trim().is_empty()); diff --git a/crates/sqlbuild-rules/src/constants.rs b/crates/sqlbuild-rules/src/constants.rs index 3a5ae371e3..ad865fb4e1 100644 --- a/crates/sqlbuild-rules/src/constants.rs +++ b/crates/sqlbuild-rules/src/constants.rs @@ -35,5 +35,25 @@ pub(crate) const SQL_RULE_PREFIX: &str = "SQBRSQL"; pub const MIN_CUSTOM_RULE_TEST_CASES_KEY: &str = "min_custom_rule_test_cases"; /// Rules whose findings name a joined relation, so an exception may cover just that relation. pub(crate) const RELATION_EXCEPTION_RULES: [&str; 1] = ["SQBRSQL038"]; +/// The layers the `____` model name grammar accepts. +pub(crate) const MODEL_NAME_LAYERS: [&str; 7] = [ + "stg", + "stg_v", + "int_clean", + "int_v", + "int_enriched", + "mart", + "mart_v", +]; /// The removed exception switch, which used to allow or forbid every escape hatch at once. pub(crate) const REMOVED_ALLOW_EXCEPTIONS: &str = "allow_exceptions"; +/// Published models must not read published models of their own domain. +pub(crate) const PUBLISHED_READ_CODE: &str = "SQBRGRAPH103"; +/// Each domain's published layer uses one model name shape. +pub(crate) const PUBLISHED_SHAPE_CODE: &str = "SQBRPROJECT107"; +/// The option naming the published layers. +pub(crate) const PUBLISHED_LAYERS_OPTION: &str = "published_layers"; +/// The option listing qualifiers that do not count towards a domain's name shape. +pub(crate) const ALLOWED_QUALIFIERS_OPTION: &str = "allowed_qualifiers"; +/// The published layers when a project does not configure them. +pub(crate) const DEFAULT_PUBLISHED_LAYERS: [&str; 2] = ["mart", "mart_v"]; diff --git a/crates/sqlbuild-rules/src/engine/tests.rs b/crates/sqlbuild-rules/src/engine/tests.rs index c217717c22..3f069465db 100644 --- a/crates/sqlbuild-rules/src/engine/tests.rs +++ b/crates/sqlbuild-rules/src/engine/tests.rs @@ -8,6 +8,8 @@ mod evaluation; mod helpers; #[path = "tests/test_incremental_rules.rs"] mod incremental_rules; +#[path = "tests/test_published_layers.rs"] +mod published_layers; #[path = "tests/test_query_facts.rs"] mod query_facts; #[path = "tests/test_relation_exceptions.rs"] diff --git a/crates/sqlbuild-rules/src/engine/tests/helpers.rs b/crates/sqlbuild-rules/src/engine/tests/helpers.rs index 47193872a6..4bba084546 100644 --- a/crates/sqlbuild-rules/src/engine/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/engine/tests/helpers.rs @@ -738,6 +738,56 @@ pub(crate) fn without_sql(models: Vec) -> Value { serde_json::to_value(stripped).unwrap_or_default() } +/// One published model JSON row, stored under its domain's `mart` folder. +fn published_model(name: &str, references: &[&str]) -> Value { + let domain: &str = name.split("__").next().unwrap_or_default(); + json!({ + "name": name, + "relative_path": format!("models/{domain}/mart/{name}.sql"), + "query_sql": "SELECT 1", + "authored_sql": "SELECT 1", + "references": references + .iter() + .map(|reference| json!({"ref_kind": "ref", "ref_name": reference})) + .collect::>(), + }) +} + +/// `(path, message and remediation)` of every fault the case's published models raise. +pub(crate) fn published_layer_faults( + test_case: &test_types::PublishedLayerTestCase, +) -> Result, String> { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let config = json!({ + "select": [test_case.code], + "rule_options": {test_case.code: test_case.options}, + "cache": {"enabled": false} + }); + let mut request: Value = + serde_json::from_str(&request(&project_dir, &config)).map_err(|error| error.to_string())?; + request["models"] = test_case + .models + .iter() + .map(|(name, references)| published_model(name, references)) + .collect(); + let result: Value = serde_json::from_str(&evaluate_json(&request.to_string())?) + .map_err(|error| error.to_string())?; + Ok(result["faults"] + .as_array() + .ok_or("faults must be an array")? + .iter() + .map(|fault| { + ( + fault["path"].as_str().unwrap_or_default().to_owned(), + format!( + "{}\n{}", + fault["message"].as_str().unwrap_or_default(), + fault["remediation"].as_str().unwrap_or_default() + ), + ) + }) + .collect()) +} /// The model path the relation-exception cases use. pub(crate) const CROSS_JOIN_MODEL_PATH: &str = "models/orders.sql"; diff --git a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs index 81853977f4..79678b5728 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_incremental_rules.rs @@ -30,6 +30,7 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRDECLARATION306", RuleScope::Project), ("SQBRGRAPH101", RuleScope::Project), ("SQBRGRAPH102", RuleScope::Model), + ("SQBRGRAPH103", RuleScope::Project), ("SQBRMODEL101", RuleScope::Model), ("SQBRMODEL102", RuleScope::Model), ("SQBRMODEL103", RuleScope::Model), @@ -40,6 +41,7 @@ const REVIEWED_SCOPES: &[(&str, RuleScope)] = &[ ("SQBRPROJECT104", RuleScope::Model), ("SQBRPROJECT105", RuleScope::Model), ("SQBRPROJECT106", RuleScope::Model), + ("SQBRPROJECT107", RuleScope::Project), ("SQBRPROJECT201", RuleScope::Project), ("SQBRPROJECT202", RuleScope::Project), ("SQBRPROJECT203", RuleScope::Project), diff --git a/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs new file mode 100644 index 0000000000..80da53a289 --- /dev/null +++ b/crates/sqlbuild-rules/src/engine/tests/test_published_layers.rs @@ -0,0 +1,170 @@ +use serde_json::json; + +use crate::engine::tests::helpers::published_layer_faults; +use crate::engine::tests::test_types::PublishedLayerTestCase; + +#[test] +fn given_published_models_when_evaluating_layer_rules_then_reads_and_shapes_match() +-> Result<(), String> { + let test_cases = [ + PublishedLayerTestCase { + description: "a consumer variant reads its own domain's published model", + code: "SQBRGRAPH103", + models: &[ + ("sales__mart__orders", &["sales__int_clean__orders"]), + ("sales__mart__orders__finance", &["sales__mart__orders"]), + ], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__finance.sql", + "sales__mart__orders__finance reads sales__mart__orders; published models must not read published models of their own domain", + )], + }, + PublishedLayerTestCase { + description: "the finding gives the intermediate move command", + code: "SQBRGRAPH103", + models: &[("sales__mart__orders__finance", &["sales__mart__orders"])], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__finance.sql", + "sqb mv sales__mart__orders__finance models/sales/intermediate/enriched/sales__int_enriched__orders__finance.sql", + )], + }, + PublishedLayerTestCase { + description: "a view of its own entity may read the table", + code: "SQBRGRAPH103", + models: &[ + ("sales__mart_v__orders", &["sales__mart__orders"]), + ( + "sales__mart_v__orders__csv_export", + &["sales__mart__orders"], + ), + ], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualified view may read the table with its qualifier", + code: "SQBRGRAPH103", + models: &[("sales__mart_v__orders__fra", &["sales__mart__orders__fra"])], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualified view may not read the table with another qualifier", + code: "SQBRGRAPH103", + models: &[("sales__mart_v__orders__fra", &["sales__mart__orders__deu"])], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart_v__orders__fra.sql", + "sales__mart_v__orders__fra reads sales__mart__orders__deu", + )], + }, + PublishedLayerTestCase { + description: "a view of another entity may not", + code: "SQBRGRAPH103", + models: &[("sales__mart_v__customers", &["sales__mart__orders"])], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart_v__customers.sql", + "sales__mart_v__customers reads sales__mart__orders", + )], + }, + PublishedLayerTestCase { + description: "reading another domain's published model is allowed", + code: "SQBRGRAPH103", + models: &[("finance__mart__revenue", &["sales__mart__orders"])], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "names outside the grammar are not checked", + code: "SQBRGRAPH103", + models: &[("orders_report", &["orders_summary"])], + options: json!({}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "configured published layers replace the defaults", + code: "SQBRGRAPH103", + models: &[("sales__mart__orders__finance", &["sales__mart__orders"])], + options: json!({"published_layers": ["mart_v"]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a qualified variant in an entity-shaped domain", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__customers", &[]), + ("sales__mart__orders__finance", &[]), + ], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__finance.sql", + "domain sales publishes 2 models as ____ and 1 as ______; sales__mart__orders__finance does not fit the majority shape ____", + )], + }, + PublishedLayerTestCase { + description: "an unqualified model in a qualified domain gets a rename command", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders__fra", &[]), + ("sales__mart__orders__deu", &[]), + ("sales__mart__customers", &[]), + ], + options: json!({}), + expected_faults: &[( + "models/sales/mart/sales__mart__customers.sql", + "sqb rename sales__mart__customers sales__mart__customers__core", + )], + }, + PublishedLayerTestCase { + description: "an allowlisted qualifier does not count towards the shape", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart_v__orders__csv_export", &[]), + ("finance__mart__revenue", &[]), + ("finance__mart__revenue__fra", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"qualifier": "csv_export", "reason": "File export views of a published entity."}, + {"domain": "finance", "qualifier": "fra", "reason": "Published once per country."} + ]}), + expected_faults: &[], + }, + PublishedLayerTestCase { + description: "a domain-scoped allowlist entry does not cover other domains", + code: "SQBRPROJECT107", + models: &[ + ("sales__mart__orders", &[]), + ("sales__mart__orders__fra", &[]), + ], + options: json!({"allowed_qualifiers": [ + {"domain": "finance", "qualifier": "fra", "reason": "Published once per country."} + ]}), + expected_faults: &[( + "models/sales/mart/sales__mart__orders__fra.sql", + "allowed_qualifiers = [{ domain = \"sales\", qualifier = \"fra\"", + )], + }, + ]; + + for test_case in test_cases { + let found: Vec<(String, String)> = published_layer_faults(&test_case)?; + + assert_eq!( + found.len(), + test_case.expected_faults.len(), + "{}: {found:?}", + test_case.description + ); + for ((path, text), (expected_path, fragment)) in found.iter().zip(test_case.expected_faults) + { + assert_eq!(path, expected_path, "{}", test_case.description); + assert!(text.contains(fragment), "{}: {text}", test_case.description); + } + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 4d2516fd8d..e631ce4e0c 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -299,6 +299,14 @@ pub(crate) struct RowCodecTestCase { pub(crate) expected_truncations_read: usize, } +pub(crate) struct PublishedLayerTestCase { + pub(crate) description: &'static str, + pub(crate) code: &'static str, + pub(crate) models: &'static [(&'static str, &'static [&'static str])], + pub(crate) options: Value, + pub(crate) expected_faults: &'static [(&'static str, &'static str)], +} + pub(crate) struct RelationExceptionTestCase { pub(crate) description: &'static str, pub(crate) exceptions: Value, diff --git a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs index 60c0c28f60..1d2ae016fc 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/constants.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/constants.rs @@ -33,7 +33,7 @@ pub(crate) const LAYOUT_RULE_PREFIXES: [&str; 2] = ["SQBRPROJECT2", "SQBRDECLARA pub(crate) const TEST_RULE_PREFIX: &str = "SQBRTEST"; pub(crate) const GRAPH_EDGE_RULE_CODE: &str = "SQBRGRAPH101"; pub(crate) const REPLACEABLE_SKILL_STATES: [&str; 3] = ["fresh", "missing", "stale"]; -pub(crate) const RULE_EXAMPLES: [(&str, &str); 26] = [ +pub(crate) const RULE_EXAMPLES: [(&str, &str); 28] = [ ( "SQBRSQL040", "Compute category_key in an input CTE, then JOIN categories AS c ON o.category_key = c.category_key.", @@ -62,10 +62,18 @@ pub(crate) const RULE_EXAMPLES: [(&str, &str); 26] = [ "SQBRGRAPH101", "Internal flow is stg -> int_clean -> int_enriched; marts are published interfaces.", ), + ( + "SQBRGRAPH103", + "sales__mart__orders and sales__mart__orders__finance both read sales__int_enriched__orders; sales__mart_v__orders may read sales__mart__orders.", + ), ( "SQBRPROJECT101", "domain__int_clean__entity or domain__mart_v__entity.", ), + ( + "SQBRPROJECT107", + "sales publishes sales__mart__orders and sales__mart__customers; the finance variant lives in finance or sales intermediate.", + ), ( "SQBRSQL038", "JOIN customers AS c ON o.customer_id = c.customer_id, or CROSS JOIN a provably one-row relation such as a one-row seed or an aggregate without GROUP BY.", diff --git a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs index 2173f679a0..d1353c4aab 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/catalogue.rs @@ -33,7 +33,9 @@ macro_rules! rule { | "SQBRPROJECT202" | "SQBRPROJECT203" | "SQBRPROJECT204" + | "SQBRPROJECT107" | "SQBRGRAPH101" + | "SQBRGRAPH103" | "SQBRTEST101" | "SQBRTEST102" | "SQBRTEST103" @@ -97,6 +99,22 @@ pub(crate) fn catalogue() -> Vec { "table dependencies must use __ref or __source", "Replace this qualified table with __ref(\"\") or __source(\"\") so it participates in the DAG.", ), + rule!( + "SQBRGRAPH103", + RuleScope::Project, + "graph", + "published-domain-isolation", + "published models must not read published models of their own domain", + "Move logic shared within a domain into its intermediate layer, or move a consumer-specific model into the consuming domain. A __mart_v__ view may read __mart__.", + ), + rule!( + "SQBRPROJECT107", + RuleScope::Project, + "repository", + "published-name-shape", + "each domain's published models must share one name shape", + "Name every published model of a domain either ____ or ______. Move consumer-specific variants into intermediate or the consuming domain, rename models into the family, or allowlist an intentional qualifier with a reason.", + ), rule!( "SQBRPROJECT101", RuleScope::Model, diff --git a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs index e2eb76d93f..d44680e3cf 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/evaluation.rs @@ -1,10 +1,12 @@ use crate::configuration::main::setting_help::setting_help; use crate::configuration::main::toml_string_array::toml_string_array; -use crate::constants::MIN_CUSTOM_RULE_TEST_CASES_KEY; use crate::constants::{ DECLARATION_DOMAIN_COMPONENTS, ENFORCED_CONTRACT, REFERENCE_KIND, SOURCE_REFERENCE_KIND, VIEW_MATERIALIZATION, }; +use crate::constants::{ + MIN_CUSTOM_RULE_TEST_CASES_KEY, MODEL_NAME_LAYERS, PUBLISHED_READ_CODE, PUBLISHED_SHAPE_CODE, +}; use crate::models::{Declaration, EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; use crate::query_facts::main::dependency_texts::dependency_texts; use crate::query_facts::main::model_shape::model_shape; @@ -262,6 +264,12 @@ pub(crate) fn evaluate_project_rules(request: &ProjectEvaluationRequest<'_>) -> if let Some(rule) = request.selected.get("SQBRGRAPH101") { model_layers::internal_forward_refs(request, rule, &faults); } + if let Some(rule) = request.selected.get(PUBLISHED_READ_CODE) { + model_layers::same_domain_published_reads(request.request, rule, &faults); + } + if let Some(rule) = request.selected.get(PUBLISHED_SHAPE_CODE) { + model_layers::published_name_shapes(request.request, rule, &faults); + } if let Some(rule) = request.selected.get("SQBRDECLARATION201") { duplicate_enums(request.request, rule, &faults); } @@ -528,11 +536,11 @@ fn select_star( #[derive(Clone)] pub(super) struct NameParts { - domain: String, + pub(super) domain: String, pub(super) layer: String, - subject: String, - source: Option, - is_view: bool, + pub(super) subject: String, + pub(super) source: Option, + pub(super) is_view: bool, } pub(super) fn parse_name(name: &str) -> Option { @@ -540,17 +548,8 @@ pub(super) fn parse_name(name: &str) -> Option { if !(3..=4).contains(&parts.len()) { return None; } - let layers = [ - "stg", - "stg_v", - "int_clean", - "int_v", - "int_enriched", - "mart", - "mart_v", - ]; if !is_lowercase_identifier(parts[0]) - || !layers.contains(&parts[1]) + || !MODEL_NAME_LAYERS.contains(&parts[1]) || !is_lowercase_identifier(parts[2]) || parts .get(3) diff --git a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs index ce66e8fcf6..8264cd1c62 100644 --- a/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs +++ b/crates/sqlbuild-rules/src/rules/_helpers/model_layers.rs @@ -1,15 +1,23 @@ -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; use crate::constants::{ - INTERMEDIATE_LAYER_DIRECTORY, MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, REFERENCE_KIND, + ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, INTERMEDIATE_LAYER_DIRECTORY, + MART_LAYER_DIRECTORY, MODEL_SCHEMA_CONFIG_KEY, PUBLISHED_LAYERS_OPTION, REFERENCE_KIND, STAGING_LAYER_DIRECTORY, }; -use crate::models::{Fault, Model, RuleMetadata}; -use crate::rules::_helpers::evaluation::parse_name; +use crate::models::{EvaluateRequest, Fault, Model, RuleMetadata, RulesConfig}; +use crate::rules::_helpers::evaluation::{parse_name, path_fault}; use crate::rules::models::{FaultCollector, ProjectEvaluationRequest}; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; const NO_CANONICAL_LAYER_FOLDER: &str = "no canonical layer folder"; +/// Where an intermediate model that replaces a published one lives. +const INTERMEDIATE_FOLDER: &str = "intermediate/enriched"; +const ENTITY_SHAPE: &str = "____"; +const QUALIFIED_SHAPE: &str = "______"; +const MODEL_FILE_SUFFIX: &str = ".sql"; pub(crate) fn explicit_model_schema(model: &Model, rule: &RuleMetadata, faults: &FaultCollector) { if model @@ -204,3 +212,329 @@ fn model_fault( remediation: remediation.unwrap_or_else(|| rule.remediation.clone()), } } + +/// The published layers configured for `code`, or the defaults. +fn published_layers(config: &RulesConfig, code: &str) -> Vec { + match config + .rule_options + .get(code) + .and_then(|options| options.get(PUBLISHED_LAYERS_OPTION)) + .and_then(Value::as_array) + { + Some(layers) => layers + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + None => DEFAULT_PUBLISHED_LAYERS.map(str::to_owned).to_vec(), + } +} + +/// The parts of a `____[__]` model name. +struct PublishedName { + domain: String, + layer: String, + subject: String, + source: Option, + is_view: bool, +} + +fn published_name(name: &str) -> Option { + let parts = parse_name(name)?; + Some(PublishedName { + domain: parts.domain, + layer: parts.layer, + subject: parts.subject, + source: parts.source, + is_view: parts.is_view, + }) +} + +/// One published model and its parsed name. +struct Published<'a> { + model: &'a Model, + name: PublishedName, +} + +fn published<'a>(model: &'a Model, layers: &[String]) -> Option> { + let name: PublishedName = published_name(&model.name)?; + layers + .contains(&name.layer) + .then_some(Published { model, name }) +} + +/// Flag every read of a same-domain published model, except a view of its own entity. +pub(crate) fn same_domain_published_reads( + request: &EvaluateRequest, + rule: &RuleMetadata, + faults: &FaultCollector, +) { + let layers: Vec = published_layers(&request.config, &rule.code); + for model in &request.models { + let Some(consumer) = published(model, &layers) else { + continue; + }; + let mut reported: BTreeSet<&str> = BTreeSet::new(); + for reference in &model.references { + if reference.ref_kind != REFERENCE_KIND || !reported.insert(&reference.ref_name) { + continue; + } + let Some(producer) = published_name(&reference.ref_name) else { + continue; + }; + if producer.domain != consumer.name.domain + || !layers.contains(&producer.layer) + || is_view_of_own_entity(&consumer.name, &producer) + { + continue; + } + faults.push(path_fault( + rule, + &model.relative_path, + format!( + "{} reads {}; published models must not read published models of their own \ + domain", + model.name, reference.ref_name + ), + read_remediation(&consumer), + )); + } + } +} + +/// `X___v__[__]` reading `X____` or the same qualifier. +fn is_view_of_own_entity(consumer: &PublishedName, producer: &PublishedName) -> bool { + consumer.is_view + && !producer.is_view + && (producer.source.is_none() || producer.source == consumer.source) + && consumer.subject == producer.subject + && consumer.layer == format!("{}_v", producer.layer) +} + +fn read_remediation(consumer: &Published<'_>) -> String { + let domain: &str = &consumer.name.domain; + let mut remediation: String = format!( + "Move the shared logic into a {domain}__int_* model that both published models read, or \ + move this model into the domain that consumes it." + ); + if let Some(command) = intermediate_move(consumer) { + remediation.push_str(&format!(" To move it into intermediate: {command}")); + } + remediation +} + +/// `sqb mv` into the domain's intermediate layer, when the model lives in a `mart` folder. +fn intermediate_move(model: &Published<'_>) -> Option { + let layer: &str = if model.name.is_view { + "int_v" + } else { + "int_enriched" + }; + let mut name: String = format!("{}__{layer}__{}", model.name.domain, model.name.subject); + if let Some(qualifier) = &model.name.source { + name.push_str("__"); + name.push_str(qualifier); + } + let parts: Vec<&str> = model.model.relative_path.split('/').collect(); + let (_, folders) = parts.split_last()?; + let mart: usize = folders + .iter() + .position(|part| *part == MART_LAYER_DIRECTORY)?; + let mut target: Vec<&str> = folders[..mart].to_vec(); + target.push(INTERMEDIATE_FOLDER); + target.extend(&folders[mart + 1..]); + Some(format!( + "sqb mv {} {}/{name}{MODEL_FILE_SUFFIX}", + model.model.name, + target.join("/") + )) +} + +/// One literal qualifier that does not count towards a domain's name shape. +struct AllowedQualifier { + domain: Option, + qualifier: String, +} + +fn allowed_qualifiers(config: &RulesConfig, code: &str) -> Vec { + let Some(entries) = config + .rule_options + .get(code) + .and_then(|options| options.get(ALLOWED_QUALIFIERS_OPTION)) + .and_then(Value::as_array) + else { + return Vec::new(); + }; + entries + .iter() + .filter_map(Value::as_object) + .filter_map(|entry| { + Some(AllowedQualifier { + domain: entry + .get("domain") + .and_then(Value::as_str) + .map(str::to_owned), + qualifier: entry.get("qualifier")?.as_str()?.to_owned(), + }) + }) + .collect() +} + +fn allowlisted(allowed: &[AllowedQualifier], name: &PublishedName) -> bool { + name.source.as_ref().is_some_and(|qualifier| { + allowed.iter().any(|entry| { + entry.qualifier == *qualifier + && entry + .domain + .as_ref() + .is_none_or(|domain| *domain == name.domain) + }) + }) +} + +/// Flag published models whose name shape differs from their domain's majority shape. +pub(crate) fn published_name_shapes( + request: &EvaluateRequest, + rule: &RuleMetadata, + faults: &FaultCollector, +) { + let layers: Vec = published_layers(&request.config, &rule.code); + let allowed: Vec = allowed_qualifiers(&request.config, &rule.code); + let mut domains: BTreeMap>> = BTreeMap::new(); + for model in &request.models { + let Some(entry) = published(model, &layers) else { + continue; + }; + if allowlisted(&allowed, &entry.name) { + continue; + } + domains + .entry(entry.name.domain.clone()) + .or_default() + .push(entry); + } + for (domain, models) in &domains { + let qualified: usize = models + .iter() + .filter(|entry| entry.name.source.is_some()) + .count(); + let entities: usize = models.len() - qualified; + if qualified == 0 || entities == 0 { + continue; + } + let majority_qualified: bool = qualified > entities; + let shape = DomainShape { + domain, + entities, + qualified, + family: models + .iter() + .filter(|entry| entry.name.source.is_some() == majority_qualified) + .collect(), + }; + for entry in models + .iter() + .filter(|entry| entry.name.source.is_some() != majority_qualified) + { + faults.push(shape_fault(rule, entry, &shape)); + } + } +} + +/// One domain's published name shapes: how many models use each, and the majority family. +struct DomainShape<'a> { + domain: &'a str, + entities: usize, + qualified: usize, + family: Vec<&'a Published<'a>>, +} + +fn shape_fault(rule: &RuleMetadata, entry: &Published<'_>, shape: &DomainShape<'_>) -> Fault { + let majority: &str = if entry.name.source.is_some() { + ENTITY_SHAPE + } else { + QUALIFIED_SHAPE + }; + let message: String = format!( + "domain {} publishes {} {} as {ENTITY_SHAPE} and {} as {QUALIFIED_SHAPE}; {} does not \ + fit the majority shape {majority}", + shape.domain, + shape.entities, + if shape.entities == 1 { + "model" + } else { + "models" + }, + shape.qualified, + entry.model.name + ); + path_fault( + rule, + &entry.model.relative_path, + message, + shape_remediation(rule, entry, &shape.family), + ) +} + +fn shape_remediation( + rule: &RuleMetadata, + entry: &Published<'_>, + family: &[&Published<'_>], +) -> String { + let domain: &str = &entry.name.domain; + let mut steps: Vec = Vec::new(); + match &entry.name.source { + Some(qualifier) => { + let mut adapt: String = format!( + "If {} adapts another published model, move that logic into {domain}__int_* or \ + into the consuming domain", + entry.model.name + ); + if let Some(command) = intermediate_move(entry) { + adapt.push_str(&format!(" ({command})")); + } + steps.push(adapt); + let mut split: String = format!( + "If {domain} really is published once per qualifier, rename its models without \ + one into the family" + ); + if let Some(model) = family.first() { + split.push_str(&format!( + " (for example: sqb rename {} {}__core)", + model.model.name, model.model.name + )); + } + steps.push(split); + steps.push(allowlist_step(rule, domain, qualifier)); + } + None => { + steps.push(format!( + "If the models with a qualifier adapt this one, move their logic into \ + {domain}__int_* or into the consuming domain" + )); + steps.push(format!( + "If {domain} really is published once per qualifier, rename this model into the \ + family (for example: sqb rename {} {}__core)", + entry.model.name, entry.model.name + )); + if let Some(qualifier) = family.first().and_then(|model| model.name.source.as_ref()) { + steps.push(allowlist_step(rule, domain, qualifier)); + } + } + } + steps + .iter() + .enumerate() + .map(|(index, step)| format!("{}. {step}.", index + 1)) + .collect::>() + .join(" ") +} + +fn allowlist_step(rule: &RuleMetadata, domain: &str, qualifier: &str) -> String { + format!( + "If {qualifier} is an intentional, structural qualifier, allowlist it with a reason in \ + {PROJECT_CONFIG_FILENAME}: [rules.rule_options.{}] {ALLOWED_QUALIFIERS_OPTION} = \ + [{{ domain = \"{domain}\", qualifier = \"{qualifier}\", reason = \"\" }}]", + rule.code + ) +} diff --git a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs index f2ad474e0c..aa1069b965 100644 --- a/crates/sqlbuild-rules/src/rules/main/builtin_options.rs +++ b/crates/sqlbuild-rules/src/rules/main/builtin_options.rs @@ -1,5 +1,8 @@ //! `builtin_options`: the options each built-in rule declares. +use crate::constants::{ + ALLOWED_QUALIFIERS_OPTION, DEFAULT_PUBLISHED_LAYERS, PUBLISHED_LAYERS_OPTION, +}; use crate::rules::models::{BuiltinOption, BuiltinOptionKind}; const fn boolean(name: &'static str, description: &'static str) -> BuiltinOption { @@ -10,6 +13,14 @@ const fn boolean(name: &'static str, description: &'static str) -> BuiltinOption } } +const fn published_layers() -> BuiltinOption { + BuiltinOption { + kind: BuiltinOptionKind::StringList(&DEFAULT_PUBLISHED_LAYERS), + name: PUBLISHED_LAYERS_OPTION, + description: "Model name layers that form each domain's published interface.", + } +} + /// The declared options of the built-in rule `code`, in declaration order. #[must_use] pub fn builtin_options(code: &str) -> Vec { @@ -42,8 +53,18 @@ pub fn builtin_options(code: &str) -> Vec { "Accept string or variant encoded date values.", ), ], + "SQBRGRAPH103" => vec![published_layers()], + "SQBRPROJECT107" => vec![ + published_layers(), + BuiltinOption { + kind: BuiltinOptionKind::TableList, + name: ALLOWED_QUALIFIERS_OPTION, + description: "Literal qualifiers that do not count towards a domain's name \ + shape, as { qualifier, reason } tables with an optional domain.", + }, + ], "SQBRTEST203" => vec![BuiltinOption { - kind: BuiltinOptionKind::StringList, + kind: BuiltinOptionKind::StringList(&[]), name: "allowed_tests", description: "Reviewed SQL test names that may assert only that empty inputs produce \ no rows; listed tests count toward min_tests_per_model.", diff --git a/crates/sqlbuild-rules/src/rules/models.rs b/crates/sqlbuild-rules/src/rules/models.rs index 0158b2f7e5..cc882b4954 100644 --- a/crates/sqlbuild-rules/src/rules/models.rs +++ b/crates/sqlbuild-rules/src/rules/models.rs @@ -76,8 +76,10 @@ impl FaultCollector { pub enum BuiltinOptionKind { /// A boolean defaulting to false. Boolean, - /// A list of strings defaulting to empty. - StringList, + /// A list of strings with the given default. + StringList(&'static [&'static str]), + /// A list of tables, each checked by the rule's own configuration validation. + TableList, } /// One option a built-in rule declares. diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 69609a0e34..4510bfa822 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -268,3 +268,14 @@ class RuleExceptionConfigCase: expected_exit: int expected_messages: tuple[str, ...] = () expected_fragments: tuple[str, ...] = () + + +@dataclass(frozen=True) +class PublishedLayerCase: + """One set of published models checked by the published-layer Rules.""" + + description: str + files: tuple[tuple[str, str], ...] + rules_toml: str + expected_findings: tuple[tuple[str, str], ...] + expected_exit: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py new file mode 100644 index 0000000000..72112b7c1e --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_published_layer_rules.py @@ -0,0 +1,114 @@ +"""E2E coverage of the published-layer Rules on the orders and finance example.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import PublishedLayerCase +from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + compile_diagnostics, + run_compile_cli, + write_model_files, +) + +_PROJECT_TOML: str = ( + 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRGRAPH103", "SQBRPROJECT107"]\n' +) +_ALLOWLIST: str = ( + "\n[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ domain = "sales", qualifier = "finance", ' + 'reason = "Finance reads a reviewed published variant." }]\n' +) +_FINANCE_PATH: str = "models/sales/mart/sales__mart__orders__finance.sql" +_INTERMEDIATE: tuple[str, str] = ( + "models/sales/intermediate/clean/sales__int_clean__orders.sql", + "MODEL (description 'Clean orders.');\nSELECT 1 AS order_id\n", +) +_ORDERS: tuple[str, str] = ( + "models/sales/mart/sales__mart__orders.sql", + "MODEL (description 'Published orders.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', +) +_CUSTOMERS: tuple[str, str] = ( + "models/sales/mart/sales__mart__customers.sql", + "MODEL (description 'Published customers.');\n" + 'SELECT o.order_id FROM __ref("sales__int_clean__orders") AS o\n', +) +_FINANCE_VARIANT: tuple[str, str] = ( + _FINANCE_PATH, + "MODEL (description 'Orders reshaped for finance.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', +) +_FINANCE_STAGING: tuple[str, str] = ( + "models/finance/staging/finance__stg_v__orders.sql", + "MODEL (description 'Finance view of orders.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders__finance") AS o\n', +) +_ORDERS_VIEW: tuple[str, str] = ( + "models/sales/mart/sales__mart_v__orders.sql", + "MODEL (description 'Orders view.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', +) +_FINANCE_FROM_PUBLISHED: tuple[str, str] = ( + "models/finance/staging/finance__stg_v__orders.sql", + "MODEL (description 'Finance view of orders.');\n" + 'SELECT o.order_id FROM __ref("sales__mart__orders") AS o\n', +) +_ORDERS_FINANCE: tuple[tuple[str, str], ...] = ( + _INTERMEDIATE, + _ORDERS, + _CUSTOMERS, + _FINANCE_VARIANT, + _FINANCE_STAGING, +) + + +@pytest.mark.parametrize( + "test_case", + [ + PublishedLayerCase( + description="a consumer variant in the producing domain's published layer", + files=_ORDERS_FINANCE, + rules_toml="", + expected_findings=(("SQBRGRAPH103", _FINANCE_PATH), ("SQBRPROJECT107", _FINANCE_PATH)), + expected_exit=1, + ), + PublishedLayerCase( + description="an allowlisted qualifier passes the shape rule but not the read rule", + files=_ORDERS_FINANCE, + rules_toml=_ALLOWLIST, + expected_findings=(("SQBRGRAPH103", _FINANCE_PATH),), + expected_exit=1, + ), + PublishedLayerCase( + description="shared logic in intermediate and a view of its own entity", + files=(_INTERMEDIATE, _ORDERS, _CUSTOMERS, _ORDERS_VIEW, _FINANCE_FROM_PUBLISHED), + rules_toml="", + expected_findings=(), + expected_exit=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_published_models_when_compiling_then_layer_rules_report_variants( + test_case: PublishedLayerCase, + tmp_path: Path, +) -> None: + write_model_files( + project_dir=tmp_path, toml=_PROJECT_TOML + test_case.rules_toml, files=test_case.files + ) + + result: subprocess.CompletedProcess[str] = run_compile_cli(tmp_path) + + findings: tuple[tuple[str, str], ...] = tuple( + sorted((str(item["code"]), str(item["path"])) for item in compile_diagnostics(result)) + ) + assert findings == test_case.expected_findings, result.stdout + result.stderr + assert result.returncode == test_case.expected_exit, result.stderr + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py index 81a9a6ec02..c7558ff763 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_catalogue.py @@ -43,12 +43,14 @@ "SQBRDECLARATION306", "SQBRGRAPH101", "SQBRGRAPH102", + "SQBRGRAPH103", "SQBRPROJECT101", "SQBRPROJECT102", "SQBRPROJECT103", "SQBRPROJECT104", "SQBRPROJECT105", "SQBRPROJECT106", + "SQBRPROJECT107", "SQBRPROJECT201", "SQBRPROJECT202", "SQBRPROJECT203", @@ -104,12 +106,14 @@ "SQBRDECLARATION306", "SQBRGRAPH101", "SQBRGRAPH102", + "SQBRGRAPH103", "SQBRPROJECT101", "SQBRPROJECT102", "SQBRPROJECT103", "SQBRPROJECT104", "SQBRPROJECT105", "SQBRPROJECT106", + "SQBRPROJECT107", "SQBRPROJECT201", "SQBRPROJECT202", "SQBRPROJECT203", diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py index 66b29d3fe3..ad73e6c27f 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/engine/test_config.py @@ -181,6 +181,19 @@ ), expected_error_pattern="sets cte without relation", ), + RulesConfigErrorTestCase( + description="unknown published layer", + source='[rules.rule_options.SQBRGRAPH103]\npublished_layers = ["gold"]\n', + expected_error_pattern="published_layers entries must be model name layers", + ), + RulesConfigErrorTestCase( + description="allowed qualifier without a reason", + source=( + "[rules.rule_options.SQBRPROJECT107]\n" + 'allowed_qualifiers = [{ qualifier = "csv_export" }]\n' + ), + expected_error_pattern="allowed_qualifiers entries must be", + ), RulesConfigErrorTestCase( description="unknown SQL test rules key", source='[rules.sql_tests]\npipeline_directory = "pipelines"\nroot = "tests"\n', From d6e61d76aa724cbf48071861d9d3942b5bf09a71 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 00:48:10 +0100 Subject: [PATCH 25/39] docs(rules): document config-only exceptions and published-layer rules Rewrite findings and exceptions for relation-level exceptions, one-row cross join inference and the removed inline suppressions and allow_exceptions setting, add the published-layer Rules page with the orders and finance example, and regenerate the docs skill. --- .../sqlbuild/references/docs/CONTENTS.md | 1 + .../docs/concepts/project-configuration.md | 13 ++- .../references/docs/concepts/rules.md | 9 +- .../rules/configuration-and-selection.md | 4 + .../concepts/rules/findings-and-exceptions.md | 89 ++++++++++++++----- .../concepts/rules/published-layer-rules.md | 81 +++++++++++++++++ .../docs/concepts/project-configuration.mdx | 13 ++- .../src/content/docs/docs/concepts/rules.mdx | 11 ++- .../rules/configuration-and-selection.mdx | 4 + .../rules/findings-and-exceptions.mdx | 79 +++++++++++----- .../concepts/rules/published-layer-rules.mdx | 78 ++++++++++++++++ website/src/sidebar.json | 1 + 12 files changed, 319 insertions(+), 64 deletions(-) create mode 100644 src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md create mode 100644 website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md index c659d00a80..0b5f43355a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md @@ -66,6 +66,7 @@ Bundled copies of every page on the SQLBuild documentation site, matching the in - [Overview](concepts/rules.md) (`concepts/rules`) - Turn repeated SQL and project review decisions into compiler-enforced requirements. - [Configuration and selection](concepts/rules/configuration-and-selection.md) (`concepts/rules/configuration-and-selection`) - Select built-in and custom Rules by exact code or family prefix. - [Findings and exceptions](concepts/rules/findings-and-exceptions.md) (`concepts/rules/findings-and-exceptions`) - Understand Rule diagnostics and record intentional exceptions safely. +- [Published-layer Rules](concepts/rules/published-layer-rules.md) (`concepts/rules/published-layer-rules`) - Keep each domain's published layer free of consumer-specific variants. - [Execution and caching](concepts/rules/execution-and-caching.md) (`concepts/rules/execution-and-caching`) - Understand Rule enforcement order, focused runs, and dependency-aware reuse. - [Testing](concepts/testing.md) (`concepts/testing`) - SQL unit tests and multi-model tests with macro support, assertions, and model chaining. - [Scenarios](concepts/scenarios.md) (`concepts/scenarios`) - End-to-end tests that build real project graphs against coherent fixture data. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index b4a40fe677..c73ead5d4d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -713,11 +713,16 @@ select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See -[Compiler-integrated Rules](rules.md) for configuration, authoring, and suppressions. +[Compiler-integrated Rules](rules.md) for configuration, authoring, and exceptions. -| Key | Default | Description | -| --- | --- | --- | -| `allow_exceptions` | `true` | When `false`, `[[rules.rule_exceptions]]` and `[[rules.rule_ignores]]` entries are configuration errors and inline `-- sqb: ignore` directives are `SQBRSQL000` errors. The project-wide `ignore` list stays allowed. See [Forbidding exceptions](rules/findings-and-exceptions.md#forbidding-exceptions). | +| Key | Description | +| --- | --- | +| `ignore` | Rule codes or families turned off for the whole project. | +| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQBRSQL038`. Stale entries fail compilation. See [Findings and exceptions](rules/findings-and-exceptions.md). | +| `[[rules.rule_ignores]]` | Rules turned off for `paths` or `selectors`, with a `reason`. | + +Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSQL000` errors +that print the entry to add, and the former `allow_exceptions` setting is a configuration error. ## Scopes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md index a512a2610d..b788e472c2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules.md @@ -209,10 +209,10 @@ need to execute SQL against controlled data. Use an audit when the answer depend A finding contains a stable code, project-relative path, line, column, explanation, and remediation. This makes the same requirement usable in a terminal, JSON output, CI annotation, or agent workflow. -Intentional departures remain explicit. Exact exceptions and path- or resource-scoped ignores -require a reason; exact exceptions are stale-checked so obsolete suppressions do not silently -accumulate. Mandatory compiler correctness cannot be suppressed. A project can forbid exceptions, -scoped ignores and inline suppressions entirely with `[rules] allow_exceptions = false`. +Intentional departures remain explicit and live only in `sqlbuild_project.toml`. Exact and +relation-level exceptions and path- or resource-scoped ignores require a reason; exceptions are +stale-checked so obsolete entries do not silently accumulate. There are no inline suppression +comments, and mandatory compiler correctness cannot be excepted at all. See [Findings and exceptions](rules/findings-and-exceptions.md) for configuration examples. @@ -234,5 +234,6 @@ and [Execution and caching](rules/execution-and-caching.md) for the Rules lifecy Select exact Rules or families and configure project-owned options. Understand diagnostics, exact exceptions, path ignores, and stale checks. + Keep each domain's published layer free of consumer-specific variants. Learn evaluation order, focused execution, and dependency-aware reuse. List, inspect, run, and generate guidance for configured Rules. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md index 5f3cc3c8ea..6a835eb92f 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/configuration-and-selection.md @@ -71,6 +71,10 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE `min_tests_per_model`. Entries that name no existing test are reported as stale. See [Empty-input tests](../testing.md#empty-input-tests). +`SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts +`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +[Published-layer Rules](published-layer-rules.md). + Unknown Rule codes, option names, and invalid option values fail configuration. ## Focused selection diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md index b833e7b2a6..6c2a500d99 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md @@ -6,9 +6,23 @@ Online: https://sqlbuild.com/docs/concepts/rules/findings-and-exceptions/ +## Contents + +- Exact exceptions +- Relation-level exceptions +- Path- and resource-scoped ignores +- No inline suppressions +- Removed settings +- Mandatory correctness +- Machine-readable findings + A Rule finding identifies an exact code, project-relative path, line, column, message, and remediation. Error findings make compilation fail and prevent artifact completion. +Every deliberate deviation from a selected Rule lives in `sqlbuild_project.toml`, never in SQL +comments. A change to the project file is something a reviewer stops to look at; a comment in a +large SQL diff is not. + ## Exact exceptions Use an exact exception for one known finding. Exceptions require a reason and are stale-checked: @@ -23,6 +37,34 @@ reason = "This example intentionally demonstrates one sampled row." An exception refers to one exact Rule code and path. If the finding disappears, SQLBuild reports the stale exception so obsolete configuration does not accumulate silently. +## Relation-level exceptions + +A file-wide exception for a deliberate cartesian product would also hide any accidental cross join +added to the same file later. `SQBRSQL038` findings name the joined relation and, inside a CTE, +that CTE, so an exception can cover exactly one join: + +```toml +[[rules.rule_exceptions]] +rule = "SQBRSQL038" +path = "models/orders/order_checks.sql" +relation = "order_policy" +cte = "policy_checks" # optional: only the join inside this top-level CTE +reason = "Every order is evaluated against every policy check." +``` + +`relation` is the joined relation as written, with `__ref`, `__seed` and `__source` calls resolved +to the referenced name. Without `cte`, the exception covers that relation anywhere in the file. +Relation-level exceptions are stale-checked like exact exceptions: an entry that matches no finding +fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be +combined with relation-level exceptions for the same code and path. + +`SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most +intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a +declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an +aggregate without `GROUP BY`, a one-row `VALUES`, `LIMIT 1`, or `QUALIFY ROW_NUMBER() OVER (...) = 1` +without a partition, carried through CTEs and subqueries that only filter or project. A join, +`UNNEST`/`FLATTEN`, set operation or `GROUP BY` in between stops the proof, and the join is reported. + ## Path- and resource-scoped ignores Use a path ignore when a documented project area intentionally follows a different convention: @@ -59,36 +101,35 @@ reason = "These intermediate SQL files intentionally preserve upstream columns." `paths` and `selectors` may be combined in one scoped ignore. Both forms require a reason. -## Inline suppressions - -Suppress one SQL Rule finding on the next SQL line with a reasoned comment: +To turn a Rule off for the whole project, add its code to `[rules] ignore`. -```sql --- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row -SELECT customer_id FROM customers LIMIT 1 -``` +## No inline suppressions -A directive without a reason, or one that no longer matches a finding, is reported as -`SQBRSQL000`. +SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` line is a `SQBRSQL000` error +that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +exact entry to add instead: -## Forbidding exceptions +```text +error[SQBRSQL000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml + --> models/orders/order_checks.sql:4:1 + = help: Remove this comment, then fix the SQBRSQL038 finding, or record it as a reviewed exception in sqlbuild_project.toml: + [[rules.rule_exceptions]] + rule = "SQBRSQL038" + path = "models/orders/order_checks.sql" + relation = "order_policy" + reason = "every order meets every policy" +``` -Set `allow_exceptions = false` to require that every selected Rule passes everywhere: +Copy the entry into `sqlbuild_project.toml`, adjust the reason for a reviewer, and delete the +comment. -```toml -[rules] -select = ["SQBRSQL", "SQBRMODEL"] -ignore = ["SQBRSQL004"] -allow_exceptions = false # default true -``` +## Removed settings -With exceptions forbidden, any `[[rules.rule_exceptions]]` or `[[rules.rule_ignores]]` entry fails -configuration loading with an error that counts the entries and shows the setting to change, so -`sqb compile` and `sqb rules run` stop before evaluating Rules. Each inline `-- sqb: ignore` -directive becomes a `SQBRSQL000` error at the directive and suppresses nothing, so the finding it -was hiding is reported too. The project-wide `ignore` list stays allowed: it is the adoption switch -for a Rule across the whole project, not a scoped escape hatch. `[[rules.graph_edge_exceptions]]`, -`[[rules.select_star_allow]]` and `[[rules.threshold_overrides]]` are not affected. +`[rules] allow_exceptions` was removed. Configured exceptions are always allowed because the project +file is the review gate, and inline suppressions no longer exist. A project that still sets +`allow_exceptions` fails configuration loading with an error that explains the removal: delete the +line, keep reviewed deviations as `[[rules.rule_exceptions]]` entries, and use `[rules] ignore` to +turn a Rule off for the whole project. ## Mandatory correctness diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md new file mode 100644 index 0000000000..248639b4a9 --- /dev/null +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/published-layer-rules.md @@ -0,0 +1,81 @@ + + +# Published-layer Rules + +> Keep each domain's published layer free of consumer-specific variants. + +Online: https://sqlbuild.com/docs/concepts/rules/published-layer-rules/ + +A domain's published layer is its interface to the rest of the project. Under pressure to make +something work, it is tempting to add a consumer-specific variant next to an existing published +model: + +```text +sales__int_clean__orders -> sales__mart__orders -> sales__mart__orders__finance -> finance__stg_v__orders +``` + +`sales__mart__orders__finance` reshapes a published model for one consumer in another domain. That +work belongs in the consumer's domain, or in the producing domain's intermediate layer. Two +built-in Rules catch it: `SQBRPROJECT107` by its name as soon as it is created, and `SQBRGRAPH103` +by its dependency even when the name looks uniform. Together they leave one fix: logic shared within +a domain belongs in its intermediate layer. + +Both Rules read model names of the form `____[__]`. Models whose +names do not follow that grammar are not checked. Both are errors and are selected with their +families (`SQBRGRAPH`, `SQBRPROJECT`) or `SQBR`. + +## SQBRGRAPH103: no published reads within a domain + +A model in a published layer must not read another published model of its own domain. The one +built-in allowance is a view of its own table: `__mart_v__[__]` may read +`__mart__`, or `__mart____` with the same qualifier. Reading another domain's published models is the point of publishing +them and is allowed. + +```text +error[SQBRGRAPH103]: sales__mart__orders__finance reads sales__mart__orders; published models must not read published models of their own domain + --> models/sales/mart/sales__mart__orders__finance.sql:1:1 + = help: Move the shared logic into a sales__int_* model that both published models read, or move this model into the domain that consumes it. To move it into intermediate: sqb mv sales__mart__orders__finance models/sales/intermediate/enriched/sales__int_enriched__orders__finance.sql +``` + +## SQBRPROJECT107: one name shape per domain + +Within one domain's published layer, every model name has the same number of segments: either all +`____` or all `______`. The models that do +not fit the domain's majority shape are reported, with the remediation: + +1. If a model with a qualifier adapts a model without one, move that logic into intermediate or into + the consumer's domain. +2. If the domain really is split by qualifier, rename the models without one into the family, for + example `sqb rename sales__mart__customers sales__mart__customers__core`. +3. If the qualifier is intentional and structural, allowlist it with a reason. + +Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left +out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain +only: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { qualifier = "csv_export", reason = "File export views of a published entity." }, + { domain = "sales", qualifier = "fra", reason = "Published once per country." }, +] +``` + +## Published layers + +Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule +to change that; each Rule reads its own option: + +```toml +[rules.rule_options.SQBRGRAPH103] +published_layers = ["mart", "mart_v"] + +[rules.rule_options.SQBRPROJECT107] +published_layers = ["mart", "mart_v"] +``` + +## Deviating + +There are no inline suppressions. Turn a Rule off for the whole project with `[rules] ignore`, or +record one reviewed model as a `[[rules.rule_exceptions]]` entry with its path and a reason. See +[Findings and exceptions](findings-and-exceptions.md). diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index d2badeca2e..800d5c4c67 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -690,11 +690,16 @@ select = ["SQBRSQL", "SQBRGRAPH", "XSQBRARCH"] Rules are opt-in. Exact codes activate individual checks and prefixes activate a family. Built-in codes begin with `SQBR`; repository-defined codes begin with `XSQBR`. See -[Compiler-integrated Rules](/docs/concepts/rules) for configuration, authoring, and suppressions. +[Compiler-integrated Rules](/docs/concepts/rules) for configuration, authoring, and exceptions. -| Key | Default | Description | -| --- | --- | --- | -| `allow_exceptions` | `true` | When `false`, `[[rules.rule_exceptions]]` and `[[rules.rule_ignores]]` entries are configuration errors and inline `-- sqb: ignore` directives are `SQBRSQL000` errors. The project-wide `ignore` list stays allowed. See [Forbidding exceptions](/docs/concepts/rules/findings-and-exceptions#forbidding-exceptions). | +| Key | Description | +| --- | --- | +| `ignore` | Rule codes or families turned off for the whole project. | +| `[[rules.rule_exceptions]]` | One reviewed exception: `rule`, `path` and `reason`, plus `relation` and optional `cte` for relation-level Rules such as `SQBRSQL038`. Stale entries fail compilation. See [Findings and exceptions](/docs/concepts/rules/findings-and-exceptions). | +| `[[rules.rule_ignores]]` | Rules turned off for `paths` or `selectors`, with a `reason`. | + +Rule exceptions live only in this file. Inline `-- sqb: ignore` comments are `SQBRSQL000` errors +that print the entry to add, and the former `allow_exceptions` setting is a configuration error. ## Scopes diff --git a/website/src/content/docs/docs/concepts/rules.mdx b/website/src/content/docs/docs/concepts/rules.mdx index ddaf3b5a6d..2b66832607 100644 --- a/website/src/content/docs/docs/concepts/rules.mdx +++ b/website/src/content/docs/docs/concepts/rules.mdx @@ -199,10 +199,10 @@ need to execute SQL against controlled data. Use an audit when the answer depend A finding contains a stable code, project-relative path, line, column, explanation, and remediation. This makes the same requirement usable in a terminal, JSON output, CI annotation, or agent workflow. -Intentional departures remain explicit. Exact exceptions and path- or resource-scoped ignores -require a reason; exact exceptions are stale-checked so obsolete suppressions do not silently -accumulate. Mandatory compiler correctness cannot be suppressed. A project can forbid exceptions, -scoped ignores and inline suppressions entirely with `[rules] allow_exceptions = false`. +Intentional departures remain explicit and live only in `sqlbuild_project.toml`. Exact and +relation-level exceptions and path- or resource-scoped ignores require a reason; exceptions are +stale-checked so obsolete entries do not silently accumulate. There are no inline suppression +comments, and mandatory compiler correctness cannot be excepted at all. See [Findings and exceptions](/docs/concepts/rules/findings-and-exceptions) for configuration examples. @@ -229,6 +229,9 @@ and [Execution and caching](/docs/concepts/rules/execution-and-caching) for the Understand diagnostics, exact exceptions, path ignores, and stale checks. + + Keep each domain's published layer free of consumer-specific variants. + Learn evaluation order, focused execution, and dependency-aware reuse. diff --git a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx index 3ef7a3e73e..070b4e38f4 100644 --- a/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx +++ b/website/src/content/docs/docs/concepts/rules/configuration-and-selection.mdx @@ -61,6 +61,10 @@ reviewed exceptions. Listed tests are not flagged and still count toward `SQBRTE `min_tests_per_model`. Entries that name no existing test are reported as stale. See [Empty-input tests](/docs/concepts/testing#empty-input-tests). +`SQBRGRAPH103` and `SQBRPROJECT107` accept `published_layers`, and `SQBRPROJECT107` accepts +`allowed_qualifiers`, a list of `{ qualifier, reason }` tables with an optional `domain`. See +[Published-layer Rules](/docs/concepts/rules/published-layer-rules). + Unknown Rule codes, option names, and invalid option values fail configuration. ## Focused selection diff --git a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx index 5928733c12..25f01b5737 100644 --- a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx +++ b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx @@ -6,6 +6,10 @@ description: "Understand Rule diagnostics and record intentional exceptions safe A Rule finding identifies an exact code, project-relative path, line, column, message, and remediation. Error findings make compilation fail and prevent artifact completion. +Every deliberate deviation from a selected Rule lives in `sqlbuild_project.toml`, never in SQL +comments. A change to the project file is something a reviewer stops to look at; a comment in a +large SQL diff is not. + ## Exact exceptions Use an exact exception for one known finding. Exceptions require a reason and are stale-checked: @@ -20,6 +24,34 @@ reason = "This example intentionally demonstrates one sampled row." An exception refers to one exact Rule code and path. If the finding disappears, SQLBuild reports the stale exception so obsolete configuration does not accumulate silently. +## Relation-level exceptions + +A file-wide exception for a deliberate cartesian product would also hide any accidental cross join +added to the same file later. `SQBRSQL038` findings name the joined relation and, inside a CTE, +that CTE, so an exception can cover exactly one join: + +```toml +[[rules.rule_exceptions]] +rule = "SQBRSQL038" +path = "models/orders/order_checks.sql" +relation = "order_policy" +cte = "policy_checks" # optional: only the join inside this top-level CTE +reason = "Every order is evaluated against every policy check." +``` + +`relation` is the joined relation as written, with `__ref`, `__seed` and `__source` calls resolved +to the referenced name. Without `cte`, the exception covers that relation anywhere in the file. +Relation-level exceptions are stale-checked like exact exceptions: an entry that matches no finding +fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be +combined with relation-level exceptions for the same code and path. + +`SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most +intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a +declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an +aggregate without `GROUP BY`, a one-row `VALUES`, `LIMIT 1`, or `QUALIFY ROW_NUMBER() OVER (...) = 1` +without a partition, carried through CTEs and subqueries that only filter or project. A join, +`UNNEST`/`FLATTEN`, set operation or `GROUP BY` in between stops the proof, and the join is reported. + ## Path- and resource-scoped ignores Use a path ignore when a documented project area intentionally follows a different convention: @@ -56,36 +88,35 @@ reason = "These intermediate SQL files intentionally preserve upstream columns." `paths` and `selectors` may be combined in one scoped ignore. Both forms require a reason. -## Inline suppressions +To turn a Rule off for the whole project, add its code to `[rules] ignore`. -Suppress one SQL Rule finding on the next SQL line with a reasoned comment: - -```sql --- sqb: ignore SQBRSQL004 because the sample intentionally keeps one row -SELECT customer_id FROM customers LIMIT 1 -``` +## No inline suppressions -A directive without a reason, or one that no longer matches a finding, is reported as -`SQBRSQL000`. +SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` line is a `SQBRSQL000` error +that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +exact entry to add instead: -## Forbidding exceptions +```text +error[SQBRSQL000]: Inline Rule suppressions are not supported; declare the exception in sqlbuild_project.toml + --> models/orders/order_checks.sql:4:1 + = help: Remove this comment, then fix the SQBRSQL038 finding, or record it as a reviewed exception in sqlbuild_project.toml: + [[rules.rule_exceptions]] + rule = "SQBRSQL038" + path = "models/orders/order_checks.sql" + relation = "order_policy" + reason = "every order meets every policy" +``` -Set `allow_exceptions = false` to require that every selected Rule passes everywhere: +Copy the entry into `sqlbuild_project.toml`, adjust the reason for a reviewer, and delete the +comment. -```toml -[rules] -select = ["SQBRSQL", "SQBRMODEL"] -ignore = ["SQBRSQL004"] -allow_exceptions = false # default true -``` +## Removed settings -With exceptions forbidden, any `[[rules.rule_exceptions]]` or `[[rules.rule_ignores]]` entry fails -configuration loading with an error that counts the entries and shows the setting to change, so -`sqb compile` and `sqb rules run` stop before evaluating Rules. Each inline `-- sqb: ignore` -directive becomes a `SQBRSQL000` error at the directive and suppresses nothing, so the finding it -was hiding is reported too. The project-wide `ignore` list stays allowed: it is the adoption switch -for a Rule across the whole project, not a scoped escape hatch. `[[rules.graph_edge_exceptions]]`, -`[[rules.select_star_allow]]` and `[[rules.threshold_overrides]]` are not affected. +`[rules] allow_exceptions` was removed. Configured exceptions are always allowed because the project +file is the review gate, and inline suppressions no longer exist. A project that still sets +`allow_exceptions` fails configuration loading with an error that explains the removal: delete the +line, keep reviewed deviations as `[[rules.rule_exceptions]]` entries, and use `[rules] ignore` to +turn a Rule off for the whole project. ## Mandatory correctness diff --git a/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx new file mode 100644 index 0000000000..a7fcb6472b --- /dev/null +++ b/website/src/content/docs/docs/concepts/rules/published-layer-rules.mdx @@ -0,0 +1,78 @@ +--- +title: "Published-layer Rules" +description: "Keep each domain's published layer free of consumer-specific variants." +--- + +A domain's published layer is its interface to the rest of the project. Under pressure to make +something work, it is tempting to add a consumer-specific variant next to an existing published +model: + +```text +sales__int_clean__orders -> sales__mart__orders -> sales__mart__orders__finance -> finance__stg_v__orders +``` + +`sales__mart__orders__finance` reshapes a published model for one consumer in another domain. That +work belongs in the consumer's domain, or in the producing domain's intermediate layer. Two +built-in Rules catch it: `SQBRPROJECT107` by its name as soon as it is created, and `SQBRGRAPH103` +by its dependency even when the name looks uniform. Together they leave one fix: logic shared within +a domain belongs in its intermediate layer. + +Both Rules read model names of the form `____[__]`. Models whose +names do not follow that grammar are not checked. Both are errors and are selected with their +families (`SQBRGRAPH`, `SQBRPROJECT`) or `SQBR`. + +## SQBRGRAPH103: no published reads within a domain + +A model in a published layer must not read another published model of its own domain. The one +built-in allowance is a view of its own table: `__mart_v__[__]` may read +`__mart__`, or `__mart____` with the same qualifier. Reading another domain's published models is the point of publishing +them and is allowed. + +```text +error[SQBRGRAPH103]: sales__mart__orders__finance reads sales__mart__orders; published models must not read published models of their own domain + --> models/sales/mart/sales__mart__orders__finance.sql:1:1 + = help: Move the shared logic into a sales__int_* model that both published models read, or move this model into the domain that consumes it. To move it into intermediate: sqb mv sales__mart__orders__finance models/sales/intermediate/enriched/sales__int_enriched__orders__finance.sql +``` + +## SQBRPROJECT107: one name shape per domain + +Within one domain's published layer, every model name has the same number of segments: either all +`____` or all `______`. The models that do +not fit the domain's majority shape are reported, with the remediation: + +1. If a model with a qualifier adapts a model without one, move that logic into intermediate or into + the consumer's domain. +2. If the domain really is split by qualifier, rename the models without one into the family, for + example `sqb rename sales__mart__customers sales__mart__customers__core`. +3. If the qualifier is intentional and structural, allowlist it with a reason. + +Allowlisted qualifiers are literal and do not count towards the shape: models that use them are left +out of the check. Each entry needs a reason, and an entry with a `domain` applies to that domain +only: + +```toml +[rules.rule_options.SQBRPROJECT107] +allowed_qualifiers = [ + { qualifier = "csv_export", reason = "File export views of a published entity." }, + { domain = "sales", qualifier = "fra", reason = "Published once per country." }, +] +``` + +## Published layers + +Both Rules treat `mart` and `mart_v` as the published layers. Set `published_layers` on either Rule +to change that; each Rule reads its own option: + +```toml +[rules.rule_options.SQBRGRAPH103] +published_layers = ["mart", "mart_v"] + +[rules.rule_options.SQBRPROJECT107] +published_layers = ["mart", "mart_v"] +``` + +## Deviating + +There are no inline suppressions. Turn a Rule off for the whole project with `[rules] ignore`, or +record one reviewed model as a `[[rules.rule_exceptions]]` entry with its path and a reason. See +[Findings and exceptions](/docs/concepts/rules/findings-and-exceptions). diff --git a/website/src/sidebar.json b/website/src/sidebar.json index 770ed3304d..443cd3b656 100644 --- a/website/src/sidebar.json +++ b/website/src/sidebar.json @@ -118,6 +118,7 @@ "docs/concepts/rules", "docs/concepts/rules/configuration-and-selection", "docs/concepts/rules/findings-and-exceptions", + "docs/concepts/rules/published-layer-rules", "docs/concepts/rules/execution-and-caching" ] }, From 2631894911d55a2c9c5999cfd5314e60861fca70 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 23:23:22 +0100 Subject: [PATCH 26/39] fix: run provider teardown on the closing thread so exit cannot abort A sink call still running when the shutdown budget ended deferred provider teardown to whichever daemon delivery thread finished last. That thread then ran the provider's teardown inside native frames; when interpreter finalization began during the teardown, the thread was ended by forced unwinding through those frames and the process aborted with SIGABRT after the command had already succeeded. Otherwise the teardown was silently lost. finalize_when_idle now waits, bounded by the sink call timeout (at most one second), for delivery to become idle and then runs the finalizer on the calling thread. Delivery threads no longer run finalizers. --- .../_helpers/event_exporting/constants.rs | 2 + .../_helpers/event_exporting/dispatcher.rs | 93 +++++++------- .../_helpers/event_exporting/scope.rs | 2 +- fensu.toml | 2 +- .../docs/concepts/observability/sinks.md | 4 +- .../commands/main/providers/_test_types.py | 7 ++ .../main/providers/test_event_exporters.py | 113 ++++++++++++++++++ .../event_exporting/classes/_test_types.py | 4 + .../event_exporting/classes/helpers.py | 5 +- .../classes/test_command_scope.py | 20 ++-- .../classes/test_dispatcher.py | 32 +++-- .../classes/test_interpreter_exit.py | 30 +++++ .../docs/concepts/observability/sinks.mdx | 4 +- 13 files changed, 242 insertions(+), 76 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs index 7e9756e372..60f5340f11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -14,5 +14,7 @@ pub(crate) const POLL_SECONDS: f64 = 0.01; pub(crate) const JOIN_GRACE_SECONDS: f64 = 0.05; /// How long shutdown and interpreter exit wait for each stopped dispatcher thread. pub(crate) const EXIT_JOIN_SECONDS: f64 = 1.0; +/// The longest a finalizer waits for sink calls still running after shutdown before it runs anyway. +pub(crate) const IDLE_WAIT_SECONDS: f64 = 1.0; /// No time left before a deadline. pub(crate) const NO_SECONDS: f64 = 0.0; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index a736ac6130..bdfd0ad7e2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -28,8 +28,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::event_exporting::constants::{ - DISPATCHER_THREAD_NAME, EXIT_JOIN_SECONDS, HEALTH_THREAD_NAME, JOIN_GRACE_SECONDS, NO_SECONDS, - NOTIFIER_THREAD_NAME, OUTPUT_SINK_THREAD_NAME, POLL_SECONDS, WORKER_ATTRIBUTE, + DISPATCHER_THREAD_NAME, EXIT_JOIN_SECONDS, HEALTH_THREAD_NAME, IDLE_WAIT_SECONDS, + JOIN_GRACE_SECONDS, NO_SECONDS, NOTIFIER_THREAD_NAME, OUTPUT_SINK_THREAD_NAME, POLL_SECONDS, + WORKER_ATTRIBUTE, }; use crate::bindings::_helpers::event_exporting::models::{ BoundEventExporter, EventExportSummary, EventExporterFailure, @@ -85,7 +86,6 @@ pub(crate) struct State { pub(crate) blocked: HashSet, pub(crate) live_invocations: HashSet, pub(crate) next_invocation: u64, - idle_finalizers: Vec>, dispatcher_running: bool, shutdown_started: bool, shutdown_complete: bool, @@ -162,13 +162,6 @@ fn remaining_seconds(deadline: Instant) -> f64 { .as_secs_f64() } -/// Run finalizers, ignoring their failures. -pub(crate) fn run_finalizers(py: Python<'_>, finalizers: Vec>) { - for finalizer in finalizers { - drop(finalizer.bind(py).call0()); - } -} - /// End a retired command-output sink thread once it has run every call queued before. fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { if let Some(calls) = retired { @@ -427,45 +420,51 @@ impl Shared { self.notify_all(py) } - /// Forget one finished invocation; once idle, run the finalizers waiting for it. + /// Forget one finished invocation and wake a caller waiting for delivery to become idle. pub(crate) fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { - let (finalizers, retired): (Vec>, Option>) = { + let retired: Option> = { let _held = self.hold(py)?; let mut state = self.locked()?; state.live_invocations.remove(&invocation); - if state.live_invocations.is_empty() && !state.dispatcher_running { - let retired: Option> = if state.shutdown_complete { - state.output_lane.take() - } else { - None - }; - (std::mem::take(&mut state.idle_finalizers), retired) - } else { - (Vec::new(), None) - } - }; - retire_output_lane(py, retired)?; - run_finalizers(py, finalizers); - Ok(()) - } - - /// Mark the dispatcher thread finished and run idle finalizers. - pub(crate) fn dispatcher_finished(&self, py: Python<'_>) -> PyResult<()> { - let finalizers: Vec> = { - let _held = self.hold(py)?; - let mut state = self.locked()?; - state.dispatcher_running = false; - let finalizers = if state.live_invocations.is_empty() { - std::mem::take(&mut state.idle_finalizers) + let retired: Option> = if Self::idle(&state) && state.shutdown_complete { + state.output_lane.take() } else { - Vec::new() + None }; drop(state); self.notify_all(py)?; - finalizers + retired }; - run_finalizers(py, finalizers); - Ok(()) + retire_output_lane(py, retired) + } + + /// Mark the dispatcher thread finished. + pub(crate) fn dispatcher_finished(&self, py: Python<'_>) -> PyResult<()> { + let _held = self.hold(py)?; + self.locked()?.dispatcher_running = false; + self.notify_all(py) + } + + /// Whether the dispatcher thread and every sink call have finished. + fn idle(state: &State) -> bool { + state.live_invocations.is_empty() && !state.dispatcher_running + } + + /// Wait until delivery is idle, the idle wait ends or interpreter exit begins. + fn await_idle(&self, py: Python<'_>) -> PyResult<()> { + let deadline: Instant = + Instant::now() + self.invocation_timeout.min(seconds(IDLE_WAIT_SECONDS)); + let _held = self.hold(py)?; + loop { + if Self::idle(&*self.locked()?) || interpreter_exiting(py)? { + return Ok(()); + } + let remaining: f64 = remaining_seconds(deadline); + if remaining <= NO_SECONDS { + return Ok(()); + } + self.wait(py, remaining.min(POLL_SECONDS))?; + } } /// Register a running invocation, returning its identity. @@ -660,7 +659,6 @@ impl EventExporterDispatcher { blocked: HashSet::new(), live_invocations: HashSet::new(), next_invocation: 0, - idle_finalizers: Vec::new(), dispatcher_running: true, shutdown_started: false, shutdown_complete: false, @@ -922,17 +920,10 @@ impl EventExporterDispatcher { Ok(summary) } - /// Run `finalizer` once the dispatcher thread and every live invocation have finished. + /// Run `finalizer` on the calling thread, never a daemon delivery thread, once delivery is idle or the idle wait ends. fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { - { - let _held = self.shared.hold(py)?; - let mut state = self.shared.locked()?; - if !state.live_invocations.is_empty() || state.dispatcher_running { - state.idle_finalizers.push(finalizer); - return Ok(()); - } - } - run_finalizers(py, vec![finalizer]); + self.shared.await_idle(py)?; + drop(finalizer.bind(py).call0()); Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs index 45396ddb7b..f3a5b2a94c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs @@ -41,7 +41,7 @@ pub(crate) struct EventExporterCommandScope { state: Mutex, } -/// Releases and force-closes a provider session once sink delivery is idle. +/// Releases and force-closes a provider session on the closing thread once sink delivery is idle. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct ProviderSessionCloser { session: Py, diff --git a/fensu.toml b/fensu.toml index 154893a523..9ef4c92771 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1224,7 +1224,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=1", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md index aa0c9d1058..33e648e3c0 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -87,7 +87,9 @@ min_severity = "warning" `shutdown_timeout` sets how long a command waits at exit for lifecycle sinks to drain queued events. It is a fixed duration from `"0s"` to `"10m"`, defaults to `"2s"`, and is accepted only under `[sinks.lifecycle]`, not per named sink. `"0s"` drops anything still queued at exit. The -setting does not change the one-second timeout for each sink call. +setting does not change the one-second timeout for each sink call. A sink call still running when +the budget ends gets up to that one second to return before providers are torn down; teardown +always runs on the command's own thread. ```toml [sinks.lifecycle] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py index 009abe88ae..a9430b328c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/_test_types.py @@ -124,3 +124,10 @@ class FilteredSinkFailureE2ETestCase: expected_runs_only: tuple[str, ...] expected_errors_only: tuple[str, ...] expected_warning: str + + +@dataclass(frozen=True) +class SlowSinkExitE2ETestCase: + description: str + command: tuple[str, ...] + expected_marker_entries: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py index b676e69f0d..79d1d9a0ea 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_event_exporters.py @@ -15,6 +15,7 @@ FilteredSinkFailureE2ETestCase, NoExporterCommandE2ETestCase, ReadOnlyProviderE2ETestCase, + SlowSinkExitE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -421,3 +422,115 @@ def failing(event: LifecycleEvent) -> None: assert errors_only == test_case.expected_errors_only assert test_case.expected_warning.format(accepted=accepted) in result.stderr assert "destination detail" not in result.stdout + result.stderr + + +@pytest.mark.parametrize( + "test_case", + ( + SlowSinkExitE2ETestCase( + description="sink call in flight when the shutdown budget ends", + command=("audit",), + expected_marker_entries=("teardown started", "teardown finished"), + ), + ), + ids=lambda case: case.description, +) +def test_given_sink_call_in_flight_at_shutdown_when_process_exits_then_teardown_completes_without_abort( + test_case: SlowSinkExitE2ETestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + marker_path: Path = tmp_path / "teardown.txt" + monkeypatch.setenv("TEARDOWN_MARKER_PATH", str(marker_path)) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="slow_sink_exit_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "slow_sink_exit_project" + adapter = "duckdb" + + [connection] + database = "slow_sink_exit_project.duckdb" + + [sinks.lifecycle] + shutdown_timeout = "0s" + """ + ).lstrip(), + "providers/destination.py": dedent( + """ + import os + import sys + import time + import types + from pathlib import Path + + from sqlbuild.providers import Provider + + + class _SlowInterpreterExit: + def __del__(self, _time=time): + _time.sleep(1.0) + + + _holder = types.ModuleType("slow_interpreter_exit") + _holder.finalizer = _SlowInterpreterExit() + sys.modules["slow_interpreter_exit"] = _holder + + + class Destination(Provider): + '''Destination whose publishes and flush are slow.''' + + def publish(self) -> None: + time.sleep(0.05) + + def teardown(self) -> None: + path = Path(os.environ["TEARDOWN_MARKER_PATH"]) + with path.open("a", encoding="utf-8") as stream: + stream.write("teardown started\\n") + time.sleep(0.5) + with path.open("a", encoding="utf-8") as stream: + stream.write("teardown finished\\n") + """ + ).lstrip(), + "sinks/publish.py": dedent( + """ + from providers.destination import Destination + from sqlbuild.sinks import LifecycleEvent, lifecycle_event_sink + + + @lifecycle_event_sink + def publish_lifecycle(event: LifecycleEvent, destination: Destination) -> None: + del event + destination.publish() + """ + ).lstrip(), + "models/orders.sql": dedent( + """ + MODEL ( + description "Test model orders.", + materialized table, + columns (order_id (audits [not_null, unique])) + ); + + SELECT range AS order_id FROM range(10) + """ + ).lstrip(), + }, + ) + built: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert built.returncode == 0, built.stdout + built.stderr + marker_path.unlink() + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", *test_case.command), project_dir=project_dir + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert ( + tuple(marker_path.read_text(encoding="utf-8").splitlines()) + == test_case.expected_marker_entries + ) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index 577e9b4db1..8bbc51e88d 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -11,6 +11,9 @@ class EventExporterDispatcherTestCase: class EventExporterTeardownTestCase: description: str expected_teardown_count: int + release_after_seconds: float + min_close_seconds: float + max_close_seconds: float @dataclass(frozen=True) @@ -47,3 +50,4 @@ class InterpreterExitTestCase: description: str script: str expected_returncode: int + expected_stderr: str = "" diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py index af644e7136..b62bc6c831 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/helpers.py @@ -1,6 +1,6 @@ from datetime import UTC, datetime from pathlib import Path -from threading import Event +from threading import Event, current_thread from typing import ClassVar from sqlbuild._native import ( @@ -19,6 +19,7 @@ class BlockingProvider(Provider): release_export: ClassVar[Event] = Event() teardown_finished: ClassVar[Event] = Event() teardown_count: ClassVar[int] = 0 + teardown_threads: ClassVar[list[str]] = [] @classmethod def reset(cls) -> None: @@ -26,6 +27,7 @@ def reset(cls) -> None: cls.release_export.clear() cls.teardown_finished.clear() cls.teardown_count = 0 + cls.teardown_threads = [] def export(self) -> None: self.entered.set() @@ -33,6 +35,7 @@ def export(self) -> None: def teardown(self) -> None: type(self).teardown_count += 1 + type(self).teardown_threads.append(current_thread().name) self.teardown_finished.set() diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py index 4f40adb060..fb66a4c64c 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_command_scope.py @@ -3,7 +3,7 @@ import time from datetime import UTC, datetime from pathlib import Path -from threading import Event +from threading import Event, Timer, current_thread import pytest @@ -31,10 +31,13 @@ @pytest.mark.parametrize( "test_case", - (EventExporterTeardownTestCase("deferred provider teardown", 1),), + ( + EventExporterTeardownTestCase("sink returns during the idle wait", 1, 0.1, 0.08, 0.9), + EventExporterTeardownTestCase("sink still hung after the idle wait", 1, 30.0, 0.9, 3.0), + ), ids=lambda case: case.description, ) -def test_given_live_timed_out_exporter_when_scope_closes_then_provider_waits_for_invocation( +def test_given_live_timed_out_exporter_when_scope_closes_then_provider_tears_down_on_closing_thread( test_case: EventExporterTeardownTestCase, tmp_path: Path, ) -> None: @@ -54,16 +57,19 @@ def test_given_live_timed_out_exporter_when_scope_closes_then_provider_waits_for ) dispatcher.enqueue(lifecycle_event()) assert BlockingProvider.entered.wait(timeout=0.2) + release: Timer = Timer(test_case.release_after_seconds, BlockingProvider.release_export.set) + release.start() started: float = time.monotonic() summary: EventExportSummary = scope.close() - assert time.monotonic() - started < 0.2 - assert summary.failed == 1 - assert BlockingProvider.teardown_count == 0 + elapsed: float = time.monotonic() - started + release.cancel() BlockingProvider.release_export.set() - assert BlockingProvider.teardown_finished.wait(timeout=0.2) + assert test_case.min_close_seconds <= elapsed < test_case.max_close_seconds + assert summary.failed == 1 assert BlockingProvider.teardown_count == test_case.expected_teardown_count + assert BlockingProvider.teardown_threads == [current_thread().name] _ = scope.close() assert BlockingProvider.teardown_count == test_case.expected_teardown_count diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 0135132e6e..4621c9e518 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -259,25 +259,26 @@ def publish(*, event: LifecycleEvent) -> None: (EventExporterDispatcherTestCase("last live invocation finalizes once", 0),), ids=lambda case: case.description, ) -def test_given_multiple_timed_out_invocations_when_registering_finalizer_then_last_return_runs_once( +def test_given_multiple_timed_out_invocations_when_finalizing_then_runs_once_on_caller_after_last( test_case: EventExporterDispatcherTestCase, ) -> None: first_entered: threading.Event = threading.Event() second_entered: threading.Event = threading.Event() first_release: threading.Event = threading.Event() second_release: threading.Event = threading.Event() - finalized: threading.Event = threading.Event() - finalizer_calls: list[str] = [] + calls: list[str] = [] def first(*, event: LifecycleEvent) -> None: del event first_entered.set() first_release.wait() + calls.append("first returned") def second(*, event: LifecycleEvent) -> None: del event second_entered.set() second_release.wait() + calls.append("second returned") dispatcher: EventExporterDispatcher = EventExporterDispatcher( exporters=( @@ -285,24 +286,29 @@ def second(*, event: LifecycleEvent) -> None: BoundEventExporter("second", second, {}), ), shutdown_timeout_seconds=0.1, - invocation_timeout_seconds=0.02, + invocation_timeout_seconds=0.3, ) dispatcher.enqueue(lifecycle_event()) - assert first_entered.wait(timeout=0.2) - assert second_entered.wait(timeout=0.2) + assert first_entered.wait(timeout=1.0) + assert second_entered.wait(timeout=1.0) summary: EventExportSummary = dispatcher.shutdown() def finalize() -> None: - finalizer_calls.append("finalized") - finalized.set() + calls.append(f"finalized on {threading.current_thread().name}") + releases: tuple[threading.Timer, ...] = ( + threading.Timer(0.05, first_release.set), + threading.Timer(0.1, second_release.set), + ) + for release in releases: + release.start() dispatcher.finalize_when_idle(finalize) - first_release.set() - assert not finalized.wait(timeout=0.02) - second_release.set() - assert finalized.wait(timeout=0.2) - assert finalizer_calls == ["finalized"] + assert calls == [ + "first returned", + "second returned", + f"finalized on {threading.current_thread().name}", + ] assert summary.delivered == test_case.expected_delivered diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py index f73489d8f4..3ccb374c2c 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_interpreter_exit.py @@ -69,6 +69,35 @@ def slow(*, event): ), 0, ), + InterpreterExitTestCase( + "provider teardown waiting for a timed-out sink", + dedent( + """ + from types import SimpleNamespace + + from sqlbuild._native import BoundEventExporter, EventExporterDispatcher + + def slow(*, event): + time.sleep(0.3) + + def teardown(): + time.sleep(0.5) + print("teardown finished", file=sys.stderr, flush=True) + + dispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("slow", slow, {}),), + shutdown_timeout_seconds=0.0, + ) + dispatcher.enqueue(SimpleNamespace(event_type="statement_submitted")) + time.sleep(0.05) + dispatcher.shutdown() + dispatcher.finalize_when_idle(teardown) + time.sleep(0.35) + """ + ), + 0, + "teardown finished", + ), InterpreterExitTestCase( "dispatcher never shut down", dedent( @@ -102,3 +131,4 @@ def test_given_live_exporter_threads_when_interpreter_finalizes_then_process_exi assert result.returncode == test_case.expected_returncode, result.stderr assert "finalizing True" in result.stderr + assert test_case.expected_stderr in result.stderr diff --git a/website/src/content/docs/docs/concepts/observability/sinks.mdx b/website/src/content/docs/docs/concepts/observability/sinks.mdx index c5c19e20a8..91aba606ad 100644 --- a/website/src/content/docs/docs/concepts/observability/sinks.mdx +++ b/website/src/content/docs/docs/concepts/observability/sinks.mdx @@ -79,7 +79,9 @@ min_severity = "warning" `shutdown_timeout` sets how long a command waits at exit for lifecycle sinks to drain queued events. It is a fixed duration from `"0s"` to `"10m"`, defaults to `"2s"`, and is accepted only under `[sinks.lifecycle]`, not per named sink. `"0s"` drops anything still queued at exit. The -setting does not change the one-second timeout for each sink call. +setting does not change the one-second timeout for each sink call. A sink call still running when +the budget ends gets up to that one second to return before providers are torn down; teardown +always runs on the command's own thread. ```toml [sinks.lifecycle] From c379b165308f725c8b333a26ab33b5e783352a63 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:50:38 +0100 Subject: [PATCH 27/39] feat: deliver lifecycle events on a long-lived thread per sink and always deliver outcomes Each lifecycle sink now has one long-lived delivery thread that calls the sink for every event in order and reports outcomes through a Python queue, instead of a new thread per event that the dispatcher waited on. The dispatcher keeps the bounded priority queue, hands each sink up to 64 attempts (never more than the queue capacity), and keeps the per-call timeout, blocking and shutdown-budget semantics. Sinks no longer wait on each other. Run and invocation outcome events are always admitted to the queue, displacing other events when it is full, and are never displaced. The final quarter of the shutdown budget is reserved for them: when it begins, other queued and waiting events are dropped and counted. --- .../src/event_export/constants.rs | 11 + .../src/event_export/main/mod.rs | 1 + .../main/queue_lifecycle_event.rs | 23 +- .../main/retain_outcome_events.rs | 12 + .../sqlbuild-core/src/event_export/models.rs | 2 + .../src/event_export/tests/helpers.rs | 18 +- .../src/event_export/tests/test_queue.rs | 103 +++- .../src/event_export/tests/test_types.rs | 18 + .../_helpers/event_exporting/constants.rs | 3 + .../_helpers/event_exporting/dispatcher.rs | 146 +++-- .../_helpers/event_exporting/registration.rs | 3 +- .../_helpers/event_exporting/workers.rs | 562 ++++++++++++------ fensu.toml | 7 +- python-boundary.toml | 2 +- .../docs/concepts/observability/sinks.md | 13 +- src/sqlbuild/_native.pyi | 3 - .../main/_invoke_lifecycle_event_sink.py | 29 +- .../event_exporting/classes/_test_types.py | 10 + .../classes/test_dispatcher.py | 94 ++- .../classes/test_priority_queue.py | 6 +- .../docs/concepts/observability/sinks.mdx | 13 +- 21 files changed, 788 insertions(+), 291 deletions(-) create mode 100644 crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs diff --git a/crates/sqlbuild-core/src/event_export/constants.rs b/crates/sqlbuild-core/src/event_export/constants.rs index c3bc6e2626..a5fe675243 100644 --- a/crates/sqlbuild-core/src/event_export/constants.rs +++ b/crates/sqlbuild-core/src/event_export/constants.rs @@ -19,6 +19,17 @@ pub const EVENT_EXPORT_KINDS: [&str; 8] = [ /// Invocation terminals, delivered only after every other queued event. pub const INVOCATION_TERMINAL_EVENT_TYPES: [&str; 2] = ["invocation_completed", "invocation_failed"]; +/// Run and invocation outcomes: always admitted, and the only events delivered in the shutdown reserve. +pub const OUTCOME_EVENT_TYPES: [&str; 6] = [ + "invocation_started", + "invocation_completed", + "invocation_failed", + "run_started", + "run_completed", + "run_failed", +]; +/// The final share of the shutdown budget kept for outcome events once other backlog is dropped. +pub const OUTCOME_RESERVE_FRACTION: f64 = 0.25; pub const RETRY_EVENT_TYPE: &str = "retry_scheduled"; pub const RETRY_EVENT_KIND: &str = "retry"; pub const RESOURCE_EVENT_PREFIX: &str = "resource"; diff --git a/crates/sqlbuild-core/src/event_export/main/mod.rs b/crates/sqlbuild-core/src/event_export/main/mod.rs index 59d4f3ce19..be3a987742 100644 --- a/crates/sqlbuild-core/src/event_export/main/mod.rs +++ b/crates/sqlbuild-core/src/event_export/main/mod.rs @@ -2,6 +2,7 @@ pub mod drop_lifecycle_attempts; pub mod export_snapshot; pub mod lifecycle_export_policy; pub mod queue_lifecycle_event; +pub mod retain_outcome_events; pub mod route_lifecycle_event; pub mod sanitized_exception_type; pub mod severity_at_least; diff --git a/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs index 1a585982c3..f8a7878ea1 100644 --- a/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs +++ b/crates/sqlbuild-core/src/event_export/main/queue_lifecycle_event.rs @@ -2,15 +2,30 @@ use crate::event_export::models::{EventQueue, QueueOutcome, QueuedEvent}; -/// Queue `event`, or reject it when every queued event has at least its priority. +/// Queue `event` or reject it when full of events at least its priority; outcome events are never displaced and always admitted. pub fn queue_lifecycle_event( mut queue: EventQueue, event: QueuedEvent, ) -> QueueOutcome { let mut displaced: Option> = None; if queue.items.len() >= queue.capacity { - let lowest: Option = queue.items.iter().map(|queued| queued.priority).min(); - let Some(lowest) = lowest.filter(|lowest| *lowest < event.priority) else { + let lowest: Option = queue + .items + .iter() + .filter(|queued| !queued.outcome) + .map(|queued| queued.priority) + .min(); + let displaceable = lowest.filter(|lowest| event.outcome || *lowest < event.priority); + let Some(lowest) = displaceable else { + if event.outcome { + queue.items.push(event); + return QueueOutcome { + queue, + inserted: true, + displaced: None, + rejected: None, + }; + } return QueueOutcome { queue, inserted: false, @@ -22,7 +37,7 @@ pub fn queue_lifecycle_event( .items .iter() .enumerate() - .filter(|(_, queued)| queued.priority == lowest) + .filter(|(_, queued)| !queued.outcome && queued.priority == lowest) .min_by_key(|(_, queued)| queued.sequence) .map(|(position, _)| position); displaced = position.map(|position| queue.items.remove(position)); diff --git a/crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs b/crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs new file mode 100644 index 0000000000..7e110abd21 --- /dev/null +++ b/crates/sqlbuild-core/src/event_export/main/retain_outcome_events.rs @@ -0,0 +1,12 @@ +//! Keep only outcome events once the shutdown reserve begins. + +use crate::event_export::models::{EventQueue, QueuedEvent}; + +/// The queue with only its outcome events, and every other event it held. +#[must_use] +pub fn retain_outcome_events(mut queue: EventQueue) -> (EventQueue, Vec>) { + let (kept, dropped): (Vec>, Vec>) = + queue.items.drain(..).partition(|queued| queued.outcome); + queue.items = kept; + (queue, dropped) +} diff --git a/crates/sqlbuild-core/src/event_export/models.rs b/crates/sqlbuild-core/src/event_export/models.rs index 66a5e2159c..9ee6750807 100644 --- a/crates/sqlbuild-core/src/event_export/models.rs +++ b/crates/sqlbuild-core/src/event_export/models.rs @@ -44,6 +44,8 @@ pub struct QueuedEvent { pub priority: u8, /// Whether the event ends the invocation and so waits behind every other queued event. pub invocation_terminal: bool, + /// Whether the event reports a run or invocation outcome, which the queue always admits. + pub outcome: bool, pub eligible_exporters: Vec, } diff --git a/crates/sqlbuild-core/src/event_export/tests/helpers.rs b/crates/sqlbuild-core/src/event_export/tests/helpers.rs index 03087a70d4..2817d06c0a 100644 --- a/crates/sqlbuild-core/src/event_export/tests/helpers.rs +++ b/crates/sqlbuild-core/src/event_export/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::event_export::models::QueuedEvent; +use crate::event_export::models::{EventQueue, QueuedEvent}; /// A queued unit event with `sequence`, `(priority, invocation terminal)` and one eligible exporter. pub(crate) fn queued( @@ -10,6 +10,22 @@ pub(crate) fn queued( item: (), priority, invocation_terminal, + outcome: false, eligible_exporters: vec![0], } } + +/// A queued non-terminal event with `sequence`, `(priority, outcome)` and one eligible exporter. +pub(crate) fn event((priority, outcome): (u8, bool), sequence: u64) -> QueuedEvent<()> { + QueuedEvent { + outcome, + ..queued(sequence, (priority, false)) + } +} + +/// The sequences `queue` holds, ascending. +pub(crate) fn sequences(queue: &EventQueue<()>) -> Vec { + let mut sequences: Vec = queue.items.iter().map(|item| item.sequence).collect(); + sequences.sort_unstable(); + sequences +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs index 95aa2ce728..2ae6361f1d 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs @@ -1,8 +1,107 @@ use crate::event_export::main::queue_lifecycle_event::queue_lifecycle_event; +use crate::event_export::main::retain_outcome_events::retain_outcome_events; use crate::event_export::main::take_lifecycle_event::take_lifecycle_event; use crate::event_export::models::EventQueue; -use crate::event_export::tests::helpers::queued; -use crate::event_export::tests::test_types::{DeliveryOrderTestCase, DisplacementTestCase}; +use crate::event_export::tests::helpers::{event, queued, sequences}; +use crate::event_export::tests::test_types::{ + DeliveryOrderTestCase, DisplacementTestCase, OutcomeAdmissionTestCase, OutcomeReserveTestCase, +}; + +#[test] +fn given_full_queue_when_outcome_or_other_event_arrives_then_outcomes_are_admitted_and_kept() { + let test_cases = [ + OutcomeAdmissionTestCase { + description: "outcome displaces an equal-priority event", + queued: &[(2, false), (2, false)], + incoming: (2, true), + expected_inserted: true, + expected_displaced: Some(0), + expected_sequences: &[1, 2], + }, + OutcomeAdmissionTestCase { + description: "outcome displaces a higher-priority event when nothing lower is queued", + queued: &[(3, false), (2, true)], + incoming: (0, true), + expected_inserted: true, + expected_displaced: Some(0), + expected_sequences: &[1, 2], + }, + OutcomeAdmissionTestCase { + description: "outcome exceeds capacity when only outcomes are queued", + queued: &[(2, true), (0, true)], + incoming: (2, true), + expected_inserted: true, + expected_displaced: None, + expected_sequences: &[0, 1, 2], + }, + OutcomeAdmissionTestCase { + description: "failure never displaces a queued outcome", + queued: &[(2, true), (0, true)], + incoming: (3, false), + expected_inserted: false, + expected_displaced: None, + expected_sequences: &[0, 1], + }, + ]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(test_case.queued.len()); + for (sequence, dimensions) in (0_u64..).zip(test_case.queued) { + queue = queue_lifecycle_event(queue, event(*dimensions, sequence)).queue; + } + let incoming_sequence: u64 = u64::try_from(test_case.queued.len()).unwrap_or(u64::MAX); + let outcome = queue_lifecycle_event(queue, event(test_case.incoming, incoming_sequence)); + assert_eq!( + ( + outcome.inserted, + outcome.displaced.as_ref().map(|event| event.sequence), + sequences(&outcome.queue), + ), + ( + test_case.expected_inserted, + test_case.expected_displaced, + test_case.expected_sequences.to_vec(), + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_mixed_backlog_when_reserve_begins_then_only_outcome_events_remain() { + let test_cases = [OutcomeReserveTestCase { + description: "other events are handed back for accounting", + queued: &[(2, false), (2, true), (0, false), (2, true)], + expected_kept: &[1, 3], + expected_dropped: &[0, 2], + }]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(8); + for (sequence, dimensions) in (0_u64..).zip(test_case.queued) { + queue = queue_lifecycle_event(queue, event(*dimensions, sequence)).queue; + } + + let (kept, dropped) = retain_outcome_events(queue); + + assert_eq!( + ( + sequences(&kept), + dropped + .iter() + .map(|item| item.sequence) + .collect::>() + ), + ( + test_case.expected_kept.to_vec(), + test_case.expected_dropped.to_vec() + ), + "{}", + test_case.description + ); + } +} #[test] fn given_full_queue_when_priority_pair_arrives_then_displacement_policy_is_exact() { diff --git a/crates/sqlbuild-core/src/event_export/tests/test_types.rs b/crates/sqlbuild-core/src/event_export/tests/test_types.rs index d3960e9af0..0338b49a40 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_types.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_types.rs @@ -58,3 +58,21 @@ pub(crate) struct SnapshotTestCase { pub(crate) expected_accepted: u64, pub(crate) expected_flush_complete: bool, } + +/// A full queue of `(priority, outcome)` events receiving one more, and what it keeps. +pub(crate) struct OutcomeAdmissionTestCase { + pub(crate) description: &'static str, + pub(crate) queued: &'static [(u8, bool)], + pub(crate) incoming: (u8, bool), + pub(crate) expected_inserted: bool, + pub(crate) expected_displaced: Option, + pub(crate) expected_sequences: &'static [u64], +} + +/// Queued `(priority, outcome)` events and what the shutdown reserve keeps and drops, by sequence. +pub(crate) struct OutcomeReserveTestCase { + pub(crate) description: &'static str, + pub(crate) queued: &'static [(u8, bool)], + pub(crate) expected_kept: &'static [u64], + pub(crate) expected_dropped: &'static [u64], +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs index 60f5340f11..6ceb9fe54e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/constants.rs @@ -5,6 +5,7 @@ pub(crate) const DISPATCHER_THREAD_NAME: &str = "sqlbuild-event-exporter-dispatc pub(crate) const NOTIFIER_THREAD_NAME: &str = "sqlbuild-event-exporter-notifier"; pub(crate) const HEALTH_THREAD_NAME: &str = "sqlbuild-event-exporter-health"; pub(crate) const OUTPUT_SINK_THREAD_NAME: &str = "sqlbuild-command-output-sink"; +/// Prefix of each exporter's long-lived delivery thread name. pub(crate) const INVOCATION_THREAD_PREFIX: &str = "sqlbuild-event-exporter-"; /// Attribute naming the worker of a long-lived dispatcher thread, for the exit hook. pub(crate) const WORKER_ATTRIBUTE: &str = "_sqlbuild_event_exporter_worker"; @@ -14,6 +15,8 @@ pub(crate) const POLL_SECONDS: f64 = 0.01; pub(crate) const JOIN_GRACE_SECONDS: f64 = 0.05; /// How long shutdown and interpreter exit wait for each stopped dispatcher thread. pub(crate) const EXIT_JOIN_SECONDS: f64 = 1.0; +/// The most attempts one exporter's delivery thread holds, at most the queue capacity; more stay queued. +pub(crate) const LANE_WINDOW: usize = 64; /// The longest a finalizer waits for sink calls still running after shutdown before it runs anyway. pub(crate) const IDLE_WAIT_SECONDS: f64 = 1.0; /// No time left before a deadline. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index bdfd0ad7e2..a1bd8c7131 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -1,6 +1,6 @@ //! `EventExporterDispatcher`: bounded priority delivery of lifecycle events on Python daemon threads. -use std::collections::{HashSet, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; @@ -8,12 +8,13 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pyfunction, pymethods, }; -use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyString}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PySet, PyString}; use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_TERMINAL_EVENT_TYPES, + OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, }; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; @@ -38,7 +39,7 @@ use crate::bindings::_helpers::event_exporting::models::{ use crate::bindings::_helpers::event_exporting::values::{ export_policy, input_error, severity_from, state_error, }; -use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, WorkerRole}; +use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, Lane, WorkerRole}; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::native_threads::constants::OUTPUT_SUPPRESSED_ATTRIBUTE; @@ -81,6 +82,18 @@ pub(crate) struct State { sequence: u64, accepting: bool, pub(crate) deadline: Option, + /// When only outcome events remain deliverable, ahead of the deadline. + pub(crate) reserve_at: Option, + pub(crate) reserved: bool, + /// Each exporter's long-lived delivery thread, once started and until retired. + pub(crate) lanes: Vec>, + /// The exporter each delivered-but-unreported attempt token belongs to. + pub(crate) tokens: HashMap, + pub(crate) next_token: u64, + /// Retired lanes, whose threads are joined before finalizers run. + pub(crate) retired: Vec, + /// Whether a timed-out sink call may still be running. + pub(crate) abandoned_calls: bool, pub(crate) stopping: bool, pub(crate) force_stop: bool, pub(crate) blocked: HashSet, @@ -103,6 +116,10 @@ pub(crate) struct State { /// What the dispatcher and its threads share. pub(crate) struct Shared { condition: Py, + /// The `SimpleQueue` every delivery thread reports `(token, error)` outcomes to. + pub(crate) results: Py, + /// The Python set of attempt tokens delivery threads must skip. + pub(crate) cancelled: Py, state: Mutex, pub(crate) invocation_timeout: Duration, shutdown_timeout: Duration, @@ -331,6 +348,34 @@ pub(crate) fn stop_event_exporter_threads(py: Python<'_>) -> PyResult<()> { Ok(()) } +impl State { + /// Count one failed attempt and queue its sanitized diagnostic, returning whether one was queued. + pub(crate) fn note_failure( + &mut self, + diagnostics: Option, + (index, kind, severity): (usize, &str, ExportSeverity), + error_type: String, + ) -> bool { + let mut exporter_name: String = String::new(); + if let Some(counts) = self.counts.get_mut(index) { + counts.failed += 1; + } + if let Some(exporter) = self.exporters.get(index) { + exporter_name.clone_from(&exporter.name); + } + if diagnostics.is_none_or(|capacity| self.failures.len() >= capacity) { + return false; + } + self.failures.push_back(EventExporterFailure { + exporter_name, + error_type, + event_kind: kind.to_owned(), + event_severity: severity, + }); + true + } +} + impl Shared { pub(crate) fn hold<'py>(&self, py: Python<'py>) -> PyResult> { let condition = self.condition.bind(py).clone(); @@ -378,7 +423,8 @@ impl Shared { queue_capacity: state.queue.capacity, drained: final_snapshot && !state.dispatcher_running - && state.live_invocations.is_empty(), + && state.live_invocations.is_empty() + && !state.abandoned_calls, }) } @@ -388,40 +434,14 @@ impl Shared { Ok(Self::snapshot(&*self.locked()?, false)) } - /// Count one failed attempt and queue its sanitized diagnostic. - pub(crate) fn record_failure( - &self, - py: Python<'_>, - (index, kind, severity): (usize, &str, ExportSeverity), - error_type: String, - ) -> PyResult<()> { - let _held = self.hold(py)?; - let mut state = self.locked()?; - let mut exporter_name: String = String::new(); - if let Some(counts) = state.counts.get_mut(index) { - counts.failed += 1; - } - if let Some(exporter) = state.exporters.get(index) { - exporter_name.clone_from(&exporter.name); - } - if !self.notifications - || self.failure_callback.is_none() - || state.failures.len() >= self.notification_capacity - { - return Ok(()); - } - state.failures.push_back(EventExporterFailure { - exporter_name, - error_type, - event_kind: kind.to_owned(), - event_severity: severity, - }); - drop(state); - self.notify_all(py) + /// The diagnostic queue capacity when failures are reported, else `None`. + pub(crate) fn diagnostics(&self) -> Option { + (self.notifications && self.failure_callback.is_some()) + .then_some(self.notification_capacity) } /// Forget one finished invocation and wake a caller waiting for delivery to become idle. - pub(crate) fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { + fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { let retired: Option> = { let _held = self.hold(py)?; let mut state = self.locked()?; @@ -450,25 +470,40 @@ impl Shared { state.live_invocations.is_empty() && !state.dispatcher_running } - /// Wait until delivery is idle, the idle wait ends or interpreter exit begins. + /// Wait until delivery and every sink call are idle, the idle wait ends or interpreter exit begins. fn await_idle(&self, py: Python<'_>) -> PyResult<()> { let deadline: Instant = Instant::now() + self.invocation_timeout.min(seconds(IDLE_WAIT_SECONDS)); - let _held = self.hold(py)?; - loop { - if Self::idle(&*self.locked()?) || interpreter_exiting(py)? { - return Ok(()); - } - let remaining: f64 = remaining_seconds(deadline); - if remaining <= NO_SECONDS { - return Ok(()); + let threads: Vec> = { + let _held = self.hold(py)?; + loop { + if interpreter_exiting(py)? { + return Ok(()); + } + let state = self.locked()?; + if Self::idle(&state) { + break state + .retired + .iter() + .map(|lane| lane.thread().clone_ref(py)) + .collect(); + } + drop(state); + let remaining: f64 = remaining_seconds(deadline); + if remaining <= NO_SECONDS { + return Ok(()); + } + self.wait(py, remaining.min(POLL_SECONDS))?; } - self.wait(py, remaining.min(POLL_SECONDS))?; + }; + for thread in threads { + join(thread.bind(py), remaining_seconds(deadline))?; } + Ok(()) } /// Register a running invocation, returning its identity. - pub(crate) fn begin_invocation(&self, py: Python<'_>) -> PyResult { + fn begin_invocation(&self, py: Python<'_>) -> PyResult { let _held = self.hold(py)?; let mut state = self.locked()?; state.next_invocation += 1; @@ -643,6 +678,8 @@ impl EventExporterDispatcher { let has_summary: bool = summary_callback.is_some(); let shared = Arc::new(Shared { condition: imported(py, "threading", "Condition")?.call0()?.unbind(), + results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), + cancelled: PySet::empty(py)?.into_any().unbind(), state: Mutex::new(State { bound: !exporters.is_none(), counts: vec![ExporterCounts::default(); bound_exporters.len()], @@ -654,6 +691,13 @@ impl EventExporterDispatcher { sequence: 0, accepting: true, deadline: None, + reserve_at: None, + reserved: false, + lanes: Vec::new(), + tokens: HashMap::new(), + next_token: 0, + retired: Vec::new(), + abandoned_calls: false, stopping: false, force_stop: false, blocked: HashSet::new(), @@ -757,6 +801,7 @@ impl EventExporterDispatcher { }, priority: policy.priority, invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), + outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), eligible_exporters: eligible, }, ); @@ -860,7 +905,14 @@ impl EventExporterDispatcher { if owns { state.shutdown_started = true; state.accepting = false; - state.deadline = Some(Instant::now() + shared.shutdown_timeout); + let started: Instant = Instant::now(); + state.deadline = Some(started + shared.shutdown_timeout); + state.reserve_at = Some( + started + + shared + .shutdown_timeout + .mul_f64(1.0 - OUTCOME_RESERVE_FRACTION), + ); state.stopping = true; state.health_stopping = true; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs index ebe2562456..40d42b4598 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs @@ -10,7 +10,7 @@ use crate::bindings::_helpers::event_exporting::models::{ BoundEventExporter, EventExportSummary, EventExporterAccounting, EventExporterCounts, EventExporterFailure, }; -use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, InvocationCompletion}; +use crate::bindings::_helpers::event_exporting::workers::DispatcherWorker; use crate::bindings::_helpers::event_exporting::{policy, scope}; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -21,7 +21,6 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - module.add_class::()?; let hook = wrap_pyfunction!(stop_event_exporter_threads, module)?; module.add_function(hook.clone())?; imported(module.py(), "atexit", "register")?.call1((hook,))?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs index 1cde088682..6e966172a8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs @@ -1,28 +1,30 @@ -//! The dispatcher's delivery, notification, health and per-invocation thread targets. +//! The dispatcher's delivery, notification and health thread targets, and each exporter's delivery lane. -use std::sync::{Arc, Mutex}; -use std::time::Instant; +use std::collections::VecDeque; +use std::sync::Arc; +use std::time::{Duration, Instant}; -use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_core::event_export::constants::TIMEOUT_ERROR_TYPE; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; +use sqlbuild_core::event_export::main::retain_outcome_events::retain_outcome_events; use sqlbuild_core::event_export::main::take_lifecycle_event::take_lifecycle_event; -use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEvent}; +use sqlbuild_core::event_export::models::{ + EventQueue, ExportSeverity, ExportSnapshot, QueuedEvent, +}; use crate::bindings::_helpers::compile_driver::python_calls::{call_with, imported}; use crate::bindings::_helpers::event_exporting::constants::{ - INVOCATION_THREAD_PREFIX, NO_SECONDS, POLL_SECONDS, + INVOCATION_THREAD_PREFIX, LANE_WINDOW, NO_SECONDS, POLL_SECONDS, }; -use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared}; +use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared, State}; use crate::bindings::_helpers::event_exporting::models::{ EventExportSummary, EventExporterFailure, }; use crate::bindings::_helpers::event_exporting::values::error_type_of; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; -use crate::bindings::_helpers::native_threads::exit_signal::interpreter_exiting; /// Which long-lived dispatcher thread a worker runs. #[derive(Clone, Copy)] @@ -39,19 +41,28 @@ pub(crate) struct DispatcherWorker { role: WorkerRole, } -/// Whether an invocation finished, and the sanitized type of what it raised. -#[derive(Default)] -struct Completion { - done: bool, - error_type: Option, +/// One attempt handed to a lane and not yet reported. +pub(crate) struct Attempt { + token: u64, + kind: String, + severity: ExportSeverity, + outcome: bool, } -/// Reports how one sink call on its own daemon thread ended. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct InvocationCompletion { - shared: Arc, - invocation: u64, - completion: Arc>, +/// One exporter's long-lived delivery thread, its call queue and the attempts it still owes. +pub(crate) struct Lane { + calls: Py, + thread: Py, + pending: VecDeque, + /// When the attempt at the front of `pending` began, approximately. + head_started: Instant, +} + +/// What one dispatch round reads from the dispatcher besides its state. +#[derive(Clone, Copy)] +pub(crate) struct Round<'a> { + shared: &'a Shared, + diagnostics: Option, } /// The next thing the notification thread delivers. @@ -62,12 +73,287 @@ enum Notification { Stop, } -/// Whether the call finished, and the sanitized type of what it raised. -fn completion_state(completion: &Mutex) -> PyResult<(bool, Option)> { - let completion = completion - .lock() - .map_err(|_| PyRuntimeError::new_err("event exporter invocation lock is poisoned"))?; - Ok((completion.done, completion.error_type.clone())) +impl Lane { + /// Start `exporter`'s delivery thread, which skips `cancelled` tokens and reports to `results`. + fn start(py: Python<'_>, exporter: &Exporter, round: Round<'_>) -> PyResult { + let calls: Bound<'_, PyAny> = imported(py, "queue", "SimpleQueue")?.call0()?; + let arguments = PyDict::new(py); + arguments.set_item("calls", &calls)?; + arguments.set_item("results", round.shared.results.bind(py))?; + arguments.set_item("cancelled", round.shared.cancelled.bind(py))?; + let thread = call_with( + &imported(py, "threading", "Thread")?, + &[ + ( + "target", + host::entry(py, UserCodeHost::InvokeLifecycleEventSink)?, + ), + ("kwargs", arguments.into_any()), + ( + "name", + PyString::new(py, &format!("{INVOCATION_THREAD_PREFIX}{}", exporter.name)) + .into_any(), + ), + ("daemon", PyBool::new(py, true).to_owned().into_any()), + ], + )?; + thread.call_method0("start")?; + Ok(Self { + calls: calls.unbind(), + thread: thread.unbind(), + pending: VecDeque::new(), + head_started: Instant::now(), + }) + } + + /// The lane's delivery thread. + pub(crate) fn thread(&self) -> &Py { + &self.thread + } + + /// Queue `None` so the thread ends after the calls queued before it. + fn retire(&self, py: Python<'_>) -> PyResult<()> { + self.calls.bind(py).call_method1("put", (py.None(),))?; + Ok(()) + } +} + +impl State { + /// Account reported attempts against their lanes, returning whether a diagnostic was queued. + fn record_reports( + &mut self, + reports: Vec<(u64, Option)>, + diagnostics: Option, + ) -> bool { + let mut notify: bool = false; + for (token, error_type) in reports { + let Some(index) = self.tokens.remove(&token) else { + continue; + }; + let Some(lane) = self.lanes.get_mut(index).and_then(Option::as_mut) else { + continue; + }; + let Some(position) = lane + .pending + .iter() + .position(|attempt| attempt.token == token) + else { + continue; + }; + let Some(attempt) = lane.pending.remove(position) else { + continue; + }; + lane.head_started = Instant::now(); + match error_type { + Some(error_type) => { + notify |= self.note_failure( + diagnostics, + (index, &attempt.kind, attempt.severity), + error_type, + ); + } + None => { + if let Some(counts) = self.counts.get_mut(index) { + counts.delivered += 1; + } + } + } + } + notify + } + + /// Skip `attempts` on their delivery thread, counting each as dropped for `index`. + fn cancel_attempts( + &mut self, + py: Python<'_>, + index: usize, + attempts: impl IntoIterator, + cancelled: &Py, + ) -> PyResult<()> { + let cancelled: &Bound<'_, PyAny> = cancelled.bind(py); + for attempt in attempts { + self.tokens.remove(&attempt.token); + cancelled.call_method1("add", (attempt.token,))?; + self.counts = drop_lifecycle_attempts(std::mem::take(&mut self.counts), &[index]); + } + Ok(()) + } + + /// Retire `index`'s lane: its running call fails as timed out and its later attempts are dropped. + fn abandon_lane(&mut self, py: Python<'_>, index: usize, round: Round<'_>) -> PyResult { + let Some(mut lane) = self.lanes.get_mut(index).and_then(Option::take) else { + return Ok(false); + }; + let mut notify: bool = false; + if let Some(running) = lane.pending.pop_front() { + self.tokens.remove(&running.token); + self.blocked.insert(index); + self.abandoned_calls = true; + notify = self.note_failure( + round.diagnostics, + (index, &running.kind, running.severity), + TIMEOUT_ERROR_TYPE.to_owned(), + ); + } + let waiting: VecDeque = std::mem::take(&mut lane.pending); + self.cancel_attempts(py, index, waiting, &round.shared.cancelled)?; + lane.retire(py)?; + self.retired.push(lane); + Ok(notify) + } + + /// Abandon every lane whose current sink call outlived `timeout`. + fn expire_lanes( + &mut self, + py: Python<'_>, + timeout: Duration, + round: Round<'_>, + ) -> PyResult { + let mut notify: bool = false; + for index in 0..self.lanes.len() { + let expired: bool = self.lanes[index].as_ref().is_some_and(|lane| { + !lane.pending.is_empty() && lane.head_started.elapsed() >= timeout + }); + if expired { + notify |= self.abandon_lane(py, index, round)?; + } + } + Ok(notify) + } + + /// Drop every queued and lane-waiting event that is not an outcome, counting each attempt. + fn reserve_for_outcomes(&mut self, py: Python<'_>, cancelled: &Py) -> PyResult<()> { + let capacity: usize = self.queue.capacity; + let (queue, dropped) = retain_outcome_events(std::mem::replace( + &mut self.queue, + EventQueue::new(capacity), + )); + self.queue = queue; + for item in &dropped { + self.counts = + drop_lifecycle_attempts(std::mem::take(&mut self.counts), &item.eligible_exporters); + } + for index in 0..self.lanes.len() { + let Some(lane) = self.lanes[index].as_mut() else { + continue; + }; + let running: Option = lane.pending.pop_front(); + let (kept, skipped): (VecDeque, VecDeque) = + lane.pending.drain(..).partition(|attempt| attempt.outcome); + lane.pending = running.into_iter().chain(kept).collect(); + self.cancel_attempts(py, index, skipped, cancelled)?; + } + drop(dropped); + Ok(()) + } + + /// Move queued events onto exporter lanes while every eligible lane has room. + fn feed(&mut self, py: Python<'_>, round: Round<'_>) -> PyResult { + let mut notify: bool = false; + while !self.queue.items.is_empty() { + let capacity: usize = self.queue.capacity; + let (queue, next) = take_lifecycle_event(std::mem::replace( + &mut self.queue, + EventQueue::new(capacity), + )); + self.queue = queue; + let Some(item) = next else { + break; + }; + let full: bool = item.eligible_exporters.iter().any(|index| { + !self.blocked.contains(index) + && self + .lanes + .get(*index) + .and_then(Option::as_ref) + .is_some_and(|lane| lane.pending.len() >= LANE_WINDOW.min(capacity)) + }); + if full { + self.queue.items.push(item); + break; + } + for index in &item.eligible_exporters { + if self.blocked.contains(index) { + self.counts = + drop_lifecycle_attempts(std::mem::take(&mut self.counts), &[*index]); + continue; + } + notify |= self.offer(py, *index, &item, round)?; + } + } + Ok(notify) + } + + /// Queue one attempt on `index`'s lane, starting the lane's thread on first use. + fn offer( + &mut self, + py: Python<'_>, + index: usize, + item: &QueuedEvent, + round: Round<'_>, + ) -> PyResult { + let Some(exporter) = self.exporters.get(index).map(Arc::clone) else { + return Ok(false); + }; + if self.lanes.len() <= index { + self.lanes.resize_with(index + 1, || None); + } + if self.lanes[index].is_none() { + match Lane::start(py, &exporter, round) { + Ok(lane) => self.lanes[index] = Some(lane), + Err(error) => { + return Ok(self.note_failure( + round.diagnostics, + (index, &item.item.kind, item.item.severity), + error_type_of(py, &error), + )); + } + } + } + let token: u64 = self.next_token; + self.next_token += 1; + let Some(lane) = self.lanes[index].as_mut() else { + return Ok(false); + }; + lane.calls.bind(py).call_method1( + "put", + (( + exporter.function.bind(py), + exporter.provider_arguments.bind(py), + item.item.event.bind(py), + token, + ),), + )?; + if lane.pending.is_empty() { + lane.head_started = Instant::now(); + } + lane.pending.push_back(Attempt { + token, + kind: item.item.kind.clone(), + severity: item.item.severity, + outcome: item.outcome, + }); + self.tokens.insert(token, index); + Ok(false) + } + + /// Whether any lane still owes a reported attempt. + fn busy(&self) -> bool { + self.lanes + .iter() + .flatten() + .any(|lane| !lane.pending.is_empty()) + } + + /// End every lane once it has run what was queued before, keeping its thread to join. + fn retire_lanes(&mut self, py: Python<'_>) -> PyResult<()> { + let lanes: Vec = self.lanes.drain(..).flatten().collect(); + for lane in lanes { + lane.retire(py)?; + self.retired.push(lane); + } + Ok(()) + } } impl DispatcherWorker { @@ -97,165 +383,78 @@ impl DispatcherWorker { } } - /// The next queued event, `Ok(None)` to keep polling and `Err(())` once delivery stops. - fn next_event(&self, py: Python<'_>) -> PyResult>, ()>> { - let shared = &self.shared; - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if Shared::past_deadline(&state) || state.force_stop { - let drained: Vec> = std::mem::take(&mut state.queue.items); - for item in &drained { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &item.eligible_exporters, - ); - } - drop(state); - drop(drained); - return Ok(Err(())); - } - if state.queue.items.is_empty() { - drop(state); - shared.wait(py, POLL_SECONDS)?; - state = shared.locked()?; - } - let capacity: usize = state.queue.capacity; - let (queue, next) = take_lifecycle_event(std::mem::replace( - &mut state.queue, - EventQueue::new(capacity), - )); - state.queue = queue; - match next { - Some(item) => Ok(Ok(Some(item))), - None if state.stopping => Ok(Err(())), - None => Ok(Ok(None)), - } - } - fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { let result = (|| -> PyResult<()> { if !self.await_bound(py)? { return Ok(()); } - while let Ok(item) = self.next_event(py)? { - if let Some(item) = item { - self.deliver(py, &item)?; - } - } + while self.dispatch_round(py)? {} Ok(()) })(); + let retired: PyResult<()> = (|| { + let _held = self.shared.hold(py)?; + self.shared.locked()?.retire_lanes(py) + })(); self.shared.dispatcher_finished(py)?; + retired?; result } - /// Offer one event to each eligible exporter in order, before the drain deadline. - fn deliver(&self, py: Python<'_>, item: &QueuedEvent) -> PyResult<()> { - let shared = &self.shared; - for (position, index) in item.eligible_exporters.iter().enumerate() { - let exporter: Arc = { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if Shared::past_deadline(&state) { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &item.eligible_exporters[position..], - ); - return Ok(()); - } - if state.blocked.contains(index) { - state.counts = - drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[*index]); - continue; - } - let Some(exporter) = state.exporters.get(*index) else { - continue; - }; - Arc::clone(exporter) - }; - self.invoke(py, (*index, exporter), item)?; - } - Ok(()) - } - - /// Call one exporter on its own daemon thread and account for its outcome. - fn invoke( - &self, - py: Python<'_>, - (index, exporter): (usize, Arc), - item: &QueuedEvent, - ) -> PyResult<()> { + /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. + fn dispatch_round(&self, py: Python<'_>) -> PyResult { let shared = &self.shared; - let failure = (index, item.item.kind.as_str(), item.item.severity); - let completion: Arc> = Arc::new(Mutex::new(Completion::default())); - let name: String = format!("{INVOCATION_THREAD_PREFIX}{}", exporter.name); - let invocation: u64 = shared.begin_invocation(py)?; - let reporter = InvocationCompletion::new(Arc::clone(shared), invocation, &completion); - let started = Py::new(py, reporter).and_then(|reporter| { - let arguments = PyDict::new(py); - arguments.set_item("function", exporter.function.bind(py))?; - arguments.set_item("provider_arguments", exporter.provider_arguments.bind(py))?; - arguments.set_item("event", item.item.event.bind(py))?; - arguments.set_item("completion", reporter)?; - let thread = call_with( - &imported(py, "threading", "Thread")?, - &[ - ( - "target", - host::entry(py, UserCodeHost::InvokeLifecycleEventSink)?, - ), - ("kwargs", arguments.into_any()), - ("name", PyString::new(py, &name).into_any()), - ("daemon", PyBool::new(py, true).to_owned().into_any()), - ], - )?; - thread.call_method0("start") - }); - if let Err(error) = started { - shared.invocation_finished(py, invocation)?; - return shared.record_failure(py, failure, error_type_of(py, &error)); - } - let completed: bool = self.await_invocation(py, &completion)?; - let (_, error_type) = completion_state(&completion)?; - if !completed { - { - let _held = shared.hold(py)?; - shared.locked()?.blocked.insert(index); + let reports: Vec<(u64, Option)> = self.reported(py)?; + let round = Round { + shared, + diagnostics: shared.diagnostics(), + }; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + let mut notify: bool = state.record_reports(reports, round.diagnostics); + let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { + let drained: Vec> = std::mem::take(&mut state.queue.items); + for item in &drained { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &item.eligible_exporters, + ); } - return shared.record_failure(py, failure, TIMEOUT_ERROR_TYPE.to_owned()); - } - if let Some(error_type) = error_type { - return shared.record_failure(py, failure, error_type); + for index in 0..state.lanes.len() { + notify |= state.abandon_lane(py, index, round)?; + } + drop(drained); + true + } else { + notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; + if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { + state.reserved = true; + state.reserve_for_outcomes(py, &shared.cancelled)?; + } + notify |= state.feed(py, round)?; + state.stopping && !state.busy() && state.queue.items.is_empty() + }; + drop(state); + if notify { + shared.notify_all(py)?; } - let _held = shared.hold(py)?; - if let Some(counts) = shared.locked()?.counts.get_mut(index) { - counts.delivered += 1; + if !stopped { + shared.wait(py, POLL_SECONDS)?; } - Ok(()) + Ok(!stopped) } - /// Wait for an invocation until its own timeout or the drain deadline, whichever is first. - fn await_invocation(&self, py: Python<'_>, completion: &Mutex) -> PyResult { - let shared = &self.shared; - let invocation_deadline: Instant = Instant::now() + shared.invocation_timeout; - let _held = shared.hold(py)?; - loop { - let state = shared.locked()?; - let effective: Instant = state.deadline.map_or(invocation_deadline, |deadline| { - deadline.min(invocation_deadline) - }); - let force_stop: bool = state.force_stop || interpreter_exiting(py)?; - drop(state); - let remaining: f64 = effective - .saturating_duration_since(Instant::now()) - .as_secs_f64(); - if remaining <= NO_SECONDS || force_stop { - return Ok(completion_state(completion)?.0); - } - shared.wait(py, remaining.min(POLL_SECONDS))?; - if completion_state(completion)?.0 { - return Ok(true); - } + /// Every `(token, sanitized error type)` delivery threads reported since the last round. + fn reported(&self, py: Python<'_>) -> PyResult)>> { + let results: &Bound<'_, PyAny> = self.shared.results.bind(py); + let mut reports: Vec<(u64, Option)> = Vec::new(); + while !results.call_method0("empty")?.is_truthy()? { + let (token, error): (u64, Bound<'_, PyAny>) = + results.call_method0("get_nowait")?.extract()?; + let error_type: Option = + (!error.is_none()).then(|| error_type_of(py, &PyErr::from_value(error))); + reports.push((token, error_type)); } + Ok(reports) } fn next_notification(&self, py: Python<'_>) -> PyResult { @@ -376,34 +575,3 @@ impl DispatcherWorker { } } } - -impl InvocationCompletion { - /// A reporter that records into `completion` and ends `invocation`. - fn new(shared: Arc, invocation: u64, completion: &Arc>) -> Self { - Self { - shared, - invocation, - completion: Arc::clone(completion), - } - } -} - -#[pymethods] -impl InvocationCompletion { - #[pyo3(signature = (error, /))] - fn __call__(&self, py: Python<'_>, error: Option<&Bound<'_, PyAny>>) -> PyResult<()> { - if let Some(error) = error { - let error_type: String = error_type_of(py, &PyErr::from_value(error.clone())); - if let Ok(mut completion) = self.completion.lock() { - completion.error_type = Some(error_type); - } - } - let finished = self.shared.invocation_finished(py, self.invocation); - let _held = self.shared.hold(py)?; - if let Ok(mut completion) = self.completion.lock() { - completion.done = true; - } - self.shared.notify_all(py)?; - finished - } -} diff --git a/fensu.toml b/fensu.toml index 9ef4c92771..5bdf278e85 100644 --- a/fensu.toml +++ b/fensu.toml @@ -754,12 +754,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/sql_test_records.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_records.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=29", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=7", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", @@ -1224,9 +1224,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/discovery/python_function_files.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/discovery/scope_snapshot.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/discovery/yaml_entries.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/models.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=9", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/sqlite.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/fingerprints/storage_io.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", diff --git a/python-boundary.toml b/python-boundary.toml index 8fbb3795d5..7f4e20791c 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -295,7 +295,7 @@ note = "Rule 3 facade: reads the decorator's definition off a user function; re- [[stays]] path = "runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" rule = 2 -note = "Rule 2 shim: the daemon delivery thread's target that calls one user lifecycle sink and reports how it ended to the native dispatcher, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." +note = "Rule 2 shim: target of each lifecycle sink's long-lived daemon delivery thread; calls the user sink for each queued event in order and reports how each call ended through a Python queue, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." [[stays]] path = "runtime/event_exporting/main/_run_command_output_sinks.py" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md index 33e648e3c0..c66d46e273 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -87,7 +87,8 @@ min_severity = "warning" `shutdown_timeout` sets how long a command waits at exit for lifecycle sinks to drain queued events. It is a fixed duration from `"0s"` to `"10m"`, defaults to `"2s"`, and is accepted only under `[sinks.lifecycle]`, not per named sink. `"0s"` drops anything still queued at exit. The -setting does not change the one-second timeout for each sink call. A sink call still running when +final quarter of the budget is reserved for run and invocation outcomes: when it begins, other +queued events are dropped and counted so the outcomes still arrive. The setting does not change the one-second timeout for each sink call. A sink call still running when the budget ends gets up to that one second to return before providers are torn down; teardown always runs on the command's own thread. @@ -102,15 +103,21 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. ## Delivery behavior -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority. +- Each sink receives events in order on its own long-lived delivery thread. +- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; + `invocation_completed` and `invocation_failed` come after every other queued event. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. + Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, + `invocation_completed`, `invocation_failed`) are always admitted and never displaced. - A failing or timed-out sink is isolated from command correctness. - `audit_completed` facts are published as each audit finishes (model audits in `sqb build` when their model finishes), not in one burst at the end of the run. - When lifecycle events are dropped or fail, the command prints one warning to stderr with the counts and affected sinks, for example `Warning: command completed successfully, but lifecycle event export was incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). Increase sinks.lifecycle.shutdown_timeout or check sink health.` - The exit code and stdout, including `--json` output, are unchanged. + A successful command then ends with + `sqb audit completed successfully; only its lifecycle event export is affected.` The exit code + and stdout, including `--json` output, are unchanged. - Provider setup and declaration errors fail before execution because the project configuration is invalid. - Providers shared by multiple sinks are set up and torn down once per command. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 19bf31f222..98a0a7e936 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3212,9 +3212,6 @@ class EventExporterDispatcher: class DispatcherWorker: def __call__(self) -> None: ... -class InvocationCompletion: - def __call__(self, error: BaseException | None, /) -> None: ... - def stop_event_exporter_threads() -> None: ... class EventExporterCommandScope: diff --git a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py index 844e0e5382..75c7d8b8f2 100644 --- a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py +++ b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py @@ -1,22 +1,27 @@ -"""Call one lifecycle-event sink on its delivery thread.""" +"""Call one lifecycle-event sink on its long-lived delivery thread.""" from collections.abc import Callable, Mapping +from queue import SimpleQueue from sqlbuild.runtime.observability.models import LifecycleEvent def invoke_lifecycle_event_sink( *, - function: Callable[..., object], - provider_arguments: Mapping[str, object], - event: LifecycleEvent, - completion: Callable[[BaseException | None], None], + calls: SimpleQueue[ + tuple[Callable[..., object], Mapping[str, object], LifecycleEvent, int] | None + ], + results: SimpleQueue[tuple[int, BaseException | None]], + cancelled: set[int], ) -> None: - """Call a user sink with one event, then report how the call ended.""" + """Call the sink with each queued event in order and report how each call ended.""" - try: - _ = function(event=event, **provider_arguments) - except BaseException as error: - _ = completion(error) - return - _ = completion(None) + for function, provider_arguments, event, token in iter(calls.get, None): + if token in cancelled: + continue + try: + _ = function(event=event, **provider_arguments) + except BaseException as error: + results.put((token, error)) + else: + results.put((token, None)) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py index 8bbc51e88d..5b534e4ed2 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/_test_types.py @@ -51,3 +51,13 @@ class InterpreterExitTestCase: script: str expected_returncode: int expected_stderr: str = "" + + +@dataclass(frozen=True) +class OutcomeDeliveryTestCase: + description: str + queue_capacity: int + backlog: int + sink_seconds: float + shutdown_timeout_seconds: float + expected_outcomes: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 4621c9e518..077d290a42 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -23,8 +23,12 @@ from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( EventExporterDispatcherTestCase, HealthIntervalTestCase, + OutcomeDeliveryTestCase, +) +from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import ( + correlated_event, + lifecycle_event, ) -from tests.unit.src.sqlbuild.runtime.event_exporting.classes.helpers import lifecycle_event @pytest.mark.parametrize( @@ -118,7 +122,7 @@ def publish(*, event: LifecycleEvent) -> None: (EventExporterDispatcherTestCase("failure isolation", 1),), ids=lambda case: case.description, ) -def test_given_multiple_exporters_when_one_fails_then_later_exporter_runs_in_order( +def test_given_multiple_exporters_when_one_fails_then_the_other_exporter_still_runs( test_case: EventExporterDispatcherTestCase, ) -> None: calls: list[str] = [] @@ -142,7 +146,7 @@ def succeeding(*, event: LifecycleEvent) -> None: summary: EventExportSummary = dispatcher.shutdown() - assert calls == ["first", "second"] + assert sorted(calls) == ["first", "second"] assert summary.delivered == test_case.expected_delivered assert summary.failed == 1 assert summary.accepted == summary.delivered + summary.failed + summary.dropped @@ -243,7 +247,7 @@ def publish(*, event: LifecycleEvent) -> None: invocation_timeout_seconds=0.2, ) for index in range(100): - dispatcher.enqueue(lifecycle_event(index)) + dispatcher.enqueue(correlated_event(index, "statement_submitted")) time.sleep(0.02) release.set() @@ -506,19 +510,19 @@ def second(*, event: LifecycleEvent) -> None: queue_capacity=1, invocation_timeout_seconds=1, ) - dispatcher.enqueue(lifecycle_event(1)) + dispatcher.enqueue(correlated_event(1, "statement_submitted")) assert entered.wait(timeout=0.2) - dispatcher.enqueue(lifecycle_event(2)) - dispatcher.enqueue(lifecycle_event(3, "invocation_failed")) + dispatcher.enqueue(correlated_event(2, "statement_submitted")) + dispatcher.enqueue(correlated_event(3, "operation_failed")) release.set() summary: EventExportSummary = dispatcher.shutdown() assert len(calls) == test_case.expected_delivered - assert calls == [ - ("first", "invocation_started"), - ("first", "invocation_failed"), - ("second", "invocation_failed"), + assert sorted(calls) == [ + ("first", "operation_failed"), + ("first", "statement_submitted"), + ("second", "operation_failed"), ] first_counts: EventExporterCounts = summary.per_exporter[0].counts second_counts: EventExporterCounts = summary.per_exporter[1].counts @@ -568,18 +572,24 @@ def test_given_sensitive_exporter_exception_when_reporting_then_only_safe_dimens test_case: EventExporterDispatcherTestCase, ) -> None: failures: list[object] = [] + reported: threading.Event = threading.Event() def failing(*, event: LifecycleEvent) -> None: del event raise RuntimeError("password=secret sql=select-sensitive destination=broker") + def report(failure: object) -> None: + failures.append(failure) + reported.set() + dispatcher: EventExporterDispatcher = EventExporterDispatcher( exporters=(BoundEventExporter("safe_name", failing, {}),), - failure_callback=failures.append, + failure_callback=report, ) dispatcher.enqueue(lifecycle_event()) dispatcher.shutdown() + assert reported.wait(timeout=1.0) assert len(failures) == test_case.expected_delivered rendered: str = repr(failures[0]) assert "RuntimeError" in rendered @@ -741,3 +751,63 @@ def summary_callback(summary: EventExportSummary) -> None: assert len(exported) == test_case.expected_delivered assert summaries[0].accepted == 1 dispatcher.shutdown() + + +@pytest.mark.parametrize( + "test_case", + ( + OutcomeDeliveryTestCase( + "saturated queue still admits outcomes", + queue_capacity=4, + backlog=100, + sink_seconds=0.0, + shutdown_timeout_seconds=5.0, + expected_outcomes=("run_completed", "invocation_completed"), + ), + OutcomeDeliveryTestCase( + "tight budget drops backlog for outcomes", + queue_capacity=1024, + backlog=300, + sink_seconds=0.02, + shutdown_timeout_seconds=0.4, + expected_outcomes=("run_completed", "invocation_completed"), + ), + ), + ids=lambda case: case.description, +) +def test_given_backlog_when_shutting_down_then_outcome_events_arrive_and_backlog_is_dropped( + test_case: OutcomeDeliveryTestCase, +) -> None: + held: threading.Event = threading.Event() + release: threading.Event = threading.Event() + delivered: list[str] = [] + + def record(*, event: LifecycleEvent) -> None: + delivered.append(event.event_type) + held.set() + _ = release.wait() + time.sleep(test_case.sink_seconds) + + dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("recorder", record, {}),), + queue_capacity=test_case.queue_capacity, + shutdown_timeout_seconds=test_case.shutdown_timeout_seconds, + invocation_timeout_seconds=5.0, + ) + dispatcher.enqueue(correlated_event(0, "run_started")) + assert held.wait(timeout=1.0) + for index in range(test_case.backlog): + dispatcher.enqueue(correlated_event(index + 1, "statement_completed")) + for offset, event_type in enumerate(test_case.expected_outcomes): + dispatcher.enqueue(correlated_event(test_case.backlog + 1 + offset, event_type)) + release.set() + + summary: EventExportSummary = dispatcher.shutdown() + + assert ( + tuple(filter(test_case.expected_outcomes.__contains__, delivered)) + == test_case.expected_outcomes + ) + assert delivered[-1] == "invocation_completed" + assert summary.dropped > 0 + assert summary.delivered + summary.dropped + summary.failed == summary.accepted diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py index 10cb6afcb4..dae71eb80e 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_priority_queue.py @@ -25,7 +25,11 @@ PriorityPairTestCase( "3 then 2", "operation_failed", "statement_completed", ("event-1",), 1 ), - PriorityPairTestCase("3 then 3", "operation_failed", "run_failed", ("event-1",), 1), + PriorityPairTestCase("3 then 3", "operation_failed", "statement_failed", ("event-1",), 1), + PriorityPairTestCase("3 then outcome 3", "operation_failed", "run_failed", ("event-2",), 1), + PriorityPairTestCase( + "outcome 2 then 3", "run_completed", "operation_failed", ("event-1",), 1 + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/observability/sinks.mdx b/website/src/content/docs/docs/concepts/observability/sinks.mdx index 91aba606ad..f58c00658b 100644 --- a/website/src/content/docs/docs/concepts/observability/sinks.mdx +++ b/website/src/content/docs/docs/concepts/observability/sinks.mdx @@ -79,7 +79,8 @@ min_severity = "warning" `shutdown_timeout` sets how long a command waits at exit for lifecycle sinks to drain queued events. It is a fixed duration from `"0s"` to `"10m"`, defaults to `"2s"`, and is accepted only under `[sinks.lifecycle]`, not per named sink. `"0s"` drops anything still queued at exit. The -setting does not change the one-second timeout for each sink call. A sink call still running when +final quarter of the budget is reserved for run and invocation outcomes: when it begins, other +queued events are dropped and counted so the outcomes still arrive. The setting does not change the one-second timeout for each sink call. A sink call still running when the budget ends gets up to that one second to return before providers are torn down; teardown always runs on the command's own thread. @@ -94,15 +95,21 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. ## Delivery behavior -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority. +- Each sink receives events in order on its own long-lived delivery thread. +- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; + `invocation_completed` and `invocation_failed` come after every other queued event. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. + Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, + `invocation_completed`, `invocation_failed`) are always admitted and never displaced. - A failing or timed-out sink is isolated from command correctness. - `audit_completed` facts are published as each audit finishes (model audits in `sqb build` when their model finishes), not in one burst at the end of the run. - When lifecycle events are dropped or fail, the command prints one warning to stderr with the counts and affected sinks, for example `Warning: command completed successfully, but lifecycle event export was incomplete: 140 of 248 events dropped, 0 failed (sink 'orders_events'). Increase sinks.lifecycle.shutdown_timeout or check sink health.` - The exit code and stdout, including `--json` output, are unchanged. + A successful command then ends with + `sqb audit completed successfully; only its lifecycle event export is affected.` The exit code + and stdout, including `--json` output, are unchanged. - Provider setup and declaration errors fail before execution because the project configuration is invalid. - Providers shared by multiple sinks are set up and torn down once per command. From a6144e940e8e9c56ae255acad35d3cd419f3ef9a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 02:50:43 +0100 Subject: [PATCH 28/39] fix: end a successful command on its own status after an incomplete-export warning The incomplete lifecycle export warning could be the last line a successful command printed. It is now followed by "sqb completed successfully; only its lifecycle event export is affected.", built by the same closing-status helper the Recall notice uses, and a failed stderr write no longer turns into a command error. --- .../src/dispatch/main/closing_status.rs | 13 ++++++++ crates/sqlbuild-cli/src/dispatch/main/mod.rs | 1 + crates/sqlbuild-cli/src/dispatch/tests.rs | 2 ++ .../src/dispatch/tests/test_closing_status.rs | 32 +++++++++++++++++++ .../src/dispatch/tests/test_types.rs | 7 ++++ .../bindings/_helpers/cli/observability.rs | 31 ++++++++++++------ .../src/bindings/_helpers/recall/constants.rs | 2 ++ .../src/bindings/_helpers/recall/session.rs | 16 ++-------- fensu.toml | 2 +- .../cli/commands/main/audit/_test_types.py | 1 + .../main/audit/test_audit_event_delivery.py | 10 +++++- 11 files changed, 92 insertions(+), 25 deletions(-) create mode 100644 crates/sqlbuild-cli/src/dispatch/main/closing_status.rs create mode 100644 crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs diff --git a/crates/sqlbuild-cli/src/dispatch/main/closing_status.rs b/crates/sqlbuild-cli/src/dispatch/main/closing_status.rs new file mode 100644 index 0000000000..53b8d3a897 --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/main/closing_status.rs @@ -0,0 +1,13 @@ +//! The closing line after an incomplete-delivery notice, so the command's own outcome is the last word. + +/// `sqb `'s outcome by exit code, naming the only thing a preceding notice affects. +#[must_use] +pub fn closing_status(command: &str, exit_code: Option, affected: &str) -> String { + match exit_code { + Some(0) => { + format!("sqb {command} completed successfully; only its {affected} is affected.\n") + } + Some(code) => format!("sqb {command} failed with exit code {code}.\n"), + None => format!("sqb {command} failed.\n"), + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/main/mod.rs b/crates/sqlbuild-cli/src/dispatch/main/mod.rs index d96f4ca211..7cc59e2303 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/mod.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/mod.rs @@ -1,3 +1,4 @@ +pub mod closing_status; pub mod command_dispatch; pub mod event_export_warning; pub mod failure_report; diff --git a/crates/sqlbuild-cli/src/dispatch/tests.rs b/crates/sqlbuild-cli/src/dispatch/tests.rs index 2844cc4653..1c47579fbf 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_closing_status.rs"] +mod closing_status; #[path = "tests/test_command_dispatch.rs"] mod command_dispatch; #[path = "tests/test_event_export_warning.rs"] diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs new file mode 100644 index 0000000000..188b2b2071 --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_closing_status.rs @@ -0,0 +1,32 @@ +use crate::dispatch::main::closing_status::closing_status; +use crate::dispatch::tests::test_types::ClosingStatusTestCase; + +#[test] +fn given_command_outcome_when_closing_a_notice_then_the_outcome_is_the_last_line() { + let test_cases = [ + ClosingStatusTestCase { + description: "success names the only affected output", + exit_code: Some(0), + expected_status: "sqb audit completed successfully; only its lifecycle event export \ + is affected.\n", + }, + ClosingStatusTestCase { + description: "failure reports the exit code", + exit_code: Some(2), + expected_status: "sqb audit failed with exit code 2.\n", + }, + ClosingStatusTestCase { + description: "unknown exit code reports failure", + exit_code: None, + expected_status: "sqb audit failed.\n", + }, + ]; + for test_case in test_cases { + assert_eq!( + closing_status("audit", test_case.exit_code, "lifecycle event export"), + test_case.expected_status, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs index a075ba8d89..7f2415c4b8 100644 --- a/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/dispatch/tests/test_types.rs @@ -18,3 +18,10 @@ pub(super) struct EventExportWarningTestCase { pub(super) command_succeeded: bool, pub(super) expected_warning: Option<&'static str>, } + +/// A command, its exit code and what a preceding notice affects, and the closing line. +pub(super) struct ClosingStatusTestCase { + pub(super) description: &'static str, + pub(super) exit_code: Option, + pub(super) expected_status: &'static str, +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index e5946c1965..a5f2f695c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -9,6 +9,7 @@ use pyo3::prelude::{ }; use pyo3::types::{PyBool, PyString, PyTuple, PyTupleMethods}; use pyo3::{IntoPyObjectExt, pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_cli::dispatch::main::closing_status::closing_status; use sqlbuild_cli::dispatch::main::event_export_warning::event_export_warning; use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; @@ -24,6 +25,7 @@ use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::native_host::user_code::UserCodeHost; use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_color; +use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::result_output::terminal_index::TerminalEventIndex; use crate::bindings::_helpers::result_output::terminal_index::terminal_event_index_scope; @@ -33,6 +35,8 @@ const DEBUG_COMMAND: &str = "debug"; const LINEAGE_COMMAND: &str = "lineage"; const INVOCATION_COMPLETED: &str = "invocation_completed"; const INVOCATION_FAILED: &str = "invocation_failed"; +/// What an incomplete-export notice says is the only thing affected. +const LIFECYCLE_EVENT_EXPORT: &str = "lifecycle event export"; const SUMMARY_FIELDS: [&str; 3] = ["queue_depth", "queue_capacity", "flush_complete"]; const COUNT_FIELDS: [&str; 5] = ["accepted", "filtered", "delivered", "failed", "dropped"]; @@ -154,6 +158,8 @@ struct Observed { recall_sinks: Option<(Py, Py)>, /// Context managers entered, innermost last. entered: Vec>, + /// The command name a closing status line reports. + command: String, } /// Log a cleanup failure without letting logging fail the command. @@ -197,18 +203,20 @@ fn export_summary(summary: &Bound<'_, PyAny>) -> PyResult { }) } -/// Print the incomplete-export warning to stderr when events were dropped or failed. -fn warn_incomplete_export(summary: &Bound<'_, PyAny>, command_succeeded: bool) -> PyResult<()> { - let py: Python<'_> = summary.py(); +/// Print the incomplete-export warning to stderr, closed by the command's success, ignoring write failures. +fn warn_incomplete_export( + summary: &Bound<'_, PyAny>, + (command, command_succeeded): (&str, bool), +) -> PyResult<()> { let Some(message) = event_export_warning(&export_summary(summary)?, command_succeeded) else { return Ok(()); }; - let options = pyo3::types::PyDict::new(py); - pyo3::types::PyDictMethods::set_item(&options, "file", py.import("sys")?.getattr("stderr")?)?; - pyo3::types::PyDictMethods::set_item(&options, "flush", true)?; - py.import("builtins")? - .getattr("print")? - .call((message,), Some(&options))?; + let closing: String = if command_succeeded { + closing_status(command, Some(0), LIFECYCLE_EVENT_EXPORT) + } else { + String::new() + }; + drop(write_stderr(summary.py(), &format!("{message}\n{closing}"))); Ok(()) } @@ -295,7 +303,7 @@ impl Observed { let succeeded: bool = self.outcome.get().succeeded.load(Ordering::SeqCst); run_cleanup( py, - warn_incomplete_export(&summary, succeeded), + warn_incomplete_export(&summary, (&self.command, succeeded)), "event_exporter_warning", ); } @@ -539,6 +547,9 @@ impl CliObservabilityScope { unsubscribe_outcome: None, recall_sinks: None, entered: Vec::new(), + command: native_field(args, "command")? + .extract::>()? + .unwrap_or_default(), }; if let Err(error) = observed.install(py, (args, self.project_dir.bind(py)), &terminal_index) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 114976c1d5..a33d548588 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -9,6 +9,8 @@ pub(crate) const NOTICE_INTERVAL_SECONDS: u64 = 24 * 60 * 60; pub(crate) const HOST_TOKEN_LENGTH: usize = 16; /// The shortest heartbeat interval the test environment variable may request. pub(crate) const MINIMUM_HEARTBEAT_SECONDS: f64 = 0.01; +/// What a Recall closing notice says is the only thing affected. +pub(crate) const RECALL_HISTORY: &str = "Recall history"; /// The compute-log stream index of stdout; every other output index is stderr. pub(crate) const STDOUT_STREAM_INDEX: usize = 0; pub(crate) const RECALL_DISABLED_CODE: &str = "C970"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 36b96eb21f..013617a07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_cli::dispatch::main::closing_status::closing_status; use sqlbuild_cli::dispatch::models::CommandFacts; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; @@ -33,7 +34,7 @@ use crate::bindings::_helpers::recall::clickhouse_store::{ use crate::bindings::_helpers::recall::console::write_stderr; use crate::bindings::_helpers::recall::constants::{ CLOSE_POLL_MS, DDL_NOTICE_PREFIX, HEARTBEAT_ENVIRONMENT, HOST_TOKEN_LENGTH, - MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, + MINIMUM_HEARTBEAT_SECONDS, NOTICE_INTERVAL_SECONDS, QUARANTINE_START_WAIT_MS, RECALL_HISTORY, STDOUT_STREAM_INDEX, }; use crate::bindings::_helpers::recall::events::escaped_text; @@ -382,17 +383,6 @@ pub(crate) fn start_invocation_recall( }) } -/// The closing sentence after a Recall notice, so the command's own outcome stays the last word. -fn final_status(command: &str, exit_code: Option) -> String { - match exit_code { - Some(0) => { - format!("sqb {command} completed successfully; only its Recall history is affected.\n") - } - Some(code) => format!("sqb {command} failed with exit code {code}.\n"), - None => format!("sqb {command} failed.\n"), - } -} - /// What Recall must tell the user about this run once delivery has finished. fn closing_notices(recall: &InvocationRecall, session: &RecallSession) -> Vec { let mut notices: Vec = session @@ -447,7 +437,7 @@ pub(crate) fn finish_invocation_recall( if !notices.is_empty() { notify( py, - Some(notices.concat() + &final_status(&recall.command, exit_code)), + Some(notices.concat() + &closing_status(&recall.command, exit_code, RECALL_HISTORY)), ); } Ok(()) diff --git a/fensu.toml b/fensu.toml index 5bdf278e85..4533303308 100644 --- a/fensu.toml +++ b/fensu.toml @@ -558,7 +558,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress_reporters.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/cli/progress_text.rs=6", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py index 309b8d22e1..f1f3f57f1b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/_test_types.py @@ -35,6 +35,7 @@ class SlowSinkDropE2ETestCase: description: str shutdown_timeout: str expected_accepted: int + expected_last_line: str @dataclass(frozen=True) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py b/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py index 7c213b7fe7..1e7c4fc1c6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/audit/test_audit_event_delivery.py @@ -111,7 +111,14 @@ def test_given_many_audits_when_running_then_audit_completed_events_arrive_durin @pytest.mark.parametrize( "test_case", - (SlowSinkDropE2ETestCase("zero drain drops queued events", "0s", 13),), + ( + SlowSinkDropE2ETestCase( + "zero drain drops queued events", + "0s", + 13, + "sqb audit completed successfully; only its lifecycle event export is affected.", + ), + ), ids=lambda case: case.description, ) def test_given_slow_sink_and_zero_drain_when_auditing_then_warns_with_counts_and_succeeds( @@ -161,6 +168,7 @@ def test_given_slow_sink_and_zero_drain_when_auditing_then_warns_with_counts_and assert test_case.expected_accepted - dropped - failed <= delivered_lines assert delivered_lines <= test_case.expected_accepted - dropped assert result.stderr.count("event export was incomplete") == 1 + assert result.stderr.splitlines()[-1] == test_case.expected_last_line @pytest.mark.parametrize( From 862cbb1da80af1915e31caa06a475cd137f3e2e9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:25:15 +0100 Subject: [PATCH 29/39] fix(snowflake): read run history in one query and give sqb cost a longer budget Slicing every run into 15-minute reads made long runs cost dozens of reads per attempt, so an eight-hour run could never finish within the 30-second budget and stayed pending. Read the whole window in one query again and split only a window that saturates or times out. The at-exit collection keeps its 30-second budget; sqb cost retries a pending run with a 5-minute budget. --- .../bindings/_helpers/cost_command/refresh.rs | 9 +- .../_helpers/cost_telemetry/snowflake.rs | 70 ++----- .../docs/concepts/adapters/snowflake.md | 18 +- src/sqlbuild/_native.pyi | 1 + .../snowflake/classes/snowflake_adapter.py | 2 + tests/unit/src/sqlbuild/cost/_test_types.py | 11 +- .../src/sqlbuild/cost/test_snowflake_scale.py | 175 ++++++++++++++++-- .../docs/docs/concepts/adapters/snowflake.mdx | 18 +- 8 files changed, 213 insertions(+), 91 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs index b090f852da..518b8f373c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_command/refresh.rs @@ -2,7 +2,7 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods}; -use pyo3::{pyfunction, wrap_pyfunction}; +use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; use sqlbuild_core::cost_command::constants::{ HISTORY_SELECTOR, LATEST_SELECTOR, REFRESHABLE_STATUSES, }; @@ -26,6 +26,9 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::replaced_keywords; +/// Collection budget of an explicit `sqb cost` refresh, which a user waits for on purpose. +const REFRESH_BUDGET_SECONDS: f64 = 300.0; + fn coverage(summary: &Bound<'_, PyAny>) -> PyResult { Ok(CostCoverage { status: native_field(summary, "status")? @@ -167,6 +170,10 @@ fn refresh(py: Python<'_>, request: &Bound<'_, PyAny>) -> PyResult<()> { ("statement_ledger_path", ledger), ("usd_per_credit", native_field(¤t, "usd_per_credit")?), ("rate_source", native_field(¤t, "rate_source")?), + ( + "budget_seconds", + REFRESH_BUDGET_SECONDS.into_pyobject(py)?.into_any(), + ), ], )?; if accepts_cost_refresh(&coverage(¤t)?, &coverage(&summary)?) != Some(true) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 5cb6da8ff2..96b5c7561b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -27,12 +27,10 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; use crate::bindings::_helpers::records::ids::RecordId; -/// Overall budget of one collection; a run's history is read in slices well inside it. +/// Default budget of one collection, which the at-exit collection keeps short. const DEADLINE_SECONDS: f64 = 30.0; /// Statement timeout of one query-history read. const QUERY_TIMEOUT_SECONDS: &str = "10"; -/// Widest end-time slice of one read; long runs are read as consecutive slices of this width. -const SLICE_SECONDS: i64 = 15 * 60; /// Narrowest slice still split in half after a statement timeout. const MIN_TIMEOUT_SPLIT_SECONDS: i64 = 60; const STATEMENT_TIMEOUT_CODE: &str = "000630"; @@ -324,28 +322,6 @@ fn row_observation<'py>( Ok(Ok(observation)) } -/// Equal slice starts of a window, each at most `SLICE_SECONDS`; one slice for a short run. -fn slice_starts<'py>( - (window_start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - run: &Bound<'py, PyAny>, -) -> PyResult>> { - let py: Python<'py> = run.py(); - let width: Bound<'py, PyAny> = window_end.sub(window_start)?; - let slice: Bound<'py, PyAny> = timedelta(py, ("seconds", SLICE_SECONDS))?; - let count: usize = if run.le(&slice)? { - 1 - } else { - width - .sub(timedelta(py, ("microseconds", 1))?)? - .floor_div(&slice)? - .extract::()? - + 1 - }; - (0..count) - .map(|index| window_start.add(width.mul(index)?.div(count)?)) - .collect() -} - /// Open-ended rows from `start`, which include running queries, or bounded windows if saturated. fn open_rows<'py>( history: &History<'py>, @@ -375,30 +351,7 @@ fn open_rows<'py>( )) } -/// Rows of the run window, read as bounded slices with an open-ended final slice. -fn sliced_rows<'py>( - history: &History<'py>, - (window_start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), - run: &Bound<'py, PyAny>, -) -> PyResult<(Vec>, bool)> { - let starts: Vec> = slice_starts((window_start, window_end), run)?; - let mut rows: Vec> = Vec::new(); - let mut saturated: bool = false; - for (index, start) in starts.iter().enumerate() { - let (slice, slice_saturated) = match starts.get(index + 1) { - Some(end) => bounded_rows(history, (start, end))?, - None => open_rows(history, (start, window_end))?, - }; - rows.extend(slice); - saturated |= slice_saturated; - } - if starts.len() == 1 { - return Ok((rows, saturated)); - } - Ok((merged_rows(rows.into_iter())?, saturated)) -} - -/// Observations of the run window: sliced history, bounded windows when saturated. +/// Observations of the run window: one open read, halved windows when saturated or timed out. fn query_observations<'py>( history: &History<'py>, (started_at, completed_at): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -408,11 +361,7 @@ fn query_observations<'py>( let skew: Bound<'py, PyAny> = timedelta(py, ("minutes", 10))?; let window_start: Bound<'py, PyAny> = started_at.sub(&skew)?; let window_end: Bound<'py, PyAny> = completed_at.add(&skew)?; - let (rows, saturated) = sliced_rows( - history, - (&window_start, &window_end), - &completed_at.sub(started_at)?, - )?; + let (rows, saturated) = open_rows(history, (&window_start, &window_end))?; let observations: Bound<'py, PyList> = PyList::empty(py); let mut observed: HashSet = HashSet::new(); let mut skipped: Vec<(String, &'static str)> = Vec::new(); @@ -442,7 +391,7 @@ fn query_observations<'py>( } /// The budget a query-history read exhausted, or `None` for any other failure. -fn budget_exhausted(py: Python<'_>, error: &PyErr) -> Option { +fn budget_exhausted(py: Python<'_>, error: &PyErr, budget_seconds: f64) -> Option { if error.is_instance_of::(py) && error .value(py) @@ -450,7 +399,7 @@ fn budget_exhausted(py: Python<'_>, error: &PyErr) -> Option { .is_ok_and(|text| text.to_string_lossy() == DEADLINE_MESSAGE) { return Some(format!( - "{DEADLINE_SECONDS:.0}-second collection deadline exceeded" + "{budget_seconds:.0}-second collection deadline exceeded" )); } statement_timeout(py, error).then(|| { @@ -603,6 +552,7 @@ struct Collection<'py> { rate_source: Bound<'py, PyAny>, attempts: i64, retry_delay_seconds: f64, + budget_seconds: f64, } fn collect<'py>( @@ -611,7 +561,7 @@ fn collect<'py>( ledger_path: &Bound<'py, PyAny>, ) -> PyResult { let py: Python<'py> = ledger_path.py(); - history.deadline = monotonic(py)? + DEADLINE_SECONDS; + history.deadline = monotonic(py)? + collection.budget_seconds; let run_id: String = collection.run_id.extract()?; let ledger: Vec> = read_ledger(ledger_path, &collection.run_id)?; let failure: Option = consume_ledger_failure(py, &run_id)?; @@ -660,7 +610,7 @@ fn collect<'py>( ) { Ok(observed) => observed, Err(error) => { - let Some(cause) = budget_exhausted(py, &error) else { + let Some(cause) = budget_exhausted(py, &error, collection.budget_seconds) else { return Err(error); }; if summary.is_some() { @@ -775,6 +725,7 @@ fn collect_snowflake_cost<'py>( "attempts", "retry_delay_seconds", "result_limit", + "budget_seconds", ], ), )?; @@ -794,6 +745,9 @@ fn collect_snowflake_cost<'py>( attempts: optional("attempts", 3_i64.into_bound_py_any(py)?)?.extract()?, retry_delay_seconds: optional("retry_delay_seconds", 1.0_f64.into_bound_py_any(py)?)? .extract()?, + budget_seconds: optional("budget_seconds", py.None().into_bound(py))? + .extract::>()? + .unwrap_or(DEADLINE_SECONDS), }; let history: History<'py> = History { connection: arguments.value("connection")?, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md index 0b104feb87..39c68946f3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/snowflake.md @@ -132,18 +132,20 @@ contract rate. Use `sqb cost`, `sqb cost latest`, `sqb cost `, or provide a versioned, decimal-safe output contract. Pending detail records are refreshed from Snowflake when inspected again. -SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` when the run finishes, in end-time slices of -at most 15 minutes, so a long run with tens of thousands of statements costs several short reads -rather than one slow one. Each read has a 10-second statement timeout; a slice that reaches it is -split in half and read again. If history still cannot be read within the 30-second collection -budget, the build is unaffected and its cost is recorded as `pending`, with a message naming the -timeout; `sqb cost ` collects it later. Any other failure is recorded as -`collection_failed` with the Snowflake error that caused it. +SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` for the whole run in one query when the run +finishes. Each read has a 10-second statement timeout; a window that reaches it, or that returns +more rows than one read allows, is split in half and read again. Collection at the end of a command +has a 30-second budget so the command exits promptly. If history cannot be read within it, the +build is unaffected and its cost is recorded as `pending`, with a message naming the timeout. +`sqb cost ` collects it later with a 5-minute budget, which completes long runs. Any other +failure is recorded as `collection_failed` with the Snowflake error that caused it. These values are attributed compute credits and estimated cost, not Snowflake-billed credits or invoice reconciliation. The estimate uses only query history visible to the executing role and does not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60-second minimum, -cloud-services credits, contract adjustments, or multi-cluster billing. Run metadata and query IDs +cloud-services credits, contract adjustments, or multi-cluster billing. A statement run with a `QUERY_TAG` +of your own is not recognised as a metadata-only `CREATE`, so it may report missing warehouse +metadata. Run metadata and query IDs are stored under `target/executions//`; that statement ledger stores only an SQL digest, not SQL text. Executed SQL artifacts are stored separately under the sensitive `target/run/` tree. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index d065ba38ad..c07bd63913 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -7581,6 +7581,7 @@ def collect_snowflake_cost( attempts: int = 3, retry_delay_seconds: float = 1.0, result_limit: int = 10000, + budget_seconds: float | None = None, ) -> RunCostSummary: ... def snowflake_query_observations( *, diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index b9a9088eea..55e7ddfdc9 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -358,6 +358,7 @@ def collect_run_cost( statement_ledger_path: Path, usd_per_credit: Decimal, rate_source: str, + budget_seconds: float | None = None, ) -> RunCostSummary: telemetry_connection_config: dict[str, object] = dict(connection_config) telemetry_connection_config["login_timeout"] = _COST_LOGIN_TIMEOUT_SECONDS @@ -381,6 +382,7 @@ def collect_run_cost( statement_ledger_path=statement_ledger_path, usd_per_credit=usd_per_credit, rate_source=rate_source, + budget_seconds=budget_seconds, ) except Exception as error: summary = RunCostSummary( diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index b6246b99c4..158999812c 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -114,7 +114,8 @@ class SnowflakeScaleTestCase: expected_query_count: int expected_message_fragments: tuple[str, ...] = () expected_max_sql_length: int = 2_000 - expected_max_slice: timedelta = timedelta(minutes=15) + expected_minimum_reads: int = 2 + expected_maximum_reads: int = 1_000 @dataclass(frozen=True) @@ -131,3 +132,11 @@ class SnowflakeCostFailureMessageTestCase: description: str expected_status: CostStatus expected_message: str + + +@dataclass(frozen=True) +class SnowflakeCostBudgetTestCase: + description: str + seconds_per_read: float + expected_status: CostStatus + expected_message_fragment: str = "" diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py index 4e68dd5f6c..7664da0998 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py @@ -3,7 +3,7 @@ from __future__ import annotations import re -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import UTC, datetime, timedelta from decimal import Decimal from pathlib import Path @@ -11,16 +11,30 @@ import pytest -from sqlbuild._native import collect_snowflake_cost +from sqlbuild._native import ( + CostCommandRequest, + CostRunRecord, + collect_snowflake_cost, + read_cost_run, + refresh_pending_cost_run, + write_cost_run, +) from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus +from tests.unit.src.sqlbuild.cli.commands._helpers.cost.helpers import write_snowflake_project from tests.unit.src.sqlbuild.cost._test_types import ( + SnowflakeCostBudgetTestCase, SnowflakeCostFailureMessageTestCase, SnowflakeLargeRunCostTestCase, SnowflakeScaleTestCase, ) -from tests.unit.src.sqlbuild.cost.helpers import history_row, run_history_rows, write_ledger +from tests.unit.src.sqlbuild.cost.helpers import ( + build_cost_run_record, + history_row, + run_history_rows, + write_ledger, +) _ORIGIN: datetime = datetime(2026, 8, 23, 1, tzinfo=UTC) _RUN_ID: str = "run-nightly" @@ -63,6 +77,16 @@ def fetchall(self) -> list[tuple[Any, ...]]: return self.rows +@dataclass +class _Clock: + """A monotonic clock that advances only when the fake history answers a read.""" + + seconds: float = 0.0 + + def monotonic(self) -> float: + return self.seconds + + @dataclass class _FakeQueryHistory: """Answers `INFORMATION_SCHEMA.QUERY_HISTORY` reads from fixed rows, like Snowflake does. @@ -75,15 +99,8 @@ class _FakeQueryHistory: rows: list[tuple[Any, ...]] timeout_window: timedelta | None = None reads: list[HistoryRead] = field(default_factory=list) - - @property - def widest_bounded_read(self) -> timedelta: - """The widest end-time window of any bounded read.""" - - return max( - (read.end - read.start for read in self.reads if read.end is not None), - default=timedelta(0), - ) + clock: _Clock = field(default_factory=_Clock) + seconds_per_read: float = 0.0 @property def longest_sql(self) -> int: @@ -102,6 +119,7 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: datetime.fromisoformat(end_match.group(1)) if end_match is not None else None ) timeout_seconds: str = kwargs["statement_params"]["STATEMENT_TIMEOUT_IN_SECONDS"] + self.clock.seconds += self.seconds_per_read self.reads.append( HistoryRead(start=start, end=end, sql_length=len(sql), timeout_seconds=timeout_seconds) ) @@ -121,7 +139,17 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: "test_case", [ SnowflakeScaleTestCase( - description="a 78-minute run of 20,000 statements is read in bounded slices", + description="an unsaturated 8-hour run is read with one query", + statement_count=5_000, + duration_minutes=480, + timeout_window=None, + expected_status=CostStatus.COMPLETE, + expected_query_count=5_000, + expected_minimum_reads=1, + expected_maximum_reads=1, + ), + SnowflakeScaleTestCase( + description="a 78-minute run of 20,000 statements splits its saturated window", statement_count=20_000, duration_minutes=78, timeout_window=None, @@ -129,7 +157,7 @@ def execute(self, sql: str, **kwargs: Any) -> _Cursor: expected_query_count=20_000, ), SnowflakeScaleTestCase( - description="slices that reach the statement timeout are split and retried", + description="a window that reaches the statement timeout is split and retried", statement_count=6_500, duration_minutes=42, timeout_window=timedelta(minutes=8), @@ -183,7 +211,8 @@ def test_given_large_run_when_collecting_snowflake_cost_then_sql_stays_bounded( assert fragment in (summary.message or "") assert history.longest_sql < test_case.expected_max_sql_length assert {read.timeout_seconds for read in history.reads} == {"10"} - assert history.widest_bounded_read <= test_case.expected_max_slice + assert test_case.expected_minimum_reads <= len(history.reads) + assert len(history.reads) <= test_case.expected_maximum_reads class _ClosingHistory(_FakeQueryHistory): @@ -292,3 +321,119 @@ def test_given_query_history_error_when_adapter_collects_cost_then_message_names assert summary.status == test_case.expected_status assert summary.message == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostBudgetTestCase( + description="the at-exit budget leaves a slow eight-hour run pending", + seconds_per_read=1.0, + expected_status=CostStatus.PENDING, + expected_message_fragment="30-second collection deadline exceeded", + ), + ], + ids=lambda case: case.description, +) +def test_given_slow_history_when_collecting_at_exit_then_run_stays_pending( + test_case: SnowflakeCostBudgetTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + duration: timedelta = timedelta(hours=8) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=2_000, duration=duration + ) + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id=_RUN_ID, rows=rows) + history: _ClosingHistory = _ClosingHistory( + rows=rows, + timeout_window=timedelta(minutes=10), + seconds_per_read=test_case.seconds_per_read, + ) + adapter: SnowflakeAdapter = SnowflakeAdapter() + monkeypatch.setattr(adapter, "connect", lambda config: history) + monkeypatch.setattr("time.monotonic", history.clock.monotonic) + + summary: RunCostSummary = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id=_RUN_ID, + started_at=_ORIGIN, + completed_at=_ORIGIN + duration, + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert summary.status == test_case.expected_status + assert test_case.expected_message_fragment in (summary.message or "") + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostBudgetTestCase( + description="sqb cost completes a slow eight-hour run left pending at exit", + seconds_per_read=1.0, + expected_status=CostStatus.COMPLETE, + ), + ], + ids=lambda case: case.description, +) +def test_given_pending_slow_run_when_refreshing_with_sqb_cost_then_larger_budget_completes_it( + test_case: SnowflakeCostBudgetTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + duration: timedelta = timedelta(hours=8) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=2_000, duration=duration + ) + ledger_dir: Path = tmp_path / "target" / "executions" / _RUN_ID + ledger_dir.mkdir(parents=True) + write_ledger(path=ledger_dir / "statements.jsonl", run_id=_RUN_ID, rows=rows) + record: CostRunRecord = build_cost_run_record(run_id=_RUN_ID, completed_at=_ORIGIN + duration) + write_cost_run( + project_dir=tmp_path, + record=replace( + record, + started_at=_ORIGIN, + cost=replace(record.cost, status=CostStatus.PENDING, resources=(), query_count=0), + ), + ) + write_snowflake_project(tmp_path) + history: _ClosingHistory = _ClosingHistory( + rows=rows, + timeout_window=timedelta(minutes=10), + seconds_per_read=test_case.seconds_per_read, + ) + adapter: SnowflakeAdapter = SnowflakeAdapter() + monkeypatch.setattr(adapter, "connect", lambda config: history) + monkeypatch.setattr("time.monotonic", history.clock.monotonic) + monkeypatch.setattr( + "sqlbuild.adapter.discovery.main.resolve_adapter.resolve_adapter", + lambda **kwargs: adapter, + ) + + refresh_pending_cost_run( + CostCommandRequest( + project_dir=tmp_path, + selector=_RUN_ID, + no_color=True, + limit=None, + no_limit=False, + sort=None, + order=None, + since=None, + until=None, + json_output=False, + json_output_path=None, + ) + ) + + refreshed: CostRunRecord | None = read_cost_run(project_dir=tmp_path, run_id=_RUN_ID) + assert refreshed is not None + assert refreshed.cost.status == test_case.expected_status + assert refreshed.cost.query_count == len(rows) + assert history.clock.seconds > 30 diff --git a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx index 3daf486fe1..829697736e 100644 --- a/website/src/content/docs/docs/concepts/adapters/snowflake.mdx +++ b/website/src/content/docs/docs/concepts/adapters/snowflake.mdx @@ -118,18 +118,20 @@ contract rate. Use `sqb cost`, `sqb cost latest`, `sqb cost `, or provide a versioned, decimal-safe output contract. Pending detail records are refreshed from Snowflake when inspected again. -SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` when the run finishes, in end-time slices of -at most 15 minutes, so a long run with tens of thousands of statements costs several short reads -rather than one slow one. Each read has a 10-second statement timeout; a slice that reaches it is -split in half and read again. If history still cannot be read within the 30-second collection -budget, the build is unaffected and its cost is recorded as `pending`, with a message naming the -timeout; `sqb cost ` collects it later. Any other failure is recorded as -`collection_failed` with the Snowflake error that caused it. +SQLBuild reads `INFORMATION_SCHEMA.QUERY_HISTORY` for the whole run in one query when the run +finishes. Each read has a 10-second statement timeout; a window that reaches it, or that returns +more rows than one read allows, is split in half and read again. Collection at the end of a command +has a 30-second budget so the command exits promptly. If history cannot be read within it, the +build is unaffected and its cost is recorded as `pending`, with a message naming the timeout. +`sqb cost ` collects it later with a 5-minute budget, which completes long runs. Any other +failure is recorded as `collection_failed` with the Snowflake error that caused it. These values are attributed compute credits and estimated cost, not Snowflake-billed credits or invoice reconciliation. The estimate uses only query history visible to the executing role and does not reconstruct invisible concurrent work, warehouse resume or idle tail, the 60-second minimum, -cloud-services credits, contract adjustments, or multi-cluster billing. Run metadata and query IDs +cloud-services credits, contract adjustments, or multi-cluster billing. A statement run with a `QUERY_TAG` +of your own is not recognised as a metadata-only `CREATE`, so it may report missing warehouse +metadata. Run metadata and query IDs are stored under `target/executions//`; that statement ledger stores only an SQL digest, not SQL text. Executed SQL artifacts are stored separately under the sensitive `target/run/` tree. From ed20a8cf0c60700b181af2428b23998d6774de0a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:28:42 +0100 Subject: [PATCH 30/39] fix(rules): tighten cross join proofs, CTE labels and directive detection A named window is never treated as unpartitioned, so QUALIFY ROW_NUMBER() OVER w = 1 no longer hides a cross join. Findings name the top-level CTE, including WITH clauses owned by set operations, so an exception copied from a finding matches it. Joins that read no named relation say only a file-wide exception can cover them and print that entry with the file's path. Inline directives are detected only inside SQL comments, so string literals never trigger SQBRSQL000. --- .../bindings/_helpers/rules/lint_pipeline.rs | 8 +- .../src/lint_text/_helpers/suppressions.rs | 147 +++++++++++++----- .../sqlbuild-rules/src/lint_text/constants.rs | 7 +- .../main/reject_inline_suppressions.rs | 4 +- .../src/lint_text/tests/test_lint_text.rs | 32 ++++ .../src/sql_lint/_helpers/join_predicates.rs | 85 +++++++--- .../sqlbuild-rules/src/sql_lint/constants.rs | 5 + .../src/sql_lint/main/fill_finding_path.rs | 9 ++ .../sqlbuild-rules/src/sql_lint/main/mod.rs | 1 + .../src/sql_lint/tests/helpers.rs | 18 ++- .../src/sql_lint/tests/test_cross_joins.rs | 72 ++++++++- .../concepts/rules/findings-and-exceptions.md | 8 +- .../main/rules/test_rule_exception_config.py | 42 +++++ .../rules/findings-and-exceptions.mdx | 8 +- 14 files changed, 368 insertions(+), 78 deletions(-) create mode 100644 crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 9c68a9a060..ea1871cbb2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -32,6 +32,7 @@ use sqlbuild_rules::lint_text::models::{ }; use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; use sqlbuild_rules::project_tree::main::project_real_paths::project_real_paths; +use sqlbuild_rules::sql_lint::main::fill_finding_path::fill_finding_path; use sqlbuild_rules::sql_lint::main::preparation::prepare; use sqlbuild_rules::sql_lint::types::PreparedSql; @@ -1166,7 +1167,7 @@ fn sentinel_spans(sites: &[InterpolationSite]) -> Vec { .collect() } -/// `rows` with a finding for every inline suppression directive, sorted by path then place. +/// `rows` with file paths in their help and a finding per inline directive, sorted by place. fn with_directive_findings( files: &[LintFile], project_dir: &Path, @@ -1185,6 +1186,11 @@ fn with_directive_findings( reject_inline_suppressions(&sources).map_err(|error| error.to_string()) }) .map_err(value_error)?; + for violation in &mut violations { + if let Some(remediation) = &mut violation.remediation { + *remediation = fill_finding_path(remediation, &paths[violation.file]); + } + } violations.extend(findings.into_iter().map(LintViolationRow::suppression)); violations.sort_by(|left, right| { ( diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs index 72c3b00555..324573671d 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -1,4 +1,4 @@ -//! Inline suppression directives, which SQLBuild rejects in favour of configured exceptions. +//! Inline suppression directives in SQL comments, rejected in favour of configured exceptions. use std::sync::LazyLock; @@ -8,13 +8,16 @@ use crate::constants::SETTING_SNIPPET_INDENT; use crate::lint_text::_helpers::lines::LineSpan; use crate::lint_text::_helpers::sites::compiled; use crate::lint_text::constants::{ - CROSS_JOIN_CODE, CROSS_JOIN_RELATION_PATTERN, DIRECTIVE_FIX_REASON, DIRECTIVE_MESSAGE, - DIRECTIVE_PATTERN, REASON_PLACEHOLDER, RULE_PLACEHOLDER, SUPPRESSION_PREFIX, + BLOCK_COMMENT_CLOSER, BLOCK_COMMENT_OPENER, COMMENT_OPENER_BYTES, CROSS_JOIN_CODE, + CROSS_JOIN_RELATION_PATTERN, DIRECTIVE_FIX_REASON, DIRECTIVE_MARKER, DIRECTIVE_MESSAGE, + DIRECTIVE_PATTERN, LINE_COMMENT_PREFIX, REASON_PLACEHOLDER, RULE_PLACEHOLDER, }; use crate::lint_text::errors::LintTextError; use crate::lint_text::models::SuppressionFinding; use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_sqltext::sql_scan::main::dialect_non_code_ranges::dialect_non_code_ranges; +use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; static DIRECTIVE: LazyLock> = LazyLock::new(|| Regex::new(DIRECTIVE_PATTERN)); @@ -29,62 +32,124 @@ struct Directive { relation: Option, } -/// The rejection finding for every directive line in one file. +/// The rejection finding for every directive inside a SQL comment of one file. pub(crate) fn directive_findings( file: usize, path: &str, - lines: &[LineSpan<'_>], + (contents, lines): (&str, &[LineSpan<'_>]), ) -> Result, LintTextError> { + if !mentions_directive(contents) { + return Ok(Vec::new()); + } let pattern: &Regex = compiled(&DIRECTIVE)?; let relation_pattern: &Regex = compiled(&CROSS_JOIN_RELATION)?; let mut findings: Vec = Vec::new(); - for (index, line) in lines.iter().enumerate() { - if !line.content.to_lowercase().contains(SUPPRESSION_PREFIX) { - continue; + for (start, end) in comment_ranges(contents) { + let body: &str = comment_body(&contents[start..end]); + let body_start: usize = start + COMMENT_OPENER_BYTES; + for captures in pattern.captures_iter(body) { + let Some(found) = captures.get(0) else { + continue; + }; + let Some(index) = line_index(contents, lines, body_start + found.start()) else { + continue; + }; + let text = |name: &str| -> Option { + captures + .name(name) + .map(|value| value.as_str().trim().to_owned()) + .filter(|value| !value.is_empty()) + }; + let code: Option = text("code"); + let relation: Option = if code.as_deref() == Some(CROSS_JOIN_CODE) { + target_line(contents, lines, (index, start)) + .and_then(|target| cross_joined_relation(relation_pattern, target)) + } else { + None + }; + let directive = Directive { + line: index + 1, + code, + reason: text("reason"), + relation, + }; + findings.push(SuppressionFinding { + file, + line: directive.line, + message: DIRECTIVE_MESSAGE.to_owned(), + remediation: remediation(&directive, path), + fault: true, + fix: None, + fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), + }); } - let captures = pattern.captures(line.content); - let text = |name: &str| -> Option { - captures - .as_ref() - .and_then(|found| found.name(name)) - .map(|found| found.as_str().trim().to_owned()) - .filter(|value| !value.is_empty()) - }; - let code: Option = text("code"); - let relation: Option = if code.as_deref() == Some(CROSS_JOIN_CODE) { - target_line(lines, index) - .and_then(|target| cross_joined_relation(relation_pattern, target)) - } else { - None - }; - let directive = Directive { - line: index + 1, - code, - reason: text("reason"), - relation, - }; - findings.push(SuppressionFinding { - file, - line: directive.line, - message: DIRECTIVE_MESSAGE.to_owned(), - remediation: remediation(&directive, path), - fault: true, - fix: None, - fix_unavailable_reason: Some(DIRECTIVE_FIX_REASON), - }); } Ok(findings) } -/// The next line that is neither blank nor a comment: the line a directive meant to cover. -fn target_line<'a>(lines: &[LineSpan<'a>], directive: usize) -> Option<&'a str> { +/// A cheap check that `contents` may hold a directive at all. +fn mentions_directive(contents: &str) -> bool { + contents + .as_bytes() + .windows(DIRECTIVE_MARKER.len()) + .any(|window| window.eq_ignore_ascii_case(DIRECTIVE_MARKER)) +} + +/// Byte ranges of the `--` line comments and `/* */` block comments, skipping quoted text. +fn comment_ranges(contents: &str) -> Vec<(usize, usize)> { + let syntax = LexicalSyntax { + line_comment_prefixes: vec![LINE_COMMENT_PREFIX.to_owned()], + ..LexicalSyntax::default() + }; + dialect_non_code_ranges(contents, &syntax) + .into_iter() + .filter(|(start, _)| { + let rest: &str = &contents[*start..]; + rest.starts_with(LINE_COMMENT_PREFIX) || rest.starts_with(BLOCK_COMMENT_OPENER) + }) + .collect() +} + +/// A comment's text without its opener and, for a block comment, its closer. +fn comment_body(comment: &str) -> &str { + let body: &str = &comment[COMMENT_OPENER_BYTES..]; + if comment.starts_with(BLOCK_COMMENT_OPENER) { + body.strip_suffix(BLOCK_COMMENT_CLOSER).unwrap_or(body) + } else { + body + } +} + +/// The zero-based line holding byte `offset` of `contents`. +fn line_index(contents: &str, lines: &[LineSpan<'_>], offset: usize) -> Option { + let position: usize = contents[..offset].chars().count(); + lines + .iter() + .position(|line| line.start <= position && position < line.end) +} + +/// The code a directive covers: code before its comment, else the next non-comment line. +fn target_line<'a>( + contents: &'a str, + lines: &[LineSpan<'a>], + (directive, comment_start): (usize, usize), +) -> Option<&'a str> { + let line: &LineSpan<'a> = lines.get(directive)?; + let line_start: usize = contents + .char_indices() + .nth(line.start) + .map_or(contents.len(), |(byte, _)| byte); + let before: &str = contents.get(line_start..comment_start).unwrap_or_default(); + if !before.trim_matches(python_isspace).is_empty() { + return Some(before); + } lines .iter() .skip(directive + 1) .map(|line| line.content) .find(|content| { let stripped: &str = content.trim_start_matches(python_isspace); - !stripped.is_empty() && !stripped.starts_with("--") + !stripped.is_empty() && !stripped.starts_with(LINE_COMMENT_PREFIX) }) } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index b76842b150..099d33f95a 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -28,7 +28,12 @@ pub(crate) const FORMAT_SITE_PATTERN: &str = r"(?i)^(?:@@|\$\{|@|(?:__cursor_end /// Diagnostic code of every inline suppression directive finding. pub const SUPPRESSION_CODE: &str = "SQBRSQL000"; -pub(crate) const SUPPRESSION_PREFIX: &str = "sqb: ignore"; +/// Text every directive contains, checked before scanning a file's comments. +pub(crate) const DIRECTIVE_MARKER: &[u8] = b"sqb:"; +pub(crate) const BLOCK_COMMENT_OPENER: &str = "/*"; +pub(crate) const BLOCK_COMMENT_CLOSER: &str = "*/"; +/// Both comment openers are two bytes long. +pub(crate) const COMMENT_OPENER_BYTES: usize = 2; pub(crate) const DIRECTIVE_PATTERN: &str = r"(?i)sqb:\s*ignore\b\s*(?P[A-Za-z0-9_-]*)(?:\s+because\s+(?P.*\S))?"; pub(crate) const CROSS_JOIN_CODE: &str = "SQBRSQL038"; diff --git a/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs b/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs index 75de3c5e87..23cf26ff3f 100644 --- a/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/main/reject_inline_suppressions.rs @@ -1,4 +1,4 @@ -//! `reject_inline_suppressions`: an error for every inline `-- sqb: ignore` directive. +//! `reject_inline_suppressions`: an error for every `sqb: ignore` directive in a SQL comment. use crate::lint_text::_helpers::lines::{LineSpan, line_spans}; use crate::lint_text::_helpers::suppressions::directive_findings; @@ -12,7 +12,7 @@ pub fn reject_inline_suppressions( let mut findings: Vec = Vec::new(); for (file, (path, contents)) in files.iter().enumerate() { let lines: Vec> = line_spans(contents); - findings.extend(directive_findings(file, path, &lines)?); + findings.extend(directive_findings(file, path, (contents, &lines))?); } Ok(findings) } diff --git a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs index db434b6463..7e0c27a01c 100644 --- a/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs +++ b/crates/sqlbuild-rules/src/lint_text/tests/test_lint_text.rs @@ -265,6 +265,38 @@ fn given_inline_directives_when_rejecting_then_each_prints_its_configured_except ], )], }, + SuppressionTestCase { + description: "a string literal mentioning a directive is not one", + contents: "select '-- sqb: ignore SQBRSQL004 because text' as note, 'sqb: ignore' as other\n", + expected_findings: vec![], + }, + SuppressionTestCase { + description: "a trailing directive after code names the join on its line", + contents: "select o.order_id\nfrom orders as o\ncross join customers as c -- sqb: ignore SQBRSQL038 because deliberate\n", + expected_findings: vec![directive_finding( + 3, + "SQBRSQL038", + &[ + "rule = \"SQBRSQL038\"", + "path = \"models/orders.sql\"", + "relation = \"customers\"", + "reason = \"deliberate\"", + ], + )], + }, + SuppressionTestCase { + description: "a directive in a block comment", + contents: "select 1\n/* sqb: ignore SQBRSQL005 because kept */\nselect 2\n", + expected_findings: vec![directive_finding( + 2, + "SQBRSQL005", + &[ + "rule = \"SQBRSQL005\"", + "path = \"models/orders.sql\"", + "reason = \"kept\"", + ], + )], + }, SuppressionTestCase { description: "a file without directives", contents: "select 1\n-- a plain comment\n", diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs index 8c936a0c80..bc841f66c0 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/join_predicates.rs @@ -3,12 +3,16 @@ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; -use polyglot_sql::expressions::{Expression, JoinKind, Literal, Select, Subquery, TableRef}; +use polyglot_sql::expressions::{Expression, JoinKind, Literal, Select, Subquery, TableRef, With}; use polyglot_sql::parser::{Parser, ParserConfig}; use polyglot_sql::tokens::{Span, Token, TokenType}; use polyglot_sql::{DialectType, ExpressionWalk}; -use crate::sql_lint::constants::{CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PREFIX, CROSS_JOIN_SUFFIX}; +use crate::constants::SETTING_SNIPPET_INDENT; +use crate::sql_lint::constants::{ + CROSS_JOIN_CTE_MARKER, CROSS_JOIN_PATH_PLACEHOLDER, CROSS_JOIN_PREFIX, + CROSS_JOIN_REASON_PLACEHOLDER, CROSS_JOIN_SUFFIX, CROSS_JOIN_UNNAMED_MESSAGE, +}; use crate::sql_lint::models::{CrossJoinRequest, LintDiagnostic, LintRuleMetadata}; const CROSS_JOIN_FIX_REASON: &str = "a keyed relationship requires user intent"; @@ -189,10 +193,14 @@ pub(super) fn cross_join_diagnostics( message: finding .relation .as_deref() - .map_or(Cow::Borrowed(rule.message), |relation| { + .map_or(Cow::Borrowed(CROSS_JOIN_UNNAMED_MESSAGE), |relation| { Cow::Owned(cross_join_message(relation, finding.cte.as_deref())) }), - remediation: Cow::Borrowed(rule.remediation), + remediation: if finding.relation.is_some() { + Cow::Borrowed(rule.remediation) + } else { + Cow::Owned(file_wide_remediation(rule.code)) + }, start: finding.span.start, end: finding.span.end, fix_unavailable_reason: Some(CROSS_JOIN_FIX_REASON), @@ -201,6 +209,24 @@ pub(super) fn cross_join_diagnostics( .collect() } +/// Help for a join no relation names: fix it, or record the file-wide exception it allows. +fn file_wide_remediation(code: &str) -> String { + let entry: String = [ + "[[rules.rule_exceptions]]".to_owned(), + format!("rule = \"{code}\""), + format!("path = \"{CROSS_JOIN_PATH_PLACEHOLDER}\""), + format!("reason = \"{CROSS_JOIN_REASON_PLACEHOLDER}\""), + ] + .iter() + .map(|line| format!("\n{SETTING_SNIPPET_INDENT}{line}")) + .collect(); + format!( + "Use a keyed JOIN, or make one side provably at most one row. This join reads no named \ + relation, so a relation-level exception cannot cover it; when the cartesian product is \ + intentional, record a reviewed file-wide exception in sqlbuild_project.toml:{entry}" + ) +} + /// The finding message naming the joined relation and, inside a CTE, that CTE. fn cross_join_message(relation: &str, cte: Option<&str>) -> String { match cte { @@ -263,7 +289,7 @@ fn multiplying_cross_joins( analysis.record_ctes(statement); } for statement in statements { - analysis.visit(statement, None); + analysis.visit(statement, None, true); } let significant: Vec = (0..tokens.len()) .filter(|&index| { @@ -304,10 +330,7 @@ impl Analysis<'_> { /// Record whether every CTE holds at most one row, in definition order. fn record_ctes(&mut self, expression: &Expression) { for node in expression.dfs() { - let Expression::Select(select) = node else { - continue; - }; - let Some(with) = &select.with else { + let Some(with) = with_clause(node) else { continue; }; for cte in &with.ctes { @@ -319,26 +342,29 @@ impl Analysis<'_> { } } - fn visit(&mut self, expression: &Expression, cte: Option<&str>) { - let Expression::Select(select) = expression else { - for child in expression.children() { - self.visit(child, cte); - } - return; - }; + /// Record joins under `expression`, labelled with the top-level CTE containing them. + fn visit(&mut self, expression: &Expression, cte: Option<&str>, top: bool) { let mut bodies: Vec<&Expression> = Vec::new(); - if let Some(with) = &select.with { + if let Some(with) = with_clause(expression) { for definition in &with.ctes { bodies.push(&definition.this); - self.visit(&definition.this, Some(&definition.alias.name)); + let label: Option<&str> = if top { + Some(&definition.alias.name) + } else { + cte + }; + self.visit(&definition.this, label, false); } } - self.record_joins(select, cte); + if let Expression::Select(select) = expression { + self.record_joins(select, cte); + } + let branches_are_top: bool = top && is_set_operation(expression); for child in expression.children() { if bodies.iter().any(|body| std::ptr::eq(*body, child)) { continue; } - self.visit(child, cte); + self.visit(child, cte, branches_are_top); } } @@ -469,6 +495,24 @@ impl Analysis<'_> { } } +/// The WITH clause a query expression owns. +fn with_clause(expression: &Expression) -> Option<&With> { + match expression { + Expression::Select(select) => select.with.as_ref(), + Expression::Union(union) => union.with.as_ref(), + Expression::Intersect(intersect) => intersect.with.as_ref(), + Expression::Except(except) => except.with.as_ref(), + _ => None, + } +} + +fn is_set_operation(expression: &Expression) -> bool { + matches!( + expression, + Expression::Union(_) | Expression::Intersect(_) | Expression::Except(_) + ) +} + fn limited_to_one_row(select: &Select) -> bool { let limited: bool = select .limit @@ -573,6 +617,7 @@ fn unpartitioned_row_number(expression: &Expression) -> bool { expression, Expression::WindowFunction(window) if matches!(window.this, Expression::RowNumber(_)) + && window.over.window_name.is_none() && window.over.partition_by.is_empty() ) } diff --git a/crates/sqlbuild-rules/src/sql_lint/constants.rs b/crates/sqlbuild-rules/src/sql_lint/constants.rs index f28e9feab8..8bec6a6bd0 100644 --- a/crates/sqlbuild-rules/src/sql_lint/constants.rs +++ b/crates/sqlbuild-rules/src/sql_lint/constants.rs @@ -54,3 +54,8 @@ pub(crate) const RELATION_MODIFIER_KEYWORDS: [&str; 43] = [ pub(crate) const CROSS_JOIN_PREFIX: &str = "CROSS JOIN with `"; pub(crate) const CROSS_JOIN_CTE_MARKER: &str = "` in CTE `"; pub(crate) const CROSS_JOIN_SUFFIX: &str = "` creates an explicit cartesian product"; +/// The `SQBRSQL038` message for a join whose relation cannot be named. +pub(crate) const CROSS_JOIN_UNNAMED_MESSAGE: &str = "CROSS JOIN creates an explicit cartesian product with an unnamed relation; only a file-wide exception (rule and path) can cover it"; +/// Stands for the finding's project-relative path until the caller that knows it fills it in. +pub(crate) const CROSS_JOIN_PATH_PLACEHOLDER: &str = ""; +pub(crate) const CROSS_JOIN_REASON_PLACEHOLDER: &str = ""; diff --git a/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs b/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs new file mode 100644 index 0000000000..d5a65801ce --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/main/fill_finding_path.rs @@ -0,0 +1,9 @@ +//! `fill_finding_path`: a SQL finding's help with its file's path written into the entry it prints. + +use crate::sql_lint::constants::CROSS_JOIN_PATH_PLACEHOLDER; + +/// `remediation` with the path placeholder of a printed exception entry replaced by `path`. +#[must_use] +pub fn fill_finding_path(remediation: &str, path: &str) -> String { + remediation.replace(CROSS_JOIN_PATH_PLACEHOLDER, path) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs index 59bc99e94c..6fdc141f76 100644 --- a/crates/sqlbuild-rules/src/sql_lint/main/mod.rs +++ b/crates/sqlbuild-rules/src/sql_lint/main/mod.rs @@ -4,6 +4,7 @@ pub mod batch_formatter; pub mod catalogue; pub mod cross_join_subject; pub mod engine; +pub mod fill_finding_path; pub mod format_requests; pub mod formatter; pub mod preparation; diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs index 614ae828c8..830abb4d14 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/helpers.rs @@ -260,6 +260,17 @@ fn is_tight_call(name: &str, sql: &str, arguments: &str) -> bool { /// The `SQBRSQL038` messages for one cross join case. pub(crate) fn cross_join_messages(test_case: &CrossJoinTestCase) -> Result, String> { + cross_join_texts(test_case, "message") +} + +/// The `SQBRSQL038` remediations for one cross join case. +pub(crate) fn cross_join_remediations( + test_case: &CrossJoinTestCase, +) -> Result, String> { + cross_join_texts(test_case, "remediation") +} + +fn cross_join_texts(test_case: &CrossJoinTestCase, field: &str) -> Result, String> { let keys: Map = test_case .one_row_relations .iter() @@ -280,11 +291,6 @@ pub(crate) fn cross_join_messages(test_case: &CrossJoinTestCase) -> Result Result<(), String> { + let test_cases = [CrossJoinTestCase { + description: "CROSS JOIN UNNEST", + sql: "SELECT o.order_id FROM orders AS o CROSS JOIN UNNEST([1, 2]) AS u(n)", + dialect: "duckdb", + one_row_relations: &[], + expected_messages: &[ + "[[rules.rule_exceptions]]\n rule = \"SQBRSQL038\"\n path = \"\"\n reason = \"\"", + ], + }]; + + for test_case in test_cases { + let remediations: Vec = cross_join_remediations(&test_case)?; + + assert_eq!(remediations.len(), 1, "{}", test_case.description); + assert!( + remediations[0].contains(test_case.expected_messages[0]), + "{}: {remediations:?}", + test_case.description + ); + } + Ok(()) +} + #[test] fn given_cross_join_messages_when_reading_their_subject_then_relation_and_cte_are_named() { let test_cases = [ diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md index 6c2a500d99..07110f3292 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/rules/findings-and-exceptions.md @@ -58,6 +58,10 @@ Relation-level exceptions are stale-checked like exact exceptions: an entry that fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be combined with relation-level exceptions for the same code and path. +A join that reads no named relation, such as `CROSS JOIN UNNEST(...)` or `CROSS JOIN LATERAL`, has +no relation to name. Its finding says so and prints the file-wide exception (rule and path) that is +the only way to cover it. + `SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an @@ -105,8 +109,8 @@ To turn a Rule off for the whole project, add its code to `[rules] ignore`. ## No inline suppressions -SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` line is a `SQBRSQL000` error -that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` or `/* sqb: ignore ... */` +comment is a `SQBRSQL000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the exact entry to add instead: ```text diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py index 731ab315d2..2d3d84d997 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_rule_exception_config.py @@ -47,6 +47,18 @@ _DIRECTIVE_SQL: str = _MODEL_SQL.format( directive=" -- sqb: ignore SQBRSQL038 because every order meets every customer\n" ) +_LITERAL_SQL: str = _PLAIN_SQL.replace( + "SELECT f.order_id, f.customer_id, f.region_id FROM final AS f", + "SELECT f.order_id, f.customer_id, f.region_id, 'sqb: ignore SQBRSQL038' AS note FROM final AS f", +) +_TRAILING_SQL: str = _PLAIN_SQL.replace( + ' CROSS JOIN __ref("customers") AS c\n', + ' CROSS JOIN __ref("customers") AS c -- sqb: ignore SQBRSQL038 because deliberate\n', +) +_UNNAMED_SQL: str = ( + "MODEL (description 'Order checks.');\n" + 'SELECT o.order_id, u.n FROM __ref("orders") AS o CROSS JOIN UNNEST([1, 2]) AS u(n)\n' +) _UPSTREAM_FILES: tuple[tuple[str, str], ...] = ( ("models/orders.sql", "MODEL (description 'Sample orders.');\nSELECT 1 AS order_id\n"), ( @@ -104,6 +116,36 @@ ' reason = "every order meets every customer"', ), ), + RuleExceptionConfigCase( + description="a string literal mentioning a directive compiles cleanly", + rules_toml=_CTE_EXCEPTIONS, + model_sql=_LITERAL_SQL, + expected_exit=0, + ), + RuleExceptionConfigCase( + description="a trailing directive after code is an error", + rules_toml=_CTE_EXCEPTIONS, + model_sql=_TRAILING_SQL, + expected_exit=1, + expected_fragments=( + "Inline Rule suppressions are not supported", + ' relation = "customers"\n reason = "deliberate"', + ), + ), + RuleExceptionConfigCase( + description="an unnamed join prints a file-wide exception for its file", + rules_toml="", + model_sql=_UNNAMED_SQL, + expected_exit=1, + expected_messages=( + "CROSS JOIN creates an explicit cartesian product with an unnamed relation; " + "only a file-wide exception (rule and path) can cover it", + ), + expected_fragments=( + '[[rules.rule_exceptions]]\n rule = "SQBRSQL038"\n' + ' path = "models/order_checks.sql"\n', + ), + ), RuleExceptionConfigCase( description="removed allow_exceptions setting fails with migration help", rules_toml="allow_exceptions = false\n", diff --git a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx index 25f01b5737..5ce22f0539 100644 --- a/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx +++ b/website/src/content/docs/docs/concepts/rules/findings-and-exceptions.mdx @@ -45,6 +45,10 @@ Relation-level exceptions are stale-checked like exact exceptions: an entry that fails compilation. `relation` is accepted only for `SQBRSQL038`, and a file-wide exception cannot be combined with relation-level exceptions for the same code and path. +A join that reads no named relation, such as `CROSS JOIN UNNEST(...)` or `CROSS JOIN LATERAL`, has +no relation to name. Its finding says so and prints the file-wide exception (rule and path) that is +the only way to cover it. + `SQBRSQL038` does not report a cross join when either side provably holds at most one row, so most intentional cross joins need no exception at all. SQLBuild infers this from the query, never from a declaration: a seed whose CSV has one data row (adding a second row brings the finding back), an @@ -92,8 +96,8 @@ To turn a Rule off for the whole project, add its code to `[rules] ignore`. ## No inline suppressions -SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` line is a `SQBRSQL000` error -that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the +SQLBuild has no inline suppression comment. An `-- sqb: ignore ...` or `/* sqb: ignore ... */` +comment is a `SQBRSQL000` error that suppresses nothing, so the finding it was meant to hide is reported too. The error prints the exact entry to add instead: ```text From fe1a67e58f9b0505f97eb455739afbce5bff60ae Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 03:31:21 +0100 Subject: [PATCH 31/39] fix: count a cancelled lifecycle attempt as dropped only when it can never run Reports are collected once per dispatch round, so a lane thread could already have finished its first call and started the next one when the dispatcher cancelled it for the shutdown reserve. The sink still received the event, but it was counted as dropped and its report was ignored, so the incomplete-export warning could claim losses that did not happen. Each delivery thread now claims a call in a dict shared with the dispatcher before running it, and the dispatcher checks and marks the same entry, both atomically under the GIL. A cancellation counts as dropped only when the call was not claimed; a claimed call stays pending and its report is accepted. Abandoned lanes count claimed calls as timed out instead of dropped. Reports are now read after taking the dispatcher condition. --- .../_helpers/event_exporting/dispatcher.rs | 8 +- .../_helpers/event_exporting/workers.rs | 71 +++++++++++------ fensu.toml | 8 +- python-boundary.toml | 2 +- .../main/_invoke_lifecycle_event_sink.py | 7 +- .../classes/test_dispatcher.py | 79 +++++++++++++++++++ 6 files changed, 141 insertions(+), 34 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index a1bd8c7131..9bdef06fc7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -8,7 +8,7 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pyfunction, pymethods, }; -use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PySet, PyString}; +use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyString}; use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, @@ -118,8 +118,8 @@ pub(crate) struct Shared { condition: Py, /// The `SimpleQueue` every delivery thread reports `(token, error)` outcomes to. pub(crate) results: Py, - /// The Python set of attempt tokens delivery threads must skip. - pub(crate) cancelled: Py, + /// Attempt tokens a delivery thread claimed (`True`) before running, or the dispatcher cancelled (`False`). + pub(crate) claims: Py, state: Mutex, pub(crate) invocation_timeout: Duration, shutdown_timeout: Duration, @@ -679,7 +679,7 @@ impl EventExporterDispatcher { let shared = Arc::new(Shared { condition: imported(py, "threading", "Condition")?.call0()?.unbind(), results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), - cancelled: PySet::empty(py)?.into_any().unbind(), + claims: PyDict::new(py).unbind(), state: Mutex::new(State { bound: !exporters.is_none(), counts: vec![ExporterCounts::default(); bound_exporters.len()], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs index 6e966172a8..ca07475d2c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs @@ -74,13 +74,13 @@ enum Notification { } impl Lane { - /// Start `exporter`'s delivery thread, which skips `cancelled` tokens and reports to `results`. + /// Start `exporter`'s delivery thread, which claims each call in `claims` and reports to `results`. fn start(py: Python<'_>, exporter: &Exporter, round: Round<'_>) -> PyResult { let calls: Bound<'_, PyAny> = imported(py, "queue", "SimpleQueue")?.call0()?; let arguments = PyDict::new(py); arguments.set_item("calls", &calls)?; arguments.set_item("results", round.shared.results.bind(py))?; - arguments.set_item("cancelled", round.shared.cancelled.bind(py))?; + arguments.set_item("claims", round.shared.claims.bind(py))?; let thread = call_with( &imported(py, "threading", "Thread")?, &[ @@ -122,11 +122,16 @@ impl State { /// Account reported attempts against their lanes, returning whether a diagnostic was queued. fn record_reports( &mut self, + py: Python<'_>, reports: Vec<(u64, Option)>, - diagnostics: Option, - ) -> bool { + round: Round<'_>, + ) -> PyResult { + let claims: &Bound<'_, PyDict> = round.shared.claims.bind(py); let mut notify: bool = false; for (token, error_type) in reports { + if claims.contains(token)? { + claims.del_item(token)?; + } let Some(index) = self.tokens.remove(&token) else { continue; }; @@ -147,7 +152,7 @@ impl State { match error_type { Some(error_type) => { notify |= self.note_failure( - diagnostics, + round.diagnostics, (index, &attempt.kind, attempt.severity), error_type, ); @@ -159,44 +164,56 @@ impl State { } } } - notify + Ok(notify) } - /// Skip `attempts` on their delivery thread, counting each as dropped for `index`. - fn cancel_attempts( + /// Cancel `attempts` the delivery thread has not claimed (checked and marked under the GIL), counting each dropped; returns the claimed. + fn cancel_unclaimed( &mut self, py: Python<'_>, index: usize, attempts: impl IntoIterator, - cancelled: &Py, - ) -> PyResult<()> { - let cancelled: &Bound<'_, PyAny> = cancelled.bind(py); + claims: &Py, + ) -> PyResult> { + let claims: &Bound<'_, PyDict> = claims.bind(py); + let mut claimed: Vec = Vec::new(); for attempt in attempts { + let already_claimed: bool = match claims.get_item(attempt.token)? { + Some(claim) => claim.is_truthy()?, + None => false, + }; + if already_claimed { + claimed.push(attempt); + continue; + } + claims.set_item(attempt.token, false)?; self.tokens.remove(&attempt.token); - cancelled.call_method1("add", (attempt.token,))?; self.counts = drop_lifecycle_attempts(std::mem::take(&mut self.counts), &[index]); } - Ok(()) + Ok(claimed) } - /// Retire `index`'s lane: its running call fails as timed out and its later attempts are dropped. + /// Retire `index`'s lane: calls it already claimed fail as timed out and the rest are dropped. fn abandon_lane(&mut self, py: Python<'_>, index: usize, round: Round<'_>) -> PyResult { let Some(mut lane) = self.lanes.get_mut(index).and_then(Option::take) else { return Ok(false); }; + let attempts: VecDeque = std::mem::take(&mut lane.pending); + let claimed: Vec = + self.cancel_unclaimed(py, index, attempts, &round.shared.claims)?; + let claims: &Bound<'_, PyDict> = round.shared.claims.bind(py); let mut notify: bool = false; - if let Some(running) = lane.pending.pop_front() { + for running in claimed { + claims.del_item(running.token)?; self.tokens.remove(&running.token); self.blocked.insert(index); self.abandoned_calls = true; - notify = self.note_failure( + notify |= self.note_failure( round.diagnostics, (index, &running.kind, running.severity), TIMEOUT_ERROR_TYPE.to_owned(), ); } - let waiting: VecDeque = std::mem::take(&mut lane.pending); - self.cancel_attempts(py, index, waiting, &round.shared.cancelled)?; lane.retire(py)?; self.retired.push(lane); Ok(notify) @@ -222,7 +239,7 @@ impl State { } /// Drop every queued and lane-waiting event that is not an outcome, counting each attempt. - fn reserve_for_outcomes(&mut self, py: Python<'_>, cancelled: &Py) -> PyResult<()> { + fn reserve_for_outcomes(&mut self, py: Python<'_>, claims: &Py) -> PyResult<()> { let capacity: usize = self.queue.capacity; let (queue, dropped) = retain_outcome_events(std::mem::replace( &mut self.queue, @@ -237,11 +254,15 @@ impl State { let Some(lane) = self.lanes[index].as_mut() else { continue; }; - let running: Option = lane.pending.pop_front(); let (kept, skipped): (VecDeque, VecDeque) = lane.pending.drain(..).partition(|attempt| attempt.outcome); - lane.pending = running.into_iter().chain(kept).collect(); - self.cancel_attempts(py, index, skipped, cancelled)?; + let claimed: Vec = self.cancel_unclaimed(py, index, skipped, claims)?; + let Some(lane) = self.lanes[index].as_mut() else { + continue; + }; + let mut pending: Vec = claimed.into_iter().chain(kept).collect(); + pending.sort_by_key(|attempt| attempt.token); + lane.pending = pending.into(); } drop(dropped); Ok(()) @@ -403,14 +424,14 @@ impl DispatcherWorker { /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. fn dispatch_round(&self, py: Python<'_>) -> PyResult { let shared = &self.shared; - let reports: Vec<(u64, Option)> = self.reported(py)?; let round = Round { shared, diagnostics: shared.diagnostics(), }; let _held = shared.hold(py)?; + let reports: Vec<(u64, Option)> = self.reported(py)?; let mut state = shared.locked()?; - let mut notify: bool = state.record_reports(reports, round.diagnostics); + let mut notify: bool = state.record_reports(py, reports, round)?; let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { let drained: Vec> = std::mem::take(&mut state.queue.items); for item in &drained { @@ -428,7 +449,7 @@ impl DispatcherWorker { notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { state.reserved = true; - state.reserve_for_outcomes(py, &shared.cancelled)?; + state.reserve_for_outcomes(py, &shared.claims)?; } notify |= state.feed(py, round)?; state.stopping && !state.busy() && state.queue.items.is_empty() diff --git a/fensu.toml b/fensu.toml index 4533303308..719415c245 100644 --- a/fensu.toml +++ b/fensu.toml @@ -498,6 +498,12 @@ symbols = [ ] reason = "Old-name steps re-read facts and physical state per move so a retry never repeats a completed step, and compatibility views are re-created in dependency order, so a chained view must read the columns of the view it selects from after that view is re-created. Each view's existence is checked right before it is redefined, and a model has only its own short chain of old names." +[[targets.python.rule_exceptions]] +rule = "FFS110" +path = "src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" +symbols = ["invoke_lifecycle_event_sink"] +reason = "The delivery thread claims each call in the dict it shares with the native dispatcher; the claim is the mutation, atomic under the GIL, so a cancelled call is never both run and counted dropped." + [[targets.python.rule_exceptions]] rule = "XSB051" path = "src/sqlbuild/executor/old_name_views/_helpers/janitor.py" @@ -759,7 +765,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/scope.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/values.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/conversion.rs=19", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/functions.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/execution_history/postgres.rs=28", diff --git a/python-boundary.toml b/python-boundary.toml index 7f4e20791c..532ec56b91 100644 --- a/python-boundary.toml +++ b/python-boundary.toml @@ -295,7 +295,7 @@ note = "Rule 3 facade: reads the decorator's definition off a user function; re- [[stays]] path = "runtime/event_exporting/main/_invoke_lifecycle_event_sink.py" rule = 2 -note = "Rule 2 shim: target of each lifecycle sink's long-lived daemon delivery thread; calls the user sink for each queued event in order and reports how each call ended through a Python queue, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." +note = "Rule 2 shim: target of each lifecycle sink's long-lived daemon delivery thread; claims each queued call in a shared dict (atomic under the GIL, so the dispatcher counts a cancellation as dropped only when the call can never run), calls the user sink in order and reports how each call ended through a Python queue, so user code never runs above native frames that interpreter finalization would abort (event_exporting/workers.rs)." [[stays]] path = "runtime/event_exporting/main/_run_command_output_sinks.py" diff --git a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py index 75c7d8b8f2..b73ee3a6c0 100644 --- a/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py +++ b/src/sqlbuild/runtime/event_exporting/main/_invoke_lifecycle_event_sink.py @@ -12,12 +12,13 @@ def invoke_lifecycle_event_sink( tuple[Callable[..., object], Mapping[str, object], LifecycleEvent, int] | None ], results: SimpleQueue[tuple[int, BaseException | None]], - cancelled: set[int], + claims: dict[int, bool], ) -> None: - """Call the sink with each queued event in order and report how each call ended.""" + """Claim and call the sink with each queued event in order and report how each call ended.""" for function, provider_arguments, event, token in iter(calls.get, None): - if token in cancelled: + if not claims.setdefault(token, True): + del claims[token] continue try: _ = function(event=event, **provider_arguments) diff --git a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py index 077d290a42..0eb28211fb 100644 --- a/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py +++ b/tests/unit/src/sqlbuild/runtime/event_exporting/classes/test_dispatcher.py @@ -2,8 +2,10 @@ import threading import time +from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime +from queue import SimpleQueue from typing import cast import pytest @@ -18,6 +20,9 @@ ) from sqlbuild.observability import LifecycleEvent from sqlbuild.runtime.event_exporting.exceptions import EventExporterInputError +from sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink import ( + invoke_lifecycle_event_sink, +) from sqlbuild.runtime.output_capture.models import BoundCommandOutputSink from sqlbuild.sinks import CommandOutputRecord, CommandOutputStream from tests.unit.src.sqlbuild.runtime.event_exporting.classes._test_types import ( @@ -811,3 +816,77 @@ def record(*, event: LifecycleEvent) -> None: assert delivered[-1] == "invocation_completed" assert summary.dropped > 0 assert summary.delivered + summary.dropped + summary.failed == summary.accepted + + +@pytest.mark.parametrize( + "test_case", + (EventExporterDispatcherTestCase("second call started before the reserve", 4),), + ids=lambda case: case.description, +) +def test_given_lane_running_its_second_call_when_reserve_begins_then_it_counts_delivered( + test_case: EventExporterDispatcherTestCase, +) -> None: + event_ids: tuple[str, ...] = ("event-1", "event-2", "event-3", "event-4") + started: dict[str, threading.Event] = {event_id: threading.Event() for event_id in event_ids} + gates: dict[str, threading.Event] = {event_id: threading.Event() for event_id in event_ids} + for event_id in ("event-1", "event-3", "event-4"): + gates[event_id].set() + release_second: threading.Event = gates["event-2"] + second_started: threading.Event = started["event-2"] + delivered: list[str] = [] + + def record(*, event: LifecycleEvent) -> None: + delivered.append(event.event_id) + started[event.event_id].set() + _ = gates[event.event_id].wait() + + dispatcher: EventExporterDispatcher = EventExporterDispatcher( + exporters=(BoundEventExporter("recorder", record, {}),), + shutdown_timeout_seconds=2.0, + invocation_timeout_seconds=5.0, + ) + dispatcher.enqueue(correlated_event(1, "statement_completed")) + dispatcher.enqueue(correlated_event(2, "statement_completed")) + dispatcher.enqueue(correlated_event(3, "run_completed")) + dispatcher.enqueue(lifecycle_event(4, "invocation_completed")) + assert second_started.wait(timeout=1.0) + release: threading.Timer = threading.Timer(1.7, release_second.set) + release.start() + + summary: EventExportSummary = dispatcher.shutdown() + + release.cancel() + assert tuple(delivered) == event_ids + assert summary.delivered == test_case.expected_delivered + assert (summary.dropped, summary.failed) == (0, 0) + + +@pytest.mark.parametrize( + "test_case", + (EventExporterDispatcherTestCase("claimed call runs, cancelled call is skipped", 1),), + ids=lambda case: case.description, +) +def test_given_claims_when_delivery_thread_runs_calls_then_only_unclaimed_cancellations_skip( + test_case: EventExporterDispatcherTestCase, +) -> None: + calls: SimpleQueue[ + tuple[Callable[..., object], Mapping[str, object], LifecycleEvent, int] | None + ] = SimpleQueue() + results: SimpleQueue[tuple[int, BaseException | None]] = SimpleQueue() + claims: dict[int, bool] = {2: False} + seen: list[str] = [] + + def record(*, event: LifecycleEvent) -> None: + seen.append(event.event_id) + assert claims.setdefault(1, False) + + calls.put((record, {}, correlated_event(1, "statement_completed"), 1)) + calls.put((record, {}, correlated_event(2, "statement_completed"), 2)) + calls.put(None) + + invoke_lifecycle_event_sink(calls=calls, results=results, claims=claims) + + assert seen == ["event-1"] + assert results.qsize() == test_case.expected_delivered + assert results.get_nowait() == (1, None) + assert claims == {1: True} From 8fff721a28ee3dd79a9360e0f124e6ad366b9965 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:08:39 +0100 Subject: [PATCH 32/39] ci: give native-building verify jobs 45 minutes --- .github/workflows/verify.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4430b01a54..9f0ea1398e 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -255,7 +255,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.checks == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -342,7 +342,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.compiler_differential == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -379,7 +379,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.compiler_differential == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -415,7 +415,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.compiler_differential == 'true' runs-on: windows-2025 - timeout-minutes: 40 + timeout-minutes: 45 defaults: run: shell: bash @@ -468,7 +468,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.format_performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -499,7 +499,7 @@ jobs: # These budgets are calibrated for the 8-vCPU Blacksmith runner. if: false runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -524,7 +524,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -572,7 +572,7 @@ jobs: if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 # The 3k and 10k cells also run the delta edit-loop guard, about five minutes at 10k. - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -717,7 +717,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -788,7 +788,7 @@ jobs: # These budgets are calibrated for the 8-vCPU Blacksmith runner. if: false runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 From d03c5d03219f0818b200c3d3ef96b50b6c73f197 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:03:09 +0100 Subject: [PATCH 33/39] fix(sqlite): switch a new database file to WAL one connection at a time Opening several connections to a new file at once could fail with A702 at once: SQLite reports the WAL switch as busy without waiting while another connection switches the same file. The switch now goes through the per-file writer gate. Also give wheel builds 75 minutes, since the Intel macOS build reached the 45-minute limit. --- .github/workflows/build-distributions.yml | 2 +- .../adapters/sqlite/classes/sqlite_adapter.py | 2 +- .../sqlite/classes/sqlite_connection.py | 15 ++++-- .../sqlbuild/adapters/sqlite/_test_types.py | 8 ++++ .../sqlbuild/adapters/sqlite/test_locking.py | 48 +++++++++++++++++++ 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index b27cad102c..eb008b7177 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -88,7 +88,7 @@ jobs: name: Build wheel (${{ matrix.platform.name }}) needs: source runs-on: ${{ matrix.platform.runner }} - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false matrix: diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 92a52a6696..1d648e01bd 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -224,7 +224,7 @@ def connect(self, config: dict[str, Any]) -> SqliteConnection: raw_connection=raw_connection, path=path, busy_timeout_seconds=busy_timeout ) if path != MEMORY_DATABASE_PATH and not read_only: - connection.execute("PRAGMA journal_mode=WAL") + connection.execute_as_writer("PRAGMA journal_mode=WAL") connection.execute(f"PRAGMA busy_timeout={int(busy_timeout * 1000)}") connection.execute("PRAGMA foreign_keys=ON") connection.execute("PRAGMA legacy_alter_table=ON") diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py index 7a1c833f18..7b2b582e33 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py @@ -40,11 +40,18 @@ def execute(self, sql: str) -> ObservedCursor: def execute_with(self, *, sql: str, parameters: Sequence[object]) -> ObservedCursor: return self._run(sql=sql, run=lambda cursor: cursor.execute(sql, parameters)) + def execute_as_writer(self, sql: str) -> ObservedCursor: + """Run a statement that needs the writer gate, such as switching a new file to WAL.""" + + return self._run(sql=sql, run=lambda cursor: cursor.execute(sql), writes=True) + def executemany(self, *, sql: str, rows: Iterable[Sequence[object]]) -> ObservedCursor: return self._run(sql=sql, run=lambda cursor: cursor.executemany(sql, rows)) - def _run(self, *, sql: str, run: Callable[[ObservedCursor], object]) -> ObservedCursor: - self._enter(sql) + def _run( + self, *, sql: str, run: Callable[[ObservedCursor], object], writes: bool = False + ) -> ObservedCursor: + self._enter(sql=sql, writes=writes) cursor: ObservedCursor = self.cursor() try: run(cursor) @@ -72,8 +79,8 @@ def close(self) -> None: finally: self._release() - def _enter(self, sql: str) -> None: - if self._holds_gate or leading_keyword(sql) in READ_STATEMENT_KEYWORDS: + def _enter(self, *, sql: str, writes: bool) -> None: + if self._holds_gate or (not writes and leading_keyword(sql) in READ_STATEMENT_KEYWORDS): return self._holds_gate = self._gate.acquire() diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py index d1d89ecfc7..803668b253 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -123,6 +123,14 @@ class SqliteInProcessWritersTestCase: expected_rows: tuple[tuple[object, ...], ...] +@dataclass(frozen=True) +class SqliteConcurrentConnectTestCase: + description: str + connections: int + attempts: int + expected_open_connections: int + + @dataclass(frozen=True) class SqliteKilledWriterTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py index 31e43e0b82..8df254a290 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py @@ -13,6 +13,7 @@ from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter from tests.integration.src.sqlbuild.adapters.sqlite._test_types import ( + SqliteConcurrentConnectTestCase, SqliteInProcessWritersTestCase, SqliteKilledWriterTestCase, SqliteLockTestCase, @@ -208,3 +209,50 @@ def test_given_writer_killed_mid_transaction_when_reopening_then_target_is_uncha assert rows == test_case.expected_rows assert tables == test_case.expected_tables + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteConcurrentConnectTestCase( + description="threads opening a new file together all switch it to WAL", + connections=4, + attempts=25, + expected_open_connections=100, + ), + ], + ids=lambda case: case.description, +) +def test_given_new_file_when_threads_connect_together_then_every_connection_opens( + test_case: SqliteConcurrentConnectTestCase, adapter: SqliteAdapter, tmp_path: Path +) -> None: + errors: list[BaseException] = [] + opened_count: int = 0 + for attempt in range(test_case.attempts): + config: dict[str, object] = {"path": str(tmp_path / f"shop_{attempt}.sqlite")} + barrier: threading.Barrier = threading.Barrier(test_case.connections) + opened: list[Any] = [] + + def connect( + config: dict[str, object] = config, + barrier: threading.Barrier = barrier, + opened: list[Any] = opened, + ) -> None: + barrier.wait(timeout=10) + try: + opened.append(adapter.connect(config)) + except BaseException as error: + errors.append(error) + + threads: tuple[threading.Thread, ...] = tuple( + threading.Thread(target=connect) for _ in range(test_case.connections) + ) + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + opened_count += len(opened) + for connection in opened: + adapter.close(connection) + + assert (opened_count, errors) == (test_case.expected_open_connections, []) From d926c9d446b426775b2a364c283d9ecdcc8524d9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:34:29 +0100 Subject: [PATCH 34/39] ci: build the Intel macOS wheel on the Apple Silicon runner The Intel runner is about twice as slow as the others and its wheel job reached 28-44 minutes. Cross-compile for x86_64 on macos-15 instead and smoke test the wheel with x64 Python under Rosetta. --- .github/workflows/build-distributions.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index b27cad102c..7102bf2402 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -100,8 +100,9 @@ jobs: runner: ubuntu-24.04-arm target: aarch64 - name: macos-x86_64 - runner: macos-15-intel + runner: macos-15 target: x86_64 + python-architecture: x64 - name: macos-arm64 runner: macos-15 target: aarch64 @@ -116,6 +117,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" + architecture: ${{ matrix.platform.python-architecture }} - uses: dtolnay/rust-toolchain@master if: inputs.prebuilt-run-id == '' with: From 6ec00765fe76ffc95e4acfe805778995e798e4ed Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:00:23 +0100 Subject: [PATCH 35/39] fix(cli): answer rules list and show natively with config-only exceptions The native rules commands no longer read the removed exception switch, and render string-list option defaults and table-list options exactly as the Python CLI does, so SQBRGRAPH103 and SQBRPROJECT107 print their published_layers and allowed_qualifiers options. A project still setting allow_exceptions, or configuring rule options, is answered by Python as before. --- .../src/native_commands/_helpers/rules.rs | 35 ++++++++++------ .../src/native_commands/constants.rs | 2 + .../src/rule_catalogue/main/mod.rs | 1 + .../rule_catalogue/main/string_tuple_repr.rs | 13 ++++++ .../src/sqlbuild/cli/entry/helpers.py | 23 ++++++++++ .../cli/entry/test_python_free_commands.py | 42 +++++++++++++++++++ 6 files changed, 104 insertions(+), 12 deletions(-) create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs index c956dd94fe..ef22baf413 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs @@ -3,12 +3,12 @@ use std::collections::BTreeMap; use std::path::Path; -use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config; use sqlbuild_rules::models::{RuleMetadata, RulesConfig}; use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::main::string_tuple_repr::string_tuple_repr; use sqlbuild_rules::rule_catalogue::models::{ CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleListing, RuleText, }; @@ -18,21 +18,31 @@ use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; use crate::native_commands::_helpers::project_facts::has_custom_rules; use crate::native_commands::constants::{ - EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, PROJECT_SUBJECT, - RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, + DICT_TYPE_NAME, EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, + PROJECT_SUBJECT, RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, RulesAction, RulesOptions}; -/// `RuleOption.boolean(...)` or `RuleOption.string_list(...)` as the catalogue validates it. +/// `RuleOption.boolean(...)`, `.string_list(...)` or a table list, as the catalogue validates it. fn option_spec(option: &BuiltinOption) -> OptionSpec { let (value_type, default, item_type) = match option.kind { BuiltinOptionKind::Boolean => (OptionKind::Bool, OptionValue::Bool(false), None), - BuiltinOptionKind::StringList => ( + BuiltinOptionKind::StringList(defaults) => ( OptionKind::Tuple, - OptionValue::Tuple(Vec::new()), + OptionValue::Tuple( + defaults + .iter() + .map(|item| OptionValue::Str((*item).to_owned())) + .collect(), + ), Some(OptionKind::Str), ), + BuiltinOptionKind::TableList => ( + OptionKind::Tuple, + OptionValue::Tuple(Vec::new()), + Some(OptionKind::Other(DICT_TYPE_NAME.to_owned())), + ), }; OptionSpec { name: option.name.to_owned(), @@ -49,15 +59,16 @@ fn option_spec(option: &BuiltinOption) -> OptionSpec { /// The option as `rules show` prints it: the `repr` of its default, which is also effective. fn option_text(option: &BuiltinOption) -> OptionText { - let default_repr: &str = match option.kind { - BuiltinOptionKind::Boolean => FALSE_REPR, - BuiltinOptionKind::StringList => EMPTY_TUPLE_REPR, + let default_repr: String = match option.kind { + BuiltinOptionKind::Boolean => FALSE_REPR.to_owned(), + BuiltinOptionKind::StringList(defaults) => string_tuple_repr(defaults), + BuiltinOptionKind::TableList => EMPTY_TUPLE_REPR.to_owned(), }; OptionText { name: option.name.to_owned(), description: option.description.to_owned(), - default_repr: default_repr.to_owned(), - effective_repr: default_repr.to_owned(), + effective_repr: default_repr.clone(), + default_repr, } } @@ -155,7 +166,7 @@ pub(crate) fn run_rules( return Err(PythonNeeded::Project); } let config: RulesConfig = load_rules_config(project_dir).map_err(|_| PythonNeeded::Failure)?; - if forbidden_exceptions(&config).is_some() || !config.rule_options.is_empty() { + if !config.rule_options.is_empty() { return Err(PythonNeeded::Project); } let rules: Vec = sorted_catalogue()?; diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 5b1dd8ff84..0d395432ff 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -63,6 +63,8 @@ pub(crate) const PROJECT_SUBJECT: &str = "project"; /// Python `repr` of the built-in option defaults: `False` and `()`. pub(crate) const FALSE_REPR: &str = "False"; pub(crate) const EMPTY_TUPLE_REPR: &str = "()"; +/// The Python type name of each `allowed_qualifiers`-style table option item. +pub(crate) const DICT_TYPE_NAME: &str = "dict"; pub(crate) const UNKNOWN_RULE_PREFIX: &str = "Unknown rule: "; /// The whitespace `int()` strips once other spaces are ASCII spaces. pub(crate) const INT_ASCII_SPACES: &str = " \t\n\x0b\x0c\r"; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs index 0c4b11601a..1cc8b483a7 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -5,3 +5,4 @@ pub mod render_skills_body; pub mod ruleset_fingerprint; pub mod skill_lock_name; pub mod sorted_catalogue_order; +pub mod string_tuple_repr; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs new file mode 100644 index 0000000000..b65087dcf0 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/string_tuple_repr.rs @@ -0,0 +1,13 @@ +//! Python `repr` of a tuple of strings, as `rules show` prints a string-list option default. + +use crate::rule_catalogue::_helpers::python_repr::string_repr; + +/// `repr(tuple(items))`: `()`, `('a',)` or `('a', 'b')`. +#[must_use] +pub fn string_tuple_repr(items: &[&str]) -> String { + let parts: Vec = items.iter().map(|item| string_repr(item)).collect(); + match parts.as_slice() { + [only] => format!("({only},)"), + _ => format!("({})", parts.join(", ")), + } +} diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 0db3751b7d..c479d5d40f 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -282,6 +282,29 @@ def prepare_custom_rules(directory: Path) -> None: (directory / "rules").mkdir() +def prepare_removed_allow_exceptions(directory: Path) -> None: + """Write the project still setting the removed `[rules] allow_exceptions`.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + "\n[rules]\nallow_exceptions = true\n", + encoding="utf-8", + ) + + +def prepare_published_layer_options(directory: Path) -> None: + """Write the project with configured published-layer Rule options.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + + "\n[rules.rule_options.SQBRPROJECT107]\n" + + 'published_layers = ["mart"]\n' + + 'allowed_qualifiers = [{ qualifier = "csv_export", reason = "Export views." }]\n', + encoding="utf-8", + ) + + def prepare_named_pipe_project(directory: Path) -> None: """Make the project file a named pipe that blocks its reader.""" diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py index fe5412e49b..747ab04902 100644 --- a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -22,6 +22,8 @@ prepare_named_pipe_project, prepare_nothing, prepare_project, + prepare_published_layer_options, + prepare_removed_allow_exceptions, prepare_stale_skills, python_starts, reset_directory, @@ -179,6 +181,34 @@ expected_exit_code=0, expected_captures=1, ), + PythonFreeTestCase( + description="rules show with a string-list option default", + args=("rules", "show", "SQBRGRAPH103"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show with a table-list option", + args=("rules", "show", "SQBRPROJECT107"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show cross join", + args=("rules", "show", "SQBRSQL038"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show inline directive rule", + args=("rules", "show", "SQBRSQL000"), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=1, + ), PythonFreeTestCase( description="rules show unknown", args=("rules", "show", "SQBRNOPE101"), @@ -262,6 +292,18 @@ def test_given_python_free_invocation_when_running_sqb_then_python_never_starts_ prepare=prepare_custom_rules, expected_python_starts=1, ), + PythonStartTestCase( + description="removed allow_exceptions setting", + args=("rules", "list"), + prepare=prepare_removed_allow_exceptions, + expected_python_starts=1, + ), + PythonStartTestCase( + description="configured rule options", + args=("rules", "show", "SQBRPROJECT107"), + prepare=prepare_published_layer_options, + expected_python_starts=1, + ), PythonStartTestCase( description="dbt passthrough help", args=("dbt", "plan", "--help"), From 355db51ad45585667a781bd4bbdf998a3136abbd Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:37:10 +0100 Subject: [PATCH 36/39] test: list concurrency among allowed target keys in the compiler golden --- tests/goldens/compiler/failure/target-unknown-key.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/goldens/compiler/failure/target-unknown-key.json b/tests/goldens/compiler/failure/target-unknown-key.json index 127cffc49b..036d4c22ef 100644 --- a/tests/goldens/compiler/failure/target-unknown-key.json +++ b/tests/goldens/compiler/failure/target-unknown-key.json @@ -8,7 +8,7 @@ "phase": "compile", "severity": "error", "code": "D001", - "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", + "message": "/sqlbuild_project.toml targets.ci contains unknown key(s): warehouse_size. Allowed keys: clone, compile_cache, concurrency, connection, database, default_table_type, defer_clone_from, defer_sources_to, diff, execution_limits, loader_schema, missing_migration_origin, owns_time_travel_retention_namespace, recall, schema, table_type_downgrade, time_travel_retention, time_travel_retention_decrease, vars, warehouses", "help": "did you mean 'warehouses'?" } ] From b96b0f3253e15e313843ffa4bb18b0e7471855fb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:02:48 +0100 Subject: [PATCH 37/39] test: clear SQLBUILD_CONCURRENCY in the target concurrency E2E tests CI sets SQLBUILD_CONCURRENCY for E2E runs, which counts as --concurrency and overrides every setting these tests check. --- .../cli/commands/main/build/test_target_concurrency.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py index 7c09f6274b..871c5bbeca 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_target_concurrency.py @@ -81,7 +81,9 @@ def test_given_target_concurrency_when_running_then_documented_precedence_applies( tmp_path: Path, test_case: TargetConcurrencyE2ETestCase, + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv("SQLBUILD_CONCURRENCY", raising=False) project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="target_concurrency", @@ -123,7 +125,9 @@ def test_given_target_concurrency_when_running_then_documented_precedence_applie def test_given_invalid_target_concurrency_when_running_then_config_error_is_coded( tmp_path: Path, test_case: TargetConcurrencyE2ETestCase, + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv("SQLBUILD_CONCURRENCY", raising=False) project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="target_concurrency", @@ -168,7 +172,9 @@ def test_given_invalid_target_concurrency_when_running_then_config_error_is_code def test_given_target_concurrency_when_planning_microbatches_then_warning_uses_it( test_case: TargetMicrobatchWarningE2ETestCase, tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: + monkeypatch.delenv("SQLBUILD_CONCURRENCY", raising=False) project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="target_microbatch", From e1a78b626e9e1f2019c60c4b66e3c6f1356f78b7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:02:48 +0100 Subject: [PATCH 38/39] ci: run the Intel macOS wheel smoke test under Rosetta The cached x64 Python is a universal binary that starts as arm64 on the Apple Silicon runner and rejects the x86_64 wheel; run the smoke commands with arch -x86_64. --- .github/workflows/build-distributions.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index f04f6dd02c..ece0f8cb8c 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -103,6 +103,7 @@ jobs: runner: macos-15 target: x86_64 python-architecture: x64 + smoke-arch: x86_64 - name: macos-arm64 runner: macos-15 target: aarch64 @@ -146,6 +147,8 @@ jobs: github-token: ${{ github.token }} - name: Smoke test installed wheels and native console entry points shell: bash + env: + SMOKE_ARCH: ${{ matrix.platform.smoke-arch }} run: | python -m venv "${RUNNER_TEMP}/wheel-smoke" if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then @@ -153,13 +156,23 @@ jobs: else smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python" fi - "${smoke_python}" -m pip install --find-links dist dist/*.whl + # A cross-compiled wheel runs under Rosetta; the cached Python is universal, so force + # the wheel's architecture or it starts as arm64 and rejects the wheel. + run_as=() + if [[ -n "${SMOKE_ARCH}" ]]; then + run_as=(arch "-${SMOKE_ARCH}") + fi + "${run_as[@]}" "${smoke_python}" -m pip install --find-links dist dist/*.whl cd "${RUNNER_TEMP}" - "${smoke_python}" -c "import sqlbuild._native" + "${run_as[@]}" "${smoke_python}" -c "import sqlbuild._native" smoke_scripts="$(dirname "${smoke_python}")" - "${smoke_scripts}/sqb" --version - "${smoke_scripts}/sqb" --help - "${smoke_scripts}/sqlbuild" --help + "${run_as[@]}" "${smoke_scripts}/sqb" --version + "${run_as[@]}" "${smoke_scripts}/sqb" --help + if [[ -n "${SMOKE_ARCH}" ]]; then + "${run_as[@]}" "${smoke_python}" "${smoke_scripts}/sqlbuild" --help + else + "${smoke_scripts}/sqlbuild" --help + fi - name: Check wheel contents and run a native command if: matrix.platform.name == 'linux-x86_64' run: | From eaa0e0f090d2ad0435e9e82296229cc654c3fcdf Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:23:35 +0100 Subject: [PATCH 39/39] ci: run the native sqlbuild entry point under Rosetta like sqb --- .github/workflows/build-distributions.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index ece0f8cb8c..e0fe2aa3ab 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -168,11 +168,7 @@ jobs: smoke_scripts="$(dirname "${smoke_python}")" "${run_as[@]}" "${smoke_scripts}/sqb" --version "${run_as[@]}" "${smoke_scripts}/sqb" --help - if [[ -n "${SMOKE_ARCH}" ]]; then - "${run_as[@]}" "${smoke_python}" "${smoke_scripts}/sqlbuild" --help - else - "${smoke_scripts}/sqlbuild" --help - fi + "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help - name: Check wheel contents and run a native command if: matrix.platform.name == 'linux-x86_64' run: |