From c9d31bdf5603537121fac9b127e4b70e94ff66d3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:43:25 +0100 Subject: [PATCH 01/17] fix: release the compute log lock before rejecting a write to a finished capture A write that reached a capture already marked complete built its rejection while still holding the storage lock. Building it imports Python, which can release the GIL; the main thread then pruning the logs waited for that lock while holding the GIL, so neither could continue and the command never exited. The lock is now released before the rejection is raised. --- .../bindings/_helpers/compute_logs/storage.rs | 2 + .../sqlbuild/runtime/compute_logs/__init__.py | 0 .../runtime/compute_logs/_test_types.py | 14 ++ .../test_storage_lock_without_python.py | 130 ++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py create mode 100644 tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py create mode 100644 tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs index 02bc5c5556..40b859097c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compute_logs/storage.rs @@ -90,6 +90,7 @@ fn exact_count(value: &Bound<'_, PyAny>, fallback: i64) -> i64 { } impl LocalFilesystemComputeLogStorage { + /// Callers wait for this lock holding the GIL, so never run Python while holding it. fn locked(&self) -> PyResult> { self.state .lock() @@ -116,6 +117,7 @@ impl LocalFilesystemComputeLogStorage { ) -> PyResult<()> { let mut state = self.locked()?; let Some(writers) = state.writers.get_mut(invocation_id) else { + drop(state); return Err(not_active(py)); }; writers[stream].write_all(data)?; diff --git a/tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py b/tests/integration/src/sqlbuild/runtime/compute_logs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py b/tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py new file mode 100644 index 0000000000..d145089911 --- /dev/null +++ b/tests/integration/src/sqlbuild/runtime/compute_logs/_test_types.py @@ -0,0 +1,14 @@ +"""Test case types for compute log storage integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class StorageLockTestCase: + """A Python program making concurrent storage calls from two threads, and what it prints.""" + + description: str + script: str + expected_stdout: str diff --git a/tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py b/tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py new file mode 100644 index 0000000000..5e9370ac95 --- /dev/null +++ b/tests/integration/src/sqlbuild/runtime/compute_logs/test_storage_lock_without_python.py @@ -0,0 +1,130 @@ +"""Compute log storage never runs Python while holding its lock, so it cannot deadlock on the GIL.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from textwrap import dedent + +import pytest + +from tests.integration.src.sqlbuild.runtime.compute_logs._test_types import ( + StorageLockTestCase, +) + +_DEADLOCK_TIMEOUT_SECONDS: int = 30 + +# A late write from a background thread reaches a capture that was already marked complete. +# Raising its rejection imports the storage exceptions, here forced to load afresh, and the +# import's file system calls release the GIL. The finder below holds that import open until the main thread is +# pruning, which takes the storage lock while holding the GIL. +_LATE_APPEND_DURING_PRUNE: str = dedent( + """ + import dataclasses + import sys + import threading + import time + from datetime import UTC, datetime + from pathlib import Path + + from sqlbuild.compute_logs import ( + COMPUTE_LOG_FORMAT_VERSION, + CaptureMetadata, + ComputeLogStream, + FinalCaptureMetadata, + LocalFilesystemComputeLogStorage, + ) + + EXCEPTIONS_MODULE = "sqlbuild.runtime.compute_logs.exceptions" + importing = threading.Event() + + + class SlowExceptionsImport: + def find_spec(self, name, path=None, target=None): + if name == EXCEPTIONS_MODULE and not importing.is_set(): + importing.set() + time.sleep(0.5) + return None + + + project_dir = Path(sys.argv[1]) + started_at = datetime.now(UTC) + metadata = CaptureMetadata( + format_version=COMPUTE_LOG_FORMAT_VERSION, + invocation_id="orders0000000000000000000000000a", + command="compile", + project_dir=str(project_dir), + started_at=started_at, + capture_date=started_at.date().isoformat(), + ) + storage = LocalFilesystemComputeLogStorage(project_dir=project_dir) + storage.start_capture(metadata) + storage.mark_complete( + FinalCaptureMetadata( + **dataclasses.asdict(metadata), + completed_at=datetime.now(UTC), + exit_code=1, + stdout_bytes=0, + stderr_bytes=0, + diagnostics_bytes=0, + ) + ) + rejected = [] + del sys.modules[EXCEPTIONS_MODULE] + sys.meta_path.insert(0, SlowExceptionsImport()) + + + def late_append(): + try: + storage.append( + invocation_id=metadata.invocation_id, + stream=ComputeLogStream.STDERR, + data=b"late customer export notice\\n", + ) + except Exception as error: + rejected.append(type(error).__name__) + + + writer = threading.Thread(target=late_append) + writer.start() + importing.wait() + storage.prune() + writer.join() + print(rejected) + """ +) + + +@pytest.mark.parametrize( + "test_case", + ( + StorageLockTestCase( + description="late append rejected while the main thread prunes", + script=_LATE_APPEND_DURING_PRUNE, + expected_stdout="['CaptureStateError']\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_rejected_append_releasing_gil_when_pruning_then_both_threads_finish( + test_case: StorageLockTestCase, tmp_path: Path +) -> None: + try: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", test_case.script, str(tmp_path)], + capture_output=True, + text=True, + timeout=_DEADLOCK_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired: + pytest.fail(f"storage calls deadlocked for {_DEADLOCK_TIMEOUT_SECONDS}s") + + assert (completed.returncode, completed.stdout) == (0, test_case.expected_stdout), ( + completed.stderr + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From c10b78caccc50c2e4bf95205f57760584f33835f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:43:25 +0100 Subject: [PATCH 02/17] test: fail a hung E2E command with every thread's stack instead of stalling the job run_sqb now gives each command ten minutes. A command still running then has its process group aborted with Python's fault handler enabled, so the failing test reports every thread's stack, and the group is killed. Standard input ends at once when a test passes none. --- Makefile | 3 +- .../cli/commands/shared/_test_types.py | 24 +++++ .../sqlbuild/cli/commands/shared/helpers.py | 78 +++++++++++++-- .../shared/test_run_with_hang_dump.py | 95 +++++++++++++++++++ 4 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py diff --git a/Makefile b/Makefile index efba5d41b3..c1725a5a3d 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,8 @@ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/snowflake \ tests/e2e/src/sqlbuild/cli/commands/main/sqlite \ tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \ - tests/e2e/src/sqlbuild/cli/commands/main/test + tests/e2e/src/sqlbuild/cli/commands/main/test \ + tests/e2e/src/sqlbuild/cli/commands/shared E2E_DUCKDB_INSPECTION_PATHS := \ tests/e2e/src/sqlbuild/cli/commands/main/clone \ diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py new file mode 100644 index 0000000000..08d61d09e7 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/_test_types.py @@ -0,0 +1,24 @@ +"""Test case types for the shared E2E command helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class HangDumpTestCase: + """A Python command that never finishes and what its hang report must show.""" + + description: str + script: str + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class FinishedCommandTestCase: + """A Python command that finishes and the result the helper must return for it.""" + + description: str + script: str + input_text: str | None + expected_result: tuple[int, str, str] diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py index 7b39be0dc4..d4cf1f3b99 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/helpers.py @@ -6,6 +6,8 @@ import os import pty import re +import shlex +import signal import struct import subprocess import termios @@ -22,6 +24,10 @@ SOURCE_LOADER_STRATEGIES_DIR: Path = ( REPO_ROOT / "tests" / "e2e" / "fixtures" / "source_loader_strategies" ) +# Far above any healthy command, so only a hang trips it: one test fails instead of the CI job. +SQB_COMMAND_TIMEOUT_SECONDS: float = 600.0 +_HANG_DUMP_GRACE_SECONDS: float = 30.0 +_HANG_OUTPUT_TAIL_CHARACTERS: int = 20_000 def prepare_waffle_shop(tmp_path: Path) -> Path: @@ -208,18 +214,78 @@ def run_sqb( process_env: dict[str, str] = dict(os.environ) process_env.update(env or {}) + process_env.setdefault("PYTHONFAULTHANDLER", "1") - return subprocess.run( - ["uv", "run", "sqb", "--project-dir", str(project_dir), *command], + return run_with_hang_dump( + argv=("uv", "run", "sqb", "--project-dir", str(project_dir), *command), cwd=working_dir or REPO_ROOT, - capture_output=True, - text=True, - input=input_text, env=process_env, - check=False, + input_text=input_text, + timeout_seconds=SQB_COMMAND_TIMEOUT_SECONDS, + ) + + +def run_with_hang_dump( + *, + argv: tuple[str, ...], + cwd: Path, + env: Mapping[str, str], + input_text: str | None, + timeout_seconds: float, +) -> subprocess.CompletedProcess[str]: + """Run `argv` to completion, or fail with every Python thread's stack once it hangs. + + Standard input carries `input_text`, or ends at once without it. The command runs in its own + process group. Past `timeout_seconds` the whole group receives + SIGABRT, so Python processes started with `PYTHONFAULTHANDLER` print all thread stacks to + stderr before they exit, and the group is then killed. + """ + + process: subprocess.Popen[str] = subprocess.Popen( + argv, + cwd=cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=dict(env), + process_group=0, + ) + try: + stdout, stderr = process.communicate(input=input_text, timeout=timeout_seconds) + except subprocess.TimeoutExpired: + stdout, stderr = _dump_and_kill_group(process=process) + raise AssertionError( + f"{shlex.join(argv)} did not finish within {timeout_seconds:.0f}s; " + "its process group was aborted to dump thread stacks.\n" + f"--- stdout (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n" + f"{stdout[-_HANG_OUTPUT_TAIL_CHARACTERS:]}\n" + f"--- stderr (last {_HANG_OUTPUT_TAIL_CHARACTERS} characters) ---\n" + f"{stderr[-_HANG_OUTPUT_TAIL_CHARACTERS:]}" + ) from None + return subprocess.CompletedProcess( + args=list(argv), returncode=process.returncode, stdout=stdout, stderr=stderr ) +def _dump_and_kill_group(*, process: subprocess.Popen[str]) -> tuple[str, str]: + _signal_group(process_group=process.pid, signal_number=signal.SIGABRT) + try: + stdout, stderr = process.communicate(timeout=_HANG_DUMP_GRACE_SECONDS) + except subprocess.TimeoutExpired: + _signal_group(process_group=process.pid, signal_number=signal.SIGKILL) + stdout, stderr = process.communicate() + _signal_group(process_group=process.pid, signal_number=signal.SIGKILL) + return stdout or "", stderr or "" + + +def _signal_group(*, process_group: int, signal_number: signal.Signals) -> None: + try: + os.killpg(process_group, signal_number) + except ProcessLookupError: + return + + def run_sqb_with_pty( *, command: tuple[str, ...], diff --git a/tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py b/tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py new file mode 100644 index 0000000000..0a2052dbcb --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/shared/test_run_with_hang_dump.py @@ -0,0 +1,95 @@ +"""A hung command fails its own test with every Python thread's stack instead of hanging the run.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.shared._test_types import ( + FinishedCommandTestCase, + HangDumpTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_with_hang_dump + + +@pytest.mark.parametrize( + "test_case", + [ + HangDumpTestCase( + description="main thread joins a worker blocked forever", + script=( + "import threading\n" + "def blocked_worker_wait():\n" + " threading.Event().wait()\n" + "def joining_main_thread():\n" + " worker = threading.Thread(target=blocked_worker_wait)\n" + " worker.start()\n" + " print('started', flush=True)\n" + " worker.join()\n" + "joining_main_thread()\n" + ), + expected_fragments=("started", "blocked_worker_wait", "joining_main_thread"), + ), + ], + ids=lambda case: case.description, +) +def test_given_hung_python_command_when_running_with_hang_dump_then_fails_with_thread_stacks( + test_case: HangDumpTestCase, tmp_path: Path +) -> None: + started: float = time.monotonic() + with pytest.raises(AssertionError) as raised: + run_with_hang_dump( + argv=(sys.executable, "-c", test_case.script), + cwd=tmp_path, + env={**os.environ, "PYTHONFAULTHANDLER": "1"}, + input_text=None, + timeout_seconds=2.0, + ) + + message: str = str(raised.value) + assert "did not finish within 2s" in message + assert [fragment in message for fragment in test_case.expected_fragments] == [True] * len( + test_case.expected_fragments + ), message + assert time.monotonic() - started < 30.0 + + +@pytest.mark.parametrize( + "test_case", + [ + FinishedCommandTestCase( + description="input reaches the command and its exit code and output return", + script="import sys; sys.stdout.write(sys.stdin.read().upper()); sys.exit(3)", + input_text="orders\n", + expected_result=(3, "ORDERS\n", ""), + ), + FinishedCommandTestCase( + description="without input the command reads end of input", + script="import sys; print(repr(sys.stdin.read()))", + input_text=None, + expected_result=(0, "''\n", ""), + ), + ], + ids=lambda case: case.description, +) +def test_given_finishing_command_when_running_with_hang_dump_then_returns_its_result( + test_case: FinishedCommandTestCase, tmp_path: Path +) -> None: + completed: subprocess.CompletedProcess[str] = run_with_hang_dump( + argv=(sys.executable, "-c", test_case.script), + cwd=tmp_path, + env=dict(os.environ), + input_text=test_case.input_text, + timeout_seconds=60.0, + ) + + assert (completed.returncode, completed.stdout, completed.stderr) == test_case.expected_result + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 3deb6a2121b6412e614026c8592a0b14443e9a67 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:10 +0100 Subject: [PATCH 03/17] refactor: share project scaffolding and rule listing text outside the Python bindings --- .../sqlbuild-core/src/init_command/errors.rs | 11 ++++ .../src/init_command/main/mod.rs | 1 + .../main/scaffold_blank_project.rs | 57 +++++++++++++++++ crates/sqlbuild-core/src/init_command/mod.rs | 1 + .../_helpers/init_command/scaffold.rs | 51 ++------------- .../bindings/_helpers/plan_command/rules.rs | 64 ++++++++----------- .../src/rule_catalogue/main/mod.rs | 1 + .../rule_catalogue/main/render_rule_list.rs | 44 +++++++++++++ .../src/rule_catalogue/models.rs | 11 ++++ fensu.toml | 2 +- 10 files changed, 157 insertions(+), 86 deletions(-) create mode 100644 crates/sqlbuild-core/src/init_command/errors.rs create mode 100644 crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs create mode 100644 crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs diff --git a/crates/sqlbuild-core/src/init_command/errors.rs b/crates/sqlbuild-core/src/init_command/errors.rs new file mode 100644 index 0000000000..752e7680d6 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/errors.rs @@ -0,0 +1,11 @@ +//! Why a blank project could not be created. + +use std::io; +use std::path::PathBuf; + +/// The I/O failure and the path it happened at, as Python's `OSError` names its `filename`. +#[derive(Debug)] +pub struct ScaffoldError { + pub error: io::Error, + pub path: PathBuf, +} diff --git a/crates/sqlbuild-core/src/init_command/main/mod.rs b/crates/sqlbuild-core/src/init_command/main/mod.rs index b8eae0db00..815db5c0cc 100644 --- a/crates/sqlbuild-core/src/init_command/main/mod.rs +++ b/crates/sqlbuild-core/src/init_command/main/mod.rs @@ -1,3 +1,4 @@ pub mod init_project_name; pub mod init_project_toml; pub mod render_init_summary; +pub mod scaffold_blank_project; diff --git a/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs b/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs new file mode 100644 index 0000000000..eb9c89ff78 --- /dev/null +++ b/crates/sqlbuild-core/src/init_command/main/scaffold_blank_project.rs @@ -0,0 +1,57 @@ +//! Create the blank project tree as Python's `pathlib` calls did. + +use std::fs::{self, OpenOptions}; +use std::io::ErrorKind; +use std::path::Path; + +use crate::constants::PROJECT_CONFIG_FILENAME; +use crate::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; +use crate::init_command::errors::ScaffoldError; + +fn failed(error: std::io::Error, path: &Path) -> ScaffoldError { + ScaffoldError { + error, + path: path.to_path_buf(), + } +} + +/// `Path.mkdir(parents=True, exist_ok=True)`. +fn make_directories(path: &Path) -> Result<(), ScaffoldError> { + match fs::create_dir(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => { + let Some(parent) = path.parent().filter(|parent| *parent != path) else { + return Err(failed(error, path)); + }; + make_directories(parent)?; + match fs::create_dir(path) { + Err(error) if !path.is_dir() => Err(failed(error, path)), + _ => Ok(()), + } + } + Err(_) if path.is_dir() => Ok(()), + Err(error) => Err(failed(error, path)), + } +} + +/// Create the resource directories, the project file, and `.gitkeep` in empty directories. +pub fn scaffold_blank_project(base_dir: &Path, project_toml: &str) -> Result<(), ScaffoldError> { + for directory in SCAFFOLD_DIRECTORIES { + make_directories(&base_dir.join(directory))?; + } + let project_file = base_dir.join(PROJECT_CONFIG_FILENAME); + fs::write(&project_file, project_toml).map_err(|error| failed(error, &project_file))?; + for directory in SCAFFOLD_DIRECTORIES { + let directory = base_dir.join(directory); + let mut entries = fs::read_dir(&directory).map_err(|error| failed(error, &directory))?; + if entries.next().is_none() { + let gitkeep = directory.join(GITKEEP_FILE); + OpenOptions::new() + .create(true) + .append(true) + .open(&gitkeep) + .map_err(|error| failed(error, &gitkeep))?; + } + } + Ok(()) +} diff --git a/crates/sqlbuild-core/src/init_command/mod.rs b/crates/sqlbuild-core/src/init_command/mod.rs index d9f45c6513..5aedb44cc0 100644 --- a/crates/sqlbuild-core/src/init_command/mod.rs +++ b/crates/sqlbuild-core/src/init_command/mod.rs @@ -1,6 +1,7 @@ //! `sqb init`: the blank project layout, its configuration, and the created summary. pub mod constants; +pub mod errors; pub mod main; #[cfg(test)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs index 8e969b0c1c..35d083baeb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/init_command/scaffold.rs @@ -1,13 +1,12 @@ //! Create the blank project tree, raising the `OSError` Python's `pathlib` would. -use std::fs::{self, OpenOptions}; -use std::io::{self, ErrorKind}; +use std::io; use std::path::Path; use crate::bindings::_helpers::records::fields::field as native_field; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python}; -use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME as PROJECT_FILE; -use sqlbuild_core::init_command::constants::{GITKEEP_FILE, SCAFFOLD_DIRECTORIES}; +use sqlbuild_core::init_command::errors::ScaffoldError; +use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_project as scaffold; /// `OSError(errno, strerror, filename)`, which Python narrows to its errno subclass. fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { @@ -35,52 +34,12 @@ fn os_error(py: Python<'_>, error: &io::Error, path: &Path) -> PyErr { } } -/// `Path.mkdir(parents=True, exist_ok=True)`. -fn make_directories(path: &Path) -> Result<(), (io::Error, std::path::PathBuf)> { - match fs::create_dir(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == ErrorKind::NotFound => { - let Some(parent) = path.parent().filter(|parent| *parent != path) else { - return Err((error, path.to_path_buf())); - }; - make_directories(parent)?; - match fs::create_dir(path) { - Err(error) if !path.is_dir() => Err((error, path.to_path_buf())), - _ => Ok(()), - } - } - Err(error) if path.is_dir() => { - drop(error); - Ok(()) - } - Err(error) => Err((error, path.to_path_buf())), - } -} - /// Create the resource directories, the project file, and `.gitkeep` in empty directories. pub(crate) fn scaffold_blank_project( py: Python<'_>, base_dir: &Path, project_toml: &str, ) -> PyResult<()> { - for directory in SCAFFOLD_DIRECTORIES { - make_directories(&base_dir.join(directory)) - .map_err(|(error, path)| os_error(py, &error, &path))?; - } - let project_file = base_dir.join(PROJECT_FILE); - fs::write(&project_file, project_toml).map_err(|error| os_error(py, &error, &project_file))?; - for directory in SCAFFOLD_DIRECTORIES { - let directory = base_dir.join(directory); - let mut entries = - fs::read_dir(&directory).map_err(|error| os_error(py, &error, &directory))?; - if entries.next().is_none() { - let gitkeep = directory.join(GITKEEP_FILE); - OpenOptions::new() - .create(true) - .append(true) - .open(&gitkeep) - .map_err(|error| os_error(py, &error, &gitkeep))?; - } - } - Ok(()) + scaffold(base_dir, project_toml) + .map_err(|ScaffoldError { error, path }| os_error(py, &error, &path)) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs index a3be4ae5df..9fcc0ef2bf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs @@ -3,8 +3,10 @@ use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python, pyfunction, }; -use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; +use pyo3::types::{PyDict, PyDictMethods, PyTuple}; use pyo3::wrap_pyfunction; +use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; +use sqlbuild_rules::rule_catalogue::models::RuleListing; use crate::bindings::_helpers::command_runtime::adapters::resolve_cli_adapter; use crate::bindings::_helpers::compile_driver::discovery::discover_project_inputs; @@ -81,47 +83,31 @@ fn run_skills( Ok(if fresh { 0 } else { 1 }) } +fn rule_listing(rule: &Bound<'_, PyAny>) -> PyResult { + let subject: Bound<'_, PyAny> = native_field(rule, "subject")?; + Ok(RuleListing { + code: native_field(rule, "code")?.str()?.extract()?, + family: native_field(rule, "family")?.extract()?, + message: native_field(rule, "message")?.str()?.extract()?, + custom: native_field(rule, "custom")?.is_truthy()?, + subject: if subject.is_none() { + None + } else { + Some(native_field(&subject, "value")?.extract()?) + }, + }) +} + fn list_rules(catalogue: &Bound<'_, PyAny>, json_output: bool) -> PyResult { - let py: Python<'_> = catalogue.py(); - if !json_output { - let mut lines: Vec = Vec::new(); - for rule in catalogue.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - lines.push(format!( - "{} {}", - native_field(&rule, "code")?.str()?, - native_field(&rule, "message")?.str()? - )); - } - print_line(py, &lines.join("\n"), false)?; - return Ok(0); - } - let rules: Bound<'_, PyList> = PyList::empty(py); + let mut rules: Vec = Vec::new(); for rule in catalogue.try_iter()? { - let rule: Bound<'_, PyAny> = rule?; - let item: Bound<'_, PyDict> = PyDict::new(py); - for field in ["code", "family", "message", "custom"] { - item.set_item(field, native_field(&rule, field)?)?; - } - let subject: Bound<'_, PyAny> = native_field(&rule, "subject")?; - item.set_item( - "subject", - if subject.is_none() { - subject - } else { - native_field(&subject, "value")? - }, - )?; - rules.append(item)?; + rules.push(rule_listing(&rule?)?); } - let payload: Bound<'_, PyDict> = PyDict::new(py); - payload.set_item("rules", rules)?; - let options: Bound<'_, PyDict> = PyDict::new(py); - options.set_item("sort_keys", true)?; - let text: String = imported(py, "json", "dumps")? - .call((payload,), Some(&options))? - .extract()?; - print_line(py, &text, false)?; + print_line( + catalogue.py(), + &render_rule_list(&rules, json_output), + false, + )?; Ok(0) } diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs index 618095dd7e..0c4b11601a 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/mod.rs @@ -1,5 +1,6 @@ pub mod format_rule_text; pub mod install_rendered_skills; +pub mod render_rule_list; pub mod render_skills_body; pub mod ruleset_fingerprint; pub mod skill_lock_name; diff --git a/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs b/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs new file mode 100644 index 0000000000..358d57d1b1 --- /dev/null +++ b/crates/sqlbuild-rules/src/rule_catalogue/main/render_rule_list.rs @@ -0,0 +1,44 @@ +//! `sqb rules list` text: one `code message` line per rule, or the sorted JSON document. + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonValue, StdlibJsonOptions}; + +use crate::rule_catalogue::models::RuleListing; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +fn rule_object(rule: &RuleListing) -> JsonValue { + JsonValue::Object(vec![ + ("code".to_owned(), text(&rule.code)), + ("family".to_owned(), text(&rule.family)), + ("message".to_owned(), text(&rule.message)), + ("custom".to_owned(), JsonValue::Bool(rule.custom)), + ( + "subject".to_owned(), + rule.subject.as_deref().map_or(JsonValue::Null, text), + ), + ]) +} + +/// The listing without its trailing newline: `json.dumps({"rules": ...}, sort_keys=True)` or lines. +#[must_use] +pub fn render_rule_list(rules: &[RuleListing], json_output: bool) -> String { + if !json_output { + return rules + .iter() + .map(|rule| format!("{} {}", rule.code, rule.message)) + .collect::>() + .join("\n"); + } + let payload: JsonValue = JsonValue::Object(vec![( + "rules".to_owned(), + JsonValue::Array(rules.iter().map(rule_object).collect()), + )]); + let options: StdlibJsonOptions = StdlibJsonOptions { + sort_keys: true, + ..StdlibJsonOptions::new(None) + }; + dumps(&payload, &JsonDialect::Stdlib(options)).unwrap_or_default() +} diff --git a/crates/sqlbuild-rules/src/rule_catalogue/models.rs b/crates/sqlbuild-rules/src/rule_catalogue/models.rs index 2ebd66fc1b..a3f5fd695c 100644 --- a/crates/sqlbuild-rules/src/rule_catalogue/models.rs +++ b/crates/sqlbuild-rules/src/rule_catalogue/models.rs @@ -85,3 +85,14 @@ pub struct SkillTarget { pub path: std::path::PathBuf, pub state: String, } + +/// One catalogue rule as `sqb rules list` prints it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct RuleListing { + pub code: String, + pub family: String, + pub message: String, + pub custom: bool, + /// The `RuleSubject` value, or `None` when the rule declares none. + pub subject: Option, +} diff --git a/fensu.toml b/fensu.toml index 154893a523..b09c54fa00 100644 --- a/fensu.toml +++ b/fensu.toml @@ -842,7 +842,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/output_capture/records.rs=16", "crates/sqlbuild-python/src/bindings/_helpers/output_capture/values.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=10", + "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/display.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/migrations.rs=1", From cf463ff3524b7cd06b4148546ce0f1c052ce0028 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:23:11 +0100 Subject: [PATCH 04/17] feat: run help, usage errors, init, skills, clean and rules list/show in sqb without Python --- Cargo.lock | 2 + Cargo.toml | 2 +- crates/sqlbuild-cli/Cargo.toml | 8 +- .../src/launcher/_helpers/dispatch.rs | 17 +- crates/sqlbuild-cli/src/lib.rs | 3 +- .../src/native_commands/_helpers/argv_host.rs | 119 +++++++ .../src/native_commands/_helpers/decision.rs | 92 +++++ .../src/native_commands/_helpers/execution.rs | 117 ++++++ .../src/native_commands/_helpers/init.rs | 101 ++++++ .../src/native_commands/_helpers/mod.rs | 10 + .../src/native_commands/_helpers/package.rs | 129 +++++++ .../src/native_commands/_helpers/paths.rs | 56 +++ .../native_commands/_helpers/project_facts.rs | 66 ++++ .../src/native_commands/_helpers/rules.rs | 169 +++++++++ .../src/native_commands/_helpers/skills.rs | 49 +++ .../src/native_commands/_helpers/terminal.rs | 93 +++++ .../src/native_commands/constants.rs | 75 ++++ .../src/native_commands/errors.rs | 16 + .../src/native_commands/main/mod.rs | 1 + .../main/run_native_invocation.rs | 52 +++ .../sqlbuild-cli/src/native_commands/mod.rs | 10 + .../src/native_commands/models.rs | 73 ++++ .../sqlbuild-cli/src/native_commands/tests.rs | 14 + .../src/native_commands/tests/helpers.rs | 28 ++ .../native_commands/tests/test_argv_host.rs | 154 ++++++++ .../native_commands/tests/test_decision.rs | 129 +++++++ .../src/native_commands/tests/test_paths.rs | 77 ++++ .../tests/test_project_facts.rs | 60 ++++ .../src/native_commands/tests/test_rules.rs | 108 ++++++ .../src/native_commands/tests/test_types.rs | 68 ++++ .../src/sqlbuild/cli/entry/_test_types.py | 31 ++ .../src/sqlbuild/cli/entry/helpers.py | 92 +++++ .../cli/entry/test_python_free_commands.py | 333 ++++++++++++++++++ 33 files changed, 2343 insertions(+), 11 deletions(-) create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/init.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/package.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/constants.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/errors.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/mod.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/models.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/helpers.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_types.rs create mode 100644 tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py diff --git a/Cargo.lock b/Cargo.lock index 3a3986c304..4907f98599 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,7 +1241,9 @@ dependencies = [ "serde", "serde_json", "sqlbuild-cache", + "sqlbuild-config", "sqlbuild-core", + "sqlbuild-rules", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index 0ffdd46f94..994caecd9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,7 +90,7 @@ ruff_text_size = "=0.0.4" rustc-hash = "2" # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "std"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" serde = { version = "1", features = ["derive"] } diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index 8daa9c84c3..cdfce87880 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "sqlbuild-cli" version.workspace = true -description = "The native `sqb` executable: reused compiles served without Python, every other command handed to the Python CLI." +description = "The native `sqb` executable: reused compiles and Python-free commands served without Python, every other command handed to the Python CLI." edition.workspace = true license-file = "LICENSE" publish.workspace = true @@ -17,18 +17,18 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true +ctrlc.workspace = true getrandom.workspace = true serde.workspace = true serde_json.workspace = true sqlbuild-cache.workspace = true +sqlbuild-config.workspace = true sqlbuild-core.workspace = true +sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true -[target.'cfg(windows)'.dependencies] -ctrlc.workspace = true - [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs index c8c85f8e54..e6f689c242 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/dispatch.rs @@ -1,4 +1,4 @@ -//! Decide how one invocation runs: natively, from a launch record, or in the Python CLI. +//! Decide how one invocation runs: from a launch record, natively, or in the Python CLI. use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -10,6 +10,7 @@ use crate::launcher::_helpers::messages::{print_version, report_missing_python_c use crate::launcher::constants::{ HANDOFF_FAILURE_EXIT_CODE, LAUNCH_RECORD_OPTION, VERSION_OPTIONS, }; +use crate::native_commands::main::run_native_invocation::run_native_invocation; use crate::replay::main::serve_reused_compile::serve_reused_compile; /// The launch record path and key for these arguments, when the directory is known. @@ -43,10 +44,16 @@ pub(crate) fn launch(arguments: Vec, started: (Instant, SystemTime)) - return HANDOFF_FAILURE_EXIT_CODE; }; let mut handed: Vec = Vec::with_capacity(arguments.len() + 1); - if let Some((path, key)) = launch_record(&program, &script, &arguments) { - if let Some(code) = serve_reused_compile(&path, &key, started) { - return exit_code(code); - } + let record: Option<(PathBuf, String)> = launch_record(&program, &script, &arguments); + if let Some((path, key)) = &record + && let Some(code) = serve_reused_compile(path, key, started) + { + return exit_code(code); + } + if let Ok(code) = run_native_invocation(&arguments, &script) { + return code; + } + if let Some((path, _)) = record { let mut option = OsString::from(LAUNCH_RECORD_OPTION); option.push(path.as_os_str()); handed.push(option); diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index 4bbb72fffb..fc36fb01fd 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -1,4 +1,4 @@ -//! The native `sqb` executable: replays reused compiles and hands other invocations to Python. +//! The native `sqb` executable: replays reused compiles, runs Python-free commands and hands the rest to Python. #![forbid(unsafe_code)] @@ -7,4 +7,5 @@ pub mod compute_logs; pub mod diff_report; pub mod dispatch; pub mod launcher; +pub mod native_commands; pub mod replay; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs new file mode 100644 index 0000000000..9e26816a90 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs @@ -0,0 +1,119 @@ +//! The parse's interpreter side without an interpreter: Python's `int`, `repr` and the version. + +use sqlbuild_core::text::main::decimal_digit_value::decimal_digit_value; +use sqlbuild_core::text::main::python_isspace::python_isspace; +use sqlbuild_core::text::main::python_str_repr::python_str_repr; + +use crate::argv::main::cli_namespace_defaults::cli_namespace_defaults; +use crate::argv::models::{NamespaceDefault, TypeFailure, Value}; +use crate::argv::types::ArgvHost; +use crate::native_commands::constants::{ + DIGIT_SEPARATOR, INT_ASCII_SPACES, INT_CONVERTER, NEGATIVE_SIGN, POSITIVE_INTEGER_CONVERTER, + POSITIVE_INTEGER_MESSAGE, POSITIVE_SIGN, PYTHON_INT_MAX_STR_DIGITS, ZERO_TEXT, +}; + +/// The ASCII text CPython parses: other decimal digits become ASCII, other spaces a space. +fn ascii_digits_and_spaces(text: &str) -> Option { + text.chars() + .map(|character| { + if character.is_ascii() { + Some(character) + } else if python_isspace(character) { + Some(' ') + } else { + decimal_digit_value(character).and_then(|digit| char::from_digit(digit, 10)) + } + }) + .collect() +} + +/// The canonical decimal text of Python's `int(text)`, or `None` where it raises `ValueError`. +pub(crate) fn python_int(text: &str) -> Option { + let ascii: String = ascii_digits_and_spaces(text)?; + let trimmed: &str = ascii.trim_matches(|character: char| INT_ASCII_SPACES.contains(character)); + let (negative, body): (bool, &str) = match trimmed.strip_prefix(NEGATIVE_SIGN) { + Some(body) => (true, body), + None => ( + false, + trimmed.strip_prefix(POSITIVE_SIGN).unwrap_or(trimmed), + ), + }; + let mut digits: String = String::with_capacity(body.len()); + let mut after_digit: bool = false; + for character in body.chars() { + if character.is_ascii_digit() { + digits.push(character); + after_digit = true; + } else if character == DIGIT_SEPARATOR && after_digit { + after_digit = false; + } else { + return None; + } + } + if !after_digit || digits.len() > PYTHON_INT_MAX_STR_DIGITS { + return None; + } + let significant: &str = digits.trim_start_matches('0'); + Some(match (significant.is_empty(), negative) { + (true, _) => ZERO_TEXT.to_owned(), + (false, true) => format!("{NEGATIVE_SIGN}{significant}"), + (false, false) => significant.to_owned(), + }) +} + +/// Converts what it can as Python would and records any converter only Python can apply. +#[derive(Debug, Default)] +pub(crate) struct NativeArgvHost { + /// Whether the parse met a converter, such as `--vars`, that needs the interpreter. + pub(crate) needs_python: bool, +} + +impl ArgvHost for NativeArgvHost { + fn convert(&mut self, converter: &str, text: &str) -> Result { + match converter { + INT_CONVERTER => python_int(text).map(Value::Int).ok_or(TypeFailure::Invalid), + POSITIVE_INTEGER_CONVERTER => { + let number: String = python_int(text).ok_or(TypeFailure::Invalid)?; + if number == ZERO_TEXT || number.starts_with(NEGATIVE_SIGN) { + return Err(TypeFailure::ArgumentType( + POSITIVE_INTEGER_MESSAGE.to_owned(), + )); + } + Ok(Value::Int(number)) + } + _ => { + self.needs_python = true; + Ok(Value::None) + } + } + } + + fn repr(&self, text: &str) -> String { + python_str_repr(text) + } + + fn program_version(&mut self) -> String { + env!("CARGO_PKG_VERSION").to_owned() + } +} + +/// The parse's starting namespace: every `CliNamespace` default. +pub(crate) fn namespace_defaults() -> Vec<(String, Value)> { + cli_namespace_defaults() + .iter() + .map(|(name, default)| { + let value: Value = match default { + NamespaceDefault::None => Value::None, + NamespaceDefault::Bool(flag) => Value::Bool(*flag), + NamespaceDefault::Int(number) => Value::Int(number.to_string()), + NamespaceDefault::Text(text) => Value::Str((*text).to_owned()), + NamespaceDefault::EmptyList => Value::List(Vec::new()), + NamespaceDefault::EmptyDict => Value::EmptyDict, + NamespaceDefault::CompileLineageFast | NamespaceDefault::ColumnLineageRich => { + Value::Host(0) + } + }; + ((*name).to_owned(), value) + }) + .collect() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs new file mode 100644 index 0000000000..30ede63f68 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/decision.rs @@ -0,0 +1,92 @@ +//! Which parsed invocations run without Python, decided from the parsed values alone. + +use std::path::PathBuf; + +use crate::argv::models::{Namespace, Value}; +use crate::dispatch::main::invocation_policy::invocation_policy; +use crate::dispatch::models::{CommandFacts, InvocationPolicy}; +use crate::native_commands::constants::{ + CLEAN_COMMAND, INIT_COMMAND, RULES_COMMAND, RULES_LIST_ACTION, RULES_SHOW_ACTION, + SKILLS_COMMAND, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{ + NativeCommand, NativeInvocation, RulesAction, RulesOptions, SkillsOptions, +}; + +fn text(namespace: &Namespace, name: &str) -> Option { + match namespace.get(name) { + Some(Value::Str(text) | Value::Path(text)) => Some(text.clone()), + _ => None, + } +} + +fn flag(namespace: &Namespace, name: &str) -> bool { + namespace.get(name) == Some(&Value::Bool(true)) +} + +/// The strings of a list attribute, or `None` when it holds anything else. +fn texts(namespace: &Namespace, name: &str) -> Option> { + match namespace.get(name) { + None | Some(Value::None) => Some(Vec::new()), + Some(Value::List(items)) => items + .iter() + .map(|item| match item { + Value::Str(text) => Some(text.clone()), + _ => None, + }) + .collect(), + Some(_) => None, + } +} + +fn rules_command(namespace: &Namespace) -> Result { + let action: RulesAction = match text(namespace, "rules_command").as_deref() { + None | Some(RULES_LIST_ACTION) => RulesAction::List, + Some(RULES_SHOW_ACTION) => RulesAction::Show, + Some(_) => return Err(PythonNeeded::Command), + }; + Ok(NativeCommand::Rules(RulesOptions { + action, + selector: text(namespace, "rules_rule_selector"), + json: flag(namespace, "json"), + })) +} + +fn native_command( + command: Option<&str>, + namespace: &Namespace, +) -> Result { + match command { + None | Some(CLEAN_COMMAND) => Ok(NativeCommand::Nothing), + Some(INIT_COMMAND) => Ok(NativeCommand::Init), + Some(SKILLS_COMMAND) => Ok(NativeCommand::Skills(SkillsOptions { + global_install: flag(namespace, "skills_global"), + targets: texts(namespace, "skills_target").ok_or(PythonNeeded::Arguments)?, + force: flag(namespace, "skills_force"), + })), + Some(RULES_COMMAND) => rules_command(namespace), + Some(_) => Err(PythonNeeded::Command), + } +} + +/// The command `namespace` runs, when it runs without Python; `--debug` always uses Python. +pub(crate) fn native_invocation(namespace: &Namespace) -> Result { + if flag(namespace, "debug") { + return Err(PythonNeeded::Command); + } + let command_name: Option = text(namespace, "command"); + let command: NativeCommand = native_command(command_name.as_deref(), namespace)?; + let policy: InvocationPolicy = invocation_policy(&CommandFacts { + command: command_name.clone(), + ..CommandFacts::default() + }); + Ok(NativeInvocation { + command, + command_name, + project_dir: text(namespace, "project_dir").map(PathBuf::from), + target: text(namespace, "target"), + creates_project: policy.creates_project, + maintains_skills: policy.maintains_skills, + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs new file mode 100644 index 0000000000..278a397f81 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -0,0 +1,117 @@ +//! Running a Python-free command: its checks, then its output with the observed services. + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use crate::argv::models::Stream; +use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::finish_capture::finish_capture; +use crate::compute_logs::main::open_log_root::open_log_root; +use crate::compute_logs::main::start_capture::start_capture; +use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::native_commands::_helpers::init::run_init; +use crate::native_commands::_helpers::package::installed_package; +use crate::native_commands::_helpers::paths::project_location; +use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; +use crate::native_commands::_helpers::rules::run_rules; +use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; +use crate::native_commands::_helpers::terminal::{exit_on_interrupt, write_text}; +use crate::native_commands::constants::UNKNOWN_COMMAND; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{ + CommandOutput, InstalledPackage, NativeCommand, NativeInvocation, ProjectLocation, +}; + +/// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. +fn open_capture( + invocation: &NativeInvocation, + project: &ProjectLocation, + started_at: SystemTime, +) -> io::Result { + let retention: i64 = i64::try_from(DEFAULT_RETENTION_COUNT).unwrap_or(i64::MAX); + let root: PathBuf = open_log_root(&project.given, None, Some(retention)) + .map_err(|error| io::Error::other(format!("{error:?}")))?; + start_capture(CaptureStart { + root, + command: invocation + .command_name + .clone() + .unwrap_or_else(|| UNKNOWN_COMMAND.to_owned()), + project_dir: project.resolved.display().to_string(), + target: invocation.target.clone(), + started_at, + }) +} + +/// Print an observed command's output after the skill notice, retaining both in its capture. +fn finish_observed( + invocation: &NativeInvocation, + (project, package): (&ProjectLocation, &InstalledPackage), + started_at: SystemTime, + output: CommandOutput, +) -> i32 { + let notice: String = if invocation.maintains_skills { + skill_freshness_notice(&project.given, package) + } else { + String::new() + }; + let capture: io::Result = open_capture(invocation, project, started_at); + let stderr: String = notice + &output.stderr; + write_text(Stream::Stderr, &stderr); + write_text(Stream::Stdout, &output.stdout); + if let Ok(capture) = capture { + let finished: io::Result<()> = finish_capture( + capture, + &CaptureEnd { + stdout: output.stdout.into_bytes(), + stderr: stderr.into_bytes(), + exit_code: i64::from(output.exit_code), + }, + ); + drop(finished); + } + output.exit_code +} + +/// What an observed command prints, unless it needs Python after all. +fn observed_output( + command: &NativeCommand, + project: &ProjectLocation, + package: &InstalledPackage, +) -> Result { + if !has_no_runtime_extensions(&project.given) { + return Err(PythonNeeded::Project); + } + match command { + NativeCommand::Nothing => Ok(CommandOutput::default()), + NativeCommand::Skills(options) => run_skills(&project.resolved, package, options), + NativeCommand::Rules(options) => run_rules(&project.resolved, options), + NativeCommand::Init => Err(PythonNeeded::Command), + } +} + +/// Run a parsed Python-free command; an error hands it to Python before any side effect. +pub(crate) fn run_command( + invocation: &NativeInvocation, + python_cli: &Path, + error_color: bool, +) -> Result { + exit_on_interrupt(); + let package: InstalledPackage = installed_package(python_cli)?; + let project: ProjectLocation = project_location(invocation.project_dir.as_deref())?; + if invocation.creates_project { + let output: CommandOutput = run_init(&project.resolved, &package, error_color)?; + write_text(Stream::Stderr, &output.stderr); + write_text(Stream::Stdout, &output.stdout); + return Ok(output.exit_code); + } + let started_at: SystemTime = SystemTime::now(); + let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; + Ok(finish_observed( + invocation, + (&project, &package), + started_at, + output, + )) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs new file mode 100644 index 0000000000..dfee4cd22b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs @@ -0,0 +1,101 @@ +//! `sqb init`: scaffold a blank project and install its skills. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::workspace::errors::WorkspaceError; +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::models::SkillInstallRequest; +use sqlbuild_core::cli_text::main::format_coded_error::format_coded_error; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; +use sqlbuild_core::init_command::errors::ScaffoldError; +use sqlbuild_core::init_command::main::init_project_name::init_project_name; +use sqlbuild_core::init_command::main::init_project_toml::init_project_toml; +use sqlbuild_core::init_command::main::render_init_summary::render_init_summary; +use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_project; + +use crate::dispatch::main::failure_report::failure_report; +use crate::dispatch::models::CommandFailure; +use crate::native_commands::_helpers::skills::home_dir; +use crate::native_commands::_helpers::terminal::stdout_supports_color; +use crate::native_commands::constants::{INIT_FAILURE_PREFIX, USER_ERROR_CLASS}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, InstalledPackage}; + +/// A failure once the project file exists, which a Python rerun would report differently. +fn failed(stderr: String) -> Result { + Ok(CommandOutput { + stdout: String::new(), + stderr, + exit_code: 1, + }) +} + +/// The project's skills could not be installed: what the Python CLI prints for its error. +fn skills_failure(error: &WorkspaceError, use_color: bool) -> Result { + let WorkspaceError::User { + code, + message, + help, + } = error + else { + return failed(format!("{INIT_FAILURE_PREFIX}{error:?}\n")); + }; + let (_, log_message) = failure_report(CommandFailure::User); + let coded: String = format_coded_error( + (code, message, help.as_deref()), + true, + CliStyle { use_color }, + ); + failed(format!( + "{log_message}\n{USER_ERROR_CLASS}: {message}\n{coded}\n\n" + )) +} + +/// Create the project; a failure before the project file exists is Python's to report. +pub(crate) fn run_init( + base_dir: &Path, + package: &InstalledPackage, + error_color: bool, +) -> Result { + let project_file: PathBuf = base_dir.join(PROJECT_CONFIG_FILENAME); + if project_file.exists() { + return Err(PythonNeeded::Failure); + } + let directory_name: &str = match base_dir.file_name() { + Some(name) => name.to_str().ok_or(PythonNeeded::Arguments)?, + None => "", + }; + let home: PathBuf = home_dir()?; + let project_name: String = init_project_name(directory_name); + if let Err(ScaffoldError { error, path }) = + scaffold_blank_project(base_dir, &init_project_toml(&project_name)) + { + if !project_file.exists() { + return Err(PythonNeeded::Failure); + } + return failed(format!( + "{INIT_FAILURE_PREFIX}{error}: {}\n", + path.display() + )); + } + if let Err(error) = install_skills(&SkillInstallRequest { + project_dir: base_dir.to_path_buf(), + skills_root: package.skills_root.clone(), + home_dir: home, + global_install: false, + requested_targets: Vec::new(), + force: false, + }) { + return skills_failure(&error, error_color); + } + Ok(CommandOutput { + stdout: render_init_summary( + &project_name, + CliStyle { + use_color: stdout_supports_color(), + }, + ), + ..CommandOutput::default() + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs new file mode 100644 index 0000000000..c01acfd621 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/mod.rs @@ -0,0 +1,10 @@ +pub(crate) mod argv_host; +pub(crate) mod decision; +pub(crate) mod execution; +pub(crate) mod init; +pub(crate) mod package; +pub(crate) mod paths; +pub(crate) mod project_facts; +pub(crate) mod rules; +pub(crate) mod skills; +pub(crate) mod terminal; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs new file mode 100644 index 0000000000..ce77e6aa57 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/package.rs @@ -0,0 +1,129 @@ +//! The installed `sqlbuild` package Python would import, found without starting Python. + +use std::path::{Path, PathBuf}; + +use crate::native_commands::constants::{ + PACKAGE_MARKER, PACKAGE_NAME, PATH_FILE_COMMENT, PATH_FILE_EXTENSION, + PATH_FILE_IMPORT_PREFIXES, PYTHON_PATH_VARIABLES, SKILLS_DIRECTORY, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::InstalledPackage; + +/// The site directories of the installation whose scripts directory holds `script`. +#[cfg(not(windows))] +fn site_directories(prefix: &Path) -> Vec { + use crate::native_commands::constants::{ + INTERPRETER_DIRECTORY_PREFIX, LIBRARY_DIRECTORY, SITE_DIRECTORIES, + }; + + let Ok(entries) = std::fs::read_dir(prefix.join(LIBRARY_DIRECTORY)) else { + return Vec::new(); + }; + let mut directories: Vec = Vec::new(); + for entry in entries.flatten() { + if !entry + .file_name() + .to_string_lossy() + .starts_with(INTERPRETER_DIRECTORY_PREFIX) + { + continue; + } + for site in SITE_DIRECTORIES { + let directory: PathBuf = entry.path().join(site); + if directory.is_dir() { + directories.push(directory); + } + } + } + directories.sort(); + directories +} + +#[cfg(windows)] +fn site_directories(prefix: &Path) -> Vec { + use crate::native_commands::constants::WINDOWS_SITE_DIRECTORY; + + let directory: PathBuf = WINDOWS_SITE_DIRECTORY + .iter() + .fold(prefix.to_path_buf(), |path, part| path.join(part)); + if directory.is_dir() { + vec![directory] + } else { + Vec::new() + } +} + +fn package_beneath(directory: &Path) -> Option { + let package: PathBuf = directory.join(PACKAGE_NAME); + package.join(PACKAGE_MARKER).is_file().then_some(package) +} + +/// The paths a site directory's `.pth` files add, in the order `site` reads them. +fn path_file_entries(site: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(site) else { + return Vec::new(); + }; + let mut files: Vec = entries + .flatten() + .map(|entry| entry.path()) + .filter(|path| { + path.extension() + .is_some_and(|extension| extension == PATH_FILE_EXTENSION) + }) + .collect(); + files.sort(); + let mut added: Vec = Vec::new(); + for file in files { + let Ok(text) = std::fs::read_to_string(&file) else { + continue; + }; + for line in text.lines() { + let line: &str = line.trim_end(); + if line.is_empty() + || line.starts_with(PATH_FILE_COMMENT) + || PATH_FILE_IMPORT_PREFIXES + .iter() + .any(|prefix| line.starts_with(prefix)) + { + continue; + } + added.push(site.join(line)); + } + } + added +} + +/// The one `sqlbuild` package the installation beside `script` holds, unless that is unclear. +pub(crate) fn installed_package(script: &Path) -> Result { + if PYTHON_PATH_VARIABLES + .iter() + .any(|name| std::env::var_os(name).is_some_and(|value| !value.is_empty())) + { + return Err(PythonNeeded::Installation); + } + let script: PathBuf = std::fs::canonicalize(script).map_err(|_| PythonNeeded::Installation)?; + let prefix: &Path = script + .parent() + .and_then(Path::parent) + .ok_or(PythonNeeded::Installation)?; + let mut found: Vec = Vec::new(); + for site in site_directories(prefix) { + let package: Option = package_beneath(&site).or_else(|| { + path_file_entries(&site) + .iter() + .find_map(|directory| package_beneath(directory)) + }); + if let Some(package) = package { + found.push(std::fs::canonicalize(&package).unwrap_or(package)); + } + } + found.dedup(); + let [package] = found.as_slice() else { + return Err(PythonNeeded::Installation); + }; + Ok(InstalledPackage { + skills_root: SKILLS_DIRECTORY + .iter() + .fold(package.clone(), |path, part| path.join(part)), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs new file mode 100644 index 0000000000..c50eea5af9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -0,0 +1,56 @@ +//! `pathlib.Path` construction and `Path.resolve()` over native paths. + +use std::path::{Component, Path, PathBuf}; + +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::ProjectLocation; + +/// `Path(text)`: repeated separators, `.` parts and trailing separators dropped. +pub(crate) fn pathlib_path(text: &Path) -> PathBuf { + let path: PathBuf = text + .components() + .filter(|component| *component != Component::CurDir) + .collect(); + if path.as_os_str().is_empty() { + return PathBuf::from("."); + } + path +} + +/// `Path.resolve()`: absolute, with every existing prefix's links followed, as `realpath` does. +pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { + let absolute: PathBuf = if path.is_absolute() { + path.to_path_buf() + } else { + cwd.join(path) + }; + if let Ok(real) = std::fs::canonicalize(&absolute) { + return real; + } + let mut resolved: PathBuf = PathBuf::new(); + for component in absolute.components() { + match component { + Component::Prefix(_) | Component::RootDir => resolved.push(component.as_os_str()), + Component::CurDir => {} + Component::ParentDir => { + resolved.pop(); + } + Component::Normal(name) => { + let candidate: PathBuf = resolved.join(name); + resolved = std::fs::canonicalize(&candidate).unwrap_or(candidate); + } + } + } + resolved +} + +/// The project directory as given and as `Path.resolve()` reads it. +pub(crate) fn project_location(given: Option<&Path>) -> Result { + let cwd: PathBuf = std::env::current_dir().map_err(|_| PythonNeeded::Installation)?; + let given: PathBuf = match given { + Some(raw) => pathlib_path(raw), + None => cwd.clone(), + }; + let resolved: PathBuf = resolved_path(&given, &cwd); + Ok(ProjectLocation { given, resolved }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs new file mode 100644 index 0000000000..e94539aa29 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -0,0 +1,66 @@ +//! Project facts that decide, before any side effect, whether a command needs Python. + +use std::path::Path; + +use sqlbuild_config::project::main::load_project_config::load_project_config; +use sqlbuild_config::project::models::{Built, ConfigPaths}; +use sqlbuild_core::constants::PROJECT_CONFIG_FILENAME; + +use crate::native_commands::constants::{ + CUSTOM_RULES_DIRECTORY, EVENT_EXPORTERS_DIRECTORY, LEGACY_LOCAL_CONFIG_FILENAME, + LEGACY_PROJECT_CONFIG_FILENAME, LIFECYCLE_SINK_PATH, LOCAL_CONFIG_FILENAME, SINKS_DIRECTORY, +}; + +fn object_field<'a>(built: &'a Built, name: &str) -> Option<&'a Built> { + match built { + Built::Object(_, fields) => fields + .iter() + .find(|(field, _)| *field == name) + .map(|(_, value)| value), + _ => None, + } +} + +/// Whether the loaded configuration names a lifecycle sink, which only Python can bind. +fn names_lifecycle_sinks(config: &Built) -> bool { + let named: Option<&Built> = LIFECYCLE_SINK_PATH + .iter() + .try_fold(config, |value, name| object_field(value, name)); + match named { + None => false, + Some(Built::Dict(entries)) => !entries.is_empty(), + Some(_) => true, + } +} + +/// The four configuration files of `project_dir`. +fn config_paths(project_dir: &Path) -> ConfigPaths { + ConfigPaths { + project: project_dir.join(PROJECT_CONFIG_FILENAME), + legacy_project: project_dir.join(LEGACY_PROJECT_CONFIG_FILENAME), + local: project_dir.join(LOCAL_CONFIG_FILENAME), + legacy_local: project_dir.join(LEGACY_LOCAL_CONFIG_FILENAME), + } +} + +/// Whether `discover_runtime_extensions` would import nothing and accept the project file. +pub(crate) fn has_no_runtime_extensions(project_dir: &Path) -> bool { + if project_dir.join(EVENT_EXPORTERS_DIRECTORY).exists() + || project_dir.join(SINKS_DIRECTORY).is_dir() + { + return false; + } + let paths: ConfigPaths = config_paths(project_dir); + if !paths.project.exists() { + return true; + } + match load_project_config(&paths) { + Ok(config) => !names_lifecycle_sinks(&config), + Err(_) => false, + } +} + +/// Whether the project defines custom rules, which Python imports. +pub(crate) fn has_custom_rules(project_dir: &Path) -> bool { + project_dir.join(CUSTOM_RULES_DIRECTORY).is_dir() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs new file mode 100644 index 0000000000..c956dd94fe --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/rules.rs @@ -0,0 +1,169 @@ +//! `sqb rules list|show` over the built-in catalogue, when no custom rule or option needs Python. + +use std::collections::BTreeMap; +use std::path::Path; + +use sqlbuild_rules::configuration::main::forbidden_exceptions::forbidden_exceptions; +use sqlbuild_rules::configuration::main::load_rules_config::load_rules_config; +use sqlbuild_rules::models::{RuleMetadata, RulesConfig}; +use sqlbuild_rules::rule_catalogue::main::format_rule_text::format_rule_text; +use sqlbuild_rules::rule_catalogue::main::render_rule_list::render_rule_list; +use sqlbuild_rules::rule_catalogue::main::sorted_catalogue_order::sorted_catalogue_order; +use sqlbuild_rules::rule_catalogue::models::{ + CatalogueRule, OptionKind, OptionSpec, OptionText, OptionValue, RuleListing, RuleText, +}; +use sqlbuild_rules::rules::main::builtin_options::builtin_options; +use sqlbuild_rules::rules::main::catalogue::catalogue; +use sqlbuild_rules::rules::models::{BuiltinOption, BuiltinOptionKind}; + +use crate::native_commands::_helpers::project_facts::has_custom_rules; +use crate::native_commands::constants::{ + EMPTY_TUPLE_REPR, FALSE_REPR, FAMILY_SUFFIX_LENGTH, MODEL_SUBJECT, PROJECT_SUBJECT, + RULE_SELECTOR_REQUIRED, RULES_USAGE_EXIT_CODE, UNKNOWN_RULE_PREFIX, +}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, RulesAction, RulesOptions}; + +/// `RuleOption.boolean(...)` or `RuleOption.string_list(...)` as the catalogue validates it. +fn option_spec(option: &BuiltinOption) -> OptionSpec { + let (value_type, default, item_type) = match option.kind { + BuiltinOptionKind::Boolean => (OptionKind::Bool, OptionValue::Bool(false), None), + BuiltinOptionKind::StringList => ( + OptionKind::Tuple, + OptionValue::Tuple(Vec::new()), + Some(OptionKind::Str), + ), + }; + OptionSpec { + name: option.name.to_owned(), + description: option.description.to_owned(), + value_type, + default, + choices: Vec::new(), + minimum: None, + maximum: None, + minimum_items: None, + item_type, + } +} + +/// The option as `rules show` prints it: the `repr` of its default, which is also effective. +fn option_text(option: &BuiltinOption) -> OptionText { + let default_repr: &str = match option.kind { + BuiltinOptionKind::Boolean => FALSE_REPR, + BuiltinOptionKind::StringList => EMPTY_TUPLE_REPR, + }; + OptionText { + name: option.name.to_owned(), + description: option.description.to_owned(), + default_repr: default_repr.to_owned(), + effective_repr: default_repr.to_owned(), + } +} + +/// The rule code without its three-digit number, as the built-in `Rule` records it. +fn family(rule: &RuleMetadata) -> String { + rule.code[..rule.code.len().saturating_sub(FAMILY_SUFFIX_LENGTH)].to_owned() +} + +fn listing(rule: &RuleMetadata) -> RuleListing { + RuleListing { + code: rule.code.clone(), + family: family(rule), + message: rule.message.clone(), + custom: false, + subject: Some( + if rule.project_wide { + PROJECT_SUBJECT + } else { + MODEL_SUBJECT + } + .to_owned(), + ), + } +} + +fn rule_text(rule: &RuleMetadata) -> RuleText { + RuleText { + code: rule.code.clone(), + family: family(rule), + slug: rule.slug.clone(), + message: rule.message.clone(), + remediation: rule.remediation.clone(), + enabled_by_default: rule.enabled_by_default, + custom: false, + source: None, + options: builtin_options(&rule.code) + .iter() + .map(option_text) + .collect(), + good_example: rule + .guidance + .as_ref() + .map(|guidance| guidance.good_example.clone()), + } +} + +/// The built-in catalogue in the order `build_rule_catalogue` sorts it. +fn sorted_catalogue() -> Result, PythonNeeded> { + let rules: Vec = catalogue(); + let views: Vec = rules + .iter() + .map(|rule| CatalogueRule { + code: rule.code.clone(), + options: builtin_options(&rule.code) + .iter() + .map(option_spec) + .collect(), + }) + .collect(); + let order: Vec = + sorted_catalogue_order(&views, &BTreeMap::new()).map_err(|_| PythonNeeded::Failure)?; + Ok(order + .into_iter() + .filter_map(|index| rules.get(index).cloned()) + .collect()) +} + +fn printed(stdout: String, exit_code: i32) -> CommandOutput { + CommandOutput { + stdout: format!("{stdout}\n"), + stderr: String::new(), + exit_code, + } +} + +fn show(selector: Option<&str>, rules: &[RuleMetadata], config: &RulesConfig) -> CommandOutput { + let Some(selector) = selector else { + return printed(RULE_SELECTOR_REQUIRED.to_owned(), RULES_USAGE_EXIT_CODE); + }; + match rules.iter().find(|rule| rule.code == selector) { + Some(rule) => printed(format_rule_text(&rule_text(rule), Some(config)), 0), + None => printed( + format!("{UNKNOWN_RULE_PREFIX}{selector}"), + RULES_USAGE_EXIT_CODE, + ), + } +} + +/// What `rules list|show` prints for `project_dir`, unless it needs Python. +pub(crate) fn run_rules( + project_dir: &Path, + options: &RulesOptions, +) -> Result { + if has_custom_rules(project_dir) { + return Err(PythonNeeded::Project); + } + let config: RulesConfig = load_rules_config(project_dir).map_err(|_| PythonNeeded::Failure)?; + if forbidden_exceptions(&config).is_some() || !config.rule_options.is_empty() { + return Err(PythonNeeded::Project); + } + let rules: Vec = sorted_catalogue()?; + Ok(match options.action { + RulesAction::List => { + let listings: Vec = rules.iter().map(listing).collect(); + printed(render_rule_list(&listings, options.json), 0) + } + RulesAction::Show => show(options.selector.as_deref(), &rules, &config), + }) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs new file mode 100644 index 0000000000..4272d40185 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs @@ -0,0 +1,49 @@ +//! `sqb skills` and the skill freshness check every observed command makes first. + +use std::path::{Path, PathBuf}; + +use sqlbuild_config::workspace::main::install_skills::install_skills; +use sqlbuild_config::workspace::main::maintain_skills::maintain_skills; +use sqlbuild_config::workspace::models::SkillInstallRequest; + +use crate::native_commands::constants::{HOME_VARIABLE, SKILL_FILES_HEADER}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, InstalledPackage, SkillsOptions}; + +/// `Path.home()` from `HOME`, unless only Python's password lookup could tell. +pub(crate) fn home_dir() -> Result { + std::env::var_os(HOME_VARIABLE) + .filter(|home| !home.is_empty()) + .map(PathBuf::from) + .ok_or(PythonNeeded::Installation) +} + +/// The packaged skill files installed into `project_dir`; Python reports any failure. +pub(crate) fn run_skills( + project_dir: &Path, + package: &InstalledPackage, + options: &SkillsOptions, +) -> Result { + let written: Vec = install_skills(&SkillInstallRequest { + project_dir: project_dir.to_path_buf(), + skills_root: package.skills_root.clone(), + home_dir: home_dir()?, + global_install: options.global_install, + requested_targets: options.targets.clone(), + force: options.force, + }) + .map_err(|_| PythonNeeded::Failure)?; + let mut stdout: String = SKILL_FILES_HEADER.to_owned(); + for path in written { + stdout.push_str(&format!(" {}\n", path.display())); + } + Ok(CommandOutput { + stdout, + ..CommandOutput::default() + }) +} + +/// The stale-skill notice, refreshing generated skills; failures stay silent, as in Python. +pub(crate) fn skill_freshness_notice(project_dir: &Path, package: &InstalledPackage) -> String { + maintain_skills(project_dir, &package.skills_root).unwrap_or_default() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs new file mode 100644 index 0000000000..3a584e7abd --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -0,0 +1,93 @@ +//! The process's terminal: its width, colour support, stream writes and Ctrl+C. + +use std::io::{self, IsTerminal, Write}; + +use crate::argv::models::Stream; +use crate::native_commands::_helpers::argv_host::python_int; +use crate::native_commands::constants::{COLUMNS_VARIABLE, FALLBACK_COLUMNS, NO_COLOR_VARIABLE}; +use crate::native_commands::errors::PythonNeeded; + +/// `int(os.environ["COLUMNS"])`, `0` when unset or not an integer; beyond 64 bits is Python's. +fn columns_variable() -> Result { + let Ok(text) = std::env::var(COLUMNS_VARIABLE) else { + return Ok(0); + }; + let Some(number) = python_int(&text) else { + return Ok(0); + }; + number.parse::().map_err(|_| PythonNeeded::Arguments) +} + +/// The width of the terminal on stdout, or the fallback when stdout is not one. +#[cfg(unix)] +fn stdout_columns() -> Result { + match rustix::termios::tcgetwinsize(io::stdout()) { + Ok(size) if size.ws_col > 0 => Ok(i64::from(size.ws_col)), + _ => Ok(FALLBACK_COLUMNS), + } +} + +/// A console's width needs the console API; only a redirected stdout has a known width. +#[cfg(not(unix))] +fn stdout_columns() -> Result { + if io::stdout().is_terminal() { + return Err(PythonNeeded::Arguments); + } + Ok(FALLBACK_COLUMNS) +} + +/// `shutil.get_terminal_size().columns`, unless only Python can tell. +pub(crate) fn terminal_columns() -> Result { + let columns: i64 = columns_variable()?; + if columns > 0 { + return Ok(columns); + } + stdout_columns() +} + +/// `stdout_supports_color`: stdout is a terminal and `NO_COLOR` is unset. +pub(crate) fn stdout_supports_color() -> bool { + std::env::var_os(NO_COLOR_VARIABLE).is_none() && io::stdout().is_terminal() +} + +/// `\n` as a Python text stream writes it on this platform. +fn console_bytes(text: &str) -> Vec { + if cfg!(windows) { + text.replace('\n', "\r\n").into_bytes() + } else { + text.as_bytes().to_vec() + } +} + +/// Write and flush one text; a closed or full stream drops it, as the parse's writes do. +pub(crate) fn write_text(stream: Stream, text: &str) { + if text.is_empty() { + return; + } + let bytes: Vec = console_bytes(text); + let written: io::Result<()> = match stream { + Stream::Stdout => { + let mut output = io::stdout().lock(); + output.write_all(&bytes).and_then(|()| output.flush()) + } + Stream::Stderr => { + let mut output = io::stderr().lock(); + output.write_all(&bytes).and_then(|()| output.flush()) + } + }; + drop(written); +} + +/// End the process with status 130 on Ctrl+C; handing off by `exec` restores the default. +#[cfg(unix)] +pub(crate) fn exit_on_interrupt() { + use crate::native_commands::constants::INTERRUPTED_EXIT_CODE; + + drop(ctrlc::set_handler(|| { + std::process::exit(INTERRUPTED_EXIT_CODE) + })); +} + +/// The Python child waited for alone handles Ctrl+C, so the native command keeps the default. +#[cfg(not(unix))] +pub(crate) fn exit_on_interrupt() {} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs new file mode 100644 index 0000000000..9c01a84a6b --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -0,0 +1,75 @@ +//! Names and limits of the Python-free command path. + +/// `type=` converters the native parse applies itself; any other needs Python. +pub(crate) const INT_CONVERTER: &str = "int"; +pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; +pub(crate) const POSITIVE_INTEGER_MESSAGE: &str = "value must be a positive integer"; +pub(crate) const NO_COLOR_OPTION: &str = "--no-color"; +pub(crate) const NO_COLOR_VARIABLE: &str = "NO_COLOR"; +pub(crate) const COLUMNS_VARIABLE: &str = "COLUMNS"; +/// `shutil.get_terminal_size`'s fallback width. +pub(crate) const FALLBACK_COLUMNS: i64 = 80; +/// Variables that change where Python imports `sqlbuild` from. +pub(crate) const PYTHON_PATH_VARIABLES: [&str; 2] = ["PYTHONPATH", "PYTHONHOME"]; +pub(crate) const PACKAGE_NAME: &str = "sqlbuild"; +pub(crate) const PACKAGE_MARKER: &str = "__init__.py"; +pub(crate) const PATH_FILE_EXTENSION: &str = "pth"; +/// `.pth` lines that `site` executes instead of adding to the search path. +pub(crate) const PATH_FILE_IMPORT_PREFIXES: [&str; 2] = ["import ", "import\t"]; +pub(crate) const PATH_FILE_COMMENT: char = '#'; +#[cfg(not(windows))] +pub(crate) const LIBRARY_DIRECTORY: &str = "lib"; +#[cfg(not(windows))] +pub(crate) const INTERPRETER_DIRECTORY_PREFIX: &str = "python3"; +#[cfg(not(windows))] +pub(crate) const SITE_DIRECTORIES: [&str; 2] = ["site-packages", "dist-packages"]; +#[cfg(windows)] +pub(crate) const WINDOWS_SITE_DIRECTORY: [&str; 2] = ["Lib", "site-packages"]; +/// The packaged skill beneath the `sqlbuild` package directory. +pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"]; +pub(crate) const HOME_VARIABLE: &str = "HOME"; +/// The exit status of a command stopped by Ctrl+C. +#[cfg(unix)] +pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; +/// The command name compute logs record for a bare `sqb`. +pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; +/// Commands with no handler: observed, then successful. +pub(crate) const CLEAN_COMMAND: &str = "clean"; +pub(crate) const INIT_COMMAND: &str = "init"; +pub(crate) const SKILLS_COMMAND: &str = "skills"; +pub(crate) const RULES_COMMAND: &str = "rules"; +pub(crate) const RULES_LIST_ACTION: &str = "list"; +pub(crate) const RULES_SHOW_ACTION: &str = "show"; +/// Project directories holding Python the command would import. +pub(crate) const EVENT_EXPORTERS_DIRECTORY: &str = "event_exporters"; +pub(crate) const SINKS_DIRECTORY: &str = "sinks"; +pub(crate) const CUSTOM_RULES_DIRECTORY: &str = "rules"; +pub(crate) const SKILL_FILES_HEADER: &str = "Updated SQLBuild skill files:\n"; +pub(crate) const RULE_SELECTOR_REQUIRED: &str = "A rule code or family prefix is required"; +/// `run_rules_command`'s exit status for a missing or unknown rule. +pub(crate) const RULES_USAGE_EXIT_CODE: i32 = 2; +pub(crate) const LEGACY_PROJECT_CONFIG_FILENAME: &str = "sqlbuild_project.yml"; +pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; +pub(crate) const LEGACY_LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.yml"; +/// The class Python's last-resort log handler names for a user error. +pub(crate) const USER_ERROR_CLASS: &str = "sqlbuild._native.CliUserError"; +pub(crate) const INIT_FAILURE_PREFIX: &str = "sqb init: "; +/// A built-in rule's family is its code without the trailing three-digit number. +pub(crate) const FAMILY_SUFFIX_LENGTH: usize = 3; +/// `RuleSubject` values. +pub(crate) const MODEL_SUBJECT: &str = "model"; +pub(crate) const PROJECT_SUBJECT: &str = "project"; +/// Python `repr` of the built-in option defaults: `False` and `()`. +pub(crate) const FALSE_REPR: &str = "False"; +pub(crate) const EMPTY_TUPLE_REPR: &str = "()"; +pub(crate) const UNKNOWN_RULE_PREFIX: &str = "Unknown rule: "; +/// The whitespace `int()` strips once other spaces are ASCII spaces. +pub(crate) const INT_ASCII_SPACES: &str = " \t\n\x0b\x0c\r"; +pub(crate) const NEGATIVE_SIGN: char = '-'; +pub(crate) const POSITIVE_SIGN: char = '+'; +pub(crate) const ZERO_TEXT: &str = "0"; +/// `sys.int_info.default_max_str_digits`: longer decimal text makes `int()` raise. +pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4300; +pub(crate) const DIGIT_SEPARATOR: char = '_'; +/// Where a loaded project configuration names its lifecycle sinks. +pub(crate) const LIFECYCLE_SINK_PATH: [&str; 3] = ["sinks", "lifecycle", "named"]; diff --git a/crates/sqlbuild-cli/src/native_commands/errors.rs b/crates/sqlbuild-cli/src/native_commands/errors.rs new file mode 100644 index 0000000000..78e1542aa8 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/errors.rs @@ -0,0 +1,16 @@ +//! Why `sqb` hands an invocation to Python after all. + +/// What made the invocation need Python, found before any side effect. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PythonNeeded { + /// The arguments are not text, or a converter or terminal fact needs the interpreter. + Arguments, + /// The command, or an option such as `--debug`, runs in Python. + Command, + /// The installed package, home directory or working directory is not known natively. + Installation, + /// The project holds Python the command imports, or files Python reports on. + Project, + /// The command failed before any side effect, and Python reports the failure. + Failure, +} diff --git a/crates/sqlbuild-cli/src/native_commands/main/mod.rs b/crates/sqlbuild-cli/src/native_commands/main/mod.rs new file mode 100644 index 0000000000..f45e2c475a --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/mod.rs @@ -0,0 +1 @@ +pub mod run_native_invocation; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs new file mode 100644 index 0000000000..222d953909 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -0,0 +1,52 @@ +//! Serve one invocation without Python when its parse and project facts allow it. + +use std::ffi::OsString; +use std::path::Path; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; +use crate::native_commands::_helpers::decision::native_invocation; +use crate::native_commands::_helpers::execution::run_command; +use crate::native_commands::_helpers::terminal::{ + stdout_supports_color, terminal_columns, write_text, +}; +use crate::native_commands::constants::NO_COLOR_OPTION; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::NativeInvocation; + +/// The exit status of an invocation `sqb` served, or why the same arguments go to Python. +pub fn run_native_invocation( + arguments: &[OsString], + python_cli: &Path, +) -> Result { + let arguments: Vec = arguments + .iter() + .map(|argument| argument.to_str().map(str::to_owned)) + .collect::>() + .ok_or(PythonNeeded::Arguments)?; + let use_color: bool = + !arguments.iter().any(|argument| argument == NO_COLOR_OPTION) && stdout_supports_color(); + let settings: ArgvSettings = ArgvSettings { + columns: terminal_columns()?, + use_color, + }; + let mut host: NativeArgvHost = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&arguments, namespace_defaults(), settings, &mut host); + if host.needs_python { + return Err(PythonNeeded::Arguments); + } + let namespace: Namespace = match outcome.parsed { + Err(code) => { + for (stream, text) in &outcome.writes { + write_text(*stream, text); + } + return Ok(i32::try_from(code).unwrap_or(1)); + } + Ok(namespace) if outcome.writes.is_empty() => namespace, + Ok(_) => return Err(PythonNeeded::Arguments), + }; + let invocation: NativeInvocation = native_invocation(&namespace)?; + run_command(&invocation, python_cli, use_color) +} diff --git a/crates/sqlbuild-cli/src/native_commands/mod.rs b/crates/sqlbuild-cli/src/native_commands/mod.rs new file mode 100644 index 0000000000..d99ca200f2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/mod.rs @@ -0,0 +1,10 @@ +//! Python-free commands `sqb` runs itself: help, usage errors and the commands needing no Python. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub mod errors; +pub mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs new file mode 100644 index 0000000000..f0ce868fa2 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -0,0 +1,73 @@ +//! Python-free commands, the invocation facts they run with and what they print. + +use std::path::PathBuf; + +/// What `sqb skills` installs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct SkillsOptions { + pub(crate) global_install: bool, + pub(crate) targets: Vec, + pub(crate) force: bool, +} + +/// Which `sqb rules` inspection runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RulesAction { + List, + Show, +} + +/// What `sqb rules list|show` prints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RulesOptions { + pub(crate) action: RulesAction, + pub(crate) selector: Option, + pub(crate) json: bool, +} + +/// A command that runs without Python. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NativeCommand { + /// `clean` and a bare `sqb`: observed, with no handler to run. + Nothing, + Init, + Skills(SkillsOptions), + Rules(RulesOptions), +} + +/// One parsed invocation `sqb` serves without Python. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NativeInvocation { + pub(crate) command: NativeCommand, + /// The parsed command name; `None` for a bare `sqb`. + pub(crate) command_name: Option, + /// `--project-dir` as given. + pub(crate) project_dir: Option, + pub(crate) target: Option, + pub(crate) creates_project: bool, + pub(crate) maintains_skills: bool, +} + +/// What a command printed and the status it ends with. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct CommandOutput { + pub(crate) stdout: String, + pub(crate) stderr: String, + pub(crate) exit_code: i32, +} + +/// Where the installed `sqlbuild` package keeps what the commands read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InstalledPackage { + /// The packaged `.agents/skills/sqlbuild` directory. + pub(crate) skills_root: PathBuf, +} + +/// The directories one command runs against. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ProjectLocation { + /// `Path(--project-dir)`, or `Path.cwd()`, unresolved. + pub(crate) given: PathBuf, + /// `Path(--project-dir).resolve()`, or `Path.cwd()`. + pub(crate) resolved: PathBuf, +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs new file mode 100644 index 0000000000..9119c141c9 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -0,0 +1,14 @@ +#[path = "tests/test_argv_host.rs"] +mod argv_host; +#[path = "tests/test_decision.rs"] +mod decision; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_paths.rs"] +mod paths; +#[path = "tests/test_project_facts.rs"] +mod project_facts; +#[path = "tests/test_rules.rs"] +mod rules; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs new file mode 100644 index 0000000000..e54ee1dd6d --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/helpers.rs @@ -0,0 +1,28 @@ +use std::path::Path; + +use crate::argv::main::parse_cli_arguments::parse_cli_arguments; +use crate::argv::models::{ArgvOutcome, ArgvSettings}; +use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; + +/// Parse `argv` onto the `CliNamespace` defaults with the native host. +pub(super) fn parse(argv: &[&str]) -> (ArgvOutcome, NativeArgvHost) { + let argv: Vec = argv.iter().map(|arg| (*arg).to_owned()).collect(); + let settings: ArgvSettings = ArgvSettings { + columns: 100, + use_color: false, + }; + let mut host: NativeArgvHost = NativeArgvHost::default(); + let outcome: ArgvOutcome = + parse_cli_arguments(&argv, namespace_defaults(), settings, &mut host); + (outcome, host) +} + +/// Create `directories` and write `files` beneath `root`. +pub(super) fn write_tree(root: &Path, directories: &[&str], files: &[(&str, &str)]) { + for directory in directories { + std::fs::create_dir_all(root.join(directory)).expect("create directory"); + } + for (relative, contents) in files { + std::fs::write(root.join(relative), contents).expect("write file"); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs new file mode 100644 index 0000000000..be011f8c40 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_argv_host.rs @@ -0,0 +1,154 @@ +use crate::argv::models::{TypeFailure, Value}; +use crate::argv::types::ArgvHost; +use crate::native_commands::_helpers::argv_host::NativeArgvHost; +use crate::native_commands::tests::test_types::ConversionTestCase; + +/// Integer expectations recorded from CPython 3.12 `int(text)`. +#[test] +fn given_converter_when_converting_natively_then_python_conversions_match() { + let test_cases = [ + ConversionTestCase { + description: "surrounding ASCII spaces", + converter: "int", + text: " 5 ", + expected_value: Ok(Value::Int("5".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "underscores between digits", + converter: "int", + text: "1_0", + expected_value: Ok(Value::Int("10".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a leading underscore", + converter: "int", + text: "_1", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a trailing underscore", + converter: "int", + text: "1_", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "doubled underscores", + converter: "int", + text: "1__0", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "an Arabic-Indic digit", + converter: "int", + text: "\u{663}", + expected_value: Ok(Value::Int("3".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a non-breaking space", + converter: "int", + text: "\u{a0}5", + expected_value: Ok(Value::Int("5".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "an ASCII separator is not stripped", + converter: "int", + text: "\u{1c}5", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a zero-width space is not a space", + converter: "int", + text: "\u{200b}7", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "negative zero", + converter: "int", + text: "-0", + expected_value: Ok(Value::Int("0".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a sign and leading zeros", + converter: "int", + text: "+007", + expected_value: Ok(Value::Int("7".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "beyond 64 bits", + converter: "int", + text: "-123456789012345678901234567890", + expected_value: Ok(Value::Int("-123456789012345678901234567890".to_owned())), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a decimal point", + converter: "int", + text: "1.5", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "empty text", + converter: "int", + text: "", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer rejects zero with its message", + converter: "_positive_integer", + text: "0", + expected_value: Err(TypeFailure::ArgumentType( + "value must be a positive integer".to_owned(), + )), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer rejects negatives", + converter: "_positive_integer", + text: "-3", + expected_value: Err(TypeFailure::ArgumentType( + "value must be a positive integer".to_owned(), + )), + expected_needs_python: false, + }, + ConversionTestCase { + description: "a positive integer that is not an integer is invalid", + converter: "_positive_integer", + text: "many", + expected_value: Err(TypeFailure::Invalid), + expected_needs_python: false, + }, + ConversionTestCase { + description: "--vars JSON needs Python", + converter: "parse_cli_vars", + text: "{}", + expected_value: Ok(Value::None), + expected_needs_python: true, + }, + ]; + + for test_case in test_cases { + let mut host: NativeArgvHost = NativeArgvHost::default(); + + let value = host.convert(test_case.converter, test_case.text); + + assert_eq!( + (value, host.needs_python), + (test_case.expected_value, test_case.expected_needs_python), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs new file mode 100644 index 0000000000..02dd4d48be --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_decision.rs @@ -0,0 +1,129 @@ +use crate::argv::models::Namespace; +use crate::native_commands::_helpers::decision::native_invocation; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{NativeCommand, RulesAction, RulesOptions, SkillsOptions}; +use crate::native_commands::tests::helpers::parse; +use crate::native_commands::tests::test_types::{DecisionTestCase, ParseTestCase}; + +#[test] +fn given_parsed_command_when_deciding_then_only_python_free_commands_run_natively() { + let test_cases = [ + DecisionTestCase { + description: "a bare sqb has no handler", + argv: &[], + expected_command: Ok(NativeCommand::Nothing), + expected_creates_project: false, + }, + DecisionTestCase { + description: "clean has no handler", + argv: &["clean"], + expected_command: Ok(NativeCommand::Nothing), + expected_creates_project: false, + }, + DecisionTestCase { + description: "init creates a project", + argv: &["init", "--project-dir", "shop"], + expected_command: Ok(NativeCommand::Init), + expected_creates_project: true, + }, + DecisionTestCase { + description: "skills with targets", + argv: &[ + "skills", "--target", "claude", "--target", "agents", "--force", + ], + expected_command: Ok(NativeCommand::Skills(SkillsOptions { + global_install: false, + targets: vec!["claude".to_owned(), "agents".to_owned()], + force: true, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules list as JSON", + argv: &["rules", "--json", "list"], + expected_command: Ok(NativeCommand::Rules(RulesOptions { + action: RulesAction::List, + selector: None, + json: true, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules show", + argv: &["rules", "show", "SQBRMODEL101"], + expected_command: Ok(NativeCommand::Rules(RulesOptions { + action: RulesAction::Show, + selector: Some("SQBRMODEL101".to_owned()), + json: false, + })), + expected_creates_project: false, + }, + DecisionTestCase { + description: "rules run evaluates the project in Python", + argv: &["rules", "run", "SQBRMODEL"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + DecisionTestCase { + description: "debug output is Python's", + argv: &["clean", "--debug"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + DecisionTestCase { + description: "compile needs Python", + argv: &["compile"], + expected_command: Err(PythonNeeded::Command), + expected_creates_project: false, + }, + ]; + + for test_case in test_cases { + let (outcome, _) = parse(test_case.argv); + let namespace: Namespace = outcome.parsed.expect(test_case.description); + + let decided = native_invocation(&namespace); + + assert_eq!( + decided + .as_ref() + .map(|invocation| invocation.command.clone()) + .map_err(|reason| *reason), + test_case.expected_command.clone(), + "{}", + test_case.description + ); + assert_eq!( + decided.is_ok_and(|invocation| invocation.creates_project), + test_case.expected_creates_project, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_command_line_when_parsing_natively_then_only_python_converters_need_python() { + let test_cases = [ + ParseTestCase { + description: "--vars JSON is parsed by Python", + argv: &["compile", "--vars", "{}"], + expected_needs_python: true, + }, + ParseTestCase { + description: "an integer option converts natively", + argv: &["build", "--concurrency", "4"], + expected_needs_python: false, + }, + ]; + + for test_case in test_cases { + let (_, host) = parse(test_case.argv); + + assert_eq!( + host.needs_python, test_case.expected_needs_python, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs new file mode 100644 index 0000000000..6183cb8ae0 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_paths.rs @@ -0,0 +1,77 @@ +use std::path::{Path, PathBuf}; + +use crate::native_commands::_helpers::paths::{pathlib_path, resolved_path}; +use crate::native_commands::tests::test_types::{PathlibTestCase, ResolveTestCase}; + +#[test] +fn given_project_dir_text_when_building_a_path_then_pathlib_normalisation_applies() { + let test_cases = [ + PathlibTestCase { + description: "a leading dot and a trailing slash drop", + text: "./shop/", + expected_path: "shop", + }, + PathlibTestCase { + description: "inner dots and repeated slashes drop", + text: "shop/.//orders", + expected_path: "shop/orders", + }, + PathlibTestCase { + description: "a lone dot stays", + text: ".", + expected_path: ".", + }, + PathlibTestCase { + description: "parent references stay", + text: "../shop", + expected_path: "../shop", + }, + ]; + + for test_case in test_cases { + assert_eq!( + pathlib_path(Path::new(test_case.text)), + PathBuf::from(test_case.expected_path), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_relative_path_when_resolving_then_existing_prefix_is_real_and_rest_is_kept() { + let test_cases = [ + ResolveTestCase { + description: "a missing directory is joined as written", + text: "new/shop", + expected_parts: &["new", "shop"], + }, + ResolveTestCase { + description: "a parent reference after a missing directory removes it", + text: "missing/../new", + expected_parts: &["new"], + }, + ResolveTestCase { + description: "the working directory itself", + text: ".", + expected_parts: &[], + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let real: PathBuf = std::fs::canonicalize(root.path()).expect("real root"); + + let resolved: PathBuf = resolved_path(Path::new(test_case.text), &real); + + assert_eq!( + resolved, + test_case + .expected_parts + .iter() + .fold(real.clone(), |path, part| path.join(part)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs new file mode 100644 index 0000000000..c7b317e626 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_project_facts.rs @@ -0,0 +1,60 @@ +use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::ProjectFactsTestCase; + +const PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n"; +const NAMED_SINK_PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n\n[sinks.lifecycle.named.audit]\nmin_severity = \"error\"\n"; + +#[test] +fn given_project_files_when_checking_runtime_extensions_then_python_owned_ones_need_python() { + let test_cases = [ + ProjectFactsTestCase { + description: "no project file", + directories: &[], + files: &[], + expected_native: true, + }, + ProjectFactsTestCase { + description: "a plain project", + directories: &[], + files: &[("sqlbuild_project.toml", PROJECT)], + expected_native: true, + }, + ProjectFactsTestCase { + description: "a sinks directory is imported", + directories: &["sinks"], + files: &[("sqlbuild_project.toml", PROJECT)], + expected_native: false, + }, + ProjectFactsTestCase { + description: "the replaced event exporters directory is rejected in Python", + directories: &["event_exporters"], + files: &[], + expected_native: false, + }, + ProjectFactsTestCase { + description: "a named lifecycle sink must be bound in Python", + directories: &[], + files: &[("sqlbuild_project.toml", NAMED_SINK_PROJECT)], + expected_native: false, + }, + ProjectFactsTestCase { + description: "an invalid project file is reported by Python", + directories: &[], + files: &[("sqlbuild_project.toml", "name = \n")], + expected_native: false, + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + write_tree(root.path(), test_case.directories, test_case.files); + + assert_eq!( + has_no_runtime_extensions(root.path()), + test_case.expected_native, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs new file mode 100644 index 0000000000..4eedd7443e --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_rules.rs @@ -0,0 +1,108 @@ +use crate::native_commands::_helpers::rules::run_rules; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{RulesAction, RulesOptions}; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::{RulesTestCase, output_start}; + +const PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n"; +const OPTIONS_PROJECT: &str = "name = \"shop\"\nadapter = \"duckdb\"\n\n[rules.rule_options.SQBRCONTRACT102]\nallow_numeric_indicators = true\n"; + +#[test] +fn given_project_when_inspecting_rules_then_builtin_catalogue_prints_unless_python_owns_it() { + let test_cases = [ + RulesTestCase { + description: "list prints code and message lines", + directories: &[], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Ok(( + "SQBRCONTRACT101 models must declare an enforced output contract\n", + 0, + )), + }, + RulesTestCase { + description: "list --json prints the sorted document", + directories: &[], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: true, + expected_output: Ok(( + "{\"rules\": [{\"code\": \"SQBRCONTRACT101\", \"custom\": false, \"family\": \"SQBRCONTRACT\", ", + 0, + )), + }, + RulesTestCase { + description: "show prints the rule", + directories: &[], + project_file: PROJECT, + action: RulesAction::Show, + selector: Some("SQBRMODEL101"), + json: false, + expected_output: Ok(( + "SQBRMODEL101: dependency-import-ctes\nFamily: SQBRMODEL\n", + 0, + )), + }, + RulesTestCase { + description: "an unknown rule is a usage failure", + directories: &[], + project_file: PROJECT, + action: RulesAction::Show, + selector: Some("SQBRNOPE101"), + json: false, + expected_output: Ok(("Unknown rule: SQBRNOPE101\n", 2)), + }, + RulesTestCase { + description: "custom rules are imported by Python", + directories: &["rules"], + project_file: PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Err(PythonNeeded::Project), + }, + RulesTestCase { + description: "configured options are rendered by Python", + directories: &[], + project_file: OPTIONS_PROJECT, + action: RulesAction::List, + selector: None, + json: false, + expected_output: Err(PythonNeeded::Project), + }, + ]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + write_tree( + root.path(), + test_case.directories, + &[("sqlbuild_project.toml", test_case.project_file)], + ); + + let output = run_rules( + root.path(), + &RulesOptions { + action: test_case.action, + selector: test_case.selector.map(str::to_owned), + json: test_case.json, + }, + ); + + let length: usize = test_case + .expected_output + .map_or(0, |(prefix, _)| prefix.chars().count()); + + assert_eq!( + output.map(|output| output_start(&output, length)), + test_case + .expected_output + .map(|(prefix, code)| (prefix.to_owned(), code)), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs new file mode 100644 index 0000000000..528aeaf4e4 --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -0,0 +1,68 @@ +use crate::argv::models::{TypeFailure, Value}; +use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::{CommandOutput, NativeCommand, RulesAction}; + +/// One converter applied to one command-line string. +pub(super) struct ConversionTestCase { + pub(super) description: &'static str, + pub(super) converter: &'static str, + pub(super) text: &'static str, + pub(super) expected_value: Result, + pub(super) expected_needs_python: bool, +} + +/// One accepted command line, the native command it runs and its invocation services. +pub(super) struct DecisionTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_command: Result, + pub(super) expected_creates_project: bool, +} + +/// One command line and whether its parse needs Python. +pub(super) struct ParseTestCase { + pub(super) description: &'static str, + pub(super) argv: &'static [&'static str], + pub(super) expected_needs_python: bool, +} + +/// One `--project-dir` string and the `pathlib.Path` it names. +pub(super) struct PathlibTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_path: &'static str, +} + +/// A path beneath a real directory and what `Path.resolve()` makes of it. +pub(super) struct ResolveTestCase { + pub(super) description: &'static str, + pub(super) text: &'static str, + pub(super) expected_parts: &'static [&'static str], +} + +/// Project directories and files, and whether an observed command can run without Python. +pub(super) struct ProjectFactsTestCase { + pub(super) description: &'static str, + pub(super) directories: &'static [&'static str], + pub(super) files: &'static [(&'static str, &'static str)], + pub(super) expected_native: bool, +} + +/// One `rules` inspection over a project and the start of what it prints. +pub(super) struct RulesTestCase { + pub(super) description: &'static str, + pub(super) directories: &'static [&'static str], + pub(super) project_file: &'static str, + pub(super) action: RulesAction, + pub(super) selector: Option<&'static str>, + pub(super) json: bool, + pub(super) expected_output: Result<(&'static str, i32), PythonNeeded>, +} + +/// The printed start and exit status of a command's output, for comparison. +pub(super) fn output_start(output: &CommandOutput, length: usize) -> (String, i32) { + ( + output.stdout.chars().take(length).collect(), + output.exit_code, + ) +} diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index 2f6143377e..a39ae9398f 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -82,3 +82,34 @@ class LaunchedTeardownTestCase: description: str args: tuple[str, ...] expected_hits: int + + +@dataclass(frozen=True) +class PythonFreeTestCase: + """One command line `sqb` serves without Python, in a directory `prepare` sets up.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int + expected_captures: int + + +@dataclass(frozen=True) +class PythonStartTestCase: + """One command line `sqb` hands to Python because of the project `prepare` writes.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_python_starts: int + + +@dataclass(frozen=True) +class InterruptTestCase: + """One Python-free command line interrupted while it waits on the project `prepare` writes.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index e81c78ba3a..09fbd33876 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -2,6 +2,7 @@ import os import re import shutil +import signal import subprocess import sys from pathlib import Path @@ -199,3 +200,94 @@ def compiled_files(project_dir: Path) -> dict[str, bytes]: path.relative_to(compiled).as_posix(): path.read_bytes() for path in sorted(compiled.rglob("*.sql")) } + + +def counting_environment(directory: Path, marker: Path) -> Path: + """Install `sqb` in an environment whose `sqlbuild-python` logs each start; return `sqb`.""" + + program: Path = counting_install(directory / "bin", marker) + site_packages: Path = ( + directory / "lib" / f"python{sys.version_info.major}.{sys.version_info.minor}" + ) / "site-packages" + site_packages.mkdir(parents=True) + _ = (site_packages / "sqlbuild_source.pth").write_text( + str(Path(sqlbuild.__file__).resolve().parent.parent) + "\n", encoding="utf-8" + ) + return program + + +def reset_directory(directory: Path) -> Path: + """An empty `directory`, removing whatever an earlier run left.""" + + shutil.rmtree(directory, ignore_errors=True) + directory.mkdir(parents=True) + return directory + + +def _capture_files(capture: Path) -> dict[str, object]: + metadata: dict[str, object] = capture_metadata(capture) + for volatile in ("invocation_id", "started_at", "completed_at", "capture_date"): + _ = metadata.pop(volatile) + streams: dict[str, bytes] = { + name: (capture / name).read_bytes() + for name in ("stdout.log", "stderr.log", "diagnostics.jsonl") + } + return {"complete": (capture / "complete").exists(), "metadata": metadata, **streams} + + +def compute_log_files(project_dir: Path) -> list[dict[str, object]]: + """Each capture's streams and metadata, without per-run identity and times.""" + + return [ + _capture_files(metadata.parent) + for metadata in sorted((project_dir / "logs").glob("*/*/metadata.json")) + ] + + +def prepare_nothing(directory: Path) -> None: + """Leave the directory empty.""" + + _ = directory + + +def prepare_project(directory: Path) -> None: + """Write the one-model project.""" + + _ = write_project(directory) + + +def prepare_custom_skill(directory: Path) -> None: + """Write a hand-written skill where `sqb init` installs the packaged one.""" + + custom: Path = directory / ".claude" / "skills" / "sqlbuild" / "SKILL.md" + custom.parent.mkdir(parents=True) + _ = custom.write_text("my own skill\n", encoding="utf-8") + + +def prepare_stale_skills(directory: Path) -> None: + """Write the project configured for Claude skills that are not installed yet.""" + + _ = write_project(directory) + _ = (directory / "sqlbuild_project.toml").write_text( + PROJECT_FILES["sqlbuild_project.toml"] + '\n[skills]\ntargets = ["claude"]\n', + encoding="utf-8", + ) + + +def prepare_custom_rules(directory: Path) -> None: + """Write the project with a custom rules directory.""" + + _ = write_project(directory) + (directory / "rules").mkdir() + + +def prepare_named_pipe_project(directory: Path) -> None: + """Make the project file a named pipe that blocks its reader.""" + + os.mkfifo(directory / "sqlbuild_project.toml") + + +def restore_default_interrupt() -> None: + """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py new file mode 100644 index 0000000000..6a787e7454 --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -0,0 +1,333 @@ +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.entry._test_types import ( + InterruptTestCase, + PythonFreeTestCase, + PythonStartTestCase, +) +from tests.integration.src.sqlbuild.cli.entry.helpers import ( + PYTHON_CLI, + LauncherRun, + compute_log_files, + counting_environment, + prepare_custom_rules, + prepare_custom_skill, + prepare_named_pipe_project, + prepare_nothing, + prepare_project, + prepare_stale_skills, + python_starts, + reset_directory, + restore_default_interrupt, + run, +) + +INTERRUPT_DELAY_SECONDS: float = 1.0 +INTERRUPT_TIMEOUT_SECONDS: float = 30.0 + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + PythonFreeTestCase( + description="top-level help", + args=("--help",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="short help after options", + args=("--no-color", "-h"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="command help", + args=("compile", "--help"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="nested command help", + args=("rules", "show", "--help"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="version among options", + args=("--no-color", "--version"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="unknown command", + args=("not-a-command",), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="invalid choice", + args=("skills", "--target", "bogus"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="invalid integer", + args=("build", "--concurrency", "many"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="post-parse rule", + args=("build", "--concurrency", "0"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="missing option value", + args=("--project-dir",), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="unrecognized argument", + args=("rules", "list", "--json"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="mutually exclusive options", + args=("lineage", "orders", "--json", "--format", "tree"), + prepare=prepare_nothing, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="bare sqb", + args=(), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="clean", + args=("clean",), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="clean outside a project", + args=("clean",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules list", + args=("rules", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules without an action", + args=("rules",), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=0, + ), + PythonFreeTestCase( + description="rules list json", + args=("rules", "--json", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show", + args=("rules", "show", "SQBRMODEL101"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show with thresholds", + args=("rules", "show", "SQBRTEST101"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules show unknown", + args=("rules", "show", "SQBRNOPE101"), + prepare=prepare_project, + expected_exit_code=2, + expected_captures=1, + ), + PythonFreeTestCase( + description="rules with stale skills", + args=("rules", "list"), + prepare=prepare_stale_skills, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="skills", + args=("skills",), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="skills for one target", + args=("skills", "--target", "claude"), + prepare=prepare_project, + expected_exit_code=0, + expected_captures=1, + ), + PythonFreeTestCase( + description="init", + args=("init",), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="init a new directory", + args=("init", "--project-dir", "shop/"), + prepare=prepare_nothing, + expected_exit_code=0, + expected_captures=0, + ), + PythonFreeTestCase( + description="init beside a custom skill", + args=("init",), + prepare=prepare_custom_skill, + expected_exit_code=1, + expected_captures=0, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_free_invocation_when_running_sqb_then_python_never_starts_and_output_matches( + test_case: PythonFreeTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = tmp_path / "workspace" + test_case.prepare(reset_directory(workspace)) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=workspace) + python_logs: list[dict[str, object]] = compute_log_files(workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native == python + assert native[0] == test_case.expected_exit_code + assert python_starts(marker) == 0 + assert compute_log_files(workspace) == python_logs + assert len(python_logs) == test_case.expected_captures + + +@pytest.mark.skipif(sys.platform == "win32", reason="the counting environment is a POSIX script") +@pytest.mark.parametrize( + "test_case", + [ + PythonStartTestCase( + description="custom rules", + args=("rules", "list"), + prepare=prepare_custom_rules, + expected_python_starts=1, + ), + PythonStartTestCase( + description="dbt passthrough help", + args=("dbt", "plan", "--help"), + prepare=prepare_nothing, + expected_python_starts=1, + ), + PythonStartTestCase( + description="debug diagnostics", + args=("clean", "--debug"), + prepare=prepare_project, + expected_python_starts=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_invocation_needing_python_when_running_sqb_then_python_runs_it( + test_case: PythonStartTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = reset_directory(tmp_path / "workspace") + test_case.prepare(workspace) + python: LauncherRun = run(PYTHON_CLI, args=test_case.args, cwd=workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run(program, args=test_case.args, cwd=workspace) + + assert native[:2] == python[:2] + assert python_starts(marker) == test_case.expected_python_starts + + +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX signals and named pipes") +@pytest.mark.parametrize( + "test_case", + [ + InterruptTestCase( + description="rules list waiting on the project file", + args=("rules", "list"), + prepare=prepare_named_pipe_project, + expected_exit_code=130, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_python( + test_case: InterruptTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + project_dir: Path = reset_directory(tmp_path / "project") + test_case.prepare(project_dir) + process: subprocess.Popen[bytes] = subprocess.Popen( + [str(program), *test_case.args], + cwd=project_dir, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=restore_default_interrupt, + ) + time.sleep(INTERRUPT_DELAY_SECONDS) + assert process.poll() is None, "sqb returned before reading the project file" + + process.send_signal(signal.SIGINT) + _, stderr = process.communicate(timeout=INTERRUPT_TIMEOUT_SECONDS) + + assert process.returncode == test_case.expected_exit_code, stderr + assert python_starts(marker) == 0 + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 986b63d4c8a0a594a755b7a841f73da94311af8b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:28:38 +0100 Subject: [PATCH 05/17] docs: note that Recall does not record commands sqb answers without Python --- .../references/docs/concepts/observability/recall.md | 5 +++-- .../src/content/docs/docs/concepts/observability/recall.mdx | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index 2b92ac0dcb..b481c1fbcf 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -41,8 +41,9 @@ of its history still succeeds, prints a warning followed by its own final status status `abandoned`. Every command is recorded except `sqb recall` itself, commands that create a project (such as -`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not -recorded either. +`sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not +recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, +`sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 69a1328e64..2511e7fb0f 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -26,8 +26,9 @@ of its history still succeeds, prints a warning followed by its own final status status `abandoned`. Every command is recorded except `sqb recall` itself, commands that create a project (such as -`sqb init`), and raw-query `sqb diff`. A `sqb compile` answered from the launcher's warm cache is not -recorded either. +`sqb init`), and raw-query `sqb diff`. Commands `sqb` answers without starting Python are not +recorded either: a `sqb compile` answered from the launcher's warm cache, and `sqb clean`, `sqb skills`, +`sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store can contain sensitive SQL and output. From 2ca8a36757fa64e0a0019528a3b666ce3bda9f31 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:19:08 +0100 Subject: [PATCH 06/17] fix: resolve native project paths and the home directory as Python does on Windows --- .../src/native_commands/_helpers/paths.rs | 33 +++++++++++++++++-- .../src/native_commands/_helpers/skills.rs | 2 +- .../src/native_commands/constants.rs | 4 +++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs index c50eea5af9..302e8a5a66 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/paths.rs @@ -2,6 +2,8 @@ use std::path::{Component, Path, PathBuf}; +use sqlbuild_rules::project_tree::main::project_real_path::project_real_path; + use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::ProjectLocation; @@ -24,8 +26,8 @@ pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { } else { cwd.join(path) }; - if let Ok(real) = std::fs::canonicalize(&absolute) { - return real; + if absolute.exists() { + return project_real_path(&absolute); } let mut resolved: PathBuf = PathBuf::new(); for component in absolute.components() { @@ -37,7 +39,11 @@ pub(crate) fn resolved_path(path: &Path, cwd: &Path) -> PathBuf { } Component::Normal(name) => { let candidate: PathBuf = resolved.join(name); - resolved = std::fs::canonicalize(&candidate).unwrap_or(candidate); + resolved = if candidate.exists() { + project_real_path(&candidate) + } else { + candidate + }; } } } @@ -52,5 +58,26 @@ pub(crate) fn project_location(given: Option<&Path>) -> Result cwd.clone(), }; let resolved: PathBuf = resolved_path(&given, &cwd); + if !spelled_as_given(&resolved, &cwd.join(&given)) { + return Err(PythonNeeded::Installation); + } Ok(ProjectLocation { given, resolved }) } + +/// Links resolve as in Python on Unix; Windows drive substitutions and mappings stay Python's. +#[cfg(unix)] +fn spelled_as_given(_resolved: &Path, _absolute: &Path) -> bool { + true +} + +/// Whether the resolved path is the absolute path as given, apart from letter case. +#[cfg(not(unix))] +fn spelled_as_given(resolved: &Path, absolute: &Path) -> bool { + let absolute: PathBuf = absolute + .components() + .filter(|component| *component != Component::CurDir) + .collect(); + resolved + .to_string_lossy() + .eq_ignore_ascii_case(&absolute.to_string_lossy()) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs index 4272d40185..ac7eee9a36 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/skills.rs @@ -10,7 +10,7 @@ use crate::native_commands::constants::{HOME_VARIABLE, SKILL_FILES_HEADER}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, InstalledPackage, SkillsOptions}; -/// `Path.home()` from `HOME`, unless only Python's password lookup could tell. +/// `Path.home()` from `HOME` (`USERPROFILE` on Windows), unless only Python could tell. pub(crate) fn home_dir() -> Result { std::env::var_os(HOME_VARIABLE) .filter(|home| !home.is_empty()) diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 9c01a84a6b..fdef90a8cf 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -27,7 +27,11 @@ pub(crate) const SITE_DIRECTORIES: [&str; 2] = ["site-packages", "dist-packages" pub(crate) const WINDOWS_SITE_DIRECTORY: [&str; 2] = ["Lib", "site-packages"]; /// The packaged skill beneath the `sqlbuild` package directory. pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"]; +/// The variable `Path.home()` reads first; without it `sqb` defers to Python's other lookups. +#[cfg(not(windows))] pub(crate) const HOME_VARIABLE: &str = "HOME"; +#[cfg(windows)] +pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; /// The exit status of a command stopped by Ctrl+C. #[cfg(unix)] pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; From 8ccb01fc426492287a993b1d466440bf78775801 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:19:27 +0100 Subject: [PATCH 07/17] fix: report lost stdout, Ctrl+C, init failures and the version as the Python CLI does --- crates/sqlbuild-cli/Cargo.toml | 4 +- .../src/compute_logs/constants.rs | 5 + .../src/compute_logs/main/abandon_capture.rs | 72 ++++++++++++ .../sqlbuild-cli/src/compute_logs/main/mod.rs | 1 + .../src/launcher/_helpers/messages.rs | 11 +- crates/sqlbuild-cli/src/launcher/constants.rs | 2 + .../src/native_commands/_helpers/argv_host.rs | 3 +- .../src/native_commands/_helpers/execution.rs | 43 ++++--- .../src/native_commands/_helpers/init.rs | 43 ++++--- .../src/native_commands/_helpers/terminal.rs | 108 ++++++++++++++---- .../src/native_commands/constants.rs | 11 +- .../main/run_native_invocation.rs | 20 ++-- .../src/native_commands/models.rs | 11 ++ .../sqlbuild-cli/src/native_commands/tests.rs | 4 + .../src/native_commands/tests/test_init.rs | 56 +++++++++ .../native_commands/tests/test_terminal.rs | 75 ++++++++++++ .../src/native_commands/tests/test_types.rs | 15 +++ .../src/sqlbuild/cli/entry/_test_types.py | 11 ++ .../src/sqlbuild/cli/entry/helpers.py | 25 +++- .../cli/entry/test_python_free_commands.py | 63 +++++++++- 20 files changed, 515 insertions(+), 68 deletions(-) create mode 100644 crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_init.rs create mode 100644 crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs diff --git a/crates/sqlbuild-cli/Cargo.toml b/crates/sqlbuild-cli/Cargo.toml index cdfce87880..791fafe765 100644 --- a/crates/sqlbuild-cli/Cargo.toml +++ b/crates/sqlbuild-cli/Cargo.toml @@ -17,7 +17,6 @@ path = "src/bin/sqlbuild.rs" [dependencies] blake3.workspace = true -ctrlc.workspace = true getrandom.workspace = true serde.workspace = true serde_json.workspace = true @@ -29,6 +28,9 @@ sqlbuild-rules.workspace = true [target.'cfg(unix)'.dependencies] rustix.workspace = true +[target.'cfg(windows)'.dependencies] +ctrlc.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/sqlbuild-cli/src/compute_logs/constants.rs b/crates/sqlbuild-cli/src/compute_logs/constants.rs index ba6664c9b8..44dfb4738b 100644 --- a/crates/sqlbuild-cli/src/compute_logs/constants.rs +++ b/crates/sqlbuild-cli/src/compute_logs/constants.rs @@ -33,3 +33,8 @@ pub const DEFAULT_LOG_DIRECTORY: &str = "logs"; pub const MONTHS_PER_YEAR: i64 = 12; pub const MAX_DAY_OF_MONTH: i64 = 31; pub const HOURS_PER_DAY: i64 = 24; +/// The debug record Python's capture logs when its streams could not be finalized. +pub const CAPTURE_FAILURE_LOGGER: &str = "sqlbuild.cli.compute_logs"; +pub const CAPTURE_FAILURE_MESSAGE: &str = "local compute log capture unavailable"; +pub const DIAGNOSTIC_PRODUCER: &str = "sqlbuild"; +pub const DIAGNOSTIC_SCHEMA_VERSION: i64 = 1; diff --git a/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs b/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs new file mode 100644 index 0000000000..1697a7896b --- /dev/null +++ b/crates/sqlbuild-cli/src/compute_logs/main/abandon_capture.rs @@ -0,0 +1,72 @@ +//! Leave a capture incomplete after its final flush failed, as Python's capture cleanup does. + +use std::io::{self, Write}; +use std::time::SystemTime; + +use sqlbuild_core::json::main::dumps::dumps; +use sqlbuild_core::json::models::{JsonDialect, JsonInteger, JsonValue, OrjsonOptions}; + +use crate::compute_logs::_helpers::timestamps::{epoch_microseconds, utc_timestamp}; +use crate::compute_logs::constants::{ + CAPTURE_FAILURE_LOGGER, CAPTURE_FAILURE_MESSAGE, DIAGNOSTIC_PRODUCER, DIAGNOSTIC_SCHEMA_VERSION, +}; +use crate::compute_logs::models::{CaptureEnd, OpenCapture}; + +fn text(value: &str) -> JsonValue { + JsonValue::String(value.to_owned()) +} + +/// The debug record `log_compute_capture_failure` writes to the capture's diagnostics. +fn capture_failure_record(invocation_id: &str, occurred_at: SystemTime) -> String { + let record: JsonValue = JsonValue::Object(vec![ + ("fields".to_owned(), JsonValue::Object(Vec::new())), + ("invocation_id".to_owned(), text(invocation_id)), + ("log_stream_id".to_owned(), JsonValue::Null), + ("logger".to_owned(), text(CAPTURE_FAILURE_LOGGER)), + ("message".to_owned(), text(CAPTURE_FAILURE_MESSAGE)), + ( + "occurred_at".to_owned(), + text(&utc_timestamp(epoch_microseconds(occurred_at))), + ), + ("operation_id".to_owned(), JsonValue::Null), + ("producer".to_owned(), text(DIAGNOSTIC_PRODUCER)), + ( + "producer_version".to_owned(), + text(env!("CARGO_PKG_VERSION")), + ), + ("resource_attempt_id".to_owned(), JsonValue::Null), + ("resource_id".to_owned(), JsonValue::Null), + ("run_id".to_owned(), JsonValue::Null), + ( + "schema_version".to_owned(), + JsonValue::Integer(JsonInteger::from(DIAGNOSTIC_SCHEMA_VERSION)), + ), + ("severity".to_owned(), text("debug")), + ("source".to_owned(), text("python_logging")), + ("statement_id".to_owned(), JsonValue::Null), + ]); + let options: OrjsonOptions = OrjsonOptions { + sort_keys: true, + ..OrjsonOptions::default() + }; + dumps(&record, &JsonDialect::Orjson(options)).unwrap_or_default() + "\n" +} + +/// Append the streams the command wrote and the capture failure, without completing it. +pub fn abandon_capture(capture: OpenCapture, end: &CaptureEnd) -> io::Result<()> { + let OpenCapture { + invocation_id, + mut streams, + .. + } = capture; + let [stdout, stderr, diagnostics] = streams.as_mut_slice() else { + return Err(io::Error::other("capture streams are not open")); + }; + stdout.write_all(&end.stdout)?; + stderr.write_all(&end.stderr)?; + diagnostics.write_all(capture_failure_record(&invocation_id, SystemTime::now()).as_bytes())?; + for stream in [stdout, stderr, diagnostics] { + stream.flush()?; + } + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs index 8e44c3eed2..60606a3a12 100644 --- a/crates/sqlbuild-cli/src/compute_logs/main/mod.rs +++ b/crates/sqlbuild-cli/src/compute_logs/main/mod.rs @@ -1,3 +1,4 @@ +pub mod abandon_capture; pub mod capture_inventory; pub mod capture_is_complete; pub mod check_completion; diff --git a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs index 8a91e6387d..6a9994f629 100644 --- a/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs +++ b/crates/sqlbuild-cli/src/launcher/_helpers/messages.rs @@ -3,11 +3,16 @@ use std::io::{self, Write}; use std::path::Path; -use crate::launcher::constants::{PROGRAM_NAME, PYTHON_CLI_SCRIPT}; +use crate::launcher::constants::{PROGRAM_NAME, PROGRAM_VERSION, PYTHON_CLI_SCRIPT}; -/// argparse's `--version` line: `sqb ` on stdout. +/// argparse's `--version` line: `sqb `. +pub(crate) fn version_line() -> String { + format!("{PROGRAM_NAME} {PROGRAM_VERSION}\n") +} + +/// Print the `--version` line on stdout. pub(crate) fn print_version() { - let _ = writeln!(io::stdout(), "{PROGRAM_NAME} {}", env!("CARGO_PKG_VERSION")); + let _ = io::stdout().write_all(version_line().as_bytes()); } /// The Python CLI script is not installed beside this program. diff --git a/crates/sqlbuild-cli/src/launcher/constants.rs b/crates/sqlbuild-cli/src/launcher/constants.rs index 82cad78763..ce9a8716eb 100644 --- a/crates/sqlbuild-cli/src/launcher/constants.rs +++ b/crates/sqlbuild-cli/src/launcher/constants.rs @@ -11,6 +11,8 @@ pub const LAUNCH_RECORD_OPTION: &str = "--sqlbuild-launch-record="; /// The program name argparse reports, fixed by the Python parser. pub const PROGRAM_NAME: &str = "sqb"; pub const VERSION_OPTIONS: [&str; 2] = ["--version", "-V"]; +/// The version `--version` prints, alone or among other options; the wheels share it. +pub const PROGRAM_VERSION: &str = env!("CARGO_PKG_VERSION"); /// Global options naming the project directory, exactly as the Python parser spells them. pub const PROJECT_DIR_OPTIONS: [&str; 2] = ["--project-dir", "--sqb-project-dir"]; /// Prefixes argparse could expand to a project directory option. diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs index 9e26816a90..4a7f8b62de 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/argv_host.rs @@ -7,6 +7,7 @@ use sqlbuild_core::text::main::python_str_repr::python_str_repr; use crate::argv::main::cli_namespace_defaults::cli_namespace_defaults; use crate::argv::models::{NamespaceDefault, TypeFailure, Value}; use crate::argv::types::ArgvHost; +use crate::launcher::constants::PROGRAM_VERSION; use crate::native_commands::constants::{ DIGIT_SEPARATOR, INT_ASCII_SPACES, INT_CONVERTER, NEGATIVE_SIGN, POSITIVE_INTEGER_CONVERTER, POSITIVE_INTEGER_MESSAGE, POSITIVE_SIGN, PYTHON_INT_MAX_STR_DIGITS, ZERO_TEXT, @@ -93,7 +94,7 @@ impl ArgvHost for NativeArgvHost { } fn program_version(&mut self) -> String { - env!("CARGO_PKG_VERSION").to_owned() + PROGRAM_VERSION.to_owned() } } diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 278a397f81..7021116ad0 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -6,6 +6,7 @@ use std::time::SystemTime; use crate::argv::models::Stream; use crate::compute_logs::constants::DEFAULT_RETENTION_COUNT; +use crate::compute_logs::main::abandon_capture::abandon_capture; use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; @@ -16,11 +17,11 @@ use crate::native_commands::_helpers::paths::project_location; use crate::native_commands::_helpers::project_facts::has_no_runtime_extensions; use crate::native_commands::_helpers::rules::run_rules; use crate::native_commands::_helpers::skills::{run_skills, skill_freshness_notice}; -use crate::native_commands::_helpers::terminal::{exit_on_interrupt, write_text}; -use crate::native_commands::constants::UNKNOWN_COMMAND; +use crate::native_commands::_helpers::terminal::{finish_lost_stdout, write_output}; +use crate::native_commands::constants::{UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, NativeCommand, NativeInvocation, ProjectLocation, + CommandOutput, InstalledPackage, LostStdout, NativeCommand, NativeInvocation, ProjectLocation, }; /// The capture Python's `dispatch_with_compute_logs` opens; a failure only loses the log. @@ -44,6 +45,19 @@ fn open_capture( }) } +/// Close the capture the way Python's does once the command's output met its reader. +fn close_capture(capture: OpenCapture, mut end: CaptureEnd, lost: Option<&LostStdout>) { + match lost { + None => drop(finish_capture(capture, &end)), + Some(LostStdout::Raised(_)) => { + end.stdout.clear(); + end.exit_code = i64::from(UNCAUGHT_EXCEPTION_STATUS); + drop(finish_capture(capture, &end)); + } + Some(LostStdout::AtExit(_) | LostStdout::Dropped) => drop(abandon_capture(capture, &end)), + } +} + /// Print an observed command's output after the skill notice, retaining both in its capture. fn finish_observed( invocation: &NativeInvocation, @@ -58,20 +72,22 @@ fn finish_observed( }; let capture: io::Result = open_capture(invocation, project, started_at); let stderr: String = notice + &output.stderr; - write_text(Stream::Stderr, &stderr); - write_text(Stream::Stdout, &output.stdout); + let lost: Option = write_output(&[ + (Stream::Stderr, stderr.as_str()), + (Stream::Stdout, output.stdout.as_str()), + ]); if let Ok(capture) = capture { - let finished: io::Result<()> = finish_capture( + close_capture( capture, - &CaptureEnd { + CaptureEnd { stdout: output.stdout.into_bytes(), stderr: stderr.into_bytes(), exit_code: i64::from(output.exit_code), }, + lost.as_ref(), ); - drop(finished); } - output.exit_code + finish_lost_stdout(lost.as_ref(), output.exit_code) } /// What an observed command prints, unless it needs Python after all. @@ -97,14 +113,15 @@ pub(crate) fn run_command( python_cli: &Path, error_color: bool, ) -> Result { - exit_on_interrupt(); let package: InstalledPackage = installed_package(python_cli)?; let project: ProjectLocation = project_location(invocation.project_dir.as_deref())?; if invocation.creates_project { let output: CommandOutput = run_init(&project.resolved, &package, error_color)?; - write_text(Stream::Stderr, &output.stderr); - write_text(Stream::Stdout, &output.stdout); - return Ok(output.exit_code); + let lost: Option = write_output(&[ + (Stream::Stderr, output.stderr.as_str()), + (Stream::Stdout, output.stdout.as_str()), + ]); + return Ok(finish_lost_stdout(lost.as_ref(), output.exit_code)); } let started_at: SystemTime = SystemTime::now(); let output: CommandOutput = observed_output(&invocation.command, &project, &package)?; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs index dfee4cd22b..ec54a0f30a 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/init.rs @@ -17,7 +17,7 @@ use sqlbuild_core::init_command::main::scaffold_blank_project::scaffold_blank_pr use crate::dispatch::main::failure_report::failure_report; use crate::dispatch::models::CommandFailure; use crate::native_commands::_helpers::skills::home_dir; -use crate::native_commands::_helpers::terminal::stdout_supports_color; +use crate::native_commands::_helpers::terminal::{error_description, stdout_supports_color}; use crate::native_commands::constants::{INIT_FAILURE_PREFIX, USER_ERROR_CLASS}; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::{CommandOutput, InstalledPackage}; @@ -31,15 +31,31 @@ fn failed(stderr: String) -> Result { }) } -/// The project's skills could not be installed: what the Python CLI prints for its error. -fn skills_failure(error: &WorkspaceError, use_color: bool) -> Result { - let WorkspaceError::User { - code, - message, - help, - } = error - else { - return failed(format!("{INIT_FAILURE_PREFIX}{error:?}\n")); +/// The project's skills could not be installed: what the Python CLI prints for its user error. +fn skills_failure( + base_dir: &Path, + error: &WorkspaceError, + use_color: bool, +) -> Result { + let (code, message, help) = match error { + WorkspaceError::User { + code, + message, + help, + } => (code, message, help), + WorkspaceError::Decode { path, .. } => { + return failed(format!( + "{INIT_FAILURE_PREFIX}cannot install SQLBuild skills: {} is not UTF-8 text\n", + path.display() + )); + } + WorkspaceError::Io(error) => { + return failed(format!( + "{INIT_FAILURE_PREFIX}cannot install SQLBuild skills into {}: {}\n", + base_dir.display(), + error_description(error) + )); + } }; let (_, log_message) = failure_report(CommandFailure::User); let coded: String = format_coded_error( @@ -75,8 +91,9 @@ pub(crate) fn run_init( return Err(PythonNeeded::Failure); } return failed(format!( - "{INIT_FAILURE_PREFIX}{error}: {}\n", - path.display() + "{INIT_FAILURE_PREFIX}cannot create {}: {}\n", + path.display(), + error_description(&error) )); } if let Err(error) = install_skills(&SkillInstallRequest { @@ -87,7 +104,7 @@ pub(crate) fn run_init( requested_targets: Vec::new(), force: false, }) { - return skills_failure(&error, error_color); + return skills_failure(base_dir, &error, error_color); } Ok(CommandOutput { stdout: render_init_summary( diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs index 3a584e7abd..44d55e4570 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -1,11 +1,15 @@ -//! The process's terminal: its width, colour support, stream writes and Ctrl+C. +//! The process's terminal: its width, colour support and stream writes. use std::io::{self, IsTerminal, Write}; use crate::argv::models::Stream; use crate::native_commands::_helpers::argv_host::python_int; -use crate::native_commands::constants::{COLUMNS_VARIABLE, FALLBACK_COLUMNS, NO_COLOR_VARIABLE}; +use crate::native_commands::constants::{ + BROKEN_PIPE_EXCEPTION, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, EXIT_FLUSH_FAILURE_STATUS, + FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PYTHON_STDOUT_BUFFER_SIZE, UNCAUGHT_EXCEPTION_STATUS, +}; use crate::native_commands::errors::PythonNeeded; +use crate::native_commands::models::LostStdout; /// `int(os.environ["COLUMNS"])`, `0` when unset or not an integer; beyond 64 bits is Python's. fn columns_variable() -> Result { @@ -59,35 +63,93 @@ fn console_bytes(text: &str) -> Vec { } } -/// Write and flush one text; a closed or full stream drops it, as the parse's writes do. -pub(crate) fn write_text(stream: Stream, text: &str) { - if text.is_empty() { - return; - } - let bytes: Vec = console_bytes(text); - let written: io::Result<()> = match stream { +fn write_stream(stream: Stream, bytes: &[u8]) -> io::Result<()> { + match stream { Stream::Stdout => { let mut output = io::stdout().lock(); - output.write_all(&bytes).and_then(|()| output.flush()) + output.write_all(bytes).and_then(|()| output.flush()) } Stream::Stderr => { let mut output = io::stderr().lock(); - output.write_all(&bytes).and_then(|()| output.flush()) + output.write_all(bytes).and_then(|()| output.flush()) } - }; - drop(written); + } } -/// End the process with status 130 on Ctrl+C; handing off by `exec` restores the default. -#[cfg(unix)] -pub(crate) fn exit_on_interrupt() { - use crate::native_commands::constants::INTERRUPTED_EXIT_CODE; +/// The system's description of an I/O failure, without Rust's `(os error N)` suffix. +pub(crate) fn error_description(error: &io::Error) -> String { + let described: String = error.to_string(); + match error.raw_os_error() { + Some(errno) => described + .strip_suffix(&format!(" (os error {errno})")) + .unwrap_or(&described) + .to_owned(), + None => described, + } +} - drop(ctrlc::set_handler(|| { - std::process::exit(INTERRUPTED_EXIT_CODE) - })); +/// The last line of the traceback Python prints for a failed write. +fn exception_line(error: &io::Error) -> String { + if error.kind() == io::ErrorKind::BrokenPipe { + return BROKEN_PIPE_EXCEPTION.to_owned(); + } + match error.raw_os_error() { + Some(errno) => format!("OSError: [Errno {errno}] {}", error_description(error)), + None => format!("OSError: {error}"), + } } -/// The Python child waited for alone handles Ctrl+C, so the native command keeps the default. -#[cfg(not(unix))] -pub(crate) fn exit_on_interrupt() {} +/// CPython's 8 KiB stdout buffer keeps up to half until exit, drops more, and fails a full write. +pub(crate) fn lost_stdout(total: usize, error: &io::Error) -> LostStdout { + if total >= PYTHON_STDOUT_BUFFER_SIZE { + LostStdout::Raised(exception_line(error)) + } else if total <= PYTHON_STDOUT_BUFFER_SIZE / 2 { + LostStdout::AtExit(exception_line(error)) + } else { + LostStdout::Dropped + } +} + +/// Write each text in order and flush it; errors on stderr are ignored, as Python's are. +pub(crate) fn write_output(writes: &[(Stream, &str)]) -> Option { + let encoded: Vec<(Stream, Vec)> = writes + .iter() + .filter(|(_, text)| !text.is_empty()) + .map(|(stream, text)| (*stream, console_bytes(text))) + .collect(); + let total: usize = encoded + .iter() + .filter(|(stream, _)| *stream == Stream::Stdout) + .map(|(_, bytes)| bytes.len()) + .sum(); + let mut lost: Option = None; + for (stream, bytes) in &encoded { + let written: io::Result<()> = match (stream, &lost) { + (Stream::Stdout, Some(_)) => Ok(()), + _ => write_stream(*stream, bytes), + }; + if let (Stream::Stdout, Err(error)) = (stream, written) { + lost = Some(lost_stdout(total, &error)); + } + } + lost +} + +/// Report lost stdout as Python does and return the exit status that results. +pub(crate) fn finish_lost_stdout(lost: Option<&LostStdout>, exit_code: i32) -> i32 { + match lost { + None | Some(LostStdout::Dropped) => exit_code, + Some(LostStdout::AtExit(exception)) => { + let text: String = format!("{EXIT_FLUSH_FAILURE}\n{exception}\n"); + drop(write_stream(Stream::Stderr, &console_bytes(&text))); + EXIT_FLUSH_FAILURE_STATUS + } + Some(LostStdout::Raised(exception)) => { + drop(write_stream( + Stream::Stderr, + &console_bytes(&format!("{exception}\n")), + )); + UNCAUGHT_EXCEPTION_STATUS + } + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index fdef90a8cf..5b1dd8ff84 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -32,9 +32,6 @@ pub(crate) const SKILLS_DIRECTORY: [&str; 3] = [".agents", "skills", "sqlbuild"] pub(crate) const HOME_VARIABLE: &str = "HOME"; #[cfg(windows)] pub(crate) const HOME_VARIABLE: &str = "USERPROFILE"; -/// The exit status of a command stopped by Ctrl+C. -#[cfg(unix)] -pub(crate) const INTERRUPTED_EXIT_CODE: i32 = 130; /// The command name compute logs record for a bare `sqb`. pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; /// Commands with no handler: observed, then successful. @@ -77,3 +74,11 @@ pub(crate) const PYTHON_INT_MAX_STR_DIGITS: usize = 4300; pub(crate) const DIGIT_SEPARATOR: char = '_'; /// Where a loaded project configuration names its lifecycle sinks. pub(crate) const LIFECYCLE_SINK_PATH: [&str; 3] = ["sinks", "lifecycle", "named"]; +/// CPython's default buffer size for a piped stdout. +pub(crate) const PYTHON_STDOUT_BUFFER_SIZE: usize = 8192; +pub(crate) const BROKEN_PIPE_EXCEPTION: &str = "BrokenPipeError: [Errno 32] Broken pipe"; +/// What Python prints when the final flush of a lost stdout fails, and the status it exits with. +pub(crate) const EXIT_FLUSH_FAILURE: &str = + "Exception ignored in: <_io.TextIOWrapper name='' mode='w' encoding='utf-8'>"; +pub(crate) const EXIT_FLUSH_FAILURE_STATUS: i32 = 120; +pub(crate) const UNCAUGHT_EXCEPTION_STATUS: i32 = 1; diff --git a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs index 222d953909..1233e99081 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/run_native_invocation.rs @@ -4,16 +4,16 @@ use std::ffi::OsString; use std::path::Path; use crate::argv::main::parse_cli_arguments::parse_cli_arguments; -use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace}; +use crate::argv::models::{ArgvOutcome, ArgvSettings, Namespace, Stream}; use crate::native_commands::_helpers::argv_host::{NativeArgvHost, namespace_defaults}; use crate::native_commands::_helpers::decision::native_invocation; use crate::native_commands::_helpers::execution::run_command; use crate::native_commands::_helpers::terminal::{ - stdout_supports_color, terminal_columns, write_text, + finish_lost_stdout, stdout_supports_color, terminal_columns, write_output, }; use crate::native_commands::constants::NO_COLOR_OPTION; use crate::native_commands::errors::PythonNeeded; -use crate::native_commands::models::NativeInvocation; +use crate::native_commands::models::{LostStdout, NativeInvocation}; /// The exit status of an invocation `sqb` served, or why the same arguments go to Python. pub fn run_native_invocation( @@ -39,10 +39,16 @@ pub fn run_native_invocation( } let namespace: Namespace = match outcome.parsed { Err(code) => { - for (stream, text) in &outcome.writes { - write_text(*stream, text); - } - return Ok(i32::try_from(code).unwrap_or(1)); + let writes: Vec<(Stream, &str)> = outcome + .writes + .iter() + .map(|(stream, text)| (*stream, text.as_str())) + .collect(); + let lost: Option = write_output(&writes); + return Ok(finish_lost_stdout( + lost.as_ref(), + i32::try_from(code).unwrap_or(1), + )); } Ok(namespace) if outcome.writes.is_empty() => namespace, Ok(_) => return Err(PythonNeeded::Arguments), diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index f0ce868fa2..9c6ece8ad5 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -71,3 +71,14 @@ pub(crate) struct ProjectLocation { /// `Path(--project-dir).resolve()`, or `Path.cwd()`. pub(crate) resolved: PathBuf, } + +/// What Python's buffered stdout does with output its reader no longer takes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum LostStdout { + /// Kept until exit, whose failed flush Python reports with this exception and status 120. + AtExit(String), + /// Dropped by a failed flush; the command's own status stands. + Dropped, + /// Raised by the write: an uncaught exception, with this last line and status 1. + Raised(String), +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests.rs b/crates/sqlbuild-cli/src/native_commands/tests.rs index 9119c141c9..2f86f371e1 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests.rs @@ -4,11 +4,15 @@ mod argv_host; mod decision; #[path = "tests/helpers.rs"] mod helpers; +#[path = "tests/test_init.rs"] +mod init; #[path = "tests/test_paths.rs"] mod paths; #[path = "tests/test_project_facts.rs"] mod project_facts; #[path = "tests/test_rules.rs"] mod rules; +#[path = "tests/test_terminal.rs"] +mod terminal; #[path = "tests/test_types.rs"] mod test_types; diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs new file mode 100644 index 0000000000..29d90d5f7f --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_init.rs @@ -0,0 +1,56 @@ +use std::path::Path; + +use crate::native_commands::_helpers::init::run_init; +use crate::native_commands::models::InstalledPackage; +use crate::native_commands::tests::helpers::write_tree; +use crate::native_commands::tests::test_types::InitFailureTestCase; + +#[cfg(unix)] +#[test] +fn given_unwritable_skill_directory_when_initialising_then_a_readable_error_names_the_path() { + let test_cases = [InitFailureTestCase { + description: "a file where the agents skill directory goes", + blocking_file: ".agents", + expected_stderr_start: "sqb init: cannot install SQLBuild skills into ", + expected_exit_code: 1, + }]; + + for test_case in test_cases { + let root = tempfile::tempdir().expect("temporary directory"); + let skills = root.path().join("packaged"); + write_tree( + root.path(), + &["packaged", "shop"], + &[("packaged/SKILL.md", "# Skill\n")], + ); + let project: &Path = &root.path().join("shop"); + write_tree( + project, + &[], + &[(test_case.blocking_file, "not a directory\n")], + ); + + let output = run_init( + project, + &InstalledPackage { + skills_root: skills, + }, + false, + ) + .expect(test_case.description); + + assert!( + output.stderr.starts_with(test_case.expected_stderr_start) + && output.stderr.contains(&project.display().to_string()) + && output.stderr.ends_with("Not a directory\n"), + "{}: {}", + test_case.description, + output.stderr + ); + assert_eq!( + output.exit_code, test_case.expected_exit_code, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs new file mode 100644 index 0000000000..19be6f26cd --- /dev/null +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_terminal.rs @@ -0,0 +1,75 @@ +use std::io; + +use crate::argv::models::Stream; +use crate::launcher::constants::{PROGRAM_NAME, PROGRAM_VERSION}; +use crate::native_commands::_helpers::terminal::lost_stdout; +use crate::native_commands::models::LostStdout; +use crate::native_commands::tests::helpers::parse; +use crate::native_commands::tests::test_types::{LostStdoutTestCase, ParseTestCase}; + +/// Thresholds recorded from CPython 3.12 writing to a pipe without a reader. +#[test] +fn given_lost_stdout_bytes_when_classifying_then_python_buffer_outcome_matches() { + let test_cases = [ + LostStdoutTestCase { + description: "half the buffer stays until the final flush", + total: 4096, + expected_lost: LostStdout::AtExit("BrokenPipeError: [Errno 32] Broken pipe".to_owned()), + }, + LostStdoutTestCase { + description: "more than half is dropped by the failed flush", + total: 4097, + expected_lost: LostStdout::Dropped, + }, + LostStdoutTestCase { + description: "a full buffer fails the write", + total: 8192, + expected_lost: LostStdout::Raised("BrokenPipeError: [Errno 32] Broken pipe".to_owned()), + }, + ]; + + for test_case in test_cases { + let error: io::Error = io::Error::from(io::ErrorKind::BrokenPipe); + + assert_eq!( + lost_stdout(test_case.total, &error), + test_case.expected_lost, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_version_among_options_when_parsing_natively_then_the_lone_version_line_prints() { + let test_cases = [ + ParseTestCase { + description: "after --no-color", + argv: &["--no-color", "--version"], + expected_needs_python: false, + }, + ParseTestCase { + description: "the short option after --debug", + argv: &["--debug", "-V"], + expected_needs_python: false, + }, + ]; + + for test_case in test_cases { + let (outcome, host) = parse(test_case.argv); + + assert_eq!( + (outcome.writes, outcome.parsed, host.needs_python), + ( + vec![( + Stream::Stdout, + format!("{PROGRAM_NAME} {PROGRAM_VERSION}\n") + )], + Err(0), + test_case.expected_needs_python, + ), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 528aeaf4e4..252ef24b95 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -66,3 +66,18 @@ pub(super) fn output_start(output: &CommandOutput, length: usize) -> (String, i3 output.exit_code, ) } + +/// Bytes a reader no longer takes and what Python's stdout buffer makes of them. +pub(super) struct LostStdoutTestCase { + pub(super) description: &'static str, + pub(super) total: usize, + pub(super) expected_lost: crate::native_commands::models::LostStdout, +} + +/// A directory `init` meets once the project file exists, and the start of its error. +pub(super) struct InitFailureTestCase { + pub(super) description: &'static str, + pub(super) blocking_file: &'static str, + pub(super) expected_stderr_start: &'static str, + pub(super) expected_exit_code: i32, +} diff --git a/tests/integration/src/sqlbuild/cli/entry/_test_types.py b/tests/integration/src/sqlbuild/cli/entry/_test_types.py index a39ae9398f..5be5472a2a 100644 --- a/tests/integration/src/sqlbuild/cli/entry/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/entry/_test_types.py @@ -113,3 +113,14 @@ class InterruptTestCase: args: tuple[str, ...] prepare: Callable[[Path], None] expected_exit_code: int + + +@dataclass(frozen=True) +class BrokenPipeTestCase: + """One Python-free command line whose stdout has no reader.""" + + description: str + args: tuple[str, ...] + prepare: Callable[[Path], None] + expected_exit_code: int + expected_stderr_end: bytes diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 09fbd33876..15825c8079 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -14,6 +14,7 @@ _ELAPSED_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d s\)") _PHASE_SECONDS: re.Pattern[bytes] = re.compile(rb"\(\d+\.\d+s\)") +_DIAGNOSTIC_IDENTITY: re.Pattern[bytes] = re.compile(rb'"(invocation_id|occurred_at)":"[^"]*"') NATIVE_SQB: Path = Path(sys.executable).with_name("sqb") PYTHON_CLI: Path = Path(sys.executable).with_name("sqlbuild-python") PROJECT_FILES: dict[str, str] = { @@ -227,9 +228,9 @@ def reset_directory(directory: Path) -> Path: def _capture_files(capture: Path) -> dict[str, object]: metadata: dict[str, object] = capture_metadata(capture) for volatile in ("invocation_id", "started_at", "completed_at", "capture_date"): - _ = metadata.pop(volatile) + _ = metadata.pop(volatile, None) streams: dict[str, bytes] = { - name: (capture / name).read_bytes() + name: _DIAGNOSTIC_IDENTITY.sub(rb'"\1":""', (capture / name).read_bytes()) for name in ("stdout.log", "stderr.log", "diagnostics.jsonl") } return {"complete": (capture / "complete").exists(), "metadata": metadata, **streams} @@ -291,3 +292,23 @@ def restore_default_interrupt() -> None: """Give a spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" _ = signal.signal(signal.SIGINT, signal.SIG_DFL) + + +def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> LauncherRun: + """Run with a stdout pipe whose reader has already closed; return exit code and stderr.""" + + reader, writer = os.pipe() + os.close(reader) + try: + completed: subprocess.CompletedProcess[bytes] = subprocess.run( + [str(program), *args], + cwd=cwd, + stdout=writer, + stderr=subprocess.PIPE, + env={**os.environ, "NO_COLOR": "1"}, + check=False, + timeout=600, + ) + finally: + os.close(writer) + return completed.returncode, b"", completed.stderr diff --git a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py index 6a787e7454..fe5412e49b 100644 --- a/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py +++ b/tests/integration/src/sqlbuild/cli/entry/test_python_free_commands.py @@ -7,6 +7,7 @@ import pytest from tests.integration.src.sqlbuild.cli.entry._test_types import ( + BrokenPipeTestCase, InterruptTestCase, PythonFreeTestCase, PythonStartTestCase, @@ -26,9 +27,11 @@ reset_directory, restore_default_interrupt, run, + run_without_reader, ) INTERRUPT_DELAY_SECONDS: float = 1.0 +TRACEBACK_LAST_LINE: bytes = b"BrokenPipeError: [Errno 32] Broken pipe\n" INTERRUPT_TIMEOUT_SECONDS: float = 30.0 @@ -299,12 +302,12 @@ def test_given_invocation_needing_python_when_running_sqb_then_python_runs_it( description="rules list waiting on the project file", args=("rules", "list"), prepare=prepare_named_pipe_project, - expected_exit_code=130, + expected_exit_code=-signal.SIGINT, ), ], ids=lambda case: case.description, ) -def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_python( +def test_given_python_free_command_when_interrupted_then_sqb_dies_by_sigint_without_python( test_case: InterruptTestCase, tmp_path: Path ) -> None: marker: Path = tmp_path / "python-starts.log" @@ -329,5 +332,61 @@ def test_given_python_free_command_when_interrupted_then_sqb_exits_130_without_p assert python_starts(marker) == 0 +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX pipes") +@pytest.mark.parametrize( + "test_case", + [ + BrokenPipeTestCase( + description="help kept until the final flush", + args=("--help",), + prepare=prepare_nothing, + expected_exit_code=120, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + BrokenPipeTestCase( + description="a short observed report kept until the final flush", + args=("rules", "show", "SQBRMODEL101"), + prepare=prepare_project, + expected_exit_code=120, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + BrokenPipeTestCase( + description="a report the failed flush drops", + args=("rules", "list"), + prepare=prepare_project, + expected_exit_code=0, + expected_stderr_end=b"", + ), + BrokenPipeTestCase( + description="a report whose write fails", + args=("rules", "--json", "list"), + prepare=prepare_project, + expected_exit_code=1, + expected_stderr_end=TRACEBACK_LAST_LINE, + ), + ], + ids=lambda case: case.description, +) +def test_given_stdout_without_reader_when_running_sqb_then_status_and_logs_match_python( + test_case: BrokenPipeTestCase, tmp_path: Path +) -> None: + marker: Path = tmp_path / "python-starts.log" + _ = marker.write_text("", encoding="utf-8") + program: Path = counting_environment(tmp_path / "environment", marker) + workspace: Path = reset_directory(tmp_path / "workspace") + test_case.prepare(workspace) + python: LauncherRun = run_without_reader(PYTHON_CLI, args=test_case.args, cwd=workspace) + python_logs: list[dict[str, object]] = compute_log_files(workspace) + test_case.prepare(reset_directory(workspace)) + + native: LauncherRun = run_without_reader(program, args=test_case.args, cwd=workspace) + + assert (native[0], python[0]) == (test_case.expected_exit_code,) * 2 + assert native[2].endswith(test_case.expected_stderr_end), native[2] + assert python[2].endswith(test_case.expected_stderr_end), python[2] + assert python_starts(marker) == 0 + assert compute_log_files(workspace) == python_logs + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From 273df0c2fa933982625451815e4288caf7181a18 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 08/17] chore(deps): bump fensu to 0.27 --- pyproject.toml | 2 +- uv.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f861ac0fc3..c26e3ebbf2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.23.0", + "fensu>=0.27.0", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/uv.lock b/uv.lock index 980908966c..10418364db 100644 --- a/uv.lock +++ b/uv.lock @@ -1345,31 +1345,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.23.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/f4/d037850c050343d2548b901a994a77783e7a02d352fe35aa2477644a185c/fensu-0.23.0.tar.gz", hash = "sha256:997c31c26e9b6ea88b2797898a5644a95ae543ba568fb5f3482efd2e17b54d61", size = 508009, upload-time = "2026-09-24T15:29:54.691Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/a3/d1eea20ab654f3c4b1b170cc8eeaf27d5092095cd74e6bc819bfff8d9312/fensu-0.27.0.tar.gz", hash = "sha256:023692a73c4ac2fdadbb167c59a58dc944c7a41b0d78a40fa260f6923a8b3e7b", size = 538895, upload-time = "2026-10-17T05:38:49.174Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/67/597b68a104ae906705e8fc27ec53f57eaad36d869a5323b71518aa77592d/fensu-0.23.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2ee20effc98db0d4e682c3f3a55b72ffca81386228c9092cab82b4bd044d1ddc", size = 4026956, upload-time = "2026-09-24T15:29:34.609Z" }, - { url = "https://files.pythonhosted.org/packages/13/0e/1e4583e15a49bc066530ea88faa7257caa131a41bf084c89eb9b34a944a7/fensu-0.23.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:39f85e019e2a66a1cfa280efc02b47740fee8270a6e4befef4885f46f8e03753", size = 3892516, upload-time = "2026-09-24T15:29:36.742Z" }, - { url = "https://files.pythonhosted.org/packages/b4/d6/a10db3b1c7f5fb667bd6e634f494d527c29637b2fb9bc8f68d13f0300482/fensu-0.23.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3002c67ba494386e1c6e94f5f2abbb4b355a174dce0ecf8ef71756c203b7d26a", size = 4119441, upload-time = "2026-09-24T15:29:38.636Z" }, - { url = "https://files.pythonhosted.org/packages/3e/a6/0ac54e58f3d0aec31b6ad6fa46bb8d3454540e7f6b066d6832880cf81fbb/fensu-0.23.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed8a57574977971b2f3c599b5beaac9bd905612aba89971b7b1697f790310e5", size = 4184904, upload-time = "2026-09-24T15:29:40.493Z" }, - { url = "https://files.pythonhosted.org/packages/99/57/7bb9c5c2b971c26bcbf1a1c10a54097925cbaff07f3c8bcb9aefad3d3865/fensu-0.23.0-cp312-abi3-win_amd64.whl", hash = "sha256:98d78a6582ea21fc456c18a31b7d01f3ff43b8ed687de9e51ef8497e54ee2514", size = 3864765, upload-time = "2026-09-24T15:29:42.311Z" }, + { url = "https://files.pythonhosted.org/packages/2c/b7/7b4053d22df1307dbaebb4ed48280bb9c3663ee649005eb6ab5e2fec28b8/fensu-0.27.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d982c4fec0d2210b78fcf67098c32ca9f8b8592f338b2a327fcd1747b59bb310", size = 4154209, upload-time = "2026-10-17T05:38:28.74Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/b7058d187e9908ec18f245c1b168c24b746633a0b13666d69052157da3eb/fensu-0.27.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c3cca4a4391028d7b07bb9ead34340f6ba0662c36ea4ef31f69b44a46847289", size = 4004244, upload-time = "2026-10-17T05:38:31.356Z" }, + { url = "https://files.pythonhosted.org/packages/f8/bc/b1eee94e617e954a2188c1802d3770b27624a97f775cbed310f54389c100/fensu-0.27.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5640ffe3cf8916b1533dfe51ca27d65bcea71c9fbd62ae5cfaa2079e8007bdaf", size = 4230195, upload-time = "2026-10-17T05:38:33.057Z" }, + { url = "https://files.pythonhosted.org/packages/7c/01/9f895e6a4427c8ee80ce5d779aceab5a2b994a156a4c1f3bf664108ab489/fensu-0.27.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d26d2713a69f2e05b07c6f778e1e2f86896e26b74247e60952018288381c04", size = 4307270, upload-time = "2026-10-17T05:38:34.827Z" }, + { url = "https://files.pythonhosted.org/packages/08/61/2a380a704a49eea91a3539935af7b2d4ad09aa1d87d787f87c1de847ae4b/fensu-0.27.0-cp312-abi3-win_amd64.whl", hash = "sha256:15a9aef82221f25812d9c4f74e782dd677c73d566980eaed60b762da13ee2b3d", size = 3977639, upload-time = "2026-10-17T05:38:36.612Z" }, ] [[package]] name = "fensu-cli" -version = "0.23.0" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4a/6f/48493231455bdaceeb637594765c15c46b59252b2526b5ee9478b0c0841c/fensu_cli-0.23.0.tar.gz", hash = "sha256:0a598d0013c662b4bc4e7785dc91f8d480665a06750c2114acbafe8de9ef3dba", size = 715137, upload-time = "2026-09-24T15:29:56.142Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/97/2540b6ea4bca959ac577502fe0c96b432c3b4e694daa1ca68df3a2a06b8a/fensu_cli-0.27.0.tar.gz", hash = "sha256:4276e7fcf7dddc4c36afc347536bf1fd7114fff246ad344544991a1edf7b4ee3", size = 760869, upload-time = "2026-10-17T05:38:50.61Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/ce/87ca84074f6ba2f278d2739505c6f6d1cde53b8b66085b373666873607a3/fensu_cli-0.23.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:4ccbf9ed3182344ef9e6fefe1d6d62f0770b825d18059bce1954347dc1d3615e", size = 7745194, upload-time = "2026-09-24T15:29:44.933Z" }, - { url = "https://files.pythonhosted.org/packages/c1/b1/21d23c39b2c8dd51e544b6546fada9bd317214a1e4a6f44b3351cc73d4be/fensu_cli-0.23.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8f8d1851c1711359ad2688a3aaa1717af8e48a2c51700a54c647082c3fe51e15", size = 7271582, upload-time = "2026-09-24T15:29:46.978Z" }, - { url = "https://files.pythonhosted.org/packages/aa/56/c190cdedf6e2afa67e58bd6cd6188853983f160e14b73494a08906e84869/fensu_cli-0.23.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d21e87ba99a9c600783c7107e2d06cce15cdce379921a8c996884c16b4c8a407", size = 7362814, upload-time = "2026-09-24T15:29:49.078Z" }, - { url = "https://files.pythonhosted.org/packages/74/13/54147ab05067512127c99df415b73667f878693bf4bd50eb1e03fcb84f76/fensu_cli-0.23.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:07ab456f0f8eb90a611e842e3ad1ddf44053b5eb5a5c69d4499f248bd21d01b9", size = 7795295, upload-time = "2026-09-24T15:29:50.828Z" }, - { url = "https://files.pythonhosted.org/packages/70/53/2c0a86d35cb37123dffd5644020cb5284169f83bd0571281e341896b8946/fensu_cli-0.23.0-py3-none-win_amd64.whl", hash = "sha256:8fa9948fd8ee64319a105f70085ddb790b4131bb5ff88636ae6d27ef51b14f57", size = 7490965, upload-time = "2026-09-24T15:29:53.097Z" }, + { url = "https://files.pythonhosted.org/packages/0b/69/9491487602985731af4a7363754970bb35baece4e27cf2a7a0a95ebe4ae7/fensu_cli-0.27.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e90c8d298904d02c639e87541ba51a31ea3912af0bc354cdfea9d95f4428fd41", size = 7935880, upload-time = "2026-10-17T05:38:38.606Z" }, + { url = "https://files.pythonhosted.org/packages/de/4e/7e84f824b3902708ce4c8399f3402fb278cba9c48ed37877d67a4bad5245/fensu_cli-0.27.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1a9f3a2fb8ec8930af9b5cd59f3f1c027cd80a1ebae1a22c7d07133b80d7c65a", size = 7457913, upload-time = "2026-10-17T05:38:40.624Z" }, + { url = "https://files.pythonhosted.org/packages/05/9f/fc20897baafdac0039faef2cc3a324b059fdfe7f4a1522bac386f1e5f8ae/fensu_cli-0.27.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf73108b3778d0d74e11857e743042f4241174ec8f16aeee7e5c8c4769e41e7b", size = 7537447, upload-time = "2026-10-17T05:38:42.834Z" }, + { url = "https://files.pythonhosted.org/packages/46/be/1fc7c87cbf320ff215dcb204df808c592fefec6a0892a1103564a1c69b76/fensu_cli-0.27.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:435dad914ec0d061d5fceb10d8b049f75cdb5c031a156efe2f8309647c744b8c", size = 8035160, upload-time = "2026-10-17T05:38:45.074Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5b/53050980866370c7a41dc7e3e61de86cf67e5848d8560e6a9d166040679a/fensu_cli-0.27.0-py3-none-win_amd64.whl", hash = "sha256:68e91a38dab7e081c4ba504e22a7709169d8d6c470137834adcad116b52fa280", size = 7652792, upload-time = "2026-10-17T05:38:47.26Z" }, ] [[package]] @@ -4350,7 +4350,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.23.0" }, + { name = "fensu", specifier = ">=0.27.0" }, { name = "ingestr", specifier = ">=1.0.0" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From 62334d9fc16db57475add1011420fda34b12acef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 09/17] chore: move function name data under the crate source --- .../semantic_validation/_helpers/function_names.rs | 2 +- .../_helpers/snowflake_functions.txt} | 0 fensu.toml | 13 +++++++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) rename crates/sqlbuild-analysis/{function_names/snowflake.txt => src/semantic_validation/_helpers/snowflake_functions.txt} (100%) diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs index fba1e4be4c..ca0337945e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/function_names.rs @@ -12,7 +12,7 @@ use std::mem::Discriminant; use std::sync::{LazyLock, OnceLock, PoisonError}; const UNKNOWN_FUNCTION: &str = "B101"; -const SNOWFLAKE_FUNCTIONS: &str = include_str!("../../../function_names/snowflake.txt"); +const SNOWFLAKE_FUNCTIONS: &str = include_str!("snowflake_functions.txt"); const DUCKDB_CATALOG: &str = "duckdb"; const PROBE_PREFIX: &str = "SELECT "; const TYPE_POSITION_KEYWORD: &str = "AS"; diff --git a/crates/sqlbuild-analysis/function_names/snowflake.txt b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/snowflake_functions.txt similarity index 100% rename from crates/sqlbuild-analysis/function_names/snowflake.txt rename to crates/sqlbuild-analysis/src/semantic_validation/_helpers/snowflake_functions.txt diff --git a/fensu.toml b/fensu.toml index 154893a523..9394b3890e 100644 --- a/fensu.toml +++ b/fensu.toml @@ -448,6 +448,19 @@ rule = "FPRSL105" path = "crates/sqlbuild-analysis/src/sql_tokens/main/builtin_function_names.rs" reason = "This test-support entry exists only for the SQL lint tests in sqlbuild-rules, which the analyzer does not count as importers." +[targets.rust.rule_options.FPRSR311] +allowed_entries = [ + "Cargo.toml", + "build.rs", + "src/", + "tests/", + "benches/", + "examples/", + "README*", + "LICENSE*", + "pyproject.toml", +] + [targets.rust.rule_options.FPRSL301] forbidden_packages = [ "sqlbuild-structure-checker", From 912815b1df275ba351e4af782e5371b691bad374 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 10/17] refactor: name large tuple types --- .../_helpers/analysis_cache.rs | 31 +- .../_helpers/compact_batch.rs | 19 +- .../_helpers/dynamic_pivot.rs | 70 +- .../analysis_session/_helpers/session.rs | 2 +- .../src/assembly/analysis_session/models.rs | 2 +- .../analysis_session/tests/helpers.rs | 2 +- .../main/locate_binding_diagnostics.rs | 26 +- .../src/binding_locations/models.rs | 11 +- .../src/binding_locations/tests/helpers.rs | 20 +- .../src/binding_locations/tests/test_types.rs | 2 +- .../sqlbuild-analysis/src/lineage/models.rs | 52 +- .../src/lineage/tests/helpers.rs | 19 +- .../lineage/tests/test_rich_lineage_panics.rs | 2 +- .../_helpers/resolved_reads.rs | 6 +- .../planner_fixtures/main/required_columns.rs | 6 +- .../src/planner_fixtures/mod.rs | 1 - .../src/planner_fixtures/models.rs | 9 +- .../src/planner_fixtures/tests/test_reads.rs | 21 +- .../src/planner_fixtures/types.rs | 4 - .../src/query_analysis/_helpers/engine.rs | 41 +- .../_helpers/metadata_checks/audits.rs | 77 ++- .../_helpers/resource_sql/diagnostics.rs | 80 ++- .../_helpers/resource_sql/fixtures.rs | 33 +- .../_helpers/resource_sql/locations.rs | 83 +-- .../src/semantic_checks/models.rs | 24 +- .../tests/test_resource_sql.rs | 6 +- .../src/semantic_usage/main.rs | 26 +- .../semantic_validation/_helpers/catalog.rs | 26 +- .../_helpers/diagnostics.rs | 28 +- .../semantic_validation/_helpers/positions.rs | 36 +- .../semantic_validation/main/diagnostics.rs | 2 +- .../src/semantic_validation/models.rs | 23 +- .../tests/test_type_locations.rs | 25 +- .../src/semantic_validation/types.rs | 10 - .../src/functions/_helpers/namespace.rs | 22 +- .../main/resolve_function_namespace.rs | 20 +- .../main/render_dbt_execution_summary.rs | 5 +- .../src/lifecycle_events/constants.rs | 602 +++++++++--------- .../src/lifecycle_events/main/catalog.rs | 16 +- .../src/lifecycle_events/types.rs | 11 +- .../src/_helpers/locations.rs | 25 +- .../_helpers/analysis/binding_catalog.rs | 55 +- .../_helpers/analysis/binding_locations.rs | 62 +- .../src/bindings/_helpers/analysis/catalog.rs | 12 +- .../_helpers/analysis/explicit_references.rs | 42 +- .../bindings/_helpers/analysis/model_reads.rs | 32 +- .../_helpers/analysis/normalization.rs | 8 +- .../analysis_session/model_analyses.rs | 104 +-- .../_helpers/analysis_session/oracle.rs | 61 +- .../analysis_session/outcome_objects.rs | 106 +-- .../analysis_session/profile_inputs.rs | 15 +- .../analysis_session/query_columns.rs | 52 +- .../_helpers/analysis_session/session.rs | 283 +++++--- .../analysis_session/stage_functions.rs | 79 +-- .../src/bindings/_helpers/boundary/mod.rs | 4 + .../bindings/_helpers/boundary/positional.rs | 114 ++++ .../_helpers/boundary/positional_records.rs | 50 ++ .../_helpers/command_runtime/invocation.rs | 37 +- .../compile_attachment/cursor_roles.rs | 3 +- .../compile_attachment/model_inputs.rs | 13 +- .../bindings/_helpers/compile_delta/image.rs | 49 +- .../_helpers/compile_driver/analysis_cache.rs | 15 +- .../compile_driver/assembly_analysis.rs | 23 +- .../compile_driver/assembly_models.rs | 2 +- .../_helpers/compile_outputs/reuse.rs | 20 +- .../compile_outputs/runtime_target.rs | 27 +- .../_helpers/compiled_project/attachment.rs | 116 ++-- .../_helpers/compiled_project/audit_loop.rs | 119 ++-- .../_helpers/compiled_project/audit_views.rs | 9 +- .../compiled_model_analysis.rs | 47 +- .../compiled_project/compiled_model_view.rs | 34 +- .../compiled_project/function_loop.rs | 103 ++- .../compiled_project/function_views.rs | 50 +- .../compiled_project/model_input_view.rs | 22 +- .../_helpers/compiled_project/model_loop.rs | 71 ++- .../compiled_project/model_loop_calls.rs | 26 +- .../compiled_project/model_loop_external.rs | 8 +- .../compiled_project/model_loop_hooks.rs | 22 +- .../compiled_project/model_loop_host.rs | 35 +- .../compiled_project/model_loop_inputs.rs | 21 +- .../compiled_project/model_loop_macros.rs | 16 +- .../_helpers/compiled_project/project.rs | 89 ++- .../_helpers/compiled_project/project_view.rs | 10 +- .../_helpers/compiled_project/records.rs | 34 +- .../compiled_project/resource_diagnostics.rs | 74 ++- .../compiled_project/scenario_loop.rs | 23 +- .../_helpers/compiled_project/seed_views.rs | 56 +- .../_helpers/compiled_project/source_views.rs | 40 +- .../compiled_project/sql_test_loop.rs | 141 ++-- .../contracts/contract_diagnostics.rs | 30 +- .../_helpers/contracts/model_contracts.rs | 63 +- .../bindings/_helpers/diff_report/query.rs | 23 +- .../_helpers/discovery/header_values.rs | 9 +- .../_helpers/graph/execution_gates.rs | 17 +- .../bindings/_helpers/graph/project_graph.rs | 72 ++- .../_helpers/graph/python_selection.rs | 59 +- .../_helpers/hand_built/compiled_model.rs | 54 +- .../_helpers/lineage/column_lineage_graph.rs | 23 +- .../_helpers/lineage/lineage_cache.rs | 46 +- .../_helpers/lineage/project_lineage.rs | 2 +- .../bindings/_helpers/load/dlt_destination.rs | 25 +- .../src/bindings/_helpers/mod.rs | 4 +- .../_helpers/model_config/model_validator.rs | 56 +- .../_helpers/plan_output/plan_json.rs | 50 +- .../planner_entries/execution_plan.rs | 37 +- .../_helpers/planner_identity/graph.rs | 25 +- .../_helpers/planner_phases/models.rs | 36 ++ .../planner_phases/output_assembly.rs | 51 +- .../planner_strategy/fixture_reads.rs | 28 +- .../planner_strategy/fixture_types.rs | 10 +- .../planner_warehouse/fingerprint_cache.rs | 35 +- .../project_assembly/declared_relations.rs | 21 +- .../project_assembly/native_resources.rs | 90 ++- .../record_behaviours/lineage_facts.rs | 23 +- .../_helpers/record_behaviours/macros.rs | 8 +- .../_helpers/refactor/refactor_facts.rs | 70 +- .../bindings/_helpers/render/authored_sql.rs | 26 +- .../_helpers/render/declaration_expansion.rs | 28 +- .../_helpers/render/macro_call_rows.rs | 79 ++- .../_helpers/render/macro_expansion.rs | 56 +- .../_helpers/render/macro_expansion_result.rs | 25 +- .../bindings/_helpers/render/macro_modules.rs | 91 ++- .../_helpers/rules/custom_rule_evidence.rs | 20 +- .../bindings/_helpers/rules/fix_session.rs | 45 +- .../bindings/_helpers/rules/fix_verdict.rs | 43 +- .../src/bindings/_helpers/rules/lint.rs | 32 +- .../bindings/_helpers/rules/lint_inputs.rs | 6 +- .../bindings/_helpers/rules/lint_pipeline.rs | 48 +- .../src/bindings/_helpers/rules/lint_text.rs | 49 +- .../bindings/_helpers/rules/relation_keys.rs | 77 ++- .../src/bindings/_helpers/rules/rows.rs | 28 +- .../src/bindings/_helpers/rules/rule_cache.rs | 25 +- .../src/bindings/_helpers/rules/sql_rules.rs | 15 +- .../scenario_command/capture_command.rs | 16 +- .../_helpers/scenario_command/presentation.rs | 20 +- .../scenario_command/snapshot_sync.rs | 14 +- .../_helpers/scenario_command/test_command.rs | 16 +- .../_helpers/scopes/compile_usages.rs | 85 +-- .../src/bindings/_helpers/scopes/placement.rs | 28 +- .../bindings/_helpers/scopes/scope_command.rs | 34 +- .../bindings/_helpers/scopes/scope_lookup.rs | 52 +- .../bindings/_helpers/scopes/scope_objects.rs | 22 +- .../bindings/_helpers/scopes/scope_reports.rs | 93 +-- .../_helpers/scopes/scope_validation.rs | 14 +- .../_helpers/scopes/scope_visibility.rs | 8 +- .../_helpers/semantic_checks/metadata.rs | 52 +- .../_helpers/semantic_checks/resource_sql.rs | 13 +- .../_helpers/semantic_shapes/shapes.rs | 16 +- .../_helpers/sql_test_glue/assembly.rs | 67 +- .../_helpers/sql_test_glue/compiled_tests.rs | 24 +- .../_helpers/sql_test_glue/plan_rows.rs | 45 +- .../_helpers/sql_test_glue/planning.rs | 96 +-- .../sqltext/declaration_references.rs | 27 +- .../_helpers/sqltext/model_headers.rs | 26 +- .../_helpers/sqltext/sql_references.rs | 55 +- .../bindings/_helpers/sqltext/static_sql.rs | 36 +- .../_helpers/type_system/normalization.rs | 33 +- crates/sqlbuild-python/src/bindings/models.rs | 47 +- crates/sqlbuild-python/src/bindings/types.rs | 13 + .../_helpers/scanning/sql_sites.rs | 31 +- .../src/engine/_helpers/rows.rs | 32 +- .../src/engine/tests/test_sql_scanning.rs | 12 +- .../src/fix_verification/main/compare.rs | 2 +- .../src/fix_verification/models.rs | 13 +- .../src/lint_text/_helpers/sql_diagnostics.rs | 27 +- .../src/lint_text/_helpers/suppressions.rs | 39 +- .../src/sql_lint/_helpers/preparation.rs | 18 +- crates/sqlbuild-rules/src/sql_lint/types.rs | 4 +- .../src/scope_index/_helpers/grants.rs | 26 +- .../src/scope_index/_helpers/ordering.rs | 37 +- .../src/scope_report/_helpers/declarations.rs | 31 +- .../src/scope_report/_helpers/projection.rs | 11 +- .../src/scope_text/_helpers/results.rs | 49 +- .../sql_interpolation/substitution.rs | 32 +- .../sqlbuild-sqltext/src/compiler/models.rs | 11 +- .../src/compiler/tests/helpers.rs | 15 +- .../compiler/tests/test_sql_interpolation.rs | 8 +- .../src/compiler/tests/test_types.rs | 4 +- crates/sqlbuild-sqltext/src/compiler/types.rs | 3 - fensu.toml | 26 +- 180 files changed, 4537 insertions(+), 2466 deletions(-) delete mode 100644 crates/sqlbuild-analysis/src/planner_fixtures/types.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs index 3b4eb35339..ce7fe35a60 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/analysis_cache.rs @@ -16,7 +16,7 @@ use crate::assembly::analysis_session::models::{ LineageRow, ModelAnalysis, ModelOutcome, ModelRequest, SessionRequest, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; const LINEAGE_NATIVE: u64 = 0; const LINEAGE_FACTS: u64 = 1; @@ -454,7 +454,16 @@ pub(crate) fn encode_outcome( encoder.flag(analysis.has_star); encoder.flag(analysis.star_resolved); encoder.uint(analysis.binding_diagnostics.len() as u64); - for (code, message, line, column, start, end, severity) in &analysis.binding_diagnostics { + for DiagnosticRow { + code, + message, + line, + column, + start, + end, + severity, + } in &analysis.binding_diagnostics + { encoder.text(code); encoder.text(message); for position in [line, column, start, end] { @@ -529,15 +538,15 @@ pub(crate) fn decode_outcome( let diagnostic_count: usize = decoder.index()?; let mut binding_diagnostics: Vec = Vec::new(); for _ in 0..diagnostic_count { - binding_diagnostics.push(( - decoder.text()?, - decoder.text()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.optional_index()?, - decoder.text()?, - )); + binding_diagnostics.push(DiagnosticRow { + code: decoder.text()?, + message: decoder.text()?, + line: decoder.optional_index()?, + column: decoder.optional_index()?, + start: decoder.optional_index()?, + end: decoder.optional_index()?, + severity: decoder.text()?, + }); } let binding_validated: bool = decoder.flag()?; decoder.finish()?; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs index 9a274defc7..8679f4480d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/compact_batch.rs @@ -23,7 +23,8 @@ use crate::assembly::analysis_session::constants::{ use crate::assembly::analysis_session::models::{ColumnFact, LineageRow}; use crate::assembly::analysis_session::types::{Pairs, Shapes}; use crate::semantic_validation::main::diagnostics::binding_diagnostics; -use crate::semantic_validation::types::{DiagnosticRow, NormalizationRequest, Relations}; +use crate::semantic_validation::models::DiagnosticRow; +use crate::semantic_validation::types::{NormalizationRequest, Relations}; const INVALID_BATCH: &str = "native compact query analysis returned an invalid batch response"; const INVALID_TEMPLATE: &str = "native compact query analysis returned an invalid template"; @@ -1424,15 +1425,15 @@ fn diagnostic_row(error: &Value) -> Result, String> { Some(Value::String(message)) if !message.is_empty() => message.clone(), _ => DEFAULT_BINDING_MESSAGE.to_owned(), }; - Ok(Some(( - code.to_owned(), + Ok(Some(DiagnosticRow { + code: code.to_owned(), message, - position(error, "line")?, - position(error, "column")?, - position(error, "start")?, - position(error, "end")?, - severity.to_owned(), - ))) + line: position(error, "line")?, + column: position(error, "column")?, + start: position(error, "start")?, + end: position(error, "end")?, + severity: severity.to_owned(), + })) } fn position(error: &Value, key: &str) -> Result, String> { diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs index b37e6c4e4b..e49ac7c85c 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/dynamic_pivot.rs @@ -192,7 +192,12 @@ fn pivot_proof( if truthy(pivot.get("unpivot")) { return failure("UNPIVOT does not produce a dynamic column family".to_owned()); } - let (pivot_columns, aggregates, dynamic, columns_valid) = components(pivot, context.dialect); + let PivotComponents { + pivot_columns, + aggregates, + dynamic, + columns_valid, + } = components(pivot, context.dialect); if !dynamic { return failure( "static pivot values must use ordinary exact column declarations".to_owned(), @@ -309,11 +314,18 @@ fn fixed_names( .collect() } +/// Python's `_pivot_components` for one pivot clause. +struct PivotComponents<'a> { + pivot_columns: Vec<&'a str>, + aggregates: Vec<&'a Value>, + /// Whether the pivot values come from a dynamic source. + dynamic: bool, + /// Whether every pivot column resolved to a name. + columns_valid: bool, +} + /// Python's `_pivot_components`: pivot columns, aggregates, dynamic values, columns valid. -fn components<'a>( - pivot: &'a Map, - dialect: &str, -) -> (Vec<&'a str>, Vec<&'a Value>, bool, bool) { +fn components<'a>(pivot: &'a Map, dialect: &str) -> PivotComponents<'a> { if SIMPLIFIED_PIVOT_DIALECTS.contains(&dialect) && truthy(pivot.get("using")) { let expressions: Vec<&Value> = dict_list(pivot.get("expressions")); let columns: Vec<&str> = expressions @@ -321,12 +333,12 @@ fn components<'a>( .filter_map(|expression| column_name(Some(expression))) .collect(); let valid: bool = columns.len() == expressions.len(); - return ( - columns, - dict_list(pivot.get("using")), - !truthy(pivot.get("fields")), - valid, - ); + return PivotComponents { + pivot_columns: columns, + aggregates: dict_list(pivot.get("using")), + dynamic: !truthy(pivot.get("fields")), + columns_valid: valid, + }; } let fields: Vec<&Value> = dict_list(pivot.get("fields")); let columns: Vec<&str> = fields @@ -335,7 +347,12 @@ fn components<'a>( .collect(); let dynamic: bool = fields.iter().any(|field| has_dynamic_values(field)); let valid: bool = columns.len() == fields.len(); - (columns, dict_list(pivot.get("expressions")), dynamic, valid) + PivotComponents { + pivot_columns: columns, + aggregates: dict_list(pivot.get("expressions")), + dynamic, + columns_valid: valid, + } } /// Python's `_has_dynamic_value_source` for one pivot field. @@ -651,13 +668,24 @@ fn failure(reason: String) -> PivotOutcome { } /// Python's `_families_equal` key for one family. -fn normalized(family: &DynamicFamily) -> (String, String, String, String, String, Option<&str>) { - ( - casefold(&family.name), - casefold(&family.pivot_column), - casefold(&family.value_column), - casefold(&family.aggregate), - casefold(&family.data_type), - family.name_pattern.as_deref(), - ) +#[derive(PartialEq, Eq)] +struct FamilyKey<'a> { + name: String, + pivot_column: String, + value_column: String, + aggregate: String, + data_type: String, + name_pattern: Option<&'a str>, +} + +/// Python's `_families_equal` key for one family. +fn normalized(family: &DynamicFamily) -> FamilyKey<'_> { + FamilyKey { + name: casefold(&family.name), + pivot_column: casefold(&family.pivot_column), + value_column: casefold(&family.value_column), + aggregate: casefold(&family.aggregate), + data_type: casefold(&family.data_type), + name_pattern: family.name_pattern.as_deref(), + } } diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs index 66f3c1196b..2451efca3d 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/_helpers/session.rs @@ -36,8 +36,8 @@ use crate::assembly::analysis_session::models::{ RelationTables, SessionModelFacts, SessionOutcome, SessionRequest, SessionStep, }; use crate::assembly::analysis_session::types::{Pairs, Shapes}; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::ProjectCatalog; -use crate::semantic_validation::types::DiagnosticRow; /// What an internal native failure of the session names. const SESSION_CONTEXT: &str = "native model analysis"; diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs index 3a625a8170..da66105b17 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/models.rs @@ -11,7 +11,7 @@ use sqlbuild_cache::store::models::NativeStore; use crate::assembly::analysis_session::_helpers::catalog_state::SessionCatalog; use crate::assembly::analysis_session::_helpers::mappings::ShapeTable; use crate::assembly::analysis_session::types::{NullabilityRuleFn, OutputSources, Pairs, Shapes}; -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; /// One `ref`, `source`, `seed`, `table_fn` or `udf` call a model's SQL makes. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs index cef7394585..321905eb3e 100644 --- a/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/assembly/analysis_session/tests/helpers.rs @@ -230,7 +230,7 @@ pub(crate) fn described(outcome: &ModelOutcome) -> Vec { analysis .binding_diagnostics .iter() - .map(|row| row.0.as_str()) + .map(|row| row.code.as_str()) .collect::>() )); lines diff --git a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs index 80221f225b..71738049d7 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/main/locate_binding_diagnostics.rs @@ -3,9 +3,11 @@ use std::collections::HashSet; use crate::binding_locations::constants::{ERROR_SEVERITY, UNKNOWN_TABLE_INTERNAL_CODE}; -use crate::binding_locations::models::{AuthoredSpan, BindingSource, LocatedDiagnostic}; +use crate::binding_locations::models::{ + AuthoredSpan, BindingSource, LocatedDiagnostic, SeenDiagnostic, +}; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::{BindingPositions, PositionInput}; -use crate::semantic_validation::types::DiagnosticRow; /// Each reported diagnostic in order; an error mapping a position is the mapper's message. pub fn locate_binding_diagnostics( @@ -17,9 +19,18 @@ pub fn locate_binding_diagnostics( .find(source.query) .map(|start| source.authored[..start].matches('\n').count()); let mut positions: Option = None; - let mut seen: HashSet<(String, String, Option, Option)> = HashSet::new(); + let mut seen: HashSet = HashSet::new(); let mut located: Vec = Vec::new(); - for (code, message, line, column, start, end, severity) in rows { + for DiagnosticRow { + code, + message, + line, + column, + start, + end, + severity, + } in rows + { if code == UNKNOWN_TABLE_INTERNAL_CODE { continue; } @@ -39,7 +50,12 @@ pub fn locate_binding_diagnostics( *column, ), }; - let key = (code.clone(), message.clone(), line, column); + let key = SeenDiagnostic { + code: code.clone(), + message: message.clone(), + line, + column, + }; if seen.contains(&key) && start.is_some() && severity == ERROR_SEVERITY { continue; } diff --git a/crates/sqlbuild-analysis/src/binding_locations/models.rs b/crates/sqlbuild-analysis/src/binding_locations/models.rs index 292ed5690a..efd854bf90 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/models.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/models.rs @@ -1,6 +1,15 @@ //! A model's authored SQL texts and the located diagnostics read from them. -use crate::semantic_validation::types::Expansion; +use crate::semantic_validation::models::Expansion; + +/// A reported diagnostic's identity: an error repeating it at a known start is dropped. +#[derive(Debug, PartialEq, Eq, Hash)] +pub(crate) struct SeenDiagnostic { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) line: Option, + pub(crate) column: Option, +} /// The texts one model's binding diagnostics are mapped back through. #[derive(Clone, Copy, Debug)] diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs index a09c0b76aa..e7dcf94764 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/helpers.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; pub(super) const MODEL: &str = "MODEL (\n description \"Orders\"\n);\n\nSELECT o.amount, o.missing\nFROM orders AS o"; @@ -11,15 +11,15 @@ fn row( span: (Option, Option), severity: &str, ) -> DiagnosticRow { - ( - code.to_owned(), - message.to_owned(), - position.0, - position.1, - span.0, - span.1, - severity.to_owned(), - ) + DiagnosticRow { + code: code.to_owned(), + message: message.to_owned(), + line: position.0, + column: position.1, + start: span.0, + end: span.1, + severity: severity.to_owned(), + } } fn missing_column() -> DiagnosticRow { diff --git a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs index 300b0e43f8..336b4dc4d1 100644 --- a/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/binding_locations/tests/test_types.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; /// `(code, line, column, (line, column, end line, end column))` of one reported diagnostic. pub(super) type LocatedCase = ( diff --git a/crates/sqlbuild-analysis/src/lineage/models.rs b/crates/sqlbuild-analysis/src/lineage/models.rs index f660fbad29..d8d4a2657b 100644 --- a/crates/sqlbuild-analysis/src/lineage/models.rs +++ b/crates/sqlbuild-analysis/src/lineage/models.rs @@ -149,15 +149,41 @@ pub enum FastLineageOutcome { UnknownDialect(String), } +/// A lineage outcome flattened into its status, columns, star flag and detail. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LineageOutcomeParts { + pub status: &'static str, + pub columns: Vec, + pub has_star: bool, + pub detail: Option, +} + +impl LineageOutcomeParts { + fn new(status: &'static str, columns: Vec, has_star: bool, detail: Option) -> Self { + Self { + status, + columns, + has_star, + detail, + } + } +} + impl FastLineageOutcome { - /// `(status, columns, has_star, detail)`: detail is the parse error or the unknown dialect. - pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + /// The status, columns and star flag; detail is the parse error or the unknown dialect. + pub fn into_parts(self) -> LineageOutcomeParts { match self { - Self::StarColumns(columns) => ("star", columns, true, None), - Self::Built { columns, has_star } => ("built", columns, has_star, None), - Self::Omitted => ("omitted", Vec::new(), false, None), - Self::Unparsed(message) => ("unparsed", Vec::new(), false, Some(message)), - Self::UnknownDialect(name) => ("unknown_dialect", Vec::new(), false, Some(name)), + Self::StarColumns(columns) => LineageOutcomeParts::new("star", columns, true, None), + Self::Built { columns, has_star } => { + LineageOutcomeParts::new("built", columns, has_star, None) + } + Self::Omitted => LineageOutcomeParts::new("omitted", Vec::new(), false, None), + Self::Unparsed(message) => { + LineageOutcomeParts::new("unparsed", Vec::new(), false, Some(message)) + } + Self::UnknownDialect(name) => { + LineageOutcomeParts::new("unknown_dialect", Vec::new(), false, Some(name)) + } } } } @@ -217,11 +243,15 @@ pub enum RichLineageOutcome { } impl RichLineageOutcome { - /// `(status, columns, has_star, detail)`: detail is the polyglot error. - pub fn into_parts(self) -> (&'static str, Vec, bool, Option) { + /// The status, columns and star flag; detail is the polyglot error. + pub fn into_parts(self) -> LineageOutcomeParts { match self { - Self::Built { columns, has_star } => ("built", columns, has_star, None), - Self::Skipped(message) => ("skipped", Vec::new(), false, Some(message)), + Self::Built { columns, has_star } => { + LineageOutcomeParts::new("built", columns, has_star, None) + } + Self::Skipped(message) => { + LineageOutcomeParts::new("skipped", Vec::new(), false, Some(message)) + } } } } diff --git a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs index 73e206171a..bf332a89da 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/helpers.rs @@ -4,8 +4,9 @@ use crate::lineage::main::build_fast_lineage::build_fast_lineage; use crate::lineage::main::build_rich_lineage::build_rich_lineage; use crate::lineage::models::RelationFingerprint; use crate::lineage::models::{ - FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageResourceType, - LineageSchemaResource, RichLineageColumn, RichLineageRequest, RichSchemaResource, + FastLineageModel, FastLineageOutcome, FastLineageRequest, LineageColumn, LineageOutcomeParts, + LineageResourceType, LineageSchemaResource, RichLineageColumn, RichLineageRequest, + RichSchemaResource, }; use crate::semantic_validation::models::ProjectCatalog; @@ -64,7 +65,12 @@ fn catalog() -> ProjectCatalog { fn described_outcome( outcome: FastLineageOutcome, ) -> (&'static str, Vec, bool, Option) { - let (status, columns, has_star, detail) = outcome.into_parts(); + let LineageOutcomeParts { + status, + columns, + has_star, + detail, + } = outcome.into_parts(); (status, described(&columns), has_star, detail) } @@ -139,7 +145,12 @@ pub(crate) fn rich_outcome( }; let mut outcomes = build_rich_lineage(&request).expect("the rich lineage pool builds"); assert_eq!(outcomes.len(), 1); - let (status, columns, has_star, detail) = outcomes.remove(0).into_parts(); + let LineageOutcomeParts { + status, + columns, + has_star, + detail, + } = outcomes.remove(0).into_parts(); ( status, columns.iter().map(described_rich_column).collect(), diff --git a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs index b4a0944961..1f4d748a01 100644 --- a/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs +++ b/crates/sqlbuild-analysis/src/lineage/tests/test_rich_lineage_panics.rs @@ -55,7 +55,7 @@ fn given_an_injected_panic_when_building_rich_lineage_then_the_request_fails() { let answered: Result, String> = outcomes.map(|outcomes| { outcomes .into_iter() - .map(|outcome| outcome.into_parts().3.unwrap_or_default()) + .map(|outcome| outcome.into_parts().detail.unwrap_or_default()) .collect() }); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs index 5e70d90bcc..8ab1fbca56 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/_helpers/resolved_reads.rs @@ -25,9 +25,9 @@ pub(crate) fn resolved_reads( let references: Vec = key .references .iter() - .map(|(kind, name, _, _)| ReferenceInput { - kind: kind.clone(), - name: name.clone(), + .map(|reference| ReferenceInput { + kind: reference.kind.clone(), + name: reference.name.clone(), }) .collect(); cache.misses += 1; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs index d2a2c86b91..4134ac1029 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/main/required_columns.rs @@ -26,14 +26,14 @@ pub fn required_columns( let mut result = FixtureReadRequirements::default(); for model in models { let mut reads = model.high_confidence.clone(); - if model.key.references.iter().any(|(kind, name, _, _)| { - let resource_type = match kind.as_str() { + if model.key.references.iter().any(|reference| { + let resource_type = match reference.kind.as_str() { "ref" => "model", "source" => "source", "seed" => "seed", _ => return false, }; - fixture_keys.contains(&(resource_type.to_owned(), name.clone())) + fixture_keys.contains(&(resource_type.to_owned(), reference.name.clone())) }) { let (updated, resolved) = resolved_reads(cache, &model.key); cache = updated; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs index 42b1da1020..b010d05a00 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/mod.rs @@ -7,4 +7,3 @@ pub mod main; pub mod models; #[cfg(test)] mod tests; -pub mod types; diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs index 703a4defa1..944b845040 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/models.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/models.rs @@ -12,7 +12,14 @@ pub struct FixtureReadSchemas<'a> { use crate::required_columns::models::ColumnRead; -use crate::planner_fixtures::types::FixtureReadReference; +/// The complete reference identity retained by the fixture read cache. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct FixtureReadReference { + pub kind: String, + pub name: String, + pub package: Option, + pub argument_count: Option, +} /// The original SQL, complete references and dialect form the read cache key. #[derive(Clone, Debug, Eq, Hash, PartialEq)] diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs index 8d19e9e57e..b7832b9526 100644 --- a/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs +++ b/crates/sqlbuild-analysis/src/planner_fixtures/tests/test_reads.rs @@ -1,6 +1,6 @@ use crate::planner_fixtures::_helpers::resolved_reads::resolved_reads; use crate::planner_fixtures::_helpers::stub_markers::stub_markers; -use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey}; +use crate::planner_fixtures::models::{FixtureReadCache, FixtureReadKey, FixtureReadReference}; use crate::planner_fixtures::tests::test_types::{FixtureMarkerTestCase, FixtureReadCacheTestCase}; #[test] @@ -50,7 +50,12 @@ fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reu for test_case in test_cases { let key = FixtureReadKey { sql: "SELECT order_id FROM __source(\"orders\")".to_owned(), - references: vec![("source".into(), "orders".into(), None, None)], + references: vec![FixtureReadReference { + kind: "source".into(), + name: "orders".into(), + package: None, + argument_count: None, + }], dialect: Some("duckdb".into()), }; let (cache, initial) = resolved_reads(FixtureReadCache::default(), &key); @@ -59,12 +64,12 @@ fn given_repeated_reads_when_resolving_then_full_reference_identity_controls_reu assert_eq!(cache.misses, 1, "{}", test_case.description); let changed = FixtureReadKey { sql: test_case.sql.to_owned(), - references: vec![( - "source".into(), - "orders".into(), - test_case.package.map(str::to_owned), - test_case.argument_count, - )], + references: vec![FixtureReadReference { + kind: "source".into(), + name: "orders".into(), + package: test_case.package.map(str::to_owned), + argument_count: test_case.argument_count, + }], dialect: test_case.dialect.map(str::to_owned), }; let (cache, reads) = resolved_reads(cache, &changed); diff --git a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs b/crates/sqlbuild-analysis/src/planner_fixtures/types.rs deleted file mode 100644 index ec569300e0..0000000000 --- a/crates/sqlbuild-analysis/src/planner_fixtures/types.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! Fixture analysis identities. - -/// The complete reference identity retained by the fixture read cache. -pub type FixtureReadReference = (String, String, Option, Option); diff --git a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs index b5def1c23c..bcd4228bfe 100644 --- a/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-analysis/src/query_analysis/_helpers/engine.rs @@ -7,7 +7,8 @@ use polyglot_sql::{ ValidationSchema, analyze_query, analyze_query_for_project_projections, }; use rayon::iter::{IntoParallelIterator, ParallelIterator}; -use serde::{Deserialize, Serialize}; +use serde::ser::SerializeTuple; +use serde::{Deserialize, Serialize, Serializer}; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Mutex, PoisonError}; @@ -1128,7 +1129,29 @@ fn analyze_project_result(request: ProjectAnalysisRequest) -> Result, u8, u8, u8, Vec<(usize, usize, usize)>); +/// One interned column fact; it serializes as the positional array the batch reader expects. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct CompactProjectColumn { + name: usize, + data_type: Option, + nullability: u8, + transform: u8, + confidence: u8, + upstream: Vec<(usize, usize, usize)>, +} + +impl Serialize for CompactProjectColumn { + fn serialize(&self, serializer: S) -> Result { + let mut fields = serializer.serialize_tuple(6)?; + fields.serialize_element(&self.name)?; + fields.serialize_element(&self.data_type)?; + fields.serialize_element(&self.nullability)?; + fields.serialize_element(&self.transform)?; + fields.serialize_element(&self.confidence)?; + fields.serialize_element(&self.upstream)?; + fields.end() + } +} #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -1222,14 +1245,14 @@ impl CompactProjectAccumulator { ) }) .collect(); - let fact = ( - self.interner.intern(column.name), - column.data_type.map(|value| self.interner.intern(value)), - nullability_code(column.nullability), - transform_code(lineage.transform_kind), - confidence_code(lineage.confidence), + let fact = CompactProjectColumn { + name: self.interner.intern(column.name), + data_type: column.data_type.map(|value| self.interner.intern(value)), + nullability: nullability_code(column.nullability), + transform: transform_code(lineage.transform_kind), + confidence: confidence_code(lineage.confidence), upstream, - ); + }; let next_index = self.facts.len(); let fact_index = *self.fact_indexes.entry(fact.clone()).or_insert_with(|| { self.facts.push(fact); diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs index 748ea8a336..c6d39d9a39 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/metadata_checks/audits.rs @@ -13,8 +13,8 @@ use crate::semantic_checks::models::{ }; use crate::semantic_validation::main::diagnostics::binding_diagnostics; use crate::semantic_validation::main::validations_json; -use crate::semantic_validation::models::{CatalogInput, Columns, ProjectCatalog}; -use crate::semantic_validation::types::{DiagnosticRow, Relations}; +use crate::semantic_validation::models::{CatalogInput, Columns, DiagnosticRow, ProjectCatalog}; +use crate::semantic_validation::types::Relations; const EXPRESSION_AUDIT: &str = "expression_is_true"; const RELATIONSHIPS_AUDIT: &str = "relationships"; @@ -107,15 +107,15 @@ fn response_rows(response: &Value) -> Result, SemanticFailure Some(Value::String(text)) if !text.is_empty() => text.clone(), _ => DEFAULT_BINDING_MESSAGE.to_owned(), }; - rows.push(( - code.to_owned(), + rows.push(DiagnosticRow { + code: code.to_owned(), message, - optional_index(error.get("line")), - optional_index(error.get("column")), - optional_index(error.get("start")), - optional_index(error.get("end")), - severity.to_owned(), - )); + line: optional_index(error.get("line")), + column: optional_index(error.get("column")), + start: optional_index(error.get("start")), + end: optional_index(error.get("end")), + severity: severity.to_owned(), + }); } Ok(rows) } @@ -162,8 +162,13 @@ fn standalone_validation( binding_diagnostics(sql, &dialect, response_rows(response)?).map_err(internal) } -/// One `(code, message, severity)` finding before it is located. -type Unlocated = (String, String, String, String); +/// One audit finding before it is located at the named column or field. +struct Unlocated { + code: String, + message: String, + severity: String, + name: String, +} fn expression_findings( expression: &str, @@ -188,14 +193,12 @@ fn expression_findings( .unwrap_or_default(); Ok(rows .into_iter() - .filter(|row| row.0 != UNKNOWN_TABLE_CODE) - .map(|row| { - ( - row.0, - format!("{EXPRESSION_AUDIT}: {}", row.1), - row.6, - expression.to_owned(), - ) + .filter(|row| row.code != UNKNOWN_TABLE_CODE) + .map(|row| Unlocated { + code: row.code, + message: format!("{EXPRESSION_AUDIT}: {}", row.message), + severity: row.severity, + name: expression.to_owned(), }) .collect()) } @@ -203,17 +206,17 @@ fn expression_findings( fn typed_findings(rows: Vec, label: &str, column: &str) -> Vec { rows.into_iter() .map(|row| { - let code = if row.6 == ERROR_SEVERITY { + let code = if row.severity == ERROR_SEVERITY { AUDIT_TYPE_CODE.to_owned() } else { - row.0 + row.code }; - ( + Unlocated { code, - format!("{label}: {}", row.1), - row.6, - column.to_owned(), - ) + message: format!("{label}: {}", row.message), + severity: row.severity, + name: column.to_owned(), + } }) .collect() } @@ -243,12 +246,12 @@ fn relationship_findings( if target_shape.is_empty() { return Ok(Vec::new()); } - return Ok(vec![( - MISSING_TARGET_COLUMN_CODE.to_owned(), - format!("relationships target has no column '{}'", audit.field), - ERROR_SEVERITY.to_owned(), - audit.field.clone(), - )]); + return Ok(vec![Unlocated { + code: MISSING_TARGET_COLUMN_CODE.to_owned(), + message: format!("relationships target has no column '{}'", audit.field), + severity: ERROR_SEVERITY.to_owned(), + name: audit.field.clone(), + }]); }; let Some((_, left)) = shape.iter().find(|(name, _)| name == column) else { return Ok(Vec::new()); @@ -337,7 +340,13 @@ pub(crate) fn audit_errors( ) -> Result, SemanticFailure> { let mut findings: Vec = Vec::new(); for audit in &model.audits { - for (code, message, severity, name) in audit_findings(audit, shape, context)? { + for Unlocated { + code, + message, + severity, + name, + } in audit_findings(audit, shape, context)? + { let (line, column) = text_position(&model.authored_sql, &name, 0)?; findings.push(AuditFinding { code, diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs index 5df90d2dcc..33ced0fba8 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/diagnostics.rs @@ -16,7 +16,8 @@ use crate::semantic_checks::_helpers::resource_sql::resources::{ MissingColumnAudit, ResourceSql, audit_label, missing_column_audits, resources, }; use crate::semantic_checks::models::{ - ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SemanticFailure, + ResourceSpan, ResourceSqlDiagnostic, ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, + SemanticFailure, }; use crate::semantic_validation::main::normalize::normalize_analysis_sql; use crate::semantic_validation::models::{Columns, NormalizationInput, ProjectCatalog}; @@ -32,7 +33,17 @@ const CURSOR_CONTEXT: &str = "Model SQL"; const ANALYSIS_CURSOR_LITERAL: &str = "CAST('2000-01-01 00:00:00' AS TIMESTAMP)"; const GENERIC_DIALECT: &str = "generic"; -type Key = (String, String, String, i64, i64, i64); +/// A finding's identity: its path, code, message, start and native order when unlocated. +#[derive(Clone, PartialEq, Eq, Hash)] +struct Key { + path: String, + code: String, + message: String, + line: i64, + column: i64, + /// The finding's native index when it has no start, else `-1`. + order: i64, +} fn kind_label(kind: ResourceSqlKind) -> &'static str { match kind { @@ -88,26 +99,26 @@ fn binding_request( /// Python's `_reported_diagnostic`: drop open-relation lookups, make parser rejections P001. fn reported(resource: &ResourceSql, finding: &Finding) -> Option { - if finding.0 != UNKNOWN_TABLE_CODE { + if finding.code != UNKNOWN_TABLE_CODE { return Some(finding.clone()); } - if finding.1.starts_with(UNKNOWN_TABLE_PREFIX) { + if finding.message.starts_with(UNKNOWN_TABLE_PREFIX) { return None; } - Some(( - SYNTAX_ERROR_CODE.to_owned(), - format!( + Some(Finding { + code: SYNTAX_ERROR_CODE.to_owned(), + message: format!( "SQL syntax error in {} '{}': {}", kind_label(resource.kind), resource.name, - finding.1 + finding.message ), - finding.2, - finding.3, - finding.4, - finding.5, - ERROR_SEVERITY.to_owned(), - )) + line: finding.line, + column: finding.column, + start: finding.start, + end: finding.end, + severity: ERROR_SEVERITY.to_owned(), + }) } /// What one resource's findings are checked against. @@ -128,31 +139,31 @@ fn resource_diagnostics( continue; }; let span = location(resource, checking.cleaned_sql, &finding)?; - let key: Key = ( - resource.path.clone(), - finding.0.clone(), - finding.1.clone(), - span.0, - span.1, - if finding.4.is_none() { + let key: Key = Key { + path: resource.path.clone(), + code: finding.code.clone(), + message: finding.message.clone(), + line: span.line, + column: span.column, + order: if finding.start.is_none() { index as i64 } else { -1 }, - ); + }; if found.iter().any(|(existing, _)| *existing == key) { continue; } let authored_text = text_from(resource, span); let authored_line = line_at(resource, span); let help = resource_sql_help( - &finding.0, - &finding.1, + &finding.code, + &finding.message, &HelpContext { kind: resource.kind, authored_text: &authored_text, authored_line: &authored_line, - column: span.1, + column: span.column, authored_body: &resource.authored_body, audit_definition: resource.audit_definition.as_deref(), shapes: &checking.request.shapes, @@ -162,16 +173,18 @@ fn resource_diagnostics( ) .map_err(semantic)?; let message = match &resource.label { - Some(label) if finding.0 != SYNTAX_ERROR_CODE => format!("{label}: {}", finding.1), - _ => finding.1.clone(), + Some(label) if finding.code != SYNTAX_ERROR_CODE => { + format!("{label}: {}", finding.message) + } + _ => finding.message.clone(), }; found.push(( key, ResourceSqlDiagnostic { kind: resource.kind, resource_name: resource.name.clone(), - is_error: finding.6 == ERROR_SEVERITY, - code: finding.0.clone(), + is_error: finding.severity == ERROR_SEVERITY, + code: finding.code.clone(), message, path: resource.path.clone(), span, @@ -199,7 +212,7 @@ fn opt_out_diagnostic( code: OPT_OUT_CODE.to_owned(), message, path: resource.path.clone(), - span: (line, column, None, None), + span: ResourceSpan::start(line, column), note: Some(note), help: Some(help), }) @@ -230,7 +243,12 @@ pub(crate) fn missing_column_diagnostic( model.name ), path: model.path.clone(), - span: (line, start, Some(line), Some(end)), + span: ResourceSpan { + line, + column: start, + end_line: Some(line), + end_column: Some(end), + }, note: None, help: Some(format!( "add {column} to the SELECT list of model '{}' or attach the {definition} audit to \ diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs index 10693cdcef..bcafbf2275 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/fixtures.rs @@ -87,14 +87,27 @@ fn empty_fixture_call_body(text: &[char], index: usize) -> Option { literal(text, after, ")", false) } -/// Every empty fixture CTE in `text`: start, prefix end, name and end. -fn empty_fixture_ctes(text: &[char]) -> Vec<(usize, usize, String, usize)> { - let mut found: Vec<(usize, usize, String, usize)> = Vec::new(); +/// One empty fixture CTE: where it starts, where its prefix ends, its name and its end. +struct EmptyFixtureCte { + start: usize, + prefix_end: usize, + name: String, + end: usize, +} + +/// Every empty fixture CTE in `text`. +fn empty_fixture_ctes(text: &[char]) -> Vec { + let mut found: Vec = Vec::new(); let mut index = 0; while index < text.len() { match empty_fixture_cte(text, index) { Some((prefix_end, name, end)) => { - found.push((index, prefix_end, name, end)); + found.push(EmptyFixtureCte { + start: index, + prefix_end, + name, + end, + }); index = end; } None => index += 1, @@ -109,11 +122,11 @@ pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { let ctes = empty_fixture_ctes(&body); let mut opened: Vec = Vec::with_capacity(body.len()); let mut copied = 0; - for (start, prefix_end, name, end) in &ctes { - opened.extend_from_slice(&body[copied..*start]); - opened.extend_from_slice(&body[*start..*prefix_end]); - opened.extend(name.chars()); - copied = *end; + for cte in &ctes { + opened.extend_from_slice(&body[copied..cte.start]); + opened.extend_from_slice(&body[cte.start..cte.prefix_end]); + opened.extend(cte.name.chars()); + copied = cte.end; } opened.extend_from_slice(&body[copied..]); let mut sql = String::with_capacity(opened.len()); @@ -128,6 +141,6 @@ pub(crate) fn prepared_test_sql(sql_body: &str) -> (String, Vec) { } } let mut relations = reference_names(&body); - relations.extend(ctes.into_iter().map(|(_, _, name, _)| name)); + relations.extend(ctes.into_iter().map(|cte| cte.name)); (sql, relations) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs index 0cc89868c2..85eadd87bb 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/_helpers/resource_sql/locations.rs @@ -4,25 +4,14 @@ use sqlbuild_core::text::main::ignorecase_key::ignorecase_key; use crate::semantic_checks::_helpers::resource_sql::resources::{ResourceSql, find_code_points}; use crate::semantic_checks::_helpers::resource_sql::scanning::word_at; -use crate::semantic_checks::models::ResourceSqlFailure; +use crate::semantic_checks::models::{ResourceSpan, ResourceSqlFailure}; use crate::semantic_validation::main::truth_test_spans::truth_test_spans; -use crate::semantic_validation::models::{BindingPositions, PositionInput}; - -/// A binding finding: code, message, line, column, start, end and severity. -pub(crate) type Finding = ( - String, - String, - Option, - Option, - Option, - Option, - String, -); +use crate::semantic_validation::models::{BindingPositions, DiagnosticRow, PositionInput}; -const UNSUPPORTED_TRUTH_TEST_CODE: &str = "B103"; +/// A binding finding: code, message, cleaned position and severity. +pub(crate) type Finding = DiagnosticRow; -/// `(line, column, end line, end column)`. -pub(crate) type Span = (i64, i64, Option, Option); +const UNSUPPORTED_TRUTH_TEST_CODE: &str = "B103"; /// Python's `_offset_location`: the 1-based line and column of a code-point offset. pub(crate) fn offset_location(text: &str, offset: usize) -> (i64, i64) { @@ -89,7 +78,7 @@ fn mapped_span( resource: &ResourceSql, cleaned_sql: &str, finding: &Finding, -) -> Result, ResourceSqlFailure> { +) -> Result, ResourceSqlFailure> { let mapper = BindingPositions::new(PositionInput { authored: resource.contents.to_string(), query: resource.authored_body.clone(), @@ -99,27 +88,36 @@ fn mapped_span( }) .map_err(ResourceSqlFailure::Normalization)?; let (line, column) = mapper - .position(finding.4, finding.2, finding.3, &finding.1) + .position( + finding.start, + finding.line, + finding.column, + &finding.message, + ) .map_err(ResourceSqlFailure::Normalization)?; let (Some(line), Some(column)) = (line, column) else { return Ok(None); }; - let (end_line, end_column) = match finding.5 { + let (end_line, end_column) = match finding.end { Some(end) => mapper .position(Some(end), None, None, "") .map_err(ResourceSqlFailure::Normalization)?, None => (None, None), }; - Ok(Some(( - line as i64, - column as i64, - usize_line(end_line), - usize_line(end_column), - ))) + Ok(Some(ResourceSpan { + line: line as i64, + column: column as i64, + end_line: usize_line(end_line), + end_column: usize_line(end_column), + })) } /// The authored span of the truth-value test a B103 finding starts at, matched by its order. -fn truth_test_span(resource: &ResourceSql, cleaned_sql: &str, start: usize) -> Option { +fn truth_test_span( + resource: &ResourceSql, + cleaned_sql: &str, + start: usize, +) -> Option { let cleaned: Vec<(usize, usize)> = truth_test_spans(cleaned_sql); let order: usize = cleaned .iter() @@ -134,7 +132,12 @@ fn truth_test_span(resource: &ResourceSql, cleaned_sql: &str, start: usize) -> O let length: usize = resource.authored_body[test_start..test_end].chars().count(); let (line, column) = offset_location(&resource.contents, offset); let (end_line, end_column) = offset_location(&resource.contents, offset + length); - Some((line, column, Some(end_line), Some(end_column))) + Some(ResourceSpan { + line, + column, + end_line: Some(end_line), + end_column: Some(end_column), + }) } /// Python's `_location`: the offending authored text, falling back to the body start. @@ -142,18 +145,18 @@ pub(crate) fn location( resource: &ResourceSql, cleaned_sql: &str, finding: &Finding, -) -> Result { +) -> Result { let body_offset = find_code_points(&resource.contents, &resource.authored_body).unwrap_or(0); - if finding.4.is_none() { + if finding.start.is_none() { let text: Vec = resource.contents.chars().collect(); - if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.1) { + if let Some(offset) = quoted_name_offset(&text, body_offset, &finding.message) { let (line, column) = offset_location(&resource.contents, offset); - return Ok((line, column, None, None)); + return Ok(ResourceSpan::start(line, column)); } } - if finding.0 == UNSUPPORTED_TRUTH_TEST_CODE + if finding.code == UNSUPPORTED_TRUTH_TEST_CODE && let Some(span) = finding - .4 + .start .and_then(|start| truth_test_span(resource, cleaned_sql, start)) { return Ok(span); @@ -162,7 +165,7 @@ pub(crate) fn location( return Ok(span); } let (line, column) = offset_location(&resource.contents, body_offset); - Ok((line, column, None, None)) + Ok(ResourceSpan::start(line, column)) } /// Code-point starts of each line of `text`. @@ -187,12 +190,12 @@ fn offset(starts: &[usize], line: i64, column: i64) -> Option { } /// Python's `_text_from`: the authored text of a span, or from its start. -pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { +pub(crate) fn text_from(resource: &ResourceSql, span: ResourceSpan) -> String { let starts = line_starts(&resource.contents); - let Some(start) = offset(&starts, span.0, span.1) else { + let Some(start) = offset(&starts, span.line, span.column) else { return String::new(); }; - let end = match (span.2, span.3) { + let end = match (span.end_line, span.end_column) { (Some(line), Some(column)) => offset(&starts, line, column), _ => None, }; @@ -204,12 +207,12 @@ pub(crate) fn text_from(resource: &ResourceSql, span: Span) -> String { } /// Python's `_line_at`: the authored line of a span, or empty text. -pub(crate) fn line_at(resource: &ResourceSql, span: Span) -> String { +pub(crate) fn line_at(resource: &ResourceSql, span: ResourceSpan) -> String { let lines: Vec<&str> = resource.contents.lines().collect(); - if span.0 < 1 { + if span.line < 1 { return String::new(); } lines - .get((span.0 - 1) as usize) + .get((span.line - 1) as usize) .map_or_else(String::new, |line| (*line).to_owned()) } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/models.rs b/crates/sqlbuild-analysis/src/semantic_checks/models.rs index d12faf7325..7674a801ce 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/models.rs @@ -445,6 +445,27 @@ pub struct ResourceSqlRequest { pub hook_files: Vec<(String, Arc)>, } +/// A resource SQL finding's authored start and, when it covers text, its end. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ResourceSpan { + pub line: i64, + pub column: i64, + pub end_line: Option, + pub end_column: Option, +} + +impl ResourceSpan { + /// A span that marks only its start. + pub fn start(line: i64, column: i64) -> Self { + Self { + line, + column, + end_line: None, + end_column: None, + } + } +} + /// A located resource SQL finding, or a rejected `sql_analysis false` with its note. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ResourceSqlDiagnostic { @@ -454,8 +475,7 @@ pub struct ResourceSqlDiagnostic { pub code: String, pub message: String, pub path: String, - /// `(line, column, end line, end column)`. - pub span: (i64, i64, Option, Option), + pub span: ResourceSpan, pub note: Option, pub help: Option, } diff --git a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs index 53b07ab184..f91795ba59 100644 --- a/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs +++ b/crates/sqlbuild-analysis/src/semantic_checks/tests/test_resource_sql.rs @@ -120,9 +120,9 @@ fn given_built_in_audit_on_missing_column_when_checking_then_reports_once_at_its ); ( diagnostic.message, - diagnostic.span.0, - diagnostic.span.1, - diagnostic.span.3.unwrap_or_default(), + diagnostic.span.line, + diagnostic.span.column, + diagnostic.span.end_column.unwrap_or_default(), ) }); assert_eq!( diff --git a/crates/sqlbuild-analysis/src/semantic_usage/main.rs b/crates/sqlbuild-analysis/src/semantic_usage/main.rs index 37a3d7e161..beeedb31e0 100644 --- a/crates/sqlbuild-analysis/src/semantic_usage/main.rs +++ b/crates/sqlbuild-analysis/src/semantic_usage/main.rs @@ -227,16 +227,26 @@ fn physical_source_name(expression: &Expression) -> Option { } } +/// The identity two usage facts share when one repeats the other. +#[derive(PartialEq, Eq, Hash)] +struct UsageKey { + context: String, + expression_sql: String, + source_name: String, + column: String, + start: Option, +} + fn dedupe(mut uses: Vec) -> Vec { - let mut seen: HashSet<(String, String, String, String, Option)> = HashSet::new(); + let mut seen: HashSet = HashSet::new(); uses.retain(|fact| { - seen.insert(( - fact.context.clone(), - fact.expression_sql.clone(), - fact.source_name.clone(), - fact.column.clone(), - fact.start, - )) + seen.insert(UsageKey { + context: fact.context.clone(), + expression_sql: fact.expression_sql.clone(), + source_name: fact.source_name.clone(), + column: fact.column.clone(), + start: fact.start, + }) }); uses } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs index a771ef8731..7f892c622e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/catalog.rs @@ -6,10 +6,10 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use crate::semantic_validation::models::{ - CatalogInput, Columns, CompactAnalysisJob, FunctionProbes, ProjectCatalog, + CatalogInput, Columns, CompactAnalysisJob, DiagnosticRow, FunctionProbes, ProjectCatalog, }; use crate::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, PreparedCompactAnalysis, Relations, + BindingRequest, NormalizationRequest, PreparedCompactAnalysis, Relations, }; use crate::semantic_validation::{ _helpers::{diagnostics, identifiers}, @@ -616,8 +616,8 @@ impl ProjectCatalog { /// The diagnostic row of `error`, naming the failure when the parser gives no message. fn messaged_row(error: ValidationError) -> DiagnosticRow { let mut row: DiagnosticRow = diagnostic_row(error); - if row.1.is_empty() { - DEFAULT_BINDING_MESSAGE.clone_into(&mut row.1); + if row.message.is_empty() { + DEFAULT_BINDING_MESSAGE.clone_into(&mut row.message); } row } @@ -627,15 +627,15 @@ pub(super) fn diagnostic_row(error: ValidationError) -> DiagnosticRow { polyglot_sql::ValidationSeverity::Error => "error", _ => "warning", }; - ( - error.code, - error.message, - error.line, - error.column, - error.start, - error.end, - severity.to_owned(), - ) + DiagnosticRow { + code: error.code, + message: error.message, + line: error.line, + column: error.column, + start: error.start, + end: error.end, + severity: severity.to_owned(), + } } fn has_exact_columns(table: &SchemaTable) -> bool { diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs index 0983e58f70..17175e9868 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/diagnostics.rs @@ -4,8 +4,8 @@ use crate::semantic_validation::_helpers::catalog::diagnostic_row; use crate::semantic_validation::_helpers::function_names; use crate::semantic_validation::_helpers::truth_tests; use crate::semantic_validation::_helpers::type_locations::locate_type_findings; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::models::FunctionProbes; -use crate::semantic_validation::types::DiagnosticRow; use polyglot_sql::expressions::Select; use polyglot_sql::{ Dialect, DialectType, Expression, ExpressionWalk, ValidationError, ValidationResult, @@ -393,21 +393,19 @@ pub(crate) fn binding_diagnostics( ) -> Result, String> { let errors: Vec = rows .into_iter() - .map( - |(code, message, line, column, start, end, severity)| ValidationError { - code, - message, - line, - column, - start, - end, - severity: if severity == ERROR { - ValidationSeverity::Error - } else { - ValidationSeverity::Warning - }, + .map(|row| ValidationError { + code: row.code, + message: row.message, + line: row.line, + column: row.column, + start: row.start, + end: row.end, + severity: if row.severity == ERROR { + ValidationSeverity::Error + } else { + ValidationSeverity::Warning }, - ) + }) .collect(); let result = map_diagnostics( sql, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs index d59718854b..0cf001abe2 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/_helpers/positions.rs @@ -2,7 +2,7 @@ use crate::semantic_validation::_helpers::diagnostics::column_pattern; use crate::semantic_validation::_helpers::normalization::normalize; -use crate::semantic_validation::models::{BindingPositions, PositionInput}; +use crate::semantic_validation::models::{BindingPositions, Expansion, PositionInput}; use regex::Regex; use std::collections::HashMap; use std::sync::{LazyLock, OnceLock}; @@ -10,7 +10,6 @@ use std::sync::{LazyLock, OnceLock}; const POPULAR_TOKEN_THRESHOLD: usize = 200; static TOKENS: LazyLock> = LazyLock::new(|| Regex::new(r"\w+|[^\w\s]").map_err(|error| error.to_string())); -type Mapping = (usize, usize, usize, usize); impl BindingPositions { pub fn new(request: PositionInput) -> Result { @@ -75,7 +74,13 @@ impl BindingPositions { .unwrap_or_else(|| *self.offsets.last().unwrap_or(&0)); for pass in self.passes.iter().rev() { let mut adjusted = mapped as isize; - for &(source_start, source_end, output_start, output_end) in pass { + for &Expansion { + source_start, + source_end, + output_start, + output_end, + } in pass + { if mapped < output_start { break; } @@ -227,28 +232,31 @@ fn align_offsets(original: &str, normalized: &str) -> Result, String> blocks.sort(); let ac = byte_char_map(original); let bc = byte_char_map(normalized); - let mut mappings: Vec = Vec::new(); + let mut mappings: Vec = Vec::new(); for (ai, bi, size) in blocks { for i in 0..size { - mappings.push(( - ac[a[ai + i].start()], - ac[a[ai + i].end()], - bc[b[bi + i].start()], - bc[b[bi + i].end()], - )); + mappings.push(Expansion { + source_start: ac[a[ai + i].start()], + source_end: ac[a[ai + i].end()], + output_start: bc[b[bi + i].start()], + output_end: bc[b[bi + i].end()], + }); } } let mut offsets = Vec::with_capacity(normalized.chars().count() + 1); let original_len = original.chars().count(); let mut index = 0; for offset in 0..=normalized.chars().count() { - while index < mappings.len() && offset > mappings[index].3 { + while index < mappings.len() && offset > mappings[index].output_end { index += 1; } offsets.push(match mappings.get(index) { - Some(&(start, end, target, _)) => { - start + offset.saturating_sub(target).min(end - start) - } + Some(&Expansion { + source_start: start, + source_end: end, + output_start: target, + .. + }) => start + offset.saturating_sub(target).min(end - start), None => original_len, }); } diff --git a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs index 2efcff0313..ae7f192632 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/main/diagnostics.rs @@ -1,4 +1,4 @@ -use crate::semantic_validation::types::DiagnosticRow; +use crate::semantic_validation::models::DiagnosticRow; pub fn binding_diagnostics( sql: &str, diff --git a/crates/sqlbuild-analysis/src/semantic_validation/models.rs b/crates/sqlbuild-analysis/src/semantic_validation/models.rs index 7f1805efcf..a24345c77a 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/models.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/models.rs @@ -1,6 +1,6 @@ //! Compile-owned native catalog, mapping state and boundary requests. -use crate::semantic_validation::types::{Expansion, PreparedCompactAnalysis, ProbeKey, Relations}; +use crate::semantic_validation::types::{PreparedCompactAnalysis, ProbeKey, Relations}; use polyglot_sql::validation::SchemaTable; use polyglot_sql::{DialectType, SchemaValidationOptions}; use rayon::ThreadPool; @@ -8,6 +8,27 @@ use std::collections::HashMap; use std::sync::atomic::AtomicUsize; use std::sync::{Arc, Mutex}; +/// One schema-validation diagnostic with its cleaned-SQL position. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiagnosticRow { + pub code: String, + pub message: String, + pub line: Option, + pub column: Option, + pub start: Option, + pub end: Option, + pub severity: String, +} + +/// One macro-expansion edit: the authored char span and the span it expanded to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Expansion { + pub source_start: usize, + pub source_end: usize, + pub output_start: usize, + pub output_end: usize, +} + #[derive(Default, Debug)] pub struct Columns(pub Vec<(String, Option)>); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs index 06c47b5f23..7bbe92b98e 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/tests/test_type_locations.rs @@ -1,4 +1,5 @@ use crate::semantic_validation::main::diagnostics::binding_diagnostics; +use crate::semantic_validation::models::DiagnosticRow; use crate::semantic_validation::tests::test_types::TypeLocationTestCase; #[test] @@ -68,16 +69,14 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex let rows = test_case .findings .iter() - .map(|(code, message)| { - ( - (*code).to_owned(), - (*message).to_owned(), - None, - None, - None, - None, - "error".to_owned(), - ) + .map(|(code, message)| DiagnosticRow { + code: (*code).to_owned(), + message: (*message).to_owned(), + line: None, + column: None, + start: None, + end: None, + severity: "error".to_owned(), }) .collect(); let mapped = binding_diagnostics(test_case.sql, test_case.dialect, rows)?; @@ -85,7 +84,7 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex .iter() .zip(test_case.expected_starts) .map(|(row, expected)| { - row.4.map_or_else(String::new, |start| { + row.start.map_or_else(String::new, |start| { test_case .sql .chars() @@ -107,7 +106,9 @@ fn given_position_less_type_findings_when_mapping_then_each_points_at_its_own_ex test_case.description ); assert!( - mapped.iter().all(|row| row.2.is_some() && row.3.is_some()), + mapped + .iter() + .all(|row| row.line.is_some() && row.column.is_some()), "{}", test_case.description ); diff --git a/crates/sqlbuild-analysis/src/semantic_validation/types.rs b/crates/sqlbuild-analysis/src/semantic_validation/types.rs index a6a8770c55..8ab94dd7a8 100644 --- a/crates/sqlbuild-analysis/src/semantic_validation/types.rs +++ b/crates/sqlbuild-analysis/src/semantic_validation/types.rs @@ -5,16 +5,6 @@ pub type Relations = HashMap; pub type BindingRequest = (String, Vec<(String, bool)>, Relations); /// One analysis SQL with its relation stubs and placeholder defaults. pub type NormalizationRequest = (String, HashMap, HashMap); -pub type DiagnosticRow = ( - String, - String, - Option, - Option, - Option, - Option, - String, -); -pub type Expansion = (usize, usize, usize, usize); pub(crate) type ProbeKey = (polyglot_sql::DialectType, String, usize); /// Compact analysis whose catalog schemas are resolved; it runs with an options-only catalog. pub(crate) type PreparedCompactAnalysis = diff --git a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs index 8a302d80e9..958cf3bd53 100644 --- a/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs +++ b/crates/sqlbuild-attachments/src/functions/_helpers/namespace.rs @@ -1,13 +1,14 @@ //! One of database or schema of a function namespace. -pub(crate) type Part = ( - Option, - Option, - Option, - Option, -); +/// The physical, logical and fingerprint names of one of database or schema. +pub(crate) struct Part { + pub(crate) physical: Option, + pub(crate) logical: Option, + pub(crate) fingerprint: Option, + pub(crate) fingerprint_logical: Option, +} -/// `(physical, logical, fingerprint, fingerprint logical)` for one of database or schema. +/// The physical, logical, fingerprint and fingerprint logical names of one namespace part. pub(crate) fn part( header: Option<&String>, default: Option<&String>, @@ -26,5 +27,10 @@ pub(crate) fn part( }; let fingerprint_logical: Option = header.or(default).cloned(); let fingerprint: Option = target.cloned().or_else(|| fingerprint_logical.clone()); - (physical, logical, fingerprint, fingerprint_logical) + Part { + physical, + logical, + fingerprint, + fingerprint_logical, + } } diff --git a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs index ae40e9d40b..74bff9d485 100644 --- a/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs +++ b/crates/sqlbuild-attachments/src/functions/main/resolve_function_namespace.rs @@ -8,26 +8,26 @@ use crate::functions::models::{FunctionLanguage, FunctionNamespace, NamespaceInp pub fn resolve_function_namespace(inputs: &NamespaceInputs) -> FunctionNamespace { let inherit: bool = inputs.language == FunctionLanguage::Sql || inputs.inherit_default_namespace; - let (database, logical_database, fingerprint_database, fingerprint_logical_database) = part( + let database = part( inputs.header_database.as_ref(), inputs.default_database.as_ref(), inputs.target_database.as_ref(), inherit, ); - let (schema, logical_schema, fingerprint_schema, fingerprint_logical_schema) = part( + let schema = part( inputs.header_schema.as_ref(), inputs.default_schema.as_ref(), inputs.target_schema.as_ref(), inherit, ); FunctionNamespace { - database, - schema, - logical_database, - logical_schema, - fingerprint_database, - fingerprint_schema, - fingerprint_logical_database, - fingerprint_logical_schema, + database: database.physical, + schema: schema.physical, + logical_database: database.logical, + logical_schema: schema.logical, + fingerprint_database: database.fingerprint, + fingerprint_schema: schema.fingerprint, + fingerprint_logical_database: database.fingerprint_logical, + fingerprint_logical_schema: schema.fingerprint_logical, } } diff --git a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs index 6c7719b1ad..251dda70cb 100644 --- a/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs +++ b/crates/sqlbuild-core/src/dbt_plan/main/render_dbt_execution_summary.rs @@ -14,7 +14,10 @@ pub fn render_dbt_execution_summary( if results.is_empty() { return None; } - let (mut passed, mut warned, mut failed, mut skipped): (i64, i64, i64, i64) = (0, 0, 0, 0); + let mut passed: i64 = 0; + let mut warned: i64 = 0; + let mut failed: i64 = 0; + let mut skipped: i64 = 0; let mut elapsed: f64 = 0.0; for result in results { let status: String = result.status.to_lowercase(); diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index fd5d643514..70e6a04f84 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -1,6 +1,6 @@ //! The versioned lifecycle event catalogue and the payload value contracts it validates. -use crate::lifecycle_events::types::CatalogEntry; +use crate::lifecycle_events::models::EventDefinition; /// The lifecycle event schema version SQLBuild produces. pub const CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION: i64 = 2; @@ -277,45 +277,51 @@ pub const OPERATION_ADAPTERS: [&str; 10] = [ ]; /// The schema version 1 catalogue in declaration order. -pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ - ("invocation_started", &[], &[], &["command"], false), - ( - "invocation_completed", - &[], - &[], - &["command", "duration_ms", "exit_code"], - true, - ), - ( - "invocation_failed", - &[], - &[], - &[ +pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [EventDefinition; 20] = [ + EventDefinition { + event_type: "invocation_started", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command"], + terminal: false, + }, + EventDefinition { + event_type: "invocation_completed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command", "duration_ms", "exit_code"], + terminal: true, + }, + EventDefinition { + event_type: "invocation_failed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &[ "command", "duration_ms", "error_code", "error_type", "exit_code", ], - true, - ), - ( - "run_started", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_started", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "configured_concurrency", "run_kind", "selected_count", "worker_count", ], - false, - ), - ( - "run_completed", - &["run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "run_completed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "fail_count", "failed_count", @@ -325,13 +331,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "run_failed", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_failed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "error_code", "error_type", @@ -343,32 +349,32 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "resource_attempt_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &["attempt_number", "resource_kind", "resource_name"], - false, - ), - ( - "resource_attempt_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &["attempt_number", "resource_kind", "resource_name"], + terminal: false, + }, + EventDefinition { + event_type: "resource_attempt_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "error_code", @@ -376,18 +382,18 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_skipped", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_skipped", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "attempt_number", "resource_kind", "resource_name", "skip_code", ], - &[ + allowed_payload: &[ "attempt_number", "duration_ms", "resource_kind", @@ -395,13 +401,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "skip_code", "skip_mode", ], - true, - ), - ( - "operation_started", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_started", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "hook_index", "hook_name", @@ -415,13 +421,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - false, - ), - ( - "operation_completed", - &["operation_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "operation_completed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "exit_code", @@ -439,13 +445,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - true, - ), - ( - "operation_failed", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_failed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "error_code", @@ -465,30 +471,30 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "strategy", "target_kind", ], - true, - ), - ( - "retry_scheduled", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "retry_scheduled", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "delay_ms", "error_type", "failed_attempt_number", "next_attempt_number", ], - &[ + allowed_payload: &[ "delay_ms", "error_code", "error_type", "failed_attempt_number", "next_attempt_number", ], - false, - ), - ( - "audit_completed", - &["run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "audit_completed", + required_correlations: &["run_id"], + required_payload: &[ "attachment_kind", "audit_definition_name", "audit_name", @@ -497,7 +503,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "run_scope_phase", "severity", ], - &[ + allowed_payload: &[ "attached_column_name", "attached_target_kind", "attached_target_name", @@ -534,13 +540,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "thresholds", "violation_count", ], - true, - ), - ( - "statement_started", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_started", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -552,13 +558,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_submitted", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_submitted", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -569,13 +575,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_heartbeat", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_heartbeat", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -588,13 +594,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_completed", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_completed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -607,13 +613,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "statement_failed", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_failed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -628,55 +634,55 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V1: [CatalogEntry; 20] = [ "sql_digest", "statement_kind", ], - true, - ), + terminal: true, + }, ]; /// The schema version 2 catalogue in declaration order. -pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ - ( - "invocation_started", - &[], - &[], - &["adapter", "command", "selection_count", "target"], - false, - ), - ( - "invocation_completed", - &[], - &[], - &["command", "duration_ms", "exit_code"], - true, - ), - ( - "invocation_failed", - &[], - &[], - &[ +pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ + EventDefinition { + event_type: "invocation_started", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["adapter", "command", "selection_count", "target"], + terminal: false, + }, + EventDefinition { + event_type: "invocation_completed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &["command", "duration_ms", "exit_code"], + terminal: true, + }, + EventDefinition { + event_type: "invocation_failed", + required_correlations: &[], + required_payload: &[], + allowed_payload: &[ "command", "duration_ms", "error_code", "error_type", "exit_code", ], - true, - ), - ( - "run_started", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_started", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "configured_concurrency", "run_kind", "selected_count", "worker_count", ], - false, - ), - ( - "run_completed", - &["run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "run_completed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "fail_count", "failed_count", @@ -686,13 +692,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "run_failed", - &["run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "run_failed", + required_correlations: &["run_id"], + required_payload: &[], + allowed_payload: &[ "duration_ms", "error_code", "error_type", @@ -704,26 +710,26 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "succeeded_count", "warn_count", ], - true, - ), - ( - "resource_attempt_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "plan_action", "plan_reason_code", "resource_kind", "resource_name", ], - false, - ), - ( - "resource_attempt_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "resource_attempt_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "plan_action", @@ -731,13 +737,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[], + allowed_payload: &[ "attempt_number", "duration_ms", "error_code", @@ -747,18 +753,18 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "resource_kind", "resource_name", ], - true, - ), - ( - "resource_attempt_skipped", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "resource_attempt_skipped", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "attempt_number", "resource_kind", "resource_name", "skip_code", ], - &[ + allowed_payload: &[ "attempt_number", "duration_ms", "plan_action", @@ -768,13 +774,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "skip_code", "skip_mode", ], - true, - ), - ( - "operation_started", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_started", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "hook_index", "hook_name", @@ -788,13 +794,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - false, - ), - ( - "operation_completed", - &["operation_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "operation_completed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "exit_code", @@ -812,13 +818,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - true, - ), - ( - "operation_failed", - &["operation_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "operation_failed", + required_correlations: &["operation_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "duration_ms", "error_code", @@ -838,30 +844,30 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "strategy", "target_kind", ], - true, - ), - ( - "retry_scheduled", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "retry_scheduled", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "delay_ms", "error_type", "failed_attempt_number", "next_attempt_number", ], - &[ + allowed_payload: &[ "delay_ms", "error_code", "error_type", "failed_attempt_number", "next_attempt_number", ], - false, - ), - ( - "audit_completed", - &["run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "audit_completed", + required_correlations: &["run_id"], + required_payload: &[ "attachment_kind", "audit_definition_name", "audit_name", @@ -870,7 +876,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "run_scope_phase", "severity", ], - &[ + allowed_payload: &[ "attached_column_name", "attached_target_kind", "attached_target_name", @@ -907,13 +913,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "thresholds", "violation_count", ], - true, - ), - ( - "statement_started", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_started", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -926,13 +932,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_submitted", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_submitted", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -944,13 +950,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_heartbeat", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_heartbeat", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -964,13 +970,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - false, - ), - ( - "statement_completed", - &["statement_id"], - &[], - &[ + terminal: false, + }, + EventDefinition { + event_type: "statement_completed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -984,13 +990,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "statement_failed", - &["statement_id"], - &[], - &[ + terminal: true, + }, + EventDefinition { + event_type: "statement_failed", + required_correlations: &["statement_id"], + required_payload: &[], + allowed_payload: &[ "adapter", "affected_rows", "batch_size", @@ -1006,12 +1012,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "sql_digest", "statement_kind", ], - true, - ), - ( - "microbatch_started", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "microbatch_started", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1024,7 +1030,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1037,12 +1043,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - false, - ), - ( - "microbatch_completed", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: false, + }, + EventDefinition { + event_type: "microbatch_completed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1055,7 +1061,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "affected_rows", "batch_count", "batch_index", @@ -1070,12 +1076,12 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - true, - ), - ( - "microbatch_failed", - &["resource_attempt_id", "resource_id", "run_id"], - &[ + terminal: true, + }, + EventDefinition { + event_type: "microbatch_failed", + required_correlations: &["resource_attempt_id", "resource_id", "run_id"], + required_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1089,7 +1095,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - &[ + allowed_payload: &[ "batch_count", "batch_index", "batch_kind", @@ -1105,6 +1111,6 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [CatalogEntry; 23] = [ "planned_cursor_end_exclusive", "planned_cursor_start", ], - true, - ), + terminal: true, + }, ]; diff --git a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs index 74d374d42a..3d8922df03 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/main/catalog.rs @@ -2,26 +2,14 @@ use crate::lifecycle_events::constants::{LIFECYCLE_EVENT_CATALOG_V1, LIFECYCLE_EVENT_CATALOG_V2}; use crate::lifecycle_events::models::EventDefinition; -use crate::lifecycle_events::types::CatalogEntry; - -fn definition(entry: &CatalogEntry) -> EventDefinition { - let (event_type, required_correlations, required_payload, allowed_payload, terminal) = *entry; - EventDefinition { - event_type, - required_correlations, - required_payload, - allowed_payload, - terminal, - } -} /// Every definition of `schema_version`, or `None` for an uncatalogued version. #[must_use] pub fn catalog(schema_version: i64) -> Option> { - let entries: &[CatalogEntry] = match schema_version { + let entries: &[EventDefinition] = match schema_version { 1 => &LIFECYCLE_EVENT_CATALOG_V1, 2 => &LIFECYCLE_EVENT_CATALOG_V2, _ => return None, }; - Some(entries.iter().map(definition).collect()) + Some(entries.to_vec()) } diff --git a/crates/sqlbuild-core/src/lifecycle_events/types.rs b/crates/sqlbuild-core/src/lifecycle_events/types.rs index 2af438e15f..7719efd781 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/types.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/types.rs @@ -1,13 +1,4 @@ -//! The shape of one catalogue entry. - -/// One catalogued event: type, required correlations, required and allowed payload, terminal. -pub(crate) type CatalogEntry = ( - &'static str, - &'static [&'static str], - &'static [&'static str], - &'static [&'static str], - bool, -); +//! The shape of one event-family payload rule. /// One event-family payload rule: the first violation in a payload, if any. pub(crate) type PayloadCheck = diff --git a/crates/sqlbuild-discovery/src/_helpers/locations.rs b/crates/sqlbuild-discovery/src/_helpers/locations.rs index e74876e9b0..14e7d4fa22 100644 --- a/crates/sqlbuild-discovery/src/_helpers/locations.rs +++ b/crates/sqlbuild-discovery/src/_helpers/locations.rs @@ -18,7 +18,13 @@ pub(crate) fn header_column_locations( let header: Vec = contents[header_start..header_end].chars().collect(); relative_locations(&header, offsets) .into_iter() - .map(|(name, relative_line, relative_column, length)| { + .map(|location| { + let RelativeLocation { + name, + line: relative_line, + column: relative_column, + length, + } = location; let line = header_line + relative_line - 1; let column = relative_column + if relative_line == 1 { @@ -39,11 +45,19 @@ pub(crate) fn header_column_locations( .collect() } +/// One header column key's line and column relative to the header, and its length. +struct RelativeLocation { + name: String, + line: usize, + column: usize, + length: usize, +} + /// Python's `_header_column_relative_locations`, including its forward-only cursor. fn relative_locations( header: &[char], offsets: &[(String, usize, usize)], -) -> Vec<(String, usize, usize, usize)> { +) -> Vec { let mut locations = Vec::with_capacity(offsets.len()); let (mut line, mut column, mut cursor) = (1, 1, 0); for (name, position, length) in offsets { @@ -63,7 +77,12 @@ fn relative_locations( } } } - locations.push((name.clone(), line, column, *length)); + locations.push(RelativeLocation { + name: name.clone(), + line, + column, + length: *length, + }); } locations } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs index 00b92ba4c4..a7db77b4eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_catalog.rs @@ -9,7 +9,10 @@ use pyo3::prelude::{ use pyo3::types::{PyBool, PyDict, PyMapping, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::types::{BindingRequest, DiagnosticRow, Relations}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; + +use crate::bindings::types::IntoPositional; +use sqlbuild_analysis::semantic_validation::types::{BindingRequest, Relations}; use crate::bindings::_helpers::analysis::conversions; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -57,7 +60,13 @@ pub(crate) struct BindingCatalog { } /// A catalog's dialect, case mode, known functions and known types. -pub(crate) type CatalogKey = (String, bool, Vec, Vec); +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct CatalogKey { + pub(crate) dialect: String, + pub(crate) quoted_ignore_case: bool, + pub(crate) known_functions: Vec, + pub(crate) known_types: Vec, +} impl BindingCatalog { /// Whether `schemas` holds exactly the relations it was created with and later given. @@ -68,9 +77,15 @@ impl BindingCatalog { /// A catalog for `key` that starts with `relations`. pub(crate) fn create( py: Python<'_>, - (dialect, quoted_ignore_case, known_functions, known_types): CatalogKey, + key: CatalogKey, relations: &Bound<'_, PyAny>, ) -> PyResult { + let CatalogKey { + dialect, + quoted_ignore_case, + known_functions, + known_types, + } = key; let input: CatalogInput = CatalogInput { dialect: dialect.clone(), quoted_ignore_case, @@ -160,12 +175,12 @@ impl BindingCatalog { )) }) }; - let key: CatalogKey = ( - field("dialect")?.extract()?, - field("quoted_ignore_case")?.extract()?, - field("known_functions")?.extract()?, - field("known_types")?.extract()?, - ); + let key: CatalogKey = CatalogKey { + dialect: field("dialect")?.extract()?, + quoted_ignore_case: field("quoted_ignore_case")?.extract()?, + known_functions: field("known_functions")?.extract()?, + known_types: field("known_types")?.extract()?, + }; Self::create(py, key, &field("relations")?) } @@ -309,12 +324,13 @@ pub(crate) fn schema_validations<'py>( let mut created: Vec<(CatalogKey, Py)> = Vec::new(); let mut groups: Vec<(Py, bool, Vec)> = Vec::new(); for (index, request) in requests.iter().enumerate() { - let key: CatalogKey = ( - native_dialect(&native_field(request, "dialect")?)?, - native_field(request, "quoted_identifiers_ignore_case")?.extract()?, - native_field(request, "known_functions")?.extract()?, - native_field(request, "known_types")?.extract()?, - ); + let key: CatalogKey = CatalogKey { + dialect: native_dialect(&native_field(request, "dialect")?)?, + quoted_ignore_case: native_field(request, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(request, "known_functions")?.extract()?, + known_types: native_field(request, "known_types")?.extract()?, + }; let explicit: Bound<'py, PyAny> = native_field(request, "catalog")?; let own: bool = !explicit.is_none(); let catalog: Py = if own { @@ -363,7 +379,14 @@ pub(crate) fn schema_validations<'py>( for (index, diagnostics) in members.into_iter().zip(rows) { let built: Vec> = diagnostics .into_iter() - .map(|row| new_record(py, RecordId::SqlBindingDiagnostic, row, None)) + .map(|row| { + new_record( + py, + RecordId::SqlBindingDiagnostic, + row.into_positional(py)?, + None, + ) + }) .collect::>()?; let keywords: Bound<'py, PyDict> = PyDict::new(py); keywords.set_item("diagnostics", PyTuple::new(py, built)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs index 4ac680f26c..52f7e0d07d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/binding_locations.rs @@ -4,27 +4,44 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; -use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::binding_locations::models::{ + AuthoredSpan, BindingSource, LocatedDiagnostic, +}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; -/// `(code, message, severity, line, column, (line, column, end line, end column))`. -type LocatedRow = ( - String, - String, - String, - Option, - Option, - Option<(usize, usize, Option, Option)>, -); +use crate::bindings::models::Positional; + +positional_record! { + into; + /// One located diagnostic as Python reads it, its span `(line, column, end line, end column)`. + pub(crate) struct LocatedRow { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) severity: String, + pub(crate) line: Option, + pub(crate) column: Option, + pub(crate) span: Option>, + } +} + +/// The expanded SQL and the expansion passes Python sends for a located model. +pub(crate) type ExpansionRow = Option<(String, Vec>>)>; /// Each reported diagnostic of `rows` through `(authored, query, cleaned)` and the expansion. #[pyfunction] pub(crate) fn locate_model_binding_diagnostics( - rows: Vec, + rows: Vec>, texts: (String, String, String), - expansion: Option<(String, Vec>)>, + expansion: ExpansionRow, ) -> PyResult> { let (authored, query, cleaned) = texts; + let rows: Vec = Positional::unwrap_all(rows); + let expansion: Option<(String, Vec>)> = expansion.map(|(expanded, passes)| { + ( + expanded, + passes.into_iter().map(Positional::unwrap_all).collect(), + ) + }); let source: BindingSource<'_> = BindingSource { authored: &authored, query: &query, @@ -39,17 +56,14 @@ pub(crate) fn locate_model_binding_diagnostics( } fn located_row(item: LocatedDiagnostic) -> LocatedRow { - let span = item - .location - .map(|span| (span.line, span.column, span.end_line, span.end_column)); - ( - item.code, - item.message, - item.severity, - item.line, - item.column, - span, - ) + LocatedRow { + code: item.code, + message: item.message, + severity: item.severity, + line: item.line, + column: item.column, + span: item.location.map(Positional), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs index 37314dacda..52c663d6b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/catalog.rs @@ -3,6 +3,7 @@ use crate::bindings::_helpers::analysis::conversions; use crate::bindings::_helpers::analysis::normalization_results::normalization_results; use crate::bindings::_helpers::boundary::panics::compiler_error; +use crate::bindings::models::Positional; use crate::bindings::models::{CatalogInput, CompactAnalysisJob, ProjectCatalog}; use crate::bindings::types::CompilerDetach; use pyo3::exceptions::PyValueError; @@ -10,8 +11,9 @@ use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use pyo3::pymethods; use pyo3::types::PyBytes; use sqlbuild_analysis::semantic_validation::models as validation; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_analysis::semantic_validation::types::{ - BindingRequest, DiagnosticRow, NormalizationRequest, Relations, + BindingRequest, NormalizationRequest, Relations, }; use std::collections::HashMap; @@ -90,9 +92,11 @@ impl ProjectCatalog { &self, py: Python<'_>, #[pyo3(from_py_with = conversions::binding_requests)] requests: Vec, - ) -> PyResult>> { - py.compiler_detach(|| self.inner.binding_results(requests)) - .map_err(compiler_error) + ) -> PyResult>>> { + let rows: Vec> = py + .compiler_detach(|| self.inner.binding_results(requests)) + .map_err(compiler_error)?; + Ok(rows.into_iter().map(Positional::wrap_all).collect()) } /// Normalize a preparation batch on the idle analysis pool, else in turn, in one detached call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs index 47c47892ed..221a27d6de 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/explicit_references.rs @@ -423,16 +423,23 @@ pub(crate) fn old_name_reference_message( old_name_message(owner_label, written, method, model_name) } -/// `(audit name, target, gated, read)` for one attached-audit ordering edge. -pub(crate) type GateRow = (String, ResourceKey, ResourceKey, ResourceKey); -/// `(hook name, gated, read, HookReadType value, path)` for one hook-read ordering edge. -pub(crate) type HookRow = ( - String, - ResourceKey, - ResourceKey, - &'static str, - Option, -); +/// One attached-audit ordering edge: the audit, its target, and the gated and read resources. +pub(crate) struct GateRow { + pub(crate) audit_name: String, + pub(crate) target: ResourceKey, + pub(crate) gated: ResourceKey, + pub(crate) read: ResourceKey, +} + +/// One hook-read ordering edge: the hook, the gated and read resources, its type and path. +pub(crate) struct HookRow { + pub(crate) hook_name: String, + pub(crate) gated: ResourceKey, + pub(crate) read: ResourceKey, + /// The `HookReadType` value. + pub(crate) hook_type: &'static str, + pub(crate) path: Option, +} /// The attached-audit gate edges and hook-read edges of one project, in Python's order. pub(crate) fn native_execution_gate_edges( @@ -452,7 +459,12 @@ pub(crate) fn native_execution_gate_edges( } fn gate_row(edge: GateEdge) -> GateRow { - (edge.audit_name, edge.target, edge.gated, edge.read) + GateRow { + audit_name: edge.audit_name, + target: edge.target, + gated: edge.gated, + read: edge.read, + } } fn hook_row(edge: HookEdge) -> HookRow { @@ -461,7 +473,13 @@ fn hook_row(edge: HookEdge) -> HookRow { HookType::Sql => "SQL", HookType::InlineSql => "inline SQL", }; - (edge.hook_name, edge.gated, edge.read, hook_type, edge.path) + HookRow { + hook_name: edge.hook_name, + gated: edge.gated, + read: edge.read, + hook_type, + path: edge.path, + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs index e1012641f5..9c449309a4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis/model_reads.rs @@ -7,8 +7,14 @@ use crate::bindings::_helpers::compiled_project::compiled_model_view::{ }; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(output, transform kind, confidence, [(resource type, resource, column)])` of one lineage output. -pub(crate) type LineageFactRow = (String, String, String, Vec<(String, String, String)>); +/// One lineage output with its transform kind and confidence values, and its sources. +pub(crate) struct LineageFactRow { + pub(crate) output_column: String, + pub(crate) transform: String, + pub(crate) confidence: String, + /// `(resource type, resource, column)` of each source. + pub(crate) sources: Vec<(String, String, String)>, +} const TRANSFORM_VALUES: [&str; 6] = [ "direct", @@ -139,12 +145,12 @@ pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult = Vec::with_capacity(rows.len()); for row in rows { - facts.push(( - row.output_column, - coded(&TRANSFORM_VALUES, row.transform_code)?, - coded(&CONFIDENCE_VALUES, row.confidence_code)?, - row.sources, - )); + facts.push(LineageFactRow { + output_column: row.output_column, + transform: coded(&TRANSFORM_VALUES, row.transform_code)?, + confidence: coded(&CONFIDENCE_VALUES, row.confidence_code)?, + sources: row.sources, + }); } return Ok(Some(facts)); } @@ -164,12 +170,12 @@ pub(crate) fn lineage_facts(model: &Bound<'_, PyAny>) -> PyResult, sql: &str, dialect: &str, - rows: Vec, -) -> PyResult> { + rows: Vec>, +) -> PyResult>> { + let rows: Vec = Positional::unwrap_all(rows); py.compiler_detach(|| { sqlbuild_analysis::semantic_validation::main::diagnostics::binding_diagnostics( sql, dialect, rows, ) }) + .map(Positional::wrap_all) .map_err(value_error) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs index 2610876a8a..840a961495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/model_analyses.rs @@ -19,11 +19,14 @@ use crate::bindings::_helpers::analysis_session::session::{ use crate::bindings::_helpers::analysis_session::stage_functions::family_rows; use crate::bindings::_helpers::attachments::cursor_intrinsics::analysis_cursor_sql; use crate::bindings::_helpers::cache::native_store::{content_digest, current_code_identity}; -use crate::bindings::_helpers::compile_driver::analysis_cache::record_analysis_cache_metrics; +use crate::bindings::_helpers::compile_driver::analysis_cache::{ + AnalysisCacheCounts, record_analysis_cache_metrics, +}; use crate::bindings::_helpers::compiled_project::model_input_view::{ AttachedSchema, NativeModelInput, attached_schema, input_model_name, input_query_sql, input_reference_rows, }; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::native_host::executor::ExecutorHost; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::enum_member; @@ -106,36 +109,46 @@ fn session_request<'py>( let (name, declared): (String, Bound<'py, PyAny>) = item?.extract()?; families.push((name, family_rows(&declared)?)); } - let (types, nullability, complete, schemas): (Shapes, Shapes, Shapes, Shapes) = - match declared_rows(request, catalog)? { - Some(rows) => rows, - None => ( - shape_rows(&request.column_types)?, - shape_rows(&request.nullability)?, - shape_rows(&request.complete_schemas)?, - shape_rows(&native_field(catalog, "schemas")?)?, - ), - }; - Ok(( - profile_dialect(profile)?, - case_sensitive_shapes(profile, &native_field(profile, "sql_analysis_dialect")?)?, - return_types(profile)?, - Some(rules), - request.rich_type_inference, - types, - nullability, - complete, - schemas, - families, - model_rows(profile.py(), &request.model_inputs)?, - )) + let shapes: RelationShapes = match declared_rows(request, catalog)? { + Some(rows) => rows, + None => RelationShapes { + types: shape_rows(&request.column_types)?, + nullability: shape_rows(&request.nullability)?, + complete: shape_rows(&request.complete_schemas)?, + schemas: shape_rows(&native_field(catalog, "schemas")?)?, + }, + }; + Ok(RequestRow { + dialect: profile_dialect(profile)?, + case_sensitive_shapes: case_sensitive_shapes( + profile, + &native_field(profile, "sql_analysis_dialect")?, + )?, + function_return_types: return_types(profile)?, + nullability_rules: Some(rules), + rich_type_inference: request.rich_type_inference, + column_types: shapes.types, + column_nullability: shapes.nullability, + complete_schemas: shapes.complete, + catalog_schemas: shapes.schemas, + dynamic_families_by_table: families, + models: model_rows(profile.py(), &request.model_inputs)?, + }) +} + +/// The column types, nullability, complete schemas and catalog schemas a session reads. +struct RelationShapes { + types: Shapes, + nullability: Shapes, + complete: Shapes, + schemas: Shapes, } /// The caller's native declaration rows when they still match the maps and catalog they mirror. fn declared_rows( request: &ModelAnalysisRequest<'_, '_>, catalog: &Bound<'_, PyAny>, -) -> PyResult> { +) -> PyResult> { let Some(declared) = request.declared else { return Ok(None); }; @@ -151,12 +164,12 @@ fn declared_rows( if !unchanged { return Ok(None); } - Ok(Some(( - declared.types.rows().clone(), - declared.nullability.rows().clone(), - complete.clone(), - complete.clone(), - ))) + Ok(Some(RelationShapes { + types: declared.types.rows().clone(), + nullability: declared.nullability.rows().clone(), + complete: complete.clone(), + schemas: complete.clone(), + })) } /// The analysis store's path and environment, or None where Python bypasses its cache. @@ -276,7 +289,13 @@ fn record_cache( python: &[Bound<'_, PyAny>], stats: Option, ) -> PyResult<()> { - let Some((hits, misses, _, failure)) = stats else { + let Some(CacheStatsRow { + hits, + misses, + failure, + .. + }) = stats + else { return record_metrics((0, 0, python.len())); }; if let Some(failure) = failure { @@ -294,7 +313,12 @@ fn record_cache( /// Record the analysis cache entries a native session read, missed and bypassed. fn record_metrics((hits, misses, bypasses): (usize, usize, usize)) -> PyResult<()> { - record_analysis_cache_metrics((0, hits, misses, bypasses)); + record_analysis_cache_metrics(AnalysisCacheCounts { + batch_hits: 0, + entry_hits: hits, + misses, + bypasses, + }); Ok(()) } @@ -334,15 +358,15 @@ fn model_rows<'py>(py: Python<'py>, model_inputs: &[Bound<'py, PyAny>]) -> PyRes Some(schema) => family_rows(&schema.dynamic_columns)?, None => Vec::new(), }; - rows.push(( - input_model_name(model_input)?, + rows.push(ModelRow { + name: input_model_name(model_input)?, query_sql, placeholders, - references, - recovers, + reference_rows: references, + recover_cte_facts: recovers, snapshot_columns, - families, - )); + dynamic_families: families, + }); } Ok(rows) } @@ -352,7 +376,7 @@ fn analysed_references(model_input: &Bound<'_, PyAny>) -> PyResult() { return Ok(input_reference_rows(model_input)? .into_iter() - .map(|(kind, name, _, _)| (kind, name)) + .map(|ReferenceRow { kind, name, .. }| (kind, name)) .collect()); } let mut references: Vec<(String, String)> = Vec::new(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs index 5d597675aa..e195edbd4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/oracle.rs @@ -10,15 +10,36 @@ use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::types::CompilerDetach; -/// `(cleaned SQL, dialect, input schemas, return types, nullability rules, recover, null filter)`. -type RecoveryRow = (String, String, Shapes, Pairs, Option, bool, bool); -/// `(deferral, types, nullability, sorted direct outputs, sorted non-null outputs)`. -type FactsRow = (Option, Pairs, Pairs, Vec, Vec); +positional_record! { + extract; + /// One query's CTE fact recovery request, as the oracle tests send it. + struct RecoveryRow { + cleaned_sql: String, + dialect: String, + input_schemas: Shapes, + function_return_types: Pairs, + nullability_rules: Option, + recover: bool, + null_filter: bool, + } +} + +positional_record! { + into; + /// The deferral, types, nullability, sorted direct outputs and sorted non-null outputs. + struct FactsRow { + deferral: Option, + types: Pairs, + nullability: Pairs, + direct_outputs: Vec, + non_null_outputs: Vec, + } +} /// The facts native enrichment recovers for one query, or the reason it defers to Python. #[pyfunction] fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult { - let ( + let RecoveryRow { cleaned_sql, dialect, input_schemas, @@ -26,7 +47,7 @@ fn _oracle_cte_fact_recovery(py: Python<'_>, request: RecoveryRow) -> PyResult, request: RecoveryRow) -> PyResult ( - None, - facts.types, - facts.nullability, - facts.direct_outputs, - facts.non_null_outputs, - ), - Err(deferral) => ( - Some(deferral), - Vec::new(), - Vec::new(), - Vec::new(), - Vec::new(), - ), + Ok(facts) => FactsRow { + deferral: None, + types: facts.types, + nullability: facts.nullability, + direct_outputs: facts.direct_outputs, + non_null_outputs: facts.non_null_outputs, + }, + Err(deferral) => FactsRow { + deferral: Some(deferral), + types: Vec::new(), + nullability: Vec::new(), + direct_outputs: Vec::new(), + non_null_outputs: Vec::new(), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs index 02ae2be7b0..4143d02422 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/outcome_objects.rs @@ -15,27 +15,45 @@ use crate::bindings::_helpers::records::ids::RecordId; const LINEAGE_NATIVE: &str = "native"; type Object<'py> = Bound<'py, PyAny>; -/// `(succeeded, columns, lineage source, lineage rows, star, resolved, bindings, validated, SQL)`. -type OutcomeObjects<'py> = ( - Object<'py>, - Object<'py>, - String, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, -); -/// `(output proven, fixed columns, families, input relations, failure reason, bare pivot)`. -type ProofObjects<'py> = ( - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, -); +positional_record! { + extract; + /// A session outcome row: its analysis, lineage, binding diagnostics and cleaned SQL. + struct OutcomeObjects<'py> { + succeeded: Object<'py>, + columns: Object<'py>, + source: String, + rows: Object<'py>, + has_star: Object<'py>, + star_resolved: Object<'py>, + raw: Object<'py>, + validated: Object<'py>, + cleaned: Object<'py>, + } +} + +positional_record! { + extract; + /// A dynamic pivot proof row. + struct ProofObjects<'py> { + output_proven: Object<'py>, + columns: Object<'py>, + families: Object<'py>, + inputs: Object<'py>, + failure_reason: Object<'py>, + bare: Object<'py>, + } +} + +positional_record! { + extract; + /// A lineage row: output column, transform and confidence codes and its sources. + struct LineageObjects<'py> { + output_column: Object<'py>, + transform_code: Object<'py>, + confidence_code: Object<'py>, + sources: Object<'py>, + } +} /// The outcome at `position` of a finished session as `ModelSqlAnalysis`, for hand-built inputs. #[pyfunction] @@ -49,7 +67,17 @@ pub(crate) fn session_outcome_analysis<'py>( .get("session")? .call_method1("outcome_rows", (request.get("position")?,))? .extract()?; - let (succeeded, columns, source, rows, has_star, star_resolved, raw, validated, cleaned): OutcomeObjects<'py> = outcome.extract()?; + let OutcomeObjects { + succeeded, + columns, + source, + rows, + has_star, + star_resolved, + raw, + validated, + cleaned, + } = outcome.extract()?; let lineage: Bound<'py, PyAny> = if source != LINEAGE_NATIVE { lineage_facts(&rows)? } else if succeeded.is_truthy()? { @@ -101,8 +129,14 @@ fn proof<'py>(row: &Bound<'py, PyAny>) -> PyResult> { if row.is_none() { return Ok(row.clone()); } - let (output_proven, columns, families, inputs, failure_reason, bare): ProofObjects<'py> = - row.extract()?; + let ProofObjects { + output_proven, + columns, + families, + inputs, + failure_reason, + bare, + } = row.extract()?; let family_class: Bound<'py, PyAny> = record_class(py, RecordId::DynamicColumnFamilyProof)?; let mut proofs: Vec> = Vec::new(); for family in families.try_iter()? { @@ -174,12 +208,12 @@ fn compact_lineage<'py>(rows: &Bound<'py, PyAny>) -> PyResult> }; let mut compact: Vec> = Vec::new(); for row in rows.try_iter()? { - let (output_column, transform_code, confidence_code, sources): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = row?.extract()?; + let LineageObjects { + output_column, + transform_code, + confidence_code, + sources, + } = row?.extract()?; let mut source_indexes: Vec> = Vec::new(); for source in sources.try_iter()? { let (resource_type, resource_name, column_name): ( @@ -225,12 +259,12 @@ fn lineage_facts<'py>(rows: &Bound<'py, PyAny>) -> PyResult> { let source_class: Bound<'py, PyAny> = record_class(py, RecordId::CompiledLineageSourceFact)?; let mut facts: Vec> = Vec::new(); for row in rows.try_iter()? { - let (output_column, transform_code, confidence_code, sources): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = row?.extract()?; + let LineageObjects { + output_column, + transform_code, + confidence_code, + sources, + } = row?.extract()?; let mut upstream: Vec> = Vec::new(); for source in sources.try_iter()? { let (resource_type, resource_name, column_name): ( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs index 7348587e6b..bdf1ddf33d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/profile_inputs.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, NATIVE_DIALECT_ALIASES, + BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, NATIVE_DIALECT_ALIASES, }; use crate::bindings::_helpers::native_host::adapter::AdapterHost; use crate::bindings::_helpers::native_host::host; @@ -135,12 +135,13 @@ pub(crate) fn profile_catalog<'py>( let py: Python<'py> = profile.py(); let catalog: BindingCatalog = BindingCatalog::create( py, - ( - dialect.to_owned(), - native_field(profile, "quoted_identifiers_ignore_case")?.extract()?, - Vec::new(), - Vec::new(), - ), + CatalogKey { + dialect: dialect.to_owned(), + quoted_ignore_case: native_field(profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: Vec::new(), + known_types: Vec::new(), + }, PyDict::new(py).as_any(), )?; Ok(Bound::new(py, catalog)?.into_any()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs index 13b2eecf79..e54f8b3274 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/query_columns.rs @@ -23,17 +23,32 @@ use crate::bindings::types::CompilerDetach; const QUERY_COLUMNS_CONTEXT: &str = "native query column analysis"; const PROJECTION_FACTS_CONTEXT: &str = "native fixture projection facts"; -/// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. -pub(crate) type QueryRow = ( - String, - Pairs, - Vec, - Vec<(String, String, String)>, - bool, - String, -); -/// `(dialect, return types, rules, types, nullability, queries, normalization dialect)`. -type RequestRow = (String, Pairs, Pairs, Shapes, Shapes, Vec, String); +positional_record! { + extract; + /// `(sql, placeholders, analysis names, lineage references, recover CTE facts, mode)`. + pub(crate) struct QueryRow { + pub(crate) sql: String, + pub(crate) placeholders: Pairs, + pub(crate) analysis_names: Vec, + pub(crate) lineage_references: Vec<(String, String, String)>, + pub(crate) recover_cte_facts: bool, + pub(crate) mode: String, + } +} + +positional_record! { + extract; + /// `(dialect, return types, rules, types, nullability, queries, normalization dialect)`. + pub(crate) struct QueryRequestRow { + pub(crate) dialect: String, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Pairs, + pub(crate) types: Shapes, + pub(crate) nullability: Shapes, + pub(crate) queries: Vec, + pub(crate) normalization_dialect: String, + } +} /// `([(name, type)], NULL literal names, quoted names)`. type ProjectionFactsRow = (Vec<(String, String)>, Vec, Vec); /// `(succeeded, [(name, type, nullability)] or None, has star)`. @@ -45,10 +60,10 @@ pub(crate) type ColumnsRow = (bool, Option, String)> pub(crate) fn infer_query_columns( py: Python<'_>, catalog: Option>, - request: RequestRow, + request: QueryRequestRow, adapter_rules: Option<(Py, Py)>, ) -> PyResult> { - let ( + let QueryRequestRow { dialect, function_return_types, nullability_rules, @@ -56,7 +71,7 @@ pub(crate) fn infer_query_columns( nullability, queries, normalization_dialect, - ) = request; + } = request; let queries: Vec = queries .into_iter() .map(query_input) @@ -99,7 +114,14 @@ pub(crate) fn query_columns_error(failure: QueryColumnsFailure) -> PyErr { } fn query_input(row: QueryRow) -> Result { - let (sql, placeholders, analysis_names, lineage_references, recover_cte_facts, mode) = row; + let QueryRow { + sql, + placeholders, + analysis_names, + lineage_references, + recover_cte_facts, + mode, + } = row; let mode: QueryColumnsMode = match mode.as_str() { "batch" => QueryColumnsMode::Batch, "reanalysis" => QueryColumnsMode::Reanalysis, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs index daffde72a8..30dd79b759 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs @@ -23,7 +23,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{ SessionStep, }; use sqlbuild_analysis::assembly::analysis_session::types::{Pairs, Shapes}; -use sqlbuild_analysis::semantic_validation::types::DiagnosticRow; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; use sqlbuild_sqltext::compiler::main::placeholder_defaults::placeholder_defaults; use sqlbuild_cache::store::models::NativeStore; @@ -38,77 +38,138 @@ use crate::bindings::_helpers::analysis_session::nullability_rules::{ }; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; +use crate::bindings::models::Positional; use crate::bindings::models::ProjectCatalog; use crate::bindings::types::CompilerDetach; /// `(name, type, nullability)`. type ColumnRow = (String, Option, String); -/// `(output column, transform code, confidence code, [(resource type, resource, column)])`. -type LineageItem = (String, u8, u8, Vec<(String, String, String)>); -/// `(name, pivot column, value column, aggregate, type, name pattern)`. -pub(crate) type FamilyRow = (String, String, String, String, String, Option); +positional_record! { + into; + /// One lineage column: output column, transform and confidence codes and its sources. + struct LineageItem { + output_column: String, + transform_code: u8, + confidence_code: u8, + /// `(resource type, resource, column)` of each source. + sources: Vec<(String, String, String)>, + } +} + +positional_record! { + extract; + /// One declared dynamic column family. + pub(crate) struct FamilyRow { + pub(crate) name: String, + pub(crate) pivot_column: String, + pub(crate) value_column: String, + pub(crate) aggregate: String, + pub(crate) data_type: String, + pub(crate) name_pattern: Option, + } +} + /// A model's name, SQL, placeholders, references, lineage, required names, flags and pivot. -pub(crate) type ModelRow = ( - String, - String, - Pairs, - Vec<(String, String)>, - bool, - Option<(String, String)>, - Vec, -); +pub(crate) struct ModelRow { + pub(crate) name: String, + pub(crate) query_sql: String, + pub(crate) placeholders: Pairs, + pub(crate) reference_rows: Vec<(String, String)>, + pub(crate) recover_cte_facts: bool, + pub(crate) snapshot_columns: Option<(String, String)>, + pub(crate) dynamic_families: Vec, +} + /// The profile settings, relation shapes and models one session analyses. -pub(crate) type RequestRow = ( - String, - bool, - Pairs, - Option, - bool, - Shapes, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, -); +pub(crate) struct RequestRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) rich_type_inference: bool, + pub(crate) column_types: Shapes, + pub(crate) column_nullability: Shapes, + pub(crate) complete_schemas: Shapes, + pub(crate) catalog_schemas: Shapes, + pub(crate) dynamic_families_by_table: Vec<(String, Vec)>, + pub(crate) models: Vec, +} + /// `(publications, failures)`. pub(crate) type StepRow = (Shapes, Vec); -/// One model's analysis, lineage source and cleaned SQL. -type OutcomeRow = ( - bool, - Option>, - &'static str, - Vec, - bool, - bool, - Vec, - bool, - String, -); -/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. -type ProofRow = ( - bool, - Vec, - Vec<(String, Option)>, - Vec, - Option, - bool, -); + +positional_record! { + into; + /// One model's analysis, lineage source and cleaned SQL. + struct OutcomeRow { + analysis_succeeded: bool, + columns: Option>, + lineage_source: &'static str, + lineage: Vec, + has_star: bool, + star_resolved: bool, + binding_diagnostics: Vec>, + binding_validated: bool, + cleaned_sql: String, + } +} + +positional_record! { + into; + /// One dynamic pivot proof. + pub(crate) struct ProofRow { + output_proven: bool, + fixed_columns: Vec, + /// `(family, inferred type)` of each family. + families: Vec<(String, Option)>, + input_relations: Vec, + failure_reason: Option, + bare_dynamic_pivot: bool, + } +} + /// One model's pivot SQL and declared families. pub(crate) type PivotModelRow = (String, Vec); -/// The dialect, relation facts and families by table every proof reads, and the models. -type PivotRequestRow = ( - String, - Shapes, - Shapes, - Shapes, - Vec<(String, Vec)>, - Vec, -); + +positional_record! { + extract; + /// The dialect, relation facts and families by table every proof reads, and the models. + pub(crate) struct PivotRequestRow { + pub(crate) dialect: String, + pub(crate) column_types: Shapes, + pub(crate) authoritative_types: Shapes, + pub(crate) column_nullability: Shapes, + pub(crate) families_by_table: Vec<(String, Vec)>, + pub(crate) models: Vec, + } +} + /// `("absent" | "deferred" | "proof", proof)`. pub(crate) type ContractRow = (&'static str, Option); -/// `(hits, misses, stored, why the store was not read or saved)`. -pub(crate) type CacheStatsRow = (usize, usize, usize, Option); + +positional_record! { + into; + /// The analysis cache's hits, misses, stored entries and why it was not read or saved. + #[derive(Clone)] + pub(crate) struct CacheStatsRow { + pub(crate) hits: usize, + pub(crate) misses: usize, + pub(crate) stored: usize, + pub(crate) failure: Option, + } +} + +positional_record! { + extract; + /// The settings and expressions one expression-source shape request infers. + pub(crate) struct ExpressionShapesRow { + pub(crate) dialect: String, + pub(crate) case_sensitive_shapes: bool, + pub(crate) function_return_types: Pairs, + pub(crate) nullability_rules: Option, + pub(crate) expressions: Vec, + } +} /// `(catalog schema additions, analysis-shape names, whether each model's analysis succeeded)`. pub(crate) type FinishRow = (Shapes, Vec, Vec); @@ -188,7 +249,12 @@ impl NativeModelAnalysisSession { let saved: Option = save_store(py, &store, path, &[]) .err() .map(|error| error.to_string()); - self.cache_stats = Some((stats.hits, stats.misses, stats.stored, saved)); + self.cache_stats = Some(CacheStatsRow { + hits: stats.hits, + misses: stats.misses, + stored: stats.stored, + failure: saved, + }); } let SessionOutcome { models, @@ -217,10 +283,10 @@ impl NativeModelAnalysisSession { } /// Each requested model's raw binding diagnostic rows, in request order. - pub(crate) fn binding_rows(&self) -> Vec> { + pub(crate) fn binding_rows(&self) -> Vec>> { self.outcomes .iter() - .map(|(outcome, _)| outcome.analysis.binding_diagnostics.clone()) + .map(|(outcome, _)| Positional::wrap_all(outcome.analysis.binding_diagnostics.clone())) .collect() } @@ -295,7 +361,14 @@ pub(crate) fn start_model_analysis_session( attach_analysis_cache(&mut session, store); cache_path = Some(path); } - Err(error) => cache_stats = Some((0, 0, 0, Some(error.to_string()))), + Err(error) => { + cache_stats = Some(CacheStatsRow { + hits: 0, + misses: 0, + stored: 0, + failure: Some(error.to_string()), + }); + } } } Ok(NativeModelAnalysisSession { @@ -315,11 +388,16 @@ pub(crate) fn start_model_analysis_session( pub(crate) fn infer_expression_source_shapes( py: Python<'_>, catalog: PyRef<'_, ProjectCatalog>, - request: (String, bool, Pairs, Option, Vec), + request: ExpressionShapesRow, adapter_rules: Option<(Py, Py)>, ) -> PyResult>> { - let (dialect, case_sensitive_shapes, function_return_types, nullability_rules, expressions) = - request; + let ExpressionShapesRow { + dialect, + case_sensitive_shapes, + function_return_types, + nullability_rules, + expressions, + } = request; let rule_failure: RuleFailure = RuleFailure::default(); let request = ExpressionShapeRequest { dialect, @@ -347,7 +425,7 @@ pub(crate) fn infer_expression_source_shapes( } fn session_request(request: RequestRow) -> SessionRequest { - let ( + let RequestRow { dialect, case_sensitive_shapes, function_return_types, @@ -359,7 +437,7 @@ fn session_request(request: RequestRow) -> SessionRequest { catalog_schemas, dynamic_families_by_table, models, - ) = request; + } = request; SessionRequest { dialect, case_sensitive_shapes, @@ -379,7 +457,7 @@ fn session_request(request: RequestRow) -> SessionRequest { } fn model_request(row: ModelRow) -> ModelRequest { - let ( + let ModelRow { name, query_sql, placeholders, @@ -387,7 +465,7 @@ fn model_request(row: ModelRow) -> ModelRequest { recover_cte_facts, snapshot_columns, dynamic_families, - ) = row; + } = row; let has_set_operation = names_set_operation(&query_sql); let facts = model_reference_facts(&reference_rows); let pivot_sql = if placeholders.is_empty() { @@ -420,13 +498,11 @@ fn column_row(column: ColumnFact) -> ColumnRow { fn lineage_items(rows: Vec) -> Vec { rows.into_iter() - .map(|row| { - ( - row.output_column, - row.transform_code, - row.confidence_code, - row.sources, - ) + .map(|row| LineageItem { + output_column: row.output_column, + transform_code: row.transform_code, + confidence_code: row.confidence_code, + sources: row.sources, }) .collect() } @@ -438,7 +514,14 @@ fn families_by_table(rows: Vec<(String, Vec)>) -> Vec<(String, Vec DynamicFamily { - let (name, pivot_column, value_column, aggregate, data_type, name_pattern) = row; + let FamilyRow { + name, + pivot_column, + value_column, + aggregate, + data_type, + name_pattern, + } = row; DynamicFamily { name, pivot_column, @@ -454,14 +537,14 @@ fn contract_row(outcome: PivotOutcome) -> ContractRow { PivotOutcome::Absent => ("absent", None), PivotOutcome::Proof(proof) => ( "proof", - Some(( - proof.output_proven, - proof.fixed_columns.into_iter().map(column_row).collect(), - proof.families, - proof.input_relations, - proof.failure_reason, - proof.bare_dynamic_pivot, - )), + Some(ProofRow { + output_proven: proof.output_proven, + fixed_columns: proof.fixed_columns.into_iter().map(column_row).collect(), + families: proof.families, + input_relations: proof.input_relations, + failure_reason: proof.failure_reason, + bare_dynamic_pivot: proof.bare_dynamic_pivot, + }), ), } } @@ -472,19 +555,19 @@ fn outcome_row(outcome: ModelOutcome, cleaned_sql: String) -> OutcomeRow { LineageFacts::Native(rows) => ("native", lineage_items(rows)), LineageFacts::NativeFacts(rows) => ("facts", lineage_items(rows)), }; - ( - analysis.analysis_succeeded, - analysis + OutcomeRow { + analysis_succeeded: analysis.analysis_succeeded, + columns: analysis .columns .map(|columns| columns.into_iter().map(column_row).collect()), - source, + lineage_source: source, lineage, - analysis.has_star, - analysis.star_resolved, - analysis.binding_diagnostics, - analysis.binding_validated, + has_star: analysis.has_star, + star_resolved: analysis.star_resolved, + binding_diagnostics: Positional::wrap_all(analysis.binding_diagnostics), + binding_validated: analysis.binding_validated, cleaned_sql, - ) + } } /// Each model's dynamic pivot proof outside any session. @@ -493,8 +576,14 @@ pub(crate) fn prove_dynamic_column_contracts( py: Python<'_>, request: PivotRequestRow, ) -> PyResult> { - let (dialect, column_types, authoritative_types, column_nullability, by_table, models) = - request; + let PivotRequestRow { + dialect, + column_types, + authoritative_types, + column_nullability, + families_by_table: by_table, + models, + } = request; let request = PivotBatchRequest { tables: PivotTables { dialect, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs index 34104f8412..d4025bf278 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/analysis_session/stage_functions.rs @@ -13,11 +13,11 @@ use crate::bindings::_helpers::analysis_session::profile_inputs::{ profile_dialect, return_types, shape_rows, }; use crate::bindings::_helpers::analysis_session::query_columns::{ - ColumnsRow, QueryRow, infer_query_columns, + ColumnsRow, QueryRequestRow, QueryRow, infer_query_columns, }; use crate::bindings::_helpers::analysis_session::session::{ - ContractRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, - infer_expression_source_shapes, prove_dynamic_column_contracts, + ContractRow, ExpressionShapesRow, FamilyRow, NativeModelAnalysisSession, PivotModelRow, + PivotRequestRow, infer_expression_source_shapes, prove_dynamic_column_contracts, }; use crate::bindings::_helpers::boundary::keyword_request::Request; use crate::bindings::_helpers::compile_driver::stage_work::{ @@ -154,13 +154,13 @@ pub(crate) fn expression_source_shapes<'py>( let shapes: Vec> = infer_expression_source_shapes( py, native, - ( + ExpressionShapesRow { dialect, - case_sensitive, - return_types(&profile)?, - Some(rules), - pending_names, - ), + case_sensitive_shapes: case_sensitive, + function_return_types: return_types(&profile)?, + nullability_rules: Some(rules), + expressions: pending_names, + }, adapter_rules, )?; for (expression, shape) in pending.keys().iter().zip(shapes) { @@ -223,17 +223,20 @@ pub(crate) fn dynamic_contract_proofs<'py>( } prove_dynamic_column_contracts( py, - ( - dialect_or_generic(&native_field(tables, "dialect")?)?, - shape_rows(&native_field(tables, "column_types_by_table")?)?, - shape_rows(&native_field( + PivotRequestRow { + dialect: dialect_or_generic(&native_field(tables, "dialect")?)?, + column_types: shape_rows(&native_field(tables, "column_types_by_table")?)?, + authoritative_types: shape_rows(&native_field( tables, "authoritative_column_types_by_table", )?)?, - shape_rows(&native_field(tables, "column_nullability_by_table")?)?, - families, - rows, - ), + column_nullability: shape_rows(&native_field( + tables, + "column_nullability_by_table", + )?)?, + families_by_table: families, + models: rows, + }, )? } else { let session: PyRef<'py, NativeModelAnalysisSession> = session.extract()?; @@ -251,14 +254,14 @@ pub(crate) fn family_rows(families: &Bound<'_, PyAny>) -> PyResult = Vec::new(); for family in families.try_iter()? { let family: Bound<'_, PyAny> = family?; - rows.push(( - native_field(&family, "name")?.extract()?, - native_field(&family, "pivot_column")?.extract()?, - native_field(&family, "value_column")?.extract()?, - native_field(&family, "aggregate")?.extract()?, - native_field(&family, "type")?.extract()?, - native_field(&family, "name_pattern")?.extract()?, - )); + rows.push(FamilyRow { + name: native_field(&family, "name")?.extract()?, + pivot_column: native_field(&family, "pivot_column")?.extract()?, + value_column: native_field(&family, "value_column")?.extract()?, + aggregate: native_field(&family, "aggregate")?.extract()?, + data_type: native_field(&family, "type")?.extract()?, + name_pattern: native_field(&family, "name_pattern")?.extract()?, + }); } Ok(rows) } @@ -310,15 +313,15 @@ pub(crate) fn query_column_rows( infer_query_columns( py, catalog, - ( + QueryRequestRow { dialect, - return_types(profile)?, - rules, - shape_rows(types)?, - shape_rows(nullability)?, - rows, + function_return_types: return_types(profile)?, + nullability_rules: rules, + types: shape_rows(types)?, + nullability: shape_rows(nullability)?, + queries: rows, normalization_dialect, - ), + }, adapter_rules, ) } @@ -359,14 +362,14 @@ fn query_row(query: &Bound<'_, PyAny>) -> PyResult { } } } - Ok(( - native_field(query, "sql")?.extract()?, + Ok(QueryRow { + sql: native_field(query, "sql")?.extract()?, placeholders, analysis_names, - lineage, - native_field(query, "recover_cte_facts")?.extract()?, - native_field(query, "mode")?.extract()?, - )) + lineage_references: lineage, + recover_cte_facts: native_field(query, "recover_cte_facts")?.extract()?, + mode: native_field(query, "mode")?.extract()?, + }) } trait OrderedDict { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs index 56be30d4df..ae146797c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/mod.rs @@ -1,6 +1,10 @@ +#[macro_use] +mod positional; + mod allocator; pub(crate) mod config_values; pub(crate) mod expected_errors; pub(crate) mod keyword_request; pub(crate) mod oracles; pub(crate) mod panics; +mod positional_records; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs new file mode 100644 index 0000000000..0485f9623e --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional.rs @@ -0,0 +1,114 @@ +//! Macros declaring named records that cross the Python boundary as positional tuples. + +/// Implement the positional traits for another crate's struct, its fields in tuple order. +macro_rules! positional_fields { + ($($mode:ident),+; $name:ident $body:tt) => { + $( + positional_fields!( + @$mode $name $body + ); + )+ + }; + (@extract $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => { + impl $crate::bindings::types::FromPositional for $name { + fn from_positional(value: &pyo3::Bound<'_, pyo3::PyAny>) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok($name { $($field),+ }) + } + } + }; + (@into $name:ident { $($field:ident : $ty:ty),+ $(,)? }) => { + impl $crate::bindings::types::IntoPositional for $name { + fn into_positional<'py>( + self, + py: pyo3::Python<'py>, + ) -> pyo3::PyResult> { + let $name { $($field),+ } = self; + pyo3::IntoPyObject::into_pyobject(($($field,)+), py) + } + } + }; +} + +/// Declare a struct naming the positions of a tuple Python sends (`extract`) or receives (`into`). +macro_rules! positional_record { + ( + $($mode:ident),+; + $(#[$meta:meta])* + $vis:vis struct $name:ident <$lifetime:lifetime> $body:tt + ) => { + $(#[$meta])* + $vis struct $name<$lifetime> $body + + $( + positional_record!( + @$mode $name <$lifetime> $body + ); + )+ + }; + ( + $($mode:ident),+; + $(#[$meta:meta])* + $vis:vis struct $name:ident $body:tt + ) => { + $(#[$meta])* + $vis struct $name $body + + $( + positional_record!( + @$mode $name $body + ); + )+ + }; + (@extract $name:ident { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<'py> pyo3::FromPyObject<'py> for $name { + fn extract_bound(value: &pyo3::Bound<'py, pyo3::PyAny>) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { $($field),+ }) + } + } + }; + (@extract $name:ident <$lifetime:lifetime> { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<$lifetime> pyo3::FromPyObject<$lifetime> for $name<$lifetime> { + fn extract_bound( + value: &pyo3::Bound<$lifetime, pyo3::PyAny>, + ) -> pyo3::PyResult { + let ($($field,)+): ($($ty,)+) = pyo3::types::PyAnyMethods::extract(value)?; + Ok(Self { $($field),+ }) + } + } + }; + (@into $name:ident { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<'py> pyo3::IntoPyObject<'py> for $name { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<'py, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject(self, py: pyo3::Python<'py>) -> Result { + pyo3::IntoPyObject::into_pyobject(($(self.$field,)+), py) + } + } + }; + (@into $name:ident <$lifetime:lifetime> { + $($(#[$field_meta:meta])* $field_vis:vis $field:ident : $ty:ty),+ $(,)? + }) => { + impl<$lifetime> pyo3::IntoPyObject<$lifetime> for $name<$lifetime> { + type Target = pyo3::types::PyTuple; + type Output = pyo3::Bound<$lifetime, pyo3::types::PyTuple>; + type Error = pyo3::PyErr; + + fn into_pyobject( + self, + py: pyo3::Python<$lifetime>, + ) -> Result { + pyo3::IntoPyObject::into_pyobject(($(self.$field,)+), py) + } + } + }; +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs new file mode 100644 index 0000000000..3162c55985 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/boundary/positional_records.rs @@ -0,0 +1,50 @@ +//! The tuple positions of records from other crates that Python exchanges as tuples. + +use sqlbuild_analysis::binding_locations::models::AuthoredSpan; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; +use sqlbuild_rules::lint_text::models::InterpolationSite; + +positional_fields! { + extract, into; + InterpolationSite { + sentinel: String, + neutralized_start: usize, + neutralized_end: usize, + original_start: usize, + original_end: usize, + original_text: String, + } +} + +positional_fields! { + extract, into; + DiagnosticRow { + code: String, + message: String, + line: Option, + column: Option, + start: Option, + end: Option, + severity: String, + } +} + +positional_fields! { + extract; + Expansion { + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, + } +} + +positional_fields! { + into; + AuthoredSpan { + line: usize, + column: usize, + end_line: Option, + end_column: Option, + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs index e14811cbdd..92558b69c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs @@ -17,13 +17,13 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; /// The project directory, discovered inputs, adapter context, colour and progress stream. -type InvocationContext<'py> = ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - bool, - Bound<'py, PyAny>, -); +struct InvocationContext<'py> { + project_dir: Bound<'py, PyAny>, + discovered: Bound<'py, PyAny>, + adapter_context: Bound<'py, PyAny>, + use_color: bool, + stream: Bound<'py, PyAny>, +} fn boolean(py: Python<'_>, value: bool) -> Bound<'_, PyAny> { PyBool::new(py, value).to_owned().into_any() @@ -97,13 +97,25 @@ fn invocation_context<'py>( let stream: Bound<'py, PyAny> = py.import("sys")? .getattr(if machine_output { "stderr" } else { "stdout" })?; - Ok((project_dir, discovered, adapter_context, use_color, stream)) + Ok(InvocationContext { + project_dir, + discovered, + adapter_context, + use_color, + stream, + }) } fn context_fields<'py>( context: &InvocationContext<'py>, ) -> PyResult)>> { - let (project_dir, discovered, adapter_context, use_color, stream) = context; + let InvocationContext { + project_dir, + discovered, + adapter_context, + use_color, + stream, + } = context; let py: Python<'py> = project_dir.py(); Ok(vec![ ("effective_project_dir", project_dir.clone()), @@ -139,7 +151,12 @@ pub(crate) fn resolve_reported_command_invocation<'py>( ) -> PyResult> { let py: Python<'py> = request.py(); let context: InvocationContext<'py> = invocation_context(request, require_connection)?; - let (_, _, adapter_context, use_color, stream) = &context; + let InvocationContext { + adapter_context, + use_color, + stream, + .. + } = &context; let mut fields: Vec<(&str, Bound<'py, PyAny>)> = context_fields(&context)?; fields.push(( "connection_progress", diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs index 7b662cb091..68a6c51b19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/cursor_roles.rs @@ -5,6 +5,7 @@ use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyString}; use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; @@ -86,7 +87,7 @@ pub(crate) fn resolve_cursor_input_roles<'py>( Err(_) => { let cursor: Bound<'py, PyAny> = mapping_get(&values, "cursor")?; if cursor.is_instance_of::() { - for (_, name, _, _) in input_reference_rows(model)? { + for ReferenceRow { name, .. } in input_reference_rows(model)? { filter.set_item(name, &cursor)?; } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs index 7b371a1eb9..ee385e3913 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/model_inputs.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ }; use crate::bindings::_helpers::compiled_project::loop_classes::model_loop_classes; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; +use crate::bindings::_helpers::compiled_project::model_loop::{LoopSwitches, NativeModelLoop}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::model_config::contract_classes::contract_classes; @@ -275,12 +275,13 @@ pub(crate) fn build_model_inputs<'py>( native_field(&syntax, "native_mapping")?.extract()?, reserved_cursor_markers(), ( - ( - native_field(&settings, "sql_analysis")?.extract()?, + LoopSwitches { + sql_analysis: native_field(&settings, "sql_analysis")?.extract()?, no_sql_validation, - defer_model_sql_validation, - native_field(&settings, "require_sql_analysis")?.extract()?, - ), + defer_sql_validation: defer_model_sql_validation, + require_sql_analysis: native_field(&settings, "require_sql_analysis")? + .extract()?, + }, project_config_path.str()?.extract()?, ), ) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs index dd32ec6f5e..6a44449683 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_delta/image.rs @@ -252,13 +252,40 @@ fn section<'a>(sections: &'a ImageSections, name: &str) -> Option<&'a [u8]> { .map(|(_, bytes)| bytes.as_slice()) } -/// The diagnostics and report lines an image keeps as JSON, being small and varied. -type StoredMisc = ( - Vec, - Option>, - Vec, - Option>, -); +/// The diagnostics and report lines an image keeps as one JSON array, being small and varied. +struct StoredMisc { + diagnostics: Vec, + text_models: Option>, + environment_names: Vec, + coverage: Option>, +} + +impl<'de> serde::Deserialize<'de> for StoredMisc { + fn deserialize>(deserializer: D) -> Result { + deserializer.deserialize_tuple(4, StoredMiscVisitor) + } +} + +/// Reads [`StoredMisc`] from its four-element array. +struct StoredMiscVisitor; + +impl<'de> serde::de::Visitor<'de> for StoredMiscVisitor { + type Value = StoredMisc; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a tuple of size 4") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let missing = |index: usize| serde::de::Error::invalid_length(index, &self); + Ok(StoredMisc { + diagnostics: seq.next_element()?.ok_or_else(|| missing(0))?, + text_models: seq.next_element()?.ok_or_else(|| missing(1))?, + environment_names: seq.next_element()?.ok_or_else(|| missing(2))?, + coverage: seq.next_element()?.ok_or_else(|| missing(3))?, + }) + } +} fn put_shapes<'a>(encoder: &mut Encoder<'a>, shapes: &'a Shapes) { encoder.count(shapes.len()); @@ -458,8 +485,12 @@ impl StoredFacts { .list(|decoder| Some((decoder.string()?, decoder.strings()?)))? .into_iter() .collect(); - let (diagnostics, text_models, environment_names, coverage): StoredMisc = - from_json(decoder.bytes()?)?; + let StoredMisc { + diagnostics, + text_models, + environment_names, + coverage, + }: StoredMisc = from_json(decoder.bytes()?)?; decoder.finish()?; Some(Self { usages, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs index 3e92952ae7..73dc556a63 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/analysis_cache.rs @@ -120,9 +120,22 @@ fn build_analysis_cache_context<'py>( ) } +/// Semantic analysis cache outcome counts for one invocation. +pub(crate) struct AnalysisCacheCounts { + pub(crate) batch_hits: usize, + pub(crate) entry_hits: usize, + pub(crate) misses: usize, + pub(crate) bypasses: usize, +} + /// Record semantic analysis cache outcomes for one invocation. pub(crate) fn record_analysis_cache_metrics( - (batch_hits, entry_hits, misses, bypasses): (usize, usize, usize, usize), + AnalysisCacheCounts { + batch_hits, + entry_hits, + misses, + bypasses, + }: AnalysisCacheCounts, ) { for (metric, value) in [ ("analysis_batch_cache_hits", batch_hits), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs index 60ec1d1a43..0f447a7cf2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_analysis.rs @@ -7,7 +7,9 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple, use sqlbuild_analysis::assembly::analysis_session::main::native_dialect::native_dialect; use sqlbuild_analysis::assembly::analysis_session::types::Pairs; -use crate::bindings::_helpers::analysis::binding_catalog::{BindingCatalog, ignores_quoted_case}; +use crate::bindings::_helpers::analysis::binding_catalog::{ + BindingCatalog, CatalogKey, ignores_quoted_case, +}; use crate::bindings::_helpers::analysis_session::model_analyses::{ ModelAnalyses, ModelAnalysisRequest, analyze_models, }; @@ -93,8 +95,12 @@ pub(crate) fn analysis_context<'py>( &native_field(inputs, "effective_settings")?.getattr("sql_analysis")?, request.skip_column_inference.bind(py), )?; - let (column_types, nullability, complete_schemas, declared): DeclaredRelations<'py> = - declared_relations(inputs.clone())?; + let DeclaredRelations { + column_types, + nullability, + complete_schemas, + declared, + }: DeclaredRelations<'py> = declared_relations(inputs.clone())?; let (column_types, nullability, complete_schemas): ( Bound<'py, PyAny>, Bound<'py, PyAny>, @@ -218,12 +224,13 @@ fn profile_with_catalog<'py>( py, BindingCatalog::create( py, - ( + CatalogKey { dialect, - native_field(&profile, "quoted_identifiers_ignore_case")?.extract()?, - native_field(&profile, "semantic_known_functions")?.extract()?, - native_field(&profile, "semantic_known_types")?.extract()?, - ), + quoted_ignore_case: native_field(&profile, "quoted_identifiers_ignore_case")? + .extract()?, + known_functions: native_field(&profile, "semantic_known_functions")?.extract()?, + known_types: native_field(&profile, "semantic_known_types")?.extract()?, + }, PyDict::new(py).as_any(), )?, )? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs index 02f990d47b..5caef70872 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs @@ -270,7 +270,7 @@ pub(crate) fn binding_rows<'py>( .flatten() { let position: usize = position.extract()?; - rows_by_name.set_item(name, rows.get(position).into_pyobject(py)?)?; + rows_by_name.set_item(name, rows.get(position).cloned().into_pyobject(py)?)?; } Ok(rows_by_name) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs index 2c61cac5ed..6563157e15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/reuse.rs @@ -24,7 +24,17 @@ use sqlbuild_cache::project_snapshot::models::SnapshotRules; use crate::bindings::_helpers::boundary::panics::compiler_guard; -type SettingsRow = (bool, Vec, Vec, Vec, Vec); +positional_record! { + extract; + /// Settings case sensitivity, names, prefixes, env files and secrets directories. + struct SettingsRow { + case_sensitive: bool, + names: Vec, + prefixes: Vec, + env_files: Vec, + secrets_dirs: Vec, + } +} type ReplayRow = (String, Vec, i64); type LaunchIdentityRow = (String, BTreeMap, Vec); /// A written artifact's digest, or a kept artifact's size and mtime. @@ -254,7 +264,13 @@ impl NativeReuseAttempt { } fn settings_inputs(row: SettingsRow) -> SettingsInputs { - let (case_sensitive, names, prefixes, env_files, secrets_dirs) = row; + let SettingsRow { + case_sensitive, + names, + prefixes, + env_files, + secrets_dirs, + } = row; SettingsInputs { case_sensitive, names, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs index 1b899e2c31..e733e70446 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_outputs/runtime_target.rs @@ -28,7 +28,13 @@ const PYTHON_CHECK_KIND: &str = "python_check"; const JSON_INDENT: i64 = 2; /// One test's identity as the test runtime target matches entries to results. -type TestKey = (String, Option, Option, Option); +#[derive(Hash, PartialEq, Eq)] +struct TestKey { + name: String, + source_path: Option, + block_index: Option, + case_name: Option, +} fn posix(path: &Bound<'_, PyAny>) -> PyResult> { if path.is_none() { @@ -42,16 +48,21 @@ fn path_object<'py>(py: Python<'py>, text: &str) -> PyResult> } fn test_key(value: &Bound<'_, PyAny>, name_field: &str) -> PyResult { - Ok(( - native_field(value, name_field)?.extract()?, - posix(&native_field(value, "source_path")?)?, - native_field(value, "block_index")?.extract()?, - native_field(value, "case_name")?.extract()?, - )) + Ok(TestKey { + name: native_field(value, name_field)?.extract()?, + source_path: posix(&native_field(value, "source_path")?)?, + block_index: native_field(value, "block_index")?.extract()?, + case_name: native_field(value, "case_name")?.extract()?, + }) } fn path_facts(test: &Bound<'_, PyAny>, model_names: Vec) -> PyResult { - let (name, source_path, block_index, case_name) = test_key(test, "name")?; + let TestKey { + name, + source_path, + block_index, + case_name, + } = test_key(test, "name")?; Ok(SqlTestPathFacts { name, source_path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs index 5adc9d2492..ea17c67eb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment.rs @@ -12,7 +12,7 @@ use crate::bindings::_helpers::compile_driver::python_calls::{ KeywordArguments, call_with, imported, }; use crate::bindings::_helpers::compiled_project::audit_loop::{AuditBlockRow, AuditFileRow}; -use crate::bindings::_helpers::compiled_project::function_loop::FunctionRow; +use crate::bindings::_helpers::compiled_project::function_loop::{FunctionRow, FunctionSettings}; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::compiled_project::resource_diagnostics::{ DiagnosticSettings, LoaderRow, resource_diagnostics, @@ -92,19 +92,19 @@ fn function_rows( items(discovered, field)? .into_iter() .map(|file| { - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "file_path")?, - file.clone().unbind(), - attr(&file, "header_values")?, - attr(&file, body)?, - if python { + Ok(FunctionRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + name: stem(&file, "file_path")?, + function_file: file.clone().unbind(), + header_values: attr(&file, "header_values")?, + body: attr(&file, body)?, + contents: if python { None } else { Some(attr(&file, "contents")?) }, - )) + }) }) .collect() } @@ -187,17 +187,17 @@ pub(crate) fn attach_resources<'py>( true, )?, ), - ( + FunctionSettings { adapter_name, validate, - ( + defaults: ( either(&defaults, ("function_database", "database"))?, either(&defaults, ("function_schema", "schema"))?, ), - target_namespace(&target_config)?, - python_functions_inherit_default_namespace, - function_kind.unbind(), - ), + target: target_namespace(&target_config)?, + inherit: python_functions_inherit_default_namespace, + kind: function_kind.unbind(), + }, )?; let mut source_rows: Vec = Vec::new(); for source_file in items(discovered_inputs, "source_files")? { @@ -241,46 +241,46 @@ fn test_case_row(case: Bound<'_, PyAny>) -> PyResult { } fn test_block_row(block: Bound<'_, PyAny>) -> PyResult { - Ok(( - block.clone().unbind(), - attr(&block, "name")?, - attr(&block, "sql_body")?, - attr(&native_field(&block, "mode")?, "value")?, - items(&block, "parameters")? + Ok(TestBlockRow { + block: block.clone().unbind(), + name: attr(&block, "name")?, + sql_body: attr(&block, "sql_body")?, + mode: attr(&native_field(&block, "mode")?, "value")?, + parameters: items(&block, "parameters")? .iter() .map(|parameter| attr(parameter, "name")) .collect::>()?, - items(&block, "cases")? + cases: items(&block, "cases")? .into_iter() .map(test_case_row) .collect::>()?, - attr(&block, "sql_body_span")?, - )) + sql_body_span: attr(&block, "sql_body_span")?, + }) } fn test_file_row(file: Bound<'_, PyAny>) -> PyResult { - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "relative_path")?, - file.clone().unbind(), - attr(&file, "contents")?, - items(&file, "blocks")? + Ok(TestFileRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + stem: stem(&file, "relative_path")?, + file: file.clone().unbind(), + contents: attr(&file, "contents")?, + blocks: items(&file, "blocks")? .into_iter() .map(test_block_row) .collect::>()?, - )) + }) } fn scenario_row(scenario: Bound<'_, PyAny>) -> PyResult { - Ok(( - text(&scenario, "file_path")?, - text(&scenario, "relative_path")?, - scenario.clone().unbind(), - attr(&scenario, "contents")?, - attr(&scenario, "sql_body")?, - attr(&scenario, "name")?, - )) + Ok(ScenarioRow { + file_path: text(&scenario, "file_path")?, + relative_path: text(&scenario, "relative_path")?, + scenario_file: scenario.clone().unbind(), + contents: attr(&scenario, "contents")?, + sql_body: attr(&scenario, "sql_body")?, + name: attr(&scenario, "name")?, + }) } /// The adapter's typed scalar renderer as `(text, None)`, or `(None, error)` for a bad value. @@ -378,31 +378,31 @@ fn audit_block_row(block: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = block.py(); let measurement: Bound<'_, PyAny> = host::entry(py, FacadeHost::AuditEvaluationMode)?.getattr(MEASUREMENT_MODE)?; - Ok(( - block.clone().unbind(), - attr(&block, "name")?, - attr(&block, "sql_body")?, - attr(&block, "evidence_sql")?, - native_field(&block, "evaluation_mode")?.eq(measurement)?, - attr(&block, "header_values")?, - )) + Ok(AuditBlockRow { + block: block.clone().unbind(), + name: attr(&block, "name")?, + sql_body: attr(&block, "sql_body")?, + evidence_sql: attr(&block, "evidence_sql")?, + measurement: native_field(&block, "evaluation_mode")?.eq(measurement)?, + header: attr(&block, "header_values")?, + }) } fn audit_file_row(file: Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = file.py(); let generic: Bound<'_, PyAny> = enum_member(py, RecordId::DeclarationKind, "AUDIT")?; - Ok(( - text(&file, "file_path")?, - text(&file, "relative_path")?, - stem(&file, "file_path")?, - file.clone().unbind(), - native_field(&file, "declaration_kind")?.is(&generic), - attr(&file, "contents")?, - items(&file, "blocks")? + Ok(AuditFileRow { + file_path: text(&file, "file_path")?, + relative_path: text(&file, "relative_path")?, + stem: stem(&file, "file_path")?, + file: file.clone().unbind(), + generic: native_field(&file, "declaration_kind")?.is(&generic), + contents: attr(&file, "contents")?, + blocks: items(&file, "blocks")? .into_iter() .map(audit_block_row) .collect::>()?, - )) + }) } /// The discovered file of one built-in generic audit, as project discovery would describe it. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs index f748dffb8e..22a82e9e5b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs @@ -61,26 +61,32 @@ const MODEL_TARGET: &str = "model"; const SOURCE_TARGET: &str = "source"; const SEED_TARGET: &str = "seed"; -/// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. -pub(crate) type AuditBlockRow = ( - Py, - Option, - String, - Option, - bool, - Py, -); - -/// An audit file's paths, stem, discovered file, generic flag, contents and blocks. -pub(crate) type AuditFileRow = ( - String, - String, - String, - Py, - bool, - String, - Vec, -); +positional_record! { + extract, into; + /// An audit block, its name, SQL body, evidence SQL, measurement flag and header values. + pub(crate) struct AuditBlockRow { + pub(crate) block: Py, + pub(crate) name: Option, + pub(crate) sql_body: String, + pub(crate) evidence_sql: Option, + pub(crate) measurement: bool, + pub(crate) header: Py, + } +} + +positional_record! { + extract, into; + /// An audit file's paths, stem, discovered file, generic flag, contents and blocks. + pub(crate) struct AuditFileRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) stem: String, + pub(crate) file: Py, + pub(crate) generic: bool, + pub(crate) contents: String, + pub(crate) blocks: Vec, + } +} /// What every audit in one compile resolves against, and the Python values it builds with. #[derive(FromPyObject)] @@ -132,8 +138,8 @@ pub(crate) fn attached_audits<'py>( settings, }; let mut audits: Vec = Vec::new(); - for file in files.iter().filter(|file| !file.4) { - for block in &file.6 { + for file in files.iter().filter(|file| !file.generic) { + for block in &file.blocks { audits.push(scope.singular(file, block)?); } } @@ -158,41 +164,41 @@ fn generic_definitions<'a, 'py>( ) -> PyResult> { let py: Python<'py> = context.host.py(); let mut project: BTreeMap> = BTreeMap::new(); - for file in files.iter().filter(|file| file.4) { - let [block] = file.6.as_slice() else { + for file in files.iter().filter(|file| file.generic) { + let [block] = file.blocks.as_slice() else { let message: String = format!( "Generic audit definition {} must contain exactly one AUDIT block", - file.1 + file.relative_path ); return Err(compile_error(context.classes, &message, None)); }; - let header: &Bound<'_, PyDict> = block.5.bind(py); - if block.4 + let header: &Bound<'_, PyDict> = block.header.bind(py); + if block.measurement && (header.contains(MEASUREMENT_HEADER_KEYS[0])? || header.contains(MEASUREMENT_HEADER_KEYS[1])?) { let message: String = format!( "Generic measurement audit definition {} must not define thresholds or \ minimum_samples in its AUDIT header; attachment owns policy", - file.1 + file.relative_path ); return Err(compile_error(context.classes, &message, None)); } - if let Some(existing) = project.get(&file.2) { + if let Some(existing) = project.get(&file.stem) { let message: String = format!( "Duplicate generic audit definition found for '{}' in {} and {}", - file.2, existing.file.1, file.1 + file.stem, existing.file.relative_path, file.relative_path ); return Err(compile_error(context.classes, &message, None)); } - project.insert(file.2.clone(), Definition { file, block }); + project.insert(file.stem.clone(), Definition { file, block }); } let mut shadows: Vec> = Vec::new(); let mut definitions: HashMap> = HashMap::new(); for builtin in builtins { - if let Some(block) = builtin.6.first() { + if let Some(block) = builtin.blocks.first() { definitions.insert( - builtin.2.clone(), + builtin.stem.clone(), Definition { file: builtin, block, @@ -202,7 +208,11 @@ fn generic_definitions<'a, 'py>( } for (name, definition) in project { if definitions.contains_key(&name) { - shadows.push(shadow_warning(context, &name, &definition.file.1)?); + shadows.push(shadow_warning( + context, + &name, + &definition.file.relative_path, + )?); } definitions.insert(name, definition); } @@ -251,8 +261,20 @@ impl<'py> AuditScope<'_, 'py> { /// One singular audit block, as `_build_singular_audit_inputs` builds it. fn singular(&self, file: &AuditFileRow, block: &AuditBlockRow) -> PyResult { let py: Python<'py> = self.context.host.py(); - let (_, relative_path, stem, audit_file, _, _, _) = file; - let (audit_block, block_name, sql_body, evidence_sql, measurement, header) = block; + let AuditFileRow { + relative_path, + stem, + file: audit_file, + .. + } = file; + let AuditBlockRow { + block: audit_block, + name: block_name, + sql_body, + evidence_sql, + measurement, + header, + } = block; let audit_name: String = block_name.clone().unwrap_or_else(|| stem.clone()); let identity: Bound<'py, PyAny> = self .settings @@ -394,7 +416,7 @@ impl<'py> AuditScope<'_, 'py> { let minimum_samples: Bound<'py, PyAny> = native_field(instance, "minimum_samples")?; let instance_severity: Option = native_field(instance, "severity")?.extract()?; let label: String = format!("{owner_file} audit '{definition_name}'"); - let measurement: bool = definition.block.4; + let measurement: bool = definition.block.measurement; if measurement { if instance_severity.is_some() { let message: String = format!( @@ -425,8 +447,8 @@ impl<'py> AuditScope<'_, 'py> { let rendering: AuditRendering = render_attached_audit(&AuditAttachment { owner_label: owner_file.to_owned(), definition_name: definition_name.clone(), - sql_body: definition.block.2.clone(), - evidence_sql: definition.block.3.clone(), + sql_body: definition.block.sql_body.clone(), + evidence_sql: definition.block.evidence_sql.clone(), implicit_arguments: implicit, explicit_arguments: argument_pairs(native_field(instance, "arguments")?.downcast()?)?, has_thresholds: !thresholds.is_none(), @@ -455,7 +477,7 @@ impl<'py> AuditScope<'_, 'py> { &format!("Audit '{definition_name}'"), )?; let calls: Vec<(&str, &str)> = reference_calls(&expanded.references); - self.checked_references(&definition.file.1, &calls)?; + self.checked_references(&definition.file.relative_path, &calls)?; let (severity, source) = rendered .policies .map_err(|message| compile_error(self.context.classes, &message, None))?; @@ -471,13 +493,13 @@ impl<'py> AuditScope<'_, 'py> { } usages.extend(expanded.usages); Ok(AttachedAudit { - audit_file: definition.file.3.clone_ref(py), - audit_block: definition.block.0.clone_ref(py), + audit_file: definition.file.file.clone_ref(py), + audit_block: definition.block.block.clone_ref(py), definition_name: definition .block - .1 + .name .clone() - .unwrap_or_else(|| definition.file.2.clone()), + .unwrap_or_else(|| definition.file.stem.clone()), sql_body: expanded.sql_body, measurement, evidence_sql: expanded.evidence_sql, @@ -508,7 +530,12 @@ impl<'py> AuditScope<'_, 'py> { identity: &Bound<'py, PyAny>, label: &str, ) -> PyResult> { - let (file_path, relative_path, _, _, _, contents, _) = file; + let AuditFileRow { + file_path, + relative_path, + contents, + .. + } = file; let declarations: Bound<'py, PyAny> = file_declarations(self.context, file_path, identity)?; let body: AuthoredExpansion<'py> = expanded_authored_sql(self.context, (sql.0, file_path), &declarations)?; @@ -682,7 +709,7 @@ impl<'py> AuditScope<'_, 'py> { fn reference_calls(references: &[ReferenceRow]) -> Vec<(&str, &str)> { references .iter() - .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .map(|ReferenceRow { kind, name, .. }| (kind.as_str(), name.as_str())) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs index 98af5da878..7ddaf506dc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs @@ -331,7 +331,14 @@ audit_getters!(NativeAuditInput, { attached .references .iter() - .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind.clone(), name.clone(), package.clone()), + ) .collect(), attached .target diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs index 8871aa9d1d..f42801729b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs @@ -14,7 +14,7 @@ use sqlbuild_analysis::assembly::analysis_session::models::{ }; use sqlbuild_analysis::binding_locations::main::locate_binding_diagnostics::locate_binding_diagnostics; use sqlbuild_analysis::binding_locations::models::{BindingSource, LocatedDiagnostic}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::semantic_validation::models::{DiagnosticRow, Expansion}; use crate::bindings::_helpers::analysis_session::session::NativeModelAnalysisSession; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -30,15 +30,18 @@ use crate::bindings::_helpers::record_behaviours::lineage_facts::{ const COMPILE_PHASE: &str = "compile"; const MODEL_RESOURCE: &str = "model"; -/// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. -pub(crate) type ProofArgument = ( - bool, - Vec, - Vec<(String, Option)>, - Vec, - Option, - bool, -); +positional_record! { + extract; + /// `(output proven, fixed columns, (family, inferred type), inputs, failure, bare pivot)`. + pub(crate) struct ProofArgument { + output_proven: bool, + fixed: Vec, + families: Vec<(String, Option)>, + input_relations: Vec, + failure_reason: Option, + bare_dynamic_pivot: bool, + } +} /// The session analysis a model was assembled from: the session and its request position. pub(crate) type SessionArgument = (Py, usize); @@ -81,8 +84,14 @@ fn inferred_row(column: &ColumnFact) -> InferredRow { } fn proof_fact(argument: ProofArgument) -> ContractProof { - let (output_proven, fixed, families, input_relations, failure_reason, bare_dynamic_pivot) = - argument; + let ProofArgument { + output_proven, + fixed, + families, + input_relations, + failure_reason, + bare_dynamic_pivot, + } = argument; ContractProof { output_proven, fixed_columns: fixed @@ -157,8 +166,18 @@ fn located(input: &ModelInputRecord, outcome: &ModelOutcome) -> PyResult> = vec![ - expansion.declaration_spans.clone(), - expansion.macro_spans.clone(), + expansion + .declaration_spans + .iter() + .copied() + .map(Expansion::from) + .collect(), + expansion + .macro_spans + .iter() + .copied() + .map(Expansion::from) + .collect(), ]; let source: BindingSource<'_> = BindingSource { authored, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs index cd7cf04bb6..fc157b5c91 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs @@ -161,6 +161,14 @@ impl Default for ViewProjections { } } +/// A model's name, compiled query, authored file, and whether it calls a function. +pub(crate) struct ModelMetadataTexts { + pub(crate) name: String, + pub(crate) query_sql: String, + pub(crate) authored_sql: String, + pub(crate) calls_functions: bool, +} + /// Deps, session analysis, failed analysis's columns, standalone proof, and macro deps. #[derive(FromPyObject)] pub(crate) struct CompiledArguments( @@ -444,17 +452,17 @@ impl NativeCompiledModel { &self, py: Python<'_>, function_kinds: &[&str], - ) -> PyResult<(String, String, String, bool)> { + ) -> PyResult { self.read(py, |record, _| { - Ok(( - model_name(record), - record.query_sql.to_string(), - record.model_file.get().contents_text().to_string(), - record + Ok(ModelMetadataTexts { + name: model_name(record), + query_sql: record.query_sql.to_string(), + authored_sql: record.model_file.get().contents_text().to_string(), + calls_functions: record .references .iter() - .any(|(kind, _, _, _)| function_kinds.contains(&kind.as_str())), - )) + .any(|reference| function_kinds.contains(&reference.kind.as_str())), + }) }) } @@ -904,12 +912,12 @@ pub(crate) fn model_graph_row( } } let folder: String = record.model_file.get().folder(py)?; - Ok(( - (MODEL_RESOURCE.to_owned(), model_name(record)), - compiled.deps.clone(), + Ok(ResourceRow { + key: (MODEL_RESOURCE.to_owned(), model_name(record)), + deps: compiled.deps.clone(), tags, - Some(folder), - )) + folder: Some(folder), + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs index ea653300d5..31748115b7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs @@ -38,7 +38,9 @@ use crate::bindings::_helpers::compiled_project::function_views::{ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, compile_error, }; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; use crate::bindings::_helpers::compiled_project::source_loop::{ @@ -55,26 +57,32 @@ const SCHEMA_KEY: &str = "schema"; /// The reference kind of a source. const SOURCE_KIND: &str = "source"; -/// A function's paths, stem, discovered file, header values, body and SQL file contents. -pub(crate) type FunctionRow = ( - String, - String, - String, - Py, - Py, - String, - Option, -); +positional_record! { + extract; + /// A function's paths, stem, discovered file, header values, body and SQL file contents. + pub(crate) struct FunctionRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) name: String, + pub(crate) function_file: Py, + pub(crate) header_values: Py, + pub(crate) body: String, + pub(crate) contents: Option, + } +} -/// Adapter, validation, raw default and target namespaces, inheritance and resource kind. -pub(crate) type FunctionSettings = ( - String, - bool, - (Option, Option), - Option<(Option, Option)>, - bool, - Py, -); +positional_record! { + extract; + /// Adapter, validation, raw default and target namespaces, inheritance and resource kind. + pub(crate) struct FunctionSettings { + pub(crate) adapter_name: String, + pub(crate) validate: bool, + pub(crate) defaults: (Option, Option), + pub(crate) target: Option<(Option, Option)>, + pub(crate) inherit: bool, + pub(crate) kind: Py, + } +} /// What every function in one compile resolves against. struct FunctionScope<'a, 'py> { @@ -97,8 +105,14 @@ pub(crate) fn attached_functions<'py>( settings: FunctionSettings, ) -> PyResult> { let py: Python<'py> = context.host.py(); - let (adapter_name, validate, (default_database, default_schema), target, inherit, kind) = - settings; + let FunctionSettings { + adapter_name, + validate, + defaults: (default_database, default_schema), + target, + inherit, + kind, + } = settings; let defaults = ( optional_template(context, default_database, "default function database")?, optional_template(context, default_schema, "default function schema")?, @@ -130,15 +144,15 @@ pub(crate) fn attached_functions<'py>( let mut attached: Vec = Vec::with_capacity(sql_rows.len() + python_rows.len()); for row in sql_rows { - if !names.insert(row.2.clone()) { - let message: String = format!("Duplicate SQL function name '{}'", row.2); + if !names.insert(row.name.clone()) { + let message: String = format!("Duplicate SQL function name '{}'", row.name); return Err(compile_error(context.classes, &message, None)); } attached.push(sql_function(&scope, row)?); } for row in python_rows { - if !names.insert(row.2.clone()) { - let message: String = format!("Duplicate function name '{}'", row.2); + if !names.insert(row.name.clone()) { + let message: String = format!("Duplicate function name '{}'", row.name); return Err(compile_error(context.classes, &message, None)); } attached.push(python_function(&scope, row)?); @@ -150,7 +164,15 @@ pub(crate) fn attached_functions<'py>( fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult { let context: &LoopContext<'_, '_> = scope.context; let py: Python<'_> = context.host.py(); - let (file_path, relative_path, name, function_file, header_values, body, contents) = row; + let FunctionRow { + file_path, + relative_path, + name, + function_file, + header_values, + body, + contents, + } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("SQL function {relative_path}"); let header: FunctionHeader = @@ -238,7 +260,14 @@ fn sql_function(scope: &FunctionScope<'_, '_>, row: FunctionRow) -> PyResult, row: FunctionRow) -> PyResult { let context: &LoopContext<'_, '_> = scope.context; let py: Python<'_> = context.host.py(); - let (_, relative_path, name, function_file, header_values, body, _) = row; + let FunctionRow { + relative_path, + name, + function_file, + header_values, + body, + .. + } = row; let header_values: Bound<'_, PyDict> = header_values.into_bound(py); let label: String = format!("Python function {relative_path}"); let header: FunctionHeader = @@ -505,7 +534,12 @@ fn function_references( context, syntax, &scanned.invalid_calls, - (None, relative_path, file_path, contents), + CallOrigin { + name: None, + relative_path, + file_path, + contents, + }, )?; let references: Vec = merge_call_site_references( scanned.references.iter().map(reference_row).collect(), @@ -520,7 +554,7 @@ fn function_references( let known: KnownResources<'_> = validator.get().known_resources(); let calls: Vec<(&str, &str)> = references .iter() - .map(|(kind, name, _, _)| (kind.as_str(), name.as_str())) + .map(|ReferenceRow { kind, name, .. }| (kind.as_str(), name.as_str())) .collect(); record_source_reads( calls @@ -585,7 +619,14 @@ pub(crate) fn checked_table_function_arities( for (owner, references) in owners { let calls: Vec<(&str, &str, Option)> = references .iter() - .map(|(kind, name, _, count)| (kind.as_str(), name.as_str(), *count)) + .map( + |ReferenceRow { + kind, + name, + call_argument_count: count, + .. + }| (kind.as_str(), name.as_str(), *count), + ) .collect(); if let Some(message) = table_function_arity_error(&owner, &calls, &counts) { return Err(compile_error(context.classes, &message, None)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs index ca5bc2aeb9..80158fa8b8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs @@ -38,13 +38,16 @@ pub(crate) struct FunctionClasses<'py> { /// `(name, type)` of an argument or return column, its types template-expanded. pub(crate) type NamedTypeRow = (String, String); -/// `(node type, name, tags, references as (kind, name, package))`. -pub(crate) type FunctionAssemblyRow = ( - &'static str, - String, - Vec, - Vec<(String, String, Option)>, -); +positional_record! { + into; + /// `(node type, name, tags, references as (kind, name, package))`. + pub(crate) struct FunctionAssemblyRow { + node_type: &'static str, + name: String, + tags: Vec, + references: Vec<(String, String, Option)>, + } +} /// `(kind, name)` of a compiled object key. pub(crate) type ObjectKeyRow = (String, String); @@ -396,16 +399,23 @@ function_getters!(NativeFunctionInput, { pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { read_function(&self.owner, self.index, py, |function, _| { let attached: &AttachedFunction = &function.attached; - Ok(( - attached.node_type(), - attached.name.clone(), - attached.tags.clone(), - attached + Ok(FunctionAssemblyRow { + node_type: attached.node_type(), + name: attached.name.clone(), + tags: attached.tags.clone(), + references: attached .references .iter() - .map(|(kind, name, package, _)| (kind.clone(), name.clone(), package.clone())) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind.clone(), name.clone(), package.clone()), + ) .collect(), - )) + }) }) } }); @@ -433,15 +443,15 @@ impl NativeCompiledFunction { .deps .as_deref() .ok_or_else(|| PyRuntimeError::new_err(NOT_ASSEMBLED))?; - Ok(( - ( + Ok(ResourceRow { + key: ( function.attached.node_type().to_owned(), function.attached.name.clone(), ), - deps.to_vec(), - function.attached.tags.clone(), - None, - )) + deps: deps.to_vec(), + tags: function.attached.tags.clone(), + folder: None, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs index bc37f09b55..9e37af932f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compiled_project::schema_facts::python_column_fac use crate::bindings::_helpers::discovery::model_file_record::{NativeModelFile, projected_text}; use crate::bindings::_helpers::model_config::header_metadata::HeaderColumns; use crate::bindings::_helpers::records::fields::field as native_field; -use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::relation_keys::{RelationKeyRow, relation_key_row}; const MODEL_RELATION_KIND: &str = "ref"; @@ -382,7 +382,13 @@ pub(crate) fn reference_tuple( cache .get_or_try_init(py, || { let mut references: Vec> = Vec::with_capacity(rows.len()); - for (kind, name, package, count) in rows { + for ReferenceRow { + kind, + name, + package, + call_argument_count: count, + } in rows + { let fields: Bound<'_, PyDict> = PyDict::new(py); fields.set_item("ref_kind", classes.reference_kind.call1((kind,))?)?; fields.set_item("ref_name", name)?; @@ -561,10 +567,10 @@ pub(crate) fn input_reference_rows(model_input: &Bound<'_, PyAny>) -> PyResult) -> PyResult { - Ok(( - native_field(reference, "ref_kind")?.str()?.extract()?, - native_field(reference, "ref_name")?.extract()?, - native_field(reference, "ref_package")?.extract()?, - native_field(reference, "call_argument_count")?.extract()?, - )) + Ok(ReferenceRow { + kind: native_field(reference, "ref_kind")?.str()?.extract()?, + name: native_field(reference, "ref_name")?.extract()?, + package: native_field(reference, "ref_package")?.extract()?, + call_argument_count: native_field(reference, "call_argument_count")?.extract()?, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs index bdea620787..e6d8cd6bb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop.rs @@ -9,7 +9,9 @@ use crate::bindings::_helpers::compiled_project::function_loop::{ }; use crate::bindings::_helpers::compiled_project::function_views::AttachedFunction; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_config::{ConfigClasses, ModelConfigs}; use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; use crate::bindings::_helpers::compiled_project::model_loop_header::{ @@ -63,10 +65,11 @@ use crate::bindings::_helpers::model_config::header_metadata::ContractClasses; use crate::bindings::_helpers::model_config::model_validator::NativeModelValidator; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, expand_declarations, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -118,8 +121,19 @@ const COMPILE_ERROR_KEY: &str = "compile_error"; /// The native validator, header contract classes, and the loop's Python classes. type LoopChecks = (Py, Py, Py); +positional_record! { + extract; + /// Whether SQL analysis runs, is skipped, is deferred, and is required. + pub(crate) struct LoopSwitches { + pub(crate) sql_analysis: bool, + pub(crate) no_sql_validation: bool, + pub(crate) defer_sql_validation: bool, + pub(crate) require_sql_analysis: bool, + } +} + /// `(analysis switches, project config path)`. -type LoopSettingsInput = ((bool, bool, bool, bool), String); +type LoopSettingsInput = (LoopSwitches, String); /// The Python classes and error factory the loop builds usages, hooks and errors with. #[derive(FromPyObject)] @@ -216,7 +230,12 @@ impl NativeModelLoop { ) -> PyResult { let (validator, contract_classes, loop_classes) = checks; let ( - (sql_analysis, no_sql_validation, defer_sql_validation, require_sql_analysis), + LoopSwitches { + sql_analysis, + no_sql_validation, + defer_sql_validation, + require_sql_analysis, + }, project_config_path, ) = settings; Python::attach(|py| index_sql_hook_definitions(loop_classes.bind(py)))?; @@ -291,19 +310,30 @@ fn prepared_rows<'py>( &sqls, )?; let mut rows: Vec = Vec::with_capacity(model_files.len()); - for (((model_file, paths), (authored, _)), (rendered, spans, reads, error)) in model_files + for (((model_file, paths), (authored, _)), interpolated) in model_files .into_iter() .zip(paths) .zip(sqls) .zip(interpolated) { + let SqlInterpolationRow { + sql: rendered, + spans, + reads, + error, + } = interpolated; let sql: String = rendered.unwrap_or(authored); let scan: Option = error.is_none().then(|| declaration_scan(&sql)); let reads: Vec<(String, String)> = reads .into_iter() .map(|(kind, name)| (kind.to_owned(), name)) .collect(); - let interpolation: InterpolationRow = (sql, reads, error, spans); + let interpolation: InterpolationRow = InterpolationRow { + sql, + reads, + error, + spans, + }; rows.push(ModelLoopRow::new((model_file, paths), scan, interpolation)); } Ok(rows) @@ -673,7 +703,12 @@ impl NativeModelLoop { context, &self.syntax, &scanned.invalid_calls, - (Some(name), relative_path, file_path, contents), + CallOrigin { + name: Some(name), + relative_path, + file_path, + contents, + }, )?; let analysis: (bool, Option>) = ( analysed && scanned.invalid_calls.is_empty(), @@ -807,7 +842,9 @@ impl NativeModelLoop { let rows: Vec<(&str, &str, bool)> = references .iter() .zip(&external) - .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .map(|(ReferenceRow { kind, name, .. }, error)| { + (kind.as_str(), name.as_str(), error.is_some()) + }) .collect(); let unmanaged: Bound<'_, PyAny> = native_field( &native_field(config, RETENTION_ATTRIBUTE)?, @@ -851,7 +888,9 @@ fn applied_interpolation( context: &LoopContext<'_, '_>, interpolation: &InterpolationRow, ) -> PyResult { - let (sql, reads, error, _) = interpolation; + let InterpolationRow { + sql, reads, error, .. + } = interpolation; record_reads(context, reads)?; match error { Some(message) => Err(compile_error(context.classes, message, None)), @@ -919,13 +958,11 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { /// Span rows as the spliced spans the scan-fault locator maps through. fn spliced_spans(rows: &[SpanRow]) -> Vec { rows.iter() - .map( - |&(source_start, source_end, output_start, output_end)| SplicedSpan { - source_start, - source_end, - output_start, - output_end, - }, - ) + .map(|row| SplicedSpan { + source_start: row.source_start, + source_end: row.source_end, + output_start: row.output_start, + output_end: row.output_end, + }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs index f80603a5f8..0403d11316 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_calls.rs @@ -19,8 +19,14 @@ use crate::bindings::_helpers::compiled_project::model_loop::{LoopContext, compi const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; const MODEL_RESOURCE: &str = "model"; -/// `(model name or none for a function, relative path, file path, contents)` of a file. -pub(crate) type CallOrigin<'a> = (Option<&'a str>, &'a str, &'a str, &'a str); +/// The model name, or none for a function, and the relative path, file path and contents of a file. +#[derive(Clone, Copy)] +pub(crate) struct CallOrigin<'a> { + pub(crate) name: Option<&'a str>, + pub(crate) relative_path: &'a str, + pub(crate) file_path: &'a str, + pub(crate) contents: &'a str, +} /// Report each rejected call in order, at its authored location when the authored text has it. pub(crate) fn report_invalid_calls( @@ -32,7 +38,7 @@ pub(crate) fn report_invalid_calls( if invalid_calls.is_empty() { return Ok(()); } - let (_, _, _, contents) = origin; + let CallOrigin { contents, .. } = origin; let authored: HashMap> = match extract_sql_references(contents, syntax) { ReferenceExtraction::Extracted(scan) => { let mut starts: HashMap> = HashMap::new(); @@ -62,7 +68,12 @@ pub(crate) fn report_invalid_call( start: Option, origin: CallOrigin<'_>, ) -> PyResult<()> { - let (name, relative_path, file_path, contents) = origin; + let CallOrigin { + name, + relative_path, + file_path, + contents, + } = origin; let py: Python<'_> = context.host.py(); let path: Bound<'_, PyAny> = context.classes.path_class.call1((relative_path,))?; let fields: Bound<'_, PyDict> = PyDict::new(py); @@ -152,7 +163,12 @@ pub(crate) fn origin_references( context, syntax, &scanned.invalid_calls, - (None, relative_path, file_path, contents), + CallOrigin { + name: None, + relative_path, + file_path, + contents, + }, )?; Ok(scanned.references) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs index d968d3fae7..e6b96c419c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_external.rs @@ -20,7 +20,13 @@ pub(crate) fn external_errors( relative_path: &str, ) -> PyResult>> { let mut errors: Vec> = Vec::with_capacity(references.len()); - for (kind, name, package, _) in references { + for ReferenceRow { + kind, + name, + package, + .. + } in references + { errors.push(if kind == DBT_REF_KIND { external_error(context, (kind, name, package.as_deref()), relative_path)? } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs index 2f27b2dd4e..eb86febd9a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_hooks.rs @@ -23,7 +23,9 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, VisibleDeclarations, compile_error, record_reads, validation_outcome, }; -use crate::bindings::_helpers::compiled_project::model_loop_calls::report_invalid_calls; +use crate::bindings::_helpers::compiled_project::model_loop_calls::{ + CallOrigin, report_invalid_calls, +}; use crate::bindings::_helpers::compiled_project::model_loop_external::external_errors; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::{ @@ -145,7 +147,9 @@ pub(crate) fn expanded_hooks<'py>( let rows: Vec<(&str, &str, bool)> = references .iter() .zip(&external) - .map(|((kind, name, _, _), error)| (kind.as_str(), name.as_str(), error.is_some())) + .map(|(ReferenceRow { kind, name, .. }, error)| { + (kind.as_str(), name.as_str(), error.is_some()) + }) .collect(); let outcome: Bound<'py, PyAny> = model_loop .validator @@ -448,12 +452,12 @@ impl<'py> HookRun<'_, '_, 'py> { self.context, &self.model_loop.syntax, &scanned.invalid_calls, - ( - Some(self.model.name), - self.model.relative_path, - self.model.file_path, - self.model.contents, - ), + CallOrigin { + name: Some(self.model.name), + relative_path: self.model.relative_path, + file_path: self.model.file_path, + contents: self.model.contents, + }, )?; self.references .borrow_mut() @@ -496,7 +500,7 @@ impl<'py> HookRun<'_, '_, 'py> { let py: Python<'py> = hooks.explicit_kinds.py(); let mut seen: Vec<(String, String)> = Vec::new(); let mut reads: Vec> = Vec::new(); - for (kind, name, _, _) in self.references.borrow().iter().skip(start) { + for ReferenceRow { kind, name, .. } in self.references.borrow().iter().skip(start) { let Some(resource_kind) = hooks.explicit_kinds.get_item(kind)? else { continue; }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs index 38a437789d..85574127c6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_host.rs @@ -25,11 +25,10 @@ use crate::bindings::_helpers::compiled_project::project::{ }; use crate::bindings::_helpers::discovery::model_file_record::{ModelFileTexts, NativeModelFile}; use crate::bindings::_helpers::records::fields::{field as native_field, replaced_keywords}; -use crate::bindings::_helpers::render::declaration_expansion::{ - ExpandedDeclarations, ScannedReference, span_rows, -}; +use crate::bindings::_helpers::render::declaration_expansion::{ExpandedDeclarations, span_rows}; use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::ExpansionOutput; +use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; use sqlbuild_discovery::model_files::models::HeaderFacts; const HEADER_KEYS_ATTRIBUTE: &str = "model_header_keys"; @@ -77,10 +76,25 @@ impl ModelLoopRow { } } } -/// `(interpolated query, template reads, error, substitution spans)`. -pub(crate) type InterpolationRow = (String, Vec<(String, String)>, Option, Vec); -/// `(kind, name, package, call argument count)`. -pub(crate) type ReferenceRow = (String, String, Option, Option); +/// A model query's interpolation: the interpolated query, template reads, error and spans. +pub(crate) struct InterpolationRow { + pub(crate) sql: String, + pub(crate) reads: Vec<(String, String)>, + pub(crate) error: Option, + /// The substitution spans. + pub(crate) spans: Vec, +} +positional_record! { + extract, into; + /// `(kind, name, package, call argument count)`. + #[derive(Clone, Debug, PartialEq, Eq)] + pub(crate) struct ReferenceRow { + pub(crate) kind: String, + pub(crate) name: String, + pub(crate) package: Option, + pub(crate) call_argument_count: Option, + } +} /// One model the native loop finished, before the host builds its Python view. pub(crate) struct FinishedModel<'py> { @@ -130,7 +144,12 @@ pub(crate) fn finished_record<'py>( }, authored_query: authored_query_sql, contents: authored_sql, - interpolation: (interpolated_sql, _, _, interpolation_spans), + interpolation: + InterpolationRow { + sql: interpolated_sql, + spans: interpolation_spans, + .. + }, model_file, .. } = row; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs index 8574f80e70..82a3173139 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_inputs.rs @@ -68,15 +68,22 @@ pub(crate) fn stripped_values<'py>( } pub(crate) fn reference_row(reference: &SqlReference) -> ReferenceRow { - ( - reference.kind.to_owned(), - reference.name.clone(), - reference.package.clone(), - reference.call_argument_count, - ) + ReferenceRow { + kind: reference.kind.to_owned(), + name: reference.name.clone(), + package: reference.package.clone(), + call_argument_count: reference.call_argument_count, + } } -pub(crate) fn reference_fact((kind, name, package, _): ReferenceRow) -> ModelReferenceFact { +pub(crate) fn reference_fact( + ReferenceRow { + kind, + name, + package, + .. + }: ReferenceRow, +) -> ModelReferenceFact { ModelReferenceFact { kind, name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs index 64a85b3c40..56eef6ba1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_macros.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::model_loop_inputs::reference_row; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, expanded, + ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; @@ -112,13 +112,13 @@ pub(crate) fn expanded_with<'py>( let py: Python<'py> = consumer.py(); let expansion: NativeMacroExpansion = NativeMacroExpansion::new( &context.classes.path_class.call1((file_path,))?, - ( - classes.loaded_macros.clone_ref(py), - PyDict::new(py).unbind(), - classes.macro_context.clone().unbind(), - declarations.clone().unbind(), - consumer.clone().unbind(), - ), + StateInput { + loaded_macros: classes.loaded_macros.clone_ref(py), + overrides: PyDict::new(py).unbind(), + macro_context: classes.macro_context.clone().unbind(), + declarations: declarations.clone().unbind(), + consumer: consumer.clone().unbind(), + }, native_field(&classes.macro_context, ENFORCE_ATTRIBUTE)?.is_truthy()?, classes.expansion.clone_ref(py), )?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs index ef803053de..6189299d1d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs @@ -49,7 +49,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::NativeModelLoop; use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_project::seed_views::{ - NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedRecord, paired_seeds, + NativeCompiledSeed, NativeSeedInput, SeedNamespace, SeedParts, SeedRecord, paired_seeds, }; use crate::bindings::_helpers::compiled_project::source_loop::{SourceRow, SourceSettings}; use crate::bindings::_helpers::compiled_project::source_views::{ @@ -70,11 +70,42 @@ const MATERIALIZED_KEY: &str = "materialized"; const TABLE_MATERIALIZATION: &str = "table"; /// `(name, type)`. -type NamedType = (String, Option); -/// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. -type ProofPy = (bool, bool, Vec, Option, Vec); -/// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. -type AnalysisPy = (bool, Vec, Vec, bool, Vec); +pub(crate) type NamedType = (String, Option); +positional_record! { + extract; + /// `(output proven, bare dynamic pivot, (family key, inferred type), failure, (family, type))`. + pub(crate) struct ProofPy { + pub(crate) output_proven: bool, + pub(crate) bare_dynamic_pivot: bool, + pub(crate) families: Vec, + pub(crate) failure_reason: Option, + pub(crate) named_families: Vec, + } +} + +positional_record! { + extract; + /// `(inferred, (name, type), nullable per column, has star, unchecked output columns)`. + pub(crate) struct AnalysisPy { + pub(crate) inferred: bool, + pub(crate) inferred_columns: Vec, + pub(crate) nullable_columns: Vec, + pub(crate) fast_lineage_has_star: bool, + pub(crate) unchecked: Vec, + } +} + +positional_record! { + extract; + /// A model's name, relative path, compiled and authored queries, and authored SQL file. + pub(crate) struct ModelTextsPy { + name: String, + relative_path: String, + query_sql: String, + authored_query_sql: String, + authored_sql: String, + } +} /// Encoded config values and the encoder's error, if any. pub(crate) type ConfigValues = (BTreeMap, Option); @@ -82,7 +113,7 @@ pub(crate) type ConfigValues = (BTreeMap, Option); /// One model as the compile-input stage produced it. #[derive(FromPyObject)] pub(crate) struct ModelInputPy<'py>( - (String, String, String, String, String), + ModelTextsPy, (Bound<'py, PyDict>, Vec, Option), Vec<(String, String, Option)>, Vec, @@ -194,7 +225,12 @@ impl NativeCompiledProject { )?); } for seed in &self.seeds { - let (name, seed_file, schema_entry, schema_file) = seed.parts(); + let SeedParts { + name, + seed_file, + schema_entry, + schema_file, + } = seed.parts(); let schema_entry: &Bound<'py, PyAny> = schema_entry.bind(py); owners.push(resource_owner( SEED_OWNER, @@ -335,19 +371,28 @@ impl NativeCompiledProject { columns: AnalysisPy, dynamic_proof: Option, ) -> bool { - let (inferred, inferred_columns, nullable_columns, fast_lineage_has_star, unchecked) = - columns; + let AnalysisPy { + inferred, + inferred_columns, + nullable_columns, + fast_lineage_has_star, + unchecked, + } = columns; let analysis: ModelAnalysisFact = ModelAnalysisFact { inferred_columns, dynamic_proof: dynamic_proof.map( - |(output_proven, bare_dynamic_pivot, families, failure_reason, named_families)| { - DynamicProofFact { - output_proven, - bare_dynamic_pivot, - families, - failure_reason, - named_families, - } + |ProofPy { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, + }| DynamicProofFact { + output_proven, + bare_dynamic_pivot, + families, + failure_reason, + named_families, }, ), inferred, @@ -620,7 +665,13 @@ impl NativeCompiledProject { pub(crate) fn model_facts(model: ModelInputPy<'_>) -> PyResult { let ModelInputPy( - (name, relative_path, query_sql, authored_query_sql, authored_sql), + ModelTextsPy { + name, + relative_path, + query_sql, + authored_query_sql, + authored_sql, + }, (values, header_keys, layer_schema), references, enum_columns, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs index a7bf443dcf..966d655754 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs @@ -576,12 +576,12 @@ fn resource_row( tag_names.push(tag?.str()?.extract()?); } } - Ok(( - key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, + Ok(ResourceRow { + key: key_row(&native_field(resource, KEY_ATTRIBUTE)?)?, deps, - tag_names, - None, - )) + tags: tag_names, + folder: None, + }) } /// `(resource type, name)` of a `CompiledObjectKey`. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs index 503d7026a5..4d095ca181 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/records.rs @@ -13,7 +13,9 @@ use crate::bindings::_helpers::compile_driver::python_calls::imported; use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::compiled_project::inputs_view::NativeInputsView; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; -use crate::bindings::_helpers::compiled_project::project::{NativeCompiledProject, model_facts}; +use crate::bindings::_helpers::compiled_project::project::{ + AnalysisPy, NativeCompiledProject, ProofPy, model_facts, +}; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::compiled_project::schema_facts::entry_schema_fact; use crate::bindings::_helpers::records::construct::{enum_member, new_record, record_class}; @@ -397,28 +399,30 @@ pub(crate) fn record_model_analyses<'py>( None } else { let families: Bound<'py, PyAny> = native_field(&proof, "families")?; - Some(( - native_field(&proof, "output_proven")?.extract()?, - native_field(&proof, "bare_dynamic_pivot")?.extract()?, - family_types(&families, true)?, - native_field(&proof, "failure_reason")?.extract()?, - family_types(&families, false)?, - )) + Some(ProofPy { + output_proven: native_field(&proof, "output_proven")?.extract()?, + bare_dynamic_pivot: native_field(&proof, "bare_dynamic_pivot")?.extract()?, + families: family_types(&families, true)?, + failure_reason: native_field(&proof, "failure_reason")?.extract()?, + named_families: family_types(&families, false)?, + }) }; - let analysis = ( - !inferred.is_none(), - rows.iter() + let analysis = AnalysisPy { + inferred: !inferred.is_none(), + inferred_columns: rows + .iter() .map(|(name, column_type, _)| (name.clone(), column_type.clone())) .collect(), - rows.iter() + nullable_columns: rows + .iter() .map(|(_, _, nullability)| nullability.eq(&nullable)) .collect::>()?, - native_field(model, "fast_lineage_has_star")?.is_truthy()?, - native_field(model, "unchecked_output_columns")? + fast_lineage_has_star: native_field(model, "fast_lineage_has_star")?.is_truthy()?, + unchecked: native_field(model, "unchecked_output_columns")? .try_iter()? .map(|column| column?.extract()) .collect::>()?, - ); + }; let name: String = native_field(model, "name")?.extract()?; transferred += usize::from( project diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs index efd61f55c2..9dd9ba24b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs @@ -21,7 +21,9 @@ use sqlbuild_analysis::resource_diagnostics::main::missing_description::missing_ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; +use crate::bindings::_helpers::compiled_project::seed_views::SeedParts; use crate::bindings::_helpers::discovery::model_file_record::NativeModelFile; use crate::bindings::_helpers::records::fields::field as native_field; @@ -36,8 +38,39 @@ const PYTHON_UDF_DECORATOR: &str = "udf"; /// A loader's description, relative path and whether that path is a YAML file. pub(crate) type LoaderRow = (Option, String, bool); -/// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. -type PythonResourceRow = (String, String, Option>, Py, String); +positional_record! { + extract; + /// A Python resource's kind, name, description (a `str` that may hold surrogates), location, help. + struct PythonResourceRow { + kind: String, + name: String, + description: Option>, + location: Py, + help: String, + } +} + +positional_record! { + extract; + /// A scenario's name, relative path, contents and authored header description. + struct ScenarioDescriptionRow { + name: String, + path: String, + contents: String, + raw: Py, + } +} + +positional_record! { + extract; + /// A SQL hook file's name, relative path, contents and description. + struct HookDescriptionRow { + name: String, + path: String, + contents: String, + description: Option, + } +} /// What the resource diagnostics read beyond the owner, and the Python values they build. #[derive(FromPyObject)] @@ -53,9 +86,9 @@ pub(crate) struct DiagnosticSettings { /// Each discovered audit factory's name, relative path and line. audit_factories: Vec<(String, String, usize)>, /// Each scenario's name, relative path, contents and authored header description. - scenarios: Vec<(String, String, String, Py)>, + scenarios: Vec, /// Each SQL hook file's name, relative path, contents and description. - sql_hooks: Vec<(String, String, String, Option)>, + sql_hooks: Vec, /// Each loader by name. loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. @@ -277,7 +310,13 @@ fn missing_descriptions<'py>( &help, )?); } - for (name, path, contents, raw) in &diagnostics.settings.scenarios { + for ScenarioDescriptionRow { + name, + path, + contents, + raw, + } in &diagnostics.settings.scenarios + { let raw: &Bound<'py, PyAny> = raw.bind(py); let description: Option = if raw.is_instance_of::() { Some(raw.extract()?) @@ -301,7 +340,11 @@ fn missing_descriptions<'py>( )?); } for seed in project.seeds() { - let (_, _, entry, schema_file) = seed.parts(); + let SeedParts { + schema_entry: entry, + schema_file, + .. + } = seed.parts(); let entry: &Bound<'py, PyAny> = entry.bind(py); let schema_file: &Bound<'py, PyAny> = schema_file.bind(py); let name: String = native_field(entry, "name")?.extract()?; @@ -324,7 +367,13 @@ fn missing_descriptions<'py>( for function in project.functions() { found.push(function_description(diagnostics, function)?); } - for (name, path, contents, description) in &diagnostics.settings.sql_hooks { + for HookDescriptionRow { + name, + path, + contents, + description, + } in &diagnostics.settings.sql_hooks + { let entry: String = format!("description \"What {name} does\""); let help: String = header_help(&HeaderHelp { kind: "hook", @@ -341,7 +390,14 @@ fn missing_descriptions<'py>( &help, )?); } - for (kind, name, description, location, help) in &diagnostics.settings.python_resources { + for PythonResourceRow { + kind, + name, + description, + location, + help, + } in &diagnostics.settings.python_resources + { let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); @@ -484,7 +540,7 @@ fn cursor_without_inputs<'py>( let kinds: Vec<&str> = record .references .iter() - .map(|(kind, _, _, _)| kind.as_str()) + .map(|ReferenceRow { kind, .. }| kind.as_str()) .collect(); let materialized: Option = config_text(&values, "materialized")?; let cursor: Option = config_text(&values, CURSOR_KEY)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs index 11af682292..2e8c4f2db4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs @@ -22,8 +22,18 @@ use crate::bindings::_helpers::compiled_project::sql_test_loop::{ FileOrigin, TestScope, assertion_targets, checked_body, source_names, unique, }; -/// A scenario file's paths, discovered file, contents, SQL body and name. -pub(crate) type ScenarioRow = (String, String, Py, String, String, String); +positional_record! { + extract; + /// A scenario file's paths, discovered file, contents, SQL body and name. + pub(crate) struct ScenarioRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) scenario_file: Py, + pub(crate) contents: String, + pub(crate) sql_body: String, + pub(crate) name: String, + } +} impl<'py> TestScope<'_, 'py> { /// Every scenario expanded, classified and checked, in discovery order. @@ -33,7 +43,14 @@ impl<'py> TestScope<'_, 'py> { fn scenario(&self, row: &ScenarioRow) -> PyResult> { let py: Python<'py> = self.context.host.py(); - let (file_path, relative_path, scenario_file, contents, sql_body, name) = row; + let ScenarioRow { + file_path, + relative_path, + scenario_file, + contents, + sql_body, + name, + } = row; let resource: Bound<'py, PyAny> = self .context .classes diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs index 4e775cf3cb..52486ab28b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs @@ -24,13 +24,24 @@ const FILE_PATH_ATTRIBUTE: &str = "file_path"; const STEM_ATTRIBUTE: &str = "stem"; const RELATIVE_PATH_ATTRIBUTE: &str = "relative_path"; -/// `(database, schema, logical database, logical schema)` of a seed's destination. -pub(crate) type SeedNamespace = ( - Option, - Option, - Option, - Option, -); +positional_record! { + extract; + /// `(database, schema, logical database, logical schema)` of a seed's destination. + pub(crate) struct SeedNamespace { + database: Option, + schema: Option, + logical_database: Option, + logical_schema: Option, + } +} + +/// A seed's name, CSV file, declaration and declaring schema file. +pub(crate) struct SeedParts<'a> { + pub(crate) name: &'a str, + pub(crate) seed_file: &'a Py, + pub(crate) schema_entry: &'a Py, + pub(crate) schema_file: &'a Py, +} /// One seed CSV paired with its declaration; the discovered objects are discovery's. pub(crate) struct SeedRecord { @@ -100,18 +111,23 @@ pub(crate) fn paired_seeds( impl SeedRecord { /// The seed's name, CSV file, declaration and declaring schema file. - pub(crate) fn parts(&self) -> (&str, &Py, &Py, &Py) { - ( - &self.name, - &self.seed_file, - &self.schema_entry, - &self.schema_file, - ) + pub(crate) fn parts(&self) -> SeedParts<'_> { + SeedParts { + name: &self.name, + seed_file: &self.seed_file, + schema_entry: &self.schema_entry, + schema_file: &self.schema_file, + } } /// Keep the destination native project assembly resolved for the seed. pub(crate) fn set_namespace(&mut self, namespace: SeedNamespace) { - let (database, schema, logical_database, logical_schema) = namespace; + let SeedNamespace { + database, + schema, + logical_database, + logical_schema, + } = namespace; self.destination = Some(DestinationFact { database, schema, @@ -201,12 +217,12 @@ impl NativeCompiledSeed { for tag in native_field(seed.schema_entry.bind(py), TAGS_ATTRIBUTE)?.try_iter()? { tags.push(tag?.str()?.extract()?); } - Ok(( - (SEED_RESOURCE.to_owned(), seed.name.clone()), - Vec::new(), + Ok(ResourceRow { + key: (SEED_RESOURCE.to_owned(), seed.name.clone()), + deps: Vec::new(), tags, - None, - )) + folder: None, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs index 48690285f0..e3537d16b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs @@ -20,6 +20,17 @@ use crate::bindings::_helpers::rules::relation_keys::audit_key_rows; const SOURCE_RESOURCE: &str = "source"; +positional_record! { + into; + /// `(name, has a loader, database, schema)` of one source. + pub(crate) struct SourceAssemblyRow { + name: String, + managed: bool, + database: Option, + schema: Option, + } +} + /// `(database, schema)` native project assembly resolved for a source, or `None`. pub(crate) type SourceNamespace = Option<(Option, Option)>; @@ -210,18 +221,15 @@ impl NativeSourceInput { /// `(name, has a loader, database, schema)`, what project assembly resolves a namespace from. #[getter] - pub(crate) fn assembly_row( - &self, - py: Python<'_>, - ) -> PyResult<(String, bool, Option, Option)> { + pub(crate) fn assembly_row(&self, py: Python<'_>) -> PyResult { read_source(&self.owner, self.index, py, |source, py| { let attached: &AttachedSource = &source.attached; - Ok(( - attached.name.clone(), - !native_field(attached.entry.bind(py), "loader")?.is_none(), - attached.database.clone(), - attached.schema.clone(), - )) + Ok(SourceAssemblyRow { + name: attached.name.clone(), + managed: !native_field(attached.entry.bind(py), "loader")?.is_none(), + database: attached.database.clone(), + schema: attached.schema.clone(), + }) }) } } @@ -241,12 +249,12 @@ impl NativeCompiledSource { /// The source's project graph row: its key, with no dependencies, tags or folder. pub(crate) fn graph_row(&self, py: Python<'_>) -> PyResult { read_source(&self.owner, self.index, py, |source, _| { - Ok(( - (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), - Vec::new(), - Vec::new(), - None, - )) + Ok(ResourceRow { + key: (SOURCE_RESOURCE.to_owned(), source.attached.name.clone()), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }) }) } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs index 07563842eb..40aa64302a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs @@ -42,7 +42,7 @@ use crate::bindings::_helpers::compiled_project::model_loop::{ LoopContext, NativeModelLoop, compile_error, helped_compile_error, }; use crate::bindings::_helpers::compiled_project::model_loop_calls::{ - origin_references, report_invalid_call, + CallOrigin, origin_references, report_invalid_call, }; use crate::bindings::_helpers::compiled_project::source_loop::{ AuthoredExpansion, expanded_authored_sql, scoped_declarations, @@ -70,19 +70,32 @@ const MACRO_VISIBILITY: &str = "macro_visibility"; /// A test case's discovered case, name and parameter values in authored order. pub(crate) type TestCaseRow = (Py, String, Vec<(String, Py)>); -/// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. -pub(crate) type TestBlockRow = ( - Py, - Option, - String, - String, - Vec, - Vec, - Option<(usize, usize)>, -); +positional_record! { + extract; + /// A test block, its name, SQL, mode, declared parameter names, cases and authored SQL span. + pub(crate) struct TestBlockRow { + pub(crate) block: Py, + pub(crate) name: Option, + pub(crate) sql_body: String, + pub(crate) mode: String, + pub(crate) parameters: Vec, + pub(crate) cases: Vec, + pub(crate) sql_body_span: Option<(usize, usize)>, + } +} -/// A test file's paths, stem, discovered file, contents and blocks. -pub(crate) type TestFileRow = (String, String, String, Py, String, Vec); +positional_record! { + extract; + /// A test file's paths, stem, discovered file, contents and blocks. + pub(crate) struct TestFileRow { + pub(crate) file_path: String, + pub(crate) relative_path: String, + pub(crate) stem: String, + pub(crate) file: Py, + pub(crate) contents: String, + pub(crate) blocks: Vec, + } +} /// What every SQL test and scenario of one compile resolves and records with. #[derive(FromPyObject)] @@ -151,11 +164,11 @@ impl<'py> TestScope<'_, 'py> { pub(crate) fn attached(&self, files: &[TestFileRow]) -> PyResult>> { let mut expanded: Vec> = Vec::new(); for file in files { - for block in &file.5 { - if block.5.is_empty() { + for block in &file.blocks { + if block.cases.is_empty() { expanded.push(self.expanded(file, block, None)?); } - for case in &block.5 { + for case in &block.cases { expanded.push(self.expanded(file, block, Some(case))?); } } @@ -178,8 +191,20 @@ impl<'py> TestScope<'_, 'py> { case: Option<&'a TestCaseRow>, ) -> PyResult> { let py: Python<'py> = self.py(); - let (file_path, relative_path, stem, _, contents, _) = file; - let (_, name, sql_body, mode, parameters, _, _) = block; + let TestFileRow { + file_path, + relative_path, + stem, + contents, + .. + } = file; + let TestBlockRow { + name, + sql_body, + mode, + parameters, + .. + } = block; let test_name: &str = name.as_deref().unwrap_or(stem); let resource: Bound<'py, PyAny> = self .context @@ -329,7 +354,7 @@ impl<'py> TestScope<'_, 'py> { let (file, block) = test; let request: SqlTestRequest = SqlTestRequest { sql: sql.to_owned(), - file_label: file.1.clone(), + file_label: file.relative_path.clone(), mode: mode.to_owned(), raw: true, authored: false, @@ -348,7 +373,7 @@ impl<'py> TestScope<'_, 'py> { .. } = &classification { - let origin: FileOrigin<'_> = (&file.0, &file.1, &file.4); + let origin: FileOrigin<'_> = (&file.file_path, &file.relative_path, &file.contents); if self.authored_invalid_calls(file, block, &actual.0)? == 0 { for cte in helpers.iter().chain([expected]) { origin_references(self.context, &self.model_loop.syntax, &cte.1, origin)?; @@ -367,8 +392,8 @@ impl<'py> TestScope<'_, 'py> { ) -> PyResult { let syntax = &self.model_loop.syntax; let request: SqlTestRequest = SqlTestRequest { - sql: block.2.clone(), - file_label: file.1.clone(), + sql: block.sql_body.clone(), + file_label: file.relative_path.clone(), mode: MODEL_MODE.to_owned(), raw: false, authored: true, @@ -394,16 +419,24 @@ impl<'py> TestScope<'_, 'py> { ReferenceExtraction::Failed(failure) => return Err(self.error(&failure.message)), }; for invalid in &scan.invalid_calls { - let Some(offset) = - authored_test_offset(&file.4, &block.2, block.6, body_start + invalid.start) - else { + let Some(offset) = authored_test_offset( + &file.contents, + &block.sql_body, + block.sql_body_span, + body_start + invalid.start, + ) else { return Ok(reported); }; report_invalid_call( self.context, invalid, Some(offset), - (None, &file.1, &file.0, &file.4), + CallOrigin { + name: None, + relative_path: &file.relative_path, + file_path: &file.file_path, + contents: &file.contents, + }, )?; reported += 1; } @@ -611,11 +644,11 @@ impl<'py> TestScope<'_, 'py> { let mut files: Vec<(&str, Vec)> = Vec::new(); let mut positions: HashMap<&str, usize> = HashMap::new(); for (index, test) in tests.iter().enumerate() { - match positions.get(test.file.1.as_str()) { + match positions.get(test.file.relative_path.as_str()) { Some(position) => files[*position].1.push(index), None => { - positions.insert(&test.file.1, files.len()); - files.push((&test.file.1, vec![index])); + positions.insert(&test.file.relative_path, files.len()); + files.push((&test.file.relative_path, vec![index])); } } } @@ -635,8 +668,8 @@ impl<'py> TestScope<'_, 'py> { let mut modes: Vec<&str> = Vec::with_capacity(indexes.len()); for index in indexes { parts.push(tests[*index].sql.clone()); - parts.push(tests[*index].block.3.clone()); - modes.push(&tests[*index].block.3); + parts.push(tests[*index].block.mode.clone()); + modes.push(&tests[*index].block.mode); } let stored: Option> = match &store { Some(store) => store @@ -662,8 +695,8 @@ impl<'py> TestScope<'_, 'py> { .iter() .map(|index| SqlTestRequest { sql: tests[*index].sql.clone(), - file_label: tests[*index].file.1.clone(), - mode: tests[*index].block.3.clone(), + file_label: tests[*index].file.relative_path.clone(), + mode: tests[*index].block.mode.clone(), raw: false, authored: false, scenario: false, @@ -713,9 +746,21 @@ impl<'py> TestScope<'_, 'py> { let offset: Option = failure .token .as_ref() - .filter(|_| sql == block.2) - .and_then(|(_, offset)| authored_test_offset(&file.4, &block.2, block.6, *offset)); - located_error(self.context, failure, (&file.1, &file.4), offset) + .filter(|_| sql == block.sql_body) + .and_then(|(_, offset)| { + authored_test_offset( + &file.contents, + &block.sql_body, + block.sql_body_span, + *offset, + ) + }); + located_error( + self.context, + failure, + (&file.relative_path, &file.contents), + offset, + ) } /// Check one classified test; `None` when a P013 problem was reported instead. @@ -725,8 +770,20 @@ impl<'py> TestScope<'_, 'py> { classification: SqlTestClassification, ) -> PyResult>> { let py: Python<'py> = self.py(); - let (file_path, relative_path, stem, test_file, contents, _) = test.file; - let (test_block, name, _, mode, _, _, _) = test.block; + let TestFileRow { + file_path, + relative_path, + stem, + file: test_file, + contents, + .. + } = test.file; + let TestBlockRow { + block: test_block, + name, + mode, + .. + } = test.block; let test_name: &str = name.as_deref().unwrap_or(stem); let origin: FileOrigin<'_> = (file_path, relative_path, contents); let payload: Bound<'py, PyAny> = match &classification { @@ -911,14 +968,14 @@ impl<'py> TestScope<'_, 'py> { ) -> PyResult<()> { let py: Python<'py> = self.py(); let located: CteLocation = sql_test_cte_location( - &file.4, - &block.2, + &file.contents, + &block.sql_body, &problem.cte_name, problem.call.as_deref(), ); let (line, column) = (located.line, located.column); let arguments: Bound<'py, PyDict> = PyDict::new(py); - arguments.set_item("path", native_field(file.3.bind(py), "relative_path")?)?; + arguments.set_item("path", native_field(file.file.bind(py), "relative_path")?)?; arguments.set_item("line", line)?; arguments.set_item("column", column)?; arguments.set_item("end_line", located.end_line)?; @@ -941,7 +998,7 @@ impl<'py> TestScope<'_, 'py> { py, [ HELPER_REFERENCE_CODE.to_owned(), - file.1.replace('\\', "/"), + file.relative_path.replace('\\', "/"), format!("{line:09}:{column:09}"), problem.message.clone(), ], diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs index 2dba2e534a..f9b65d4618 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/contract_diagnostics.rs @@ -56,10 +56,16 @@ fn output_column_location<'py>( } let implicit: bool = native_field(model, "extract_implicit_alias_columns")?.is_truthy()?; let rows: Vec = location_rows(model_output_columns(&authored, implicit)); - let Some((_, line, column, end_line, end_column)) = rows + let Some(ColumnLocation { + line, + column, + end_line, + end_column, + .. + }) = rows .into_iter() .rev() - .find(|(name, ..)| name == column_name) + .find(|location| location.name == column_name) else { return Ok(None); }; @@ -126,7 +132,16 @@ fn contract_diagnostic<'py>( row: DiagnosticRow, ) -> PyResult> { let py: Python<'py> = model.py(); - let (code, is_error, message, column_name, declared_index, output_column, related, help) = row; + let DiagnosticRow { + code, + is_error, + message, + column_name, + declared_index, + output_column, + related, + help, + } = row; let severity: &str = if is_error { "error" } else { "warning" }; keyword_call( &record_class(py, RecordId::CompilerDiagnostic)?, @@ -228,7 +243,14 @@ pub(crate) fn promotion_conflict_diagnostics<'py>( let diagnostic_type: Bound<'py, PyAny> = record_class(py, RecordId::CompilerDiagnostic)?; let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; let mut diagnostics: Vec> = Vec::with_capacity(conflicts.len()); - for (name, relative_path, code, message, help) in conflicts { + for ConflictRow { + name, + relative_path, + code, + message, + help, + } in conflicts + { diagnostics.push(keyword_call( &diagnostic_type, vec![ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs index 4b840c0e71..d3d532a8c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/contracts/model_contracts.rs @@ -20,17 +20,18 @@ use crate::bindings::_helpers::compiled_project::project::NativeCompiledProject; use crate::bindings::_helpers::type_system::normalization::normalization_error; use crate::bindings::types::CompilerDetach; -/// `(code, is_error, message, column, declared_index, output_column, related, help)`. -pub(crate) type DiagnosticRow = ( - &'static str, - bool, - String, - Option, - Option, - Option, - Option<(String, String)>, - String, -); +/// One contract diagnostic: its code, severity, message, column, location, related output and help. +#[derive(Clone)] +pub(crate) struct DiagnosticRow { + pub(crate) code: &'static str, + pub(crate) is_error: bool, + pub(crate) message: String, + pub(crate) column_name: Option, + pub(crate) declared_index: Option, + pub(crate) output_column: Option, + pub(crate) related: Option<(String, String)>, + pub(crate) help: String, +} /// Evaluate the column contract of every model the compile's owner recorded, in its order. pub(crate) fn evaluate_project_model_contracts( @@ -133,8 +134,14 @@ fn fact_dynamic_proof(proof: &DynamicProofFact) -> DynamicContractProof { } } -/// `(model name, model path, code, message, help)`. -pub(crate) type ConflictRow = (String, String, &'static str, String, String); +/// One K011 conflict: the model name and path, and the code, message and help. +pub(crate) struct ConflictRow { + pub(crate) name: String, + pub(crate) relative_path: String, + pub(crate) code: &'static str, + pub(crate) message: String, + pub(crate) help: String, +} /// The K011 conflicts of enforced contracts under immediate table promotion, by model name. pub(crate) fn project_promotion_conflicts( @@ -166,29 +173,29 @@ pub(crate) fn project_promotion_conflicts( .into_iter() .map(|conflict| { let model: &CompiledModelFacts = &models[conflict.model_index]; - ( - model.name.clone(), - model.relative_path.clone(), - conflict.code, - conflict.message, - conflict.help, - ) + ConflictRow { + name: model.name.clone(), + relative_path: model.relative_path.clone(), + code: conflict.code, + message: conflict.message, + help: conflict.help, + } }) .collect()) } fn diagnostic_row(diagnostic: ContractDiagnostic) -> DiagnosticRow { let (declared_index, output_column) = diagnostic.location.into_parts(); - ( - diagnostic.code, - diagnostic.severity == ContractSeverity::Error, - diagnostic.message, - diagnostic.column_name, + DiagnosticRow { + code: diagnostic.code, + is_error: diagnostic.severity == ContractSeverity::Error, + message: diagnostic.message, + column_name: diagnostic.column_name, declared_index, output_column, - diagnostic + related: diagnostic .related_output .map(|related| (related.column_name, related.message)), - diagnostic.help, - ) + help: diagnostic.help, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs index 405832ce4d..bd8dd810b2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs @@ -162,6 +162,17 @@ fn read_only_query( Ok(normalized.to_owned()) } +positional_record! { + extract; + /// The target name, target, database and schema a raw-query diff resolves. + struct ResolvedNamespace<'py> { + target_name: Bound<'py, PyAny>, + _target: Bound<'py, PyAny>, + database: Bound<'py, PyAny>, + schema: Bound<'py, PyAny>, + } +} + /// Resolve raw SQL, the active target namespace, the adapter and its connection. pub(crate) fn prepare_query_diff<'py>( request: &Bound<'py, PyAny>, @@ -206,12 +217,12 @@ pub(crate) fn prepare_query_diff<'py>( )?), ) .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; - let (target_name, _target, database, schema): ( - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - Bound<'py, PyAny>, - ) = namespace.extract()?; + let ResolvedNamespace { + target_name, + database, + schema, + .. + }: ResolvedNamespace<'py> = namespace.extract()?; if schema.is_none() { return Err(failure( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs index 605837e2c8..623fd46ff4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/discovery/header_values.rs @@ -103,7 +103,14 @@ fn source_locations<'py>( ) -> PyResult> { let py: Python<'py> = relative_path.py(); let locations: Bound<'py, PyDict> = PyDict::new(py); - for (name, line, column, end_line, end_column) in rows { + for ColumnLocation { + name, + line, + column, + end_line, + end_column, + } in rows + { let arguments: Bound<'py, PyDict> = PyDict::new(py); arguments.set_item("path", relative_path)?; arguments.set_item("line", line)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs index 2ebfe2d8b7..ac8620b4b1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/execution_gates.rs @@ -39,7 +39,13 @@ fn attached_audit_gate_edges<'py>( let cache: KeyCache<'py> = KeyCache::new(py)?; let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::AttachedAuditGateEdge)?; let mut edges: Vec> = Vec::with_capacity(gates.len()); - for (audit_name, target, gated, read) in gates { + for GateRow { + audit_name, + target, + gated, + read, + } in gates + { edges.push(keyword_call( &edge_type, &[ @@ -65,7 +71,14 @@ fn hook_read_edges<'py>( let edge_type: Bound<'py, PyAny> = record_class(py, RecordId::HookReadEdge)?; let path_type: Bound<'py, PyAny> = py.import(PATHLIB_MODULE)?.getattr("Path")?; let mut edges: Vec> = Vec::with_capacity(hooks.len()); - for (hook_name, gated, read, kind, path) in hooks { + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + path, + } in hooks + { let hook_path: Bound<'py, PyAny> = match path { None => py.None().into_bound(py), Some(path) => path_type.call1((path,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs index 62e7a80269..6e880c4db6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/project_graph.rs @@ -24,17 +24,28 @@ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::graph::views::lineage_graph_views; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(key, deps, tags, model folder)`. -pub(crate) type ResourceRow = (ObjectKey, Vec, Vec, Option); +positional_record! { + extract; + /// `(key, deps, tags, model folder)`. + pub(crate) struct ResourceRow { + pub(crate) key: ObjectKey, + pub(crate) deps: Vec, + pub(crate) tags: Vec, + pub(crate) folder: Option, + } +} type Edges = Vec<(ObjectKey, Vec)>; -/// `(names, upstream, downstream, tags, paths)` in Python's dict order. -type IndexesRow = ( - Vec<(String, ObjectKey)>, - Edges, - Edges, - Vec<(String, Vec)>, - Vec<(ObjectKey, String)>, -); +positional_record! { + extract, into; + /// `(names, upstream, downstream, tags, paths)` in Python's dict order. + struct IndexesRow { + names: Vec<(String, ObjectKey)>, + upstream: Edges, + downstream: Edges, + tags: Vec<(String, Vec)>, + paths: Vec<(ObjectKey, String)>, + } +} /// `(unique id, neighbour unique ids)` in graph order. type UniqueIdMap = Vec<(String, Vec)>; /// `PlannerInputError`'s `(code, message, help)`. @@ -167,12 +178,19 @@ pub(crate) fn project_graph(resources: Vec) -> PyResult = resources .into_iter() - .map(|(key, deps, tags, folder)| GraphResource { - key, - deps, - tags, - folder, - }) + .map( + |ResourceRow { + key, + deps, + tags, + folder, + }| GraphResource { + key, + deps, + tags, + folder, + }, + ) .collect(); Ok(NativeProjectGraph::new(build_project_graph(&resources))) }) @@ -190,7 +208,13 @@ impl NativeProjectGraph { #[staticmethod] fn from_indexes(indexes: IndexesRow) -> PyResult { compiler_guard(|| { - let (names, upstream, downstream, tags, paths) = indexes; + let IndexesRow { + names, + upstream, + downstream, + tags, + paths, + } = indexes; Ok(Self::new(graph_from_indexes(GraphIndexes { names, upstream, @@ -204,13 +228,13 @@ impl NativeProjectGraph { /// `(names, upstream, downstream, tags, paths)` for Python's `ProjectGraph`. fn indexes(&self) -> IndexesRow { let graph: &ProjectGraph = &self.graph; - ( - graph.names.clone(), - graph.upstream.clone(), - graph.downstream.clone(), - graph.tags.clone(), - graph.paths.clone(), - ) + IndexesRow { + names: graph.names.clone(), + upstream: graph.upstream.clone(), + downstream: graph.downstream.clone(), + tags: graph.tags.clone(), + paths: graph.paths.clone(), + } } /// The key's direct upstream lineage keys, in authored dependency order. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 44539a45cf..63aa2fd6ee 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -24,16 +24,28 @@ use crate::bindings::_helpers::records::fields::mapping_items; use crate::bindings::_helpers::records::ids::ErrorId; type NameRows = Vec<(String, Vec)>; -/// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. -type InventoryRow = (String, Vec, String, Vec); -/// `(edges, upstream, downstream, tags, paths)` in Python's dict order. -type IndexesRow = ( - Vec<(String, String)>, - NameRows, - NameRows, - NameRows, - Vec<(String, String)>, -); +positional_record! { + extract; + /// `(name, tags, POSIX relative path, upstream node names)` of one discovered node. + struct InventoryRow { + name: String, + tags: Vec, + relative_path: String, + upstream: Vec, + } +} + +positional_record! { + into; + /// `(edges, upstream, downstream, tags, paths)` in Python's dict order. + struct IndexesRow { + edges: Vec<(String, String)>, + upstream: NameRows, + downstream: NameRows, + tags: NameRows, + paths: Vec<(String, String)>, + } +} /// The `PlannerInputError` Python raises for a selector failure. fn planner_error(py: Python<'_>, error: SelectorError) -> PyErr { @@ -189,12 +201,19 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { compiler_guard(|| { let nodes: Vec = nodes .into_iter() - .map(|(name, tags, relative_path, upstream)| InventoryNode { - name, - tags, - relative_path, - upstream, - }) + .map( + |InventoryRow { + name, + tags, + relative_path, + upstream, + }| InventoryNode { + name, + tags, + relative_path, + upstream, + }, + ) .collect(); let InventoryIndexes { edges, @@ -203,7 +222,13 @@ fn python_node_graph_indexes(nodes: Vec) -> PyResult { tags, paths, } = graph_indexes(&nodes); - Ok((edges, upstream, downstream, tags, paths)) + Ok(IndexesRow { + edges, + upstream, + downstream, + tags, + paths, + }) }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs index c950f2c2f7..d2b894de74 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/hand_built/compiled_model.rs @@ -1,15 +1,19 @@ //! A `CompiledModel` for a model input built outside the native compile-input stage. -use crate::bindings::_helpers::analysis::binding_locations::locate_model_binding_diagnostics; +use crate::bindings::_helpers::analysis::binding_locations::{ + ExpansionRow, LocatedRow, locate_model_binding_diagnostics, +}; use crate::bindings::_helpers::analysis_session::outcome_objects::dynamic_contract_proof; use crate::bindings::_helpers::analysis_session::outcome_objects::session_outcome_analysis; use crate::bindings::_helpers::records::construct::{enum_from_value, enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::ids::RecordId; +use crate::bindings::models::Positional; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyDict, PyList, PyString, PyTuple}; -use sqlbuild_analysis::semantic_validation::types::{DiagnosticRow, Expansion}; +use sqlbuild_analysis::semantic_validation::models::DiagnosticRow; type Object<'py> = Bound<'py, PyAny>; @@ -264,53 +268,43 @@ fn located<'py>( )? .into_any() }; - let located: Object<'py> = locate_model_binding_diagnostics( - PyList::new(py, rows)?.extract::>()?, + let located: Vec = locate_model_binding_diagnostics( + PyList::new(py, rows)?.extract::>>()?, texts.extract::<(String, String, String)>()?, - expansion_row.extract::>)>>()?, - ) - .and_then(|value| pyo3::IntoPyObjectExt::into_bound_py_any(value, py))?; + expansion_row.extract::()?, + )?; let diagnostic_class: Object<'py> = record_class(py, RecordId::CompilerDiagnostic)?; let location_class: Object<'py> = record_class(py, RecordId::SourceLocation)?; let phase: Object<'py> = enum_member(py, RecordId::DiagnosticPhase, "COMPILE")?; let model: Object<'py> = enum_member(py, RecordId::CompiledResourceType, "MODEL")?; let name: Object<'py> = native_field(&model_file, "file_path")?.getattr("stem")?; let mut built: Vec> = Vec::new(); - for row in located.try_iter()? { - let (code, message, severity, line, column, span): ( - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - Object<'py>, - ) = row?.extract()?; - let location: Object<'py> = if span.is_none() { - span - } else { - call( + for row in located { + let location: Object<'py> = match row.span { + None => py.None().into_bound(py), + Some(Positional(span)) => call( &location_class, &[ ("path", path.clone()), - ("line", span.get_item(0)?), - ("column", span.get_item(1)?), - ("end_line", span.get_item(2)?), - ("end_column", span.get_item(3)?), + ("line", span.line.into_bound_py_any(py)?), + ("column", span.column.into_bound_py_any(py)?), + ("end_line", span.end_line.into_bound_py_any(py)?), + ("end_column", span.end_column.into_bound_py_any(py)?), ], - )? + )?, }; built.push(call( &diagnostic_class, &[ ("phase", phase.clone()), - ("severity", severity), - ("code", code), - ("message", message), + ("severity", row.severity.into_bound_py_any(py)?), + ("code", row.code.into_bound_py_any(py)?), + ("message", row.message.into_bound_py_any(py)?), ("resource_type", model.clone()), ("resource_name", name.clone()), ("path", path.clone()), - ("line", line), - ("column", column), + ("line", row.line.into_bound_py_any(py)?), + ("column", row.column.into_bound_py_any(py)?), ("location", location), ], )?); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs index 91df11f629..e588c0c463 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs @@ -58,6 +58,17 @@ enum EdgeParts { }, } +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: i64, + confidence_code: i64, + sources: Bound<'py, PyAny>, + } +} + /// Indexed edge records, materialised edges shared between equal records. struct EdgeRecords { keys: Vec, @@ -442,12 +453,12 @@ impl RecordBuilder { || native_field(columns, "resource_name_indexes")?.is_truthy()?; let mut members = CodeMembers::default(); for row in native_field(columns, "rows")?.try_iter()? { - let (name_index, transform_code, confidence_code, sources): ( - Bound<'_, PyAny>, - i64, - i64, - Bound<'_, PyAny>, - ) = row?.extract()?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row?.extract()?; let output_column: String = pool.get_item(name_index)?.extract()?; for source in sources.try_iter()? { let (kind, name, column): (Bound<'_, PyAny>, Bound<'_, PyAny>, Bound<'_, PyAny>) = diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs index a83d2f1dd9..e17daf4485 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_cache.rs @@ -70,8 +70,27 @@ const SCHEMA_SQL: &str = " ); "; -type ResourceRow = (String, String, Option, Option); -type EdgeRow = (String, String, String, String); +positional_record! { + extract; + /// One cached resource: its type, name, relative path and qualified name. + struct ResourceRow { + resource_type: String, + name: String, + relative_path: Option, + qualified_name: Option, + } +} + +positional_record! { + extract; + /// One cached lineage edge: its upstream and downstream types and names. + struct EdgeRow { + upstream_type: String, + upstream_name: String, + downstream_type: String, + downstream_name: String, + } +} type TripleRow = (String, String, String); fn cache_path(project_dir: &Path) -> PathBuf { @@ -178,7 +197,13 @@ fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { return Ok(None); } let mut nodes: HashMap = HashMap::new(); - for (resource_type, name, relative_path, qualified_name) in resources { + for ResourceRow { + resource_type, + name, + relative_path, + qualified_name, + } in resources + { let key: LineageKey = LineageKey { resource_type, name, @@ -201,7 +226,13 @@ fn read_index(connection: &Bound<'_, PyAny>) -> PyResult> { }; let mut upstream: Groups = Groups::new(); let mut downstream: Groups = Groups::new(); - for (upstream_type, upstream_name, downstream_type, downstream_name) in edges { + for EdgeRow { + upstream_type, + upstream_name, + downstream_type, + downstream_name, + } in edges + { let (Some(parent), Some(child)) = ( known(upstream_type, upstream_name), known(downstream_type, downstream_name), @@ -379,12 +410,15 @@ fn write_index( "INSERT INTO lookups VALUES (?, ?, ?)", PyList::new(py, lookups)?, )?; - let mut edges: Vec<(&str, &str, &str, &str)> = Vec::new(); + let mut edges: Vec> = Vec::new(); for (child, parents) in &index.upstream { for parent in parents { let ((parent_type, parent_name), (child_type, child_name)) = (key_parts(parent), key_parts(child)); - edges.push((parent_type, parent_name, child_type, child_name)); + edges.push(PyTuple::new( + py, + [parent_type, parent_name, child_type, child_name], + )?); } } insert( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs index 49c73bacfc..a5fca8538c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/lineage/project_lineage.rs @@ -237,7 +237,7 @@ fn fast_lineage<'py>( if model.expands_star { let star_columns: Vec = outcomes .next() - .map(|outcome| outcome.into_parts().1) + .map(|outcome| outcome.into_parts().columns) .unwrap_or_default(); let mut expanded: Vec> = columns.try_iter()?.collect::>()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs index 1447ef485b..0255c9390e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs @@ -20,13 +20,16 @@ use crate::bindings::_helpers::records::fields::mapping_get_or; const DLT_ERROR: ThirdPartyHost = ThirdPartyHost::DltIntegrationError; -/// `(factory, arguments, options, dataset_name)` of one dlt destination. -type DestinationCall<'py> = ( - &'static str, - Bound<'py, PyDict>, - Bound<'py, PyDict>, - Bound<'py, PyAny>, -); +positional_record! { + into; + /// `(factory, arguments, options, dataset_name)` of one dlt destination. + struct DestinationCall<'py> { + factory: &'static str, + arguments: Bound<'py, PyDict>, + options: Bound<'py, PyDict>, + dataset: Bound<'py, PyAny>, + } +} /// A `DltIntegrationError`, or the failure to construct one. fn dlt_error(py: Python<'_>, message: String) -> PyErr { @@ -110,12 +113,12 @@ fn dlt_destination_arguments<'py>( DltDataset::Given => dataset_name.clone(), DltDataset::Default(name) => PyString::new(py, name).into_any(), }; - Ok(( - plan.factory, - arguments(&plan.arguments, connection_config)?, + Ok(DestinationCall { + factory: plan.factory, + arguments: arguments(&plan.arguments, connection_config)?, options, dataset, - )) + }) } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs index 0db8c1a73f..06dd803e7b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/mod.rs @@ -1,10 +1,12 @@ +#[macro_use] +pub(crate) mod boundary; + pub(crate) mod analysis; pub(crate) mod analysis_session; pub(crate) mod attachments; pub(crate) mod audit_command; pub(crate) mod auditing; pub(crate) mod authored_values; -pub(crate) mod boundary; pub(crate) mod build_command; pub(crate) mod build_progress; pub(crate) mod cache; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs index 68a0ba0b93..2e4d8740f9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/model_config/model_validator.rs @@ -24,17 +24,28 @@ const RETURNS_KEY: &str = "returns"; /// The only key of a table function's `returns` mapping. const TABLE_RETURNS_KEY: &str = "table"; -/// Model, seed, source, function and table function names. -type ResourceNames = ( - HashSet, - HashSet, - HashSet, - HashSet, - HashSet, -); - -/// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. -type ModelFacts = (Vec<(String, String, bool)>, Option>, bool, bool); +positional_record! { + extract; + /// Model, seed, source, function and table function names. + struct ResourceNames { + models: HashSet, + seeds: HashSet, + sources: HashSet, + functions: HashSet, + table_functions: HashSet, + } +} + +positional_record! { + extract; + /// References `(kind, name, rejected)`, schema columns, unmanaged retention and table type. + struct ModelFacts { + references: Vec<(String, String, bool)>, + declared_columns: Option>, + retention_unmanaged: bool, + table_type_declared: bool, + } +} /// The project facts every model's validation reads, captured once per compile. #[pyclass(module = "sqlbuild._native", frozen)] @@ -83,7 +94,13 @@ fn discovered_names(inputs: &Bound<'_, PyAny>) -> PyResult { functions.insert(name); } functions.extend(each(inputs, "python_function_files", model_stem)?); - Ok((models, seeds, sources, functions, table_functions)) + Ok(ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + }) } /// Whether a SQL function's header `returns` is a mapping whose only key is `table`. @@ -126,7 +143,13 @@ impl NativeModelValidator { custom_materializations: HashSet, microbatch_concurrency: bool, ) -> PyResult { - let (models, seeds, sources, functions, table_functions) = names; + let ResourceNames { + models, + seeds, + sources, + functions, + table_functions, + } = names; Ok(Self { project: ProjectValidationFacts { custom_materializations, @@ -150,7 +173,12 @@ impl NativeModelValidator { ) -> PyResult> { compiler_guard(|| { let (model_name, query_sql, relative_path) = model; - let (references, declared_columns, retention_unmanaged, table_type_declared) = facts; + let ModelFacts { + references, + declared_columns, + retention_unmanaged, + table_type_declared, + } = facts; let references: Vec = references .into_iter() .map(|(kind, name, externally_rejected)| ModelReference { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs index 64a3add7ec..f3d60591e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs @@ -443,13 +443,13 @@ fn python_entry_json<'py>(entry: &Bound<'py, PyAny>) -> PyResult = ( - String, - (String, String, String), - Bound<'py, PyAny>, - Bound<'py, PyAny>, -); +/// One provider usage: provider, `(kind, name, parameter)` consumer, annotation class and module. +struct Usage<'py> { + provider: String, + consumer: (String, String, String), + class_name: Bound<'py, PyAny>, + module: Bound<'py, PyAny>, +} fn providers_json<'py>( plan: &Bound<'py, PyAny>, @@ -458,37 +458,45 @@ fn providers_json<'py>( let py: Python<'py> = plan.py(); let mut usages: Vec> = Vec::new(); for usage in items(&native_field(plan, "provider_usages")?)? { - usages.push(( - text(&native_field(&usage, "provider_name")?)?, - ( + usages.push(Usage { + provider: text(&native_field(&usage, "provider_name")?)?, + consumer: ( text(&native_field(&usage, "consumer_kind")?)?, text(&native_field(&usage, "consumer_name")?)?, text(&native_field(&usage, "parameter_name")?)?, ), - native_field(&usage, "annotation_class_name")?, - native_field(&usage, "annotation_module")?, - )); + class_name: native_field(&usage, "annotation_class_name")?, + module: native_field(&usage, "annotation_module")?, + }); } for entry in python_entries { let kind: String = text(&enum_value(entry, "kind")?)?; let name: String = text(&native_field(entry, "name")?)?; for usage in items(&native_field(entry, "provider_usages")?)? { - usages.push(( - text(&native_field(&usage, "provider_name")?)?, - ( + usages.push(Usage { + provider: text(&native_field(&usage, "provider_name")?)?, + consumer: ( kind.clone(), name.clone(), text(&native_field(&usage, "parameter_name")?)?, ), - native_field(&usage, "annotation_class_name")?, - native_field(&usage, "annotation_module")?, - )); + class_name: native_field(&usage, "annotation_class_name")?, + module: native_field(&usage, "annotation_module")?, + }); } } - usages.sort_by(|left, right| (&left.0, &left.1).cmp(&(&right.0, &right.1))); + usages.sort_by(|left, right| { + (&left.provider, &left.consumer).cmp(&(&right.provider, &right.consumer)) + }); let providers: Bound<'py, PyList> = PyList::empty(py); let mut current: Option<(String, Bound<'py, PyList>)> = None; - for (provider, (kind, name, parameter), class_name, module) in usages { + for Usage { + provider, + consumer: (kind, name, parameter), + class_name, + module, + } in usages + { if current.as_ref().is_none_or(|(open, _)| *open != provider) { let used_by: Bound<'py, PyList> = PyList::empty(py); providers.append(dict( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs index 61289d856c..d2e2734323 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_entries/execution_plan.rs @@ -23,6 +23,9 @@ use crate::bindings::_helpers::planner_phases::identities::{ use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerRuntime; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; +use crate::bindings::_helpers::planner_phases::models::{ + PlanOutputContext, PlanWarningContext, PlannedModelEntries, StoragePolicyContext, +}; use crate::bindings::_helpers::planner_phases::output_assembly::{ add_plan_metadata, add_plan_warnings, add_storage_policies, base_plan_output, }; @@ -158,19 +161,37 @@ fn planned_output<'py>( )?; let planned_sql_tests = test_planning.result(py)?.into_bound(py); let output = base_plan_output( - (runtime, &warehouse, &identities, overrides), - ( - &inspection_scope, - &reconciliation, + PlanOutputContext { + runtime, + warehouse: &warehouse, + identities: &identities, + overrides, + }, + PlannedModelEntries { + execution_scope: &inspection_scope, + reconciliation: &reconciliation, entries, - &source_freshness, - ), + source_freshness: &source_freshness, + }, &planned_sql_tests, )?; - let output = add_storage_policies(output, (runtime, &warehouse, scopes, &policies))?; + let output = add_storage_policies( + output, + StoragePolicyContext { + runtime, + warehouse: &warehouse, + scopes, + policies: &policies, + }, + )?; let output = add_plan_warnings( output, - (scopes, &warehouse, &identities, &stale_warning_changes), + PlanWarningContext { + scopes, + warehouse: &warehouse, + identities: &identities, + stale_warning_changes: &stale_warning_changes, + }, (&source_freshness, &policies), )?; let plan_output = add_plan_metadata(output, &source_freshness, &policies)?.build(py)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs index d97d516142..35ebf4b34e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_identity/graph.rs @@ -142,11 +142,11 @@ pub(crate) fn build_execution_upstream_deps<'py>( let (gates, hooks): (Vec, Vec) = native_execution_gate_edges(project)?; let cache: KeyCache<'py> = KeyCache::new(py)?; let mut extra: Vec<(Bound<'py, PyAny>, Bound<'py, PyAny>)> = Vec::new(); - for (_, _, gated, read) in &gates { - extra.push((cache.key(gated)?, cache.key(read)?)); + for gate in &gates { + extra.push((cache.key(&gate.gated)?, cache.key(&gate.read)?)); } - for (_, gated, read, _, _) in &hooks { - extra.push((cache.key(gated)?, cache.key(read)?)); + for hook in &hooks { + extra.push((cache.key(&hook.gated)?, cache.key(&hook.read)?)); } for (gated, read) in extra { let gated_upstream: Bound<'py, PyAny> = @@ -192,7 +192,13 @@ pub(crate) fn build_execution_edge_origins<'py>( .call_method1("upstream", (gated.clone(),))? .contains(read.clone()) }; - for (audit_name, target, gated, read) in &gates { + for GateRow { + audit_name, + target, + gated, + read, + } in &gates + { if is_lineage(gated, read)? { continue; } @@ -204,7 +210,14 @@ pub(crate) fn build_execution_edge_origins<'py>( ), )?; } - for (hook_name, gated, read, kind, _) in &hooks { + for HookRow { + hook_name, + gated, + read, + hook_type: kind, + .. + } in &hooks + { if is_lineage(gated, read)? { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs index dcb5eee1d3..ced5ea9dd5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs @@ -26,6 +26,42 @@ pub(crate) struct ModelEntryResults<'py> { pub(crate) warnings: Vec>, } +/// The runtime, warehouse state, identities and overrides the base plan output reads. +#[derive(Debug)] +pub(crate) struct PlanOutputContext<'a, 'py> { + pub(crate) runtime: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) identities: &'a Bound<'py, PyAny>, + pub(crate) overrides: &'a Bound<'py, PyAny>, +} + +/// The execution scope, reconciled changes, planned entries and source freshness of a plan. +#[derive(Debug)] +pub(crate) struct PlannedModelEntries<'a, 'py> { + pub(crate) execution_scope: &'a Bound<'py, PyAny>, + pub(crate) reconciliation: &'a ChangeReconciliation<'py>, + pub(crate) entries: ModelEntryResults<'py>, + pub(crate) source_freshness: &'a Bound<'py, PyAny>, +} + +/// The runtime, warehouse state, scopes and policies storage policy planning reads. +#[derive(Debug)] +pub(crate) struct StoragePolicyContext<'a, 'py> { + pub(crate) runtime: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) scopes: &'a Bound<'py, PyAny>, + pub(crate) policies: &'a Bound<'py, PyAny>, +} + +/// The scopes, warehouse state, identities and changes stale-selection warnings read. +#[derive(Debug)] +pub(crate) struct PlanWarningContext<'a, 'py> { + pub(crate) scopes: &'a Bound<'py, PyAny>, + pub(crate) warehouse: &'a Bound<'py, PyAny>, + pub(crate) identities: &'a Bound<'py, PyAny>, + pub(crate) stale_warning_changes: &'a Bound<'py, PyAny>, +} + /// Resolved execution environment shared by planner phases. #[pyclass(module = "sqlbuild._native", frozen, get_all)] #[derive(Debug)] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs index 0dfe501080..71e3e09c26 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_phases/output_assembly.rs @@ -10,7 +10,8 @@ use crate::bindings::_helpers::planner_phases::contexts::class_field; use crate::bindings::_helpers::planner_phases::models::PlannerIdentityContext; use crate::bindings::_helpers::planner_phases::models::PlannerScopeResolution; use crate::bindings::_helpers::planner_phases::models::{ - ChangeReconciliation, ModelEntryResults, PlanOutputFields, + PlanOutputContext, PlanOutputFields, PlanWarningContext, PlannedModelEntries, + StoragePolicyContext, }; use crate::bindings::_helpers::planner_phases::objects::object; use crate::bindings::_helpers::planner_phases::objects::{ @@ -69,18 +70,18 @@ impl<'py> PlanOutputFields<'py> { /// The plan output's base fields from the planned entries, relations and source freshness. pub(crate) fn base_plan_output<'py>( - (runtime, warehouse, identities, overrides): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), - (execution_scope, reconciliation, entries, source_freshness): ( - &Bound<'py, PyAny>, - &ChangeReconciliation<'py>, - ModelEntryResults<'py>, - &Bound<'py, PyAny>, - ), + PlanOutputContext { + runtime, + warehouse, + identities, + overrides, + }: PlanOutputContext<'_, 'py>, + PlannedModelEntries { + execution_scope, + reconciliation, + entries, + source_freshness, + }: PlannedModelEntries<'_, 'py>, planned_sql_tests: &Bound<'py, PyAny>, ) -> PyResult> { let py = runtime.py(); @@ -163,12 +164,12 @@ pub(crate) fn with_column_rename_hints<'py>( /// Add migration fingerprints, rename hints, migration, table-type and retention entries. pub(crate) fn add_storage_policies<'py>( mut output: PlanOutputFields<'py>, - (runtime, warehouse, scopes, policies): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + StoragePolicyContext { + runtime, + warehouse, + scopes, + policies, + }: StoragePolicyContext<'_, 'py>, ) -> PyResult> { let py = runtime.py(); let mut entries = output.field("model_entries")?; @@ -242,12 +243,12 @@ pub(crate) fn add_storage_policies<'py>( /// Add unknown source freshness warnings and, with selection diagnostics, stale-selection warnings. pub(crate) fn add_plan_warnings<'py>( mut output: PlanOutputFields<'py>, - (scopes, warehouse, identities, stale_warning_changes): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - ), + PlanWarningContext { + scopes, + warehouse, + identities, + stale_warning_changes, + }: PlanWarningContext<'_, 'py>, (source_freshness, policies): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult> { let py = scopes.py(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs index 71d74dd497..bd33f206d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_reads.rs @@ -6,16 +6,17 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::planner_fixtures::main::required_columns::required_columns; use sqlbuild_analysis::planner_fixtures::main::star_keys::star_keys; use sqlbuild_analysis::planner_fixtures::models::{ - FixtureReadKey, FixtureReadModel, FixtureReadRequirements, FixtureReadSchemas, - FixtureStarRelation, + FixtureReadKey, FixtureReadModel, FixtureReadReference, FixtureReadRequirements, + FixtureReadSchemas, FixtureStarRelation, }; -use sqlbuild_analysis::planner_fixtures::types::FixtureReadReference; use sqlbuild_analysis::required_columns::models::ColumnRead; use sqlbuild_core::panics::main::catch_compiler_panic::catch_compiler_panic; use std::collections::{HashMap, HashSet}; use crate::bindings::_helpers::analysis::model_reads; -use crate::bindings::_helpers::analysis::model_reads::{lineage_facts, proven_inputs, star_flags}; +use crate::bindings::_helpers::analysis::model_reads::{ + LineageFactRow, lineage_facts, proven_inputs, star_flags, +}; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::planner_strategy::constants::HIGH_CONFIDENCE; use crate::bindings::_helpers::planner_strategy::fixture_context::work_counts; @@ -63,12 +64,12 @@ pub(crate) fn required_fixture_columns( let mut references: Vec = Vec::new(); for reference in native_field(&model, "references")?.try_iter()? { let reference = reference?; - references.push(( - native_field(&reference, "ref_kind")?.extract()?, - native_field(&reference, "ref_name")?.extract()?, - native_field(&reference, "ref_package")?.extract()?, - native_field(&reference, "call_argument_count")?.extract()?, - )); + references.push(FixtureReadReference { + kind: native_field(&reference, "ref_kind")?.extract()?, + name: native_field(&reference, "ref_name")?.extract()?, + package: native_field(&reference, "ref_package")?.extract()?, + argument_count: native_field(&reference, "call_argument_count")?.extract()?, + }); } models.push(FixtureReadModel { name: name.clone(), @@ -102,7 +103,12 @@ pub(crate) fn required_fixture_columns( pub(crate) fn lineage_reads(model: &Bound<'_, PyAny>) -> PyResult> { let mut reads: Vec = Vec::new(); - for (_, _, confidence, sources) in lineage_facts(model)?.unwrap_or_default() { + for LineageFactRow { + confidence, + sources, + .. + } in lineage_facts(model)?.unwrap_or_default() + { if confidence != HIGH_CONFIDENCE { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs index 3a6a1dd758..1fe436cfdf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_strategy/fixture_types.rs @@ -9,7 +9,7 @@ use sqlbuild_analysis::planner_fixtures::models::{ use sqlbuild_core::text::main::python_casefold::python_casefold; -use crate::bindings::_helpers::analysis::model_reads::lineage_facts; +use crate::bindings::_helpers::analysis::model_reads::{LineageFactRow, lineage_facts}; use crate::bindings::_helpers::planner_strategy::constants::{ DIRECT_TRANSFORM, MODEL_REFERENCE_KIND, }; @@ -88,7 +88,13 @@ pub(crate) fn expected_fixture_types<'a>( pub(crate) fn projections(model: &Bound<'_, PyAny>) -> PyResult> { let mut projections: Vec = Vec::new(); - for (output_column, transform, _, sources) in lineage_facts(model)?.unwrap_or_default() { + for LineageFactRow { + output_column, + transform, + sources, + .. + } in lineage_facts(model)?.unwrap_or_default() + { if transform != DIRECT_TRANSFORM { continue; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs index 4059788791..16afa8f495 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs @@ -31,7 +31,14 @@ const INSERT_SQL: &str = "INSERT OR REPLACE INTO migration_fingerprint (cache_key, payload) VALUES (?, ?)"; const SELECT_SQL: &str = "SELECT payload FROM migration_fingerprint WHERE cache_key = ?"; -type MemoKey = (String, String, Option, Vec<(String, String)>); +/// A fingerprint request: query SQL, metadata JSON, dialect and visible renames. +#[derive(Hash, PartialEq, Eq)] +struct MemoKey { + query_sql: String, + metadata_json: String, + dialect: Option, + visible_renames: Vec<(String, String)>, +} static FINGERPRINT_WORK: PyOnceLock> = PyOnceLock::new(); @@ -160,10 +167,10 @@ impl MigrationFingerprintCache { } fingerprint_cache_key(&FingerprintCacheKey { shared_digest: self.shared_digest.clone(), - query_sql: key.0.clone(), - metadata_json: key.1.clone(), - dialect: key.2.clone(), - visible_renames: key.3.clone(), + query_sql: key.query_sql.clone(), + metadata_json: key.metadata_json.clone(), + dialect: key.dialect.clone(), + visible_renames: key.visible_renames.clone(), }) .map_err(|error| PyValueError::new_err(format!("{error:?}"))) } @@ -339,7 +346,12 @@ impl MigrationFingerprintCache { visible.push((name, identity)); } } - let key: MemoKey = (query_sql, metadata_json, dialect, visible); + let key: MemoKey = MemoKey { + query_sql, + metadata_json, + dialect, + visible_renames: visible, + }; if let Some(found) = self.fingerprints.get(&key) { return Ok(found.clone()); } @@ -348,9 +360,14 @@ impl MigrationFingerprintCache { Some(stored) => stored, None => { count_computation(py, &key)?; - let identities: BTreeMap = key.3.iter().cloned().collect(); - let computed: Option = - fingerprint(&key.0, &key.1, &identities, key.2.as_deref())?; + let identities: BTreeMap = + key.visible_renames.iter().cloned().collect(); + let computed: Option = fingerprint( + &key.query_sql, + &key.metadata_json, + &identities, + key.dialect.as_deref(), + )?; self.remember(&cache_key, computed.as_deref())?; computed } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs index effacd0eb4..f308830cb4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs @@ -27,13 +27,13 @@ const UNKNOWN_MEMBER: &str = "UNKNOWN"; /// `str(None)`, the text of an exact declaration without a type. const NONE_TEXT: &str = "None"; -/// The three per-relation declaration maps the analysis starts from. -pub(crate) type DeclaredRelations<'py> = ( - Bound<'py, PyDict>, - Bound<'py, PyDict>, - Bound<'py, PyDict>, - DeclaredShapes, -); +/// The three per-relation declaration maps the analysis starts from, and the declared shapes. +pub(crate) struct DeclaredRelations<'py> { + pub(crate) column_types: Bound<'py, PyDict>, + pub(crate) nullability: Bound<'py, PyDict>, + pub(crate) complete_schemas: Bound<'py, PyDict>, + pub(crate) declared: DeclaredShapes, +} /// Relation shapes as rows in a dict's insertion order, each value as `str(value)`. #[derive(Debug, Clone, Default)] @@ -327,5 +327,10 @@ pub(crate) fn declared_relations<'py>( declared.complete(&maps, &name, (&columns, true))?; } let (types, nulls, complete) = maps; - Ok((types, nulls, complete, declared.shapes)) + Ok(DeclaredRelations { + column_types: types, + nullability: nulls, + complete_schemas: complete, + declared: declared.shapes, + }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs index 267a7b8c1d..0fceed1307 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/project_assembly/native_resources.rs @@ -20,6 +20,7 @@ use crate::bindings::_helpers::authored_values::project_vars::project_var_text; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::_helpers::compile_driver::python_calls::{imported, keywords}; use crate::bindings::_helpers::compiled_project::model_input_view::input_reference_rows; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::model_config::template_data::record_reads; use crate::bindings::_helpers::project_assembly::models::{ NativeProjectResources, NativeResourceFailure, @@ -43,13 +44,42 @@ const CONTEXT_READ: &str = "ctx"; const NATIVE_FAILURE_PREFIX: &str = "NativeCompilerError: "; /// `(kind, name, package)`. -type ReferenceRow = (String, String, Option); -/// `(name, managed, database, schema)`. -type SourceRow = (String, bool, Option, Option); -/// `(kind, name, tags, references)`. -type FunctionRow = (String, String, Vec, Vec); +type ResourceReferenceRow = (String, String, Option); /// `(references, attached (kind, name))`. -type AuditRow = (Vec, Option<(String, String)>); +type AuditRow = (Vec, Option<(String, String)>); + +positional_record! { + extract; + /// `(name, managed, database, schema)`. + struct SourceRow { + name: String, + managed: bool, + database: Option, + schema: Option, + } +} + +positional_record! { + extract; + /// `(kind, name, tags, references)`. + struct FunctionRow { + kind: String, + name: String, + tags: Vec, + references: Vec, + } +} + +positional_record! { + into; + /// `(database, schema, logical database, logical schema)` of one seed. + struct NamespaceRow { + database: Option, + schema: Option, + logical_database: Option, + logical_schema: Option, + } +} /// The request's named variables and the Python exceptions opaque ones raise, by index. type RequestVariables<'py> = (Vec<(String, Variable)>, Vec>); @@ -143,7 +173,7 @@ fn environment_rows( .collect() } -fn references(rows: Vec) -> Vec { +fn references(rows: Vec) -> Vec { rows.into_iter() .map(|(kind, name, package)| Reference { kind, @@ -166,9 +196,16 @@ fn tags(value: &Bound<'_, PyAny>) -> PyResult> { fn model_facts(model_input: &Bound<'_, PyAny>, checks: &Bound<'_, PyAny>) -> PyResult { let model_file: Bound<'_, PyAny> = native_field(model_input, "model_file")?; - let reference_rows: Vec = input_reference_rows(model_input)? + let reference_rows: Vec = input_reference_rows(model_input)? .into_iter() - .map(|(kind, name, package, _)| (kind, name, package)) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| (kind, name, package), + ) .collect(); let mut syntax_checks: Vec = Vec::new(); for check in checks.try_iter()? { @@ -280,7 +317,12 @@ fn project_request<'py>( let sources: Vec = rows("source_inputs")? .into_iter() .map(|row| { - let (name, managed, database, schema): SourceRow = row.extract()?; + let SourceRow { + name, + managed, + database, + schema, + }: SourceRow = row.extract()?; Ok(SourceFacts { name, managed, @@ -292,7 +334,12 @@ fn project_request<'py>( let functions: Vec = rows("sql_function_inputs")? .into_iter() .map(|row| { - let (kind, name, tags, reference_rows): FunctionRow = row.extract()?; + let FunctionRow { + kind, + name, + tags, + references: reference_rows, + }: FunctionRow = row.extract()?; Ok(FunctionFacts { kind, name, @@ -383,20 +430,13 @@ fn failure_object<'py>( .into_any()) } -fn namespace_row( - namespace: Namespace, -) -> ( - Option, - Option, - Option, - Option, -) { - ( - namespace.database, - namespace.schema, - namespace.logical_database, - namespace.logical_schema, - ) +fn namespace_row(namespace: Namespace) -> NamespaceRow { + NamespaceRow { + database: namespace.database, + schema: namespace.schema, + logical_database: namespace.logical_database, + logical_schema: namespace.logical_schema, + } } /// `NativeProjectResources` for the native result, replaying its template reads first. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs index bbba79328d..32f93faf4f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs @@ -70,16 +70,27 @@ pub(crate) fn facts_resource_name( Ok(resource_name(instance, &index)?.unbind()) } +positional_record! { + extract; + /// One compact lineage row: output name index, transform and confidence codes, sources. + struct CompactColumnRow<'py> { + name_index: Bound<'py, PyAny>, + transform_code: usize, + confidence_code: usize, + sources: Bound<'py, PyAny>, + } +} + /// One compact row projected to a `CompiledLineageColumnFact`. fn project_row(facts: &Bound<'_, PyAny>, row: &Bound<'_, PyAny>) -> PyResult> { let py: Python<'_> = facts.py(); let pool: Bound<'_, PyAny> = native_field(facts, "string_pool")?; - let (name_index, transform_code, confidence_code, sources): ( - Bound<'_, PyAny>, - usize, - usize, - Bound<'_, PyAny>, - ) = row.extract()?; + let CompactColumnRow { + name_index, + transform_code, + confidence_code, + sources, + }: CompactColumnRow<'_> = row.extract()?; let mut upstream: Vec> = Vec::new(); for source in sources.try_iter()? { let source: Bound<'_, PyTuple> = source?.cast_into::()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs index 7e6704149a..a2fe347ae6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/macros.rs @@ -131,12 +131,12 @@ fn resource_rows( for resource in resources.try_iter()? { let resource: Bound<'_, PyAny> = resource?; let (tags, folder) = describe(&resource)?; - rows.push(( - graph_key(&native_field(&resource, "key")?)?, - graph_keys(&native_field(&resource, "deps")?)?, + rows.push(ResourceRow { + key: graph_key(&native_field(&resource, "key")?)?, + deps: graph_keys(&native_field(&resource, "deps")?)?, tags, folder, - )); + }); } Ok(rows) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs index df38754cff..666f5985c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/refactor/refactor_facts.rs @@ -196,27 +196,61 @@ fn block_bodies(blocks: &Bound<'_, PyAny>) -> PyResult> { .collect() } +/// One discovered file group: its role, attribute, body field and whether it has blocks. +struct FileGroup { + role: SqlFileRole, + attribute: &'static str, + body: Option<&'static str>, + blocks: bool, +} + fn authored_files(discovered: &Bound<'_, PyAny>) -> PyResult> { - let groups: [(SqlFileRole, &str, Option<&str>, bool); 6] = [ - (SqlFileRole::Test, "test_files", None, true), - ( - SqlFileRole::Scenario, - "scenario_files", - Some("sql_body"), - false, - ), - (SqlFileRole::Audit, "audit_files", None, true), - (SqlFileRole::Hook, "sql_hook_files", Some("sql_body"), false), - ( - SqlFileRole::Function, - "sql_function_files", - Some("body_sql"), - false, - ), - (SqlFileRole::Schema, "model_schema_files", None, false), + let groups: [FileGroup; 6] = [ + FileGroup { + role: SqlFileRole::Test, + attribute: "test_files", + body: None, + blocks: true, + }, + FileGroup { + role: SqlFileRole::Scenario, + attribute: "scenario_files", + body: Some("sql_body"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Audit, + attribute: "audit_files", + body: None, + blocks: true, + }, + FileGroup { + role: SqlFileRole::Hook, + attribute: "sql_hook_files", + body: Some("sql_body"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Function, + attribute: "sql_function_files", + body: Some("body_sql"), + blocks: false, + }, + FileGroup { + role: SqlFileRole::Schema, + attribute: "model_schema_files", + body: None, + blocks: false, + }, ]; let mut files: Vec = Vec::new(); - for (role, attribute, body, blocks) in groups { + for FileGroup { + role, + attribute, + body, + blocks, + } in groups + { for item in native_field(discovered, attribute)?.try_iter()? { let item: Bound<'_, PyAny> = item?; let texts: Vec = match (body, blocks) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs index 498e99529b..2f52c18305 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/authored_sql.rs @@ -1,7 +1,7 @@ //! Authored SQL expanded through project vars, declarations and macros, with each pass's spans. use pyo3::prelude::{ - Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, + Bound, IntoPyObject, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; @@ -16,15 +16,18 @@ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, NativeAttr, RecordId}; use crate::bindings::_helpers::records::registry; use crate::bindings::_helpers::render::declaration_expansion::{ - DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, SpanRow, + DeclarationExpansion, ExpandedDeclarations, ExpansionClasses, ExpansionConsumer, expand_declarations, span_rows, }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::render::macro_call_rows::span_row; use crate::bindings::_helpers::render::macro_expansion_result::{ExpansionRequest, macro_output}; use crate::bindings::_helpers::sqltext::declaration_references::declaration_scan; -use crate::bindings::_helpers::sqltext::static_sql::{InterpolationSources, interpolation_batch}; +use crate::bindings::_helpers::sqltext::static_sql::{ + InterpolationRow, InterpolationSources, interpolation_batch, +}; const DECLARATION_FIELDS: [&str; 4] = [ "enums", @@ -53,7 +56,13 @@ fn substituted<'py>( let text: String = sql.extract()?; let path: String = file_path.str()?.extract()?; let rows = interpolation_batch(sources, &[(text.clone(), path)])?; - let Some((rendered, spans, reads, error)) = rows.into_iter().next() else { + let Some(InterpolationRow { + sql: rendered, + spans, + reads, + error, + }) = rows.into_iter().next() + else { return Ok((text, Vec::new())); }; record_reads(py, reads.iter().map(|(kind, name)| (*kind, name.as_str())))?; @@ -69,7 +78,14 @@ fn substituted<'py>( fn span_objects<'py>(py: Python<'py>, spans: &[SpanRow]) -> PyResult> { let objects: Vec> = spans .iter() - .map(|span| new_record(py, RecordId::ExpansionSpan, *span, None)) + .map(|span| { + new_record( + py, + RecordId::ExpansionSpan, + IntoPyObject::into_pyobject(*span, py)?, + None, + ) + }) .collect::>()?; PyTuple::new(py, objects) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs index 57f1fa829d..57a9ea530c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/declaration_expansion.rs @@ -1,5 +1,6 @@ //! Enum and constant references expanded natively, for model queries and every other caller. +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use pyo3::FromPyObject; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyTuple}; @@ -12,6 +13,7 @@ use sqlbuild_analysis::compiled_project::models::{ }; use crate::bindings::_helpers::render::declaration_values::ConstantRendering; +use crate::bindings::_helpers::sqltext::declaration_references::ScannedReference; const ENUM_REFERENCE_KIND: u8 = 0; const ENUM_KIND: &str = "enum"; @@ -39,11 +41,6 @@ const THROUGH: &str = "through"; const CONSUMER: &str = "consumer"; const ENUM_MEMBER: &str = "enum_member"; -/// `(kind, name, member, start, end)` for one natively scanned reference. -pub(crate) type ScannedReference = (u8, String, Option, usize, usize); -/// `(source start, source end, output start, output end)`. -pub(crate) type SpanRow = (usize, usize, usize, usize); - /// The Python classes expansion builds usages and errors from. #[derive(FromPyObject)] #[pyo3(from_item_all)] @@ -82,7 +79,14 @@ pub(crate) fn expand_declarations<'py>( ) -> PyResult> { let mut splices: Vec = Vec::with_capacity(request.references.len()); let mut usages: Vec> = Vec::new(); - for (kind, name, member, start, end) in request.references { + for ScannedReference { + kind, + name, + member, + start, + end, + } in request.references + { let (replacement, visibility) = if *kind == ENUM_REFERENCE_KIND { enum_text(request, name, member.as_deref())? } else { @@ -281,13 +285,11 @@ pub(crate) fn span_rows(spliced: &SplicedSql) -> Vec { spliced .spans .iter() - .map(|span| { - ( - span.source_start, - span.source_end, - span.output_start, - span.output_end, - ) + .map(|span| SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs index 33941bf858..fb1a6f7986 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_rows.rs @@ -1,28 +1,71 @@ //! Python rows of macro call sites and splice spans. +use sqlbuild_analysis::semantic_validation::models::Expansion; use sqlbuild_render::macro_calls::models::{MacroCallSite, MacroSpliceSpan}; +use sqlbuild_sqltext::compiler::models::CharSpan; -/// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. -pub(crate) type SiteRow = (usize, usize, usize, String, Option>, bool); -/// One substitution span: source start and end, output start and end. -pub(crate) type SpanRow = (usize, usize, usize, usize); +positional_record! { + into; + /// One call site: start, arguments start, end, name, tree names if scanned and whether typed reference text appears. + pub(crate) struct SiteRow { + start: usize, + args_start: usize, + end: usize, + name: String, + tree_names: Option>, + typed_reference_text: bool, + } +} +positional_record! { + extract, into; + /// One substitution span: source start and end, output start and end. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub(crate) struct SpanRow { + pub(crate) source_start: usize, + pub(crate) source_end: usize, + pub(crate) output_start: usize, + pub(crate) output_end: usize, + } +} + +impl From for SpanRow { + fn from(span: CharSpan) -> Self { + Self { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } + } +} + +impl From for Expansion { + fn from(span: SpanRow) -> Self { + Self { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } + } +} pub(crate) fn site_row(site: MacroCallSite) -> SiteRow { - ( - site.start, - site.args_start, - site.end, - site.name, - site.tree_names, - site.typed_reference_text, - ) + SiteRow { + start: site.start, + args_start: site.args_start, + end: site.end, + name: site.name, + tree_names: site.tree_names, + typed_reference_text: site.typed_reference_text, + } } pub(crate) fn span_row(span: MacroSpliceSpan) -> SpanRow { - ( - span.source_start, - span.source_end, - span.output_start, - span.output_end, - ) + SpanRow { + source_start: span.source_start, + source_end: span.source_end, + output_start: span.output_start, + output_end: span.output_end, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs index c44bc2db8f..c605e8c5ab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs @@ -33,14 +33,17 @@ const RELATION_SUFFIX: &str = "__"; const CONSTANT_KIND: &str = "constant"; const RELATIONSHIP_CONSUMER_KINDS: [&str; 2] = ["test", "scenario"]; -/// `(sql, spans, argument references, dependencies, usages)` of one expansion. -type ExpansionRow<'py> = ( - String, - Vec, - Vec, - Vec>, - Vec>, -); +positional_record! { + into; + /// `(sql, spans, argument references, dependencies, usages)` of one expansion. + struct ExpansionRow { + sql: String, + spans: Vec, + argument_references: Vec, + dependencies: Vec>, + usages: Vec>, + } +} /// The Python classes, enums and helpers an expansion builds facts, errors and contexts from. #[derive(FromPyObject)] @@ -87,8 +90,17 @@ pub(crate) struct NativeMacroExpansion { pub(crate) facts: RefCell, } -/// The `(macros, overrides, context, declarations, consumer)` one expansion reads. -type StateInput = (Py, Py, Py, Py, Py); +positional_record! { + extract; + /// The `(macros, overrides, context, declarations, consumer)` one expansion reads. + pub(crate) struct StateInput { + pub(crate) loaded_macros: Py, + pub(crate) overrides: Py, + pub(crate) macro_context: Py, + pub(crate) declarations: Py, + pub(crate) consumer: Py, + } +} #[pymethods] impl NativeMacroExpansion { @@ -99,7 +111,13 @@ impl NativeMacroExpansion { enforce_explicit: bool, classes: Py, ) -> PyResult { - let (loaded_macros, overrides, macro_context, declarations, consumer) = state; + let StateInput { + loaded_macros, + overrides, + macro_context, + declarations, + consumer, + } = state; Ok(Self { file_path: file_path.clone().unbind(), file_label: file_path.str()?.to_string(), @@ -119,15 +137,15 @@ impl NativeMacroExpansion { slf: &Bound<'py, Self>, sql: &Bound<'py, PyString>, bridge: &Bound<'py, PyAny>, - ) -> PyResult> { + ) -> PyResult { let output: ExpansionOutput = expanded(slf, sql.to_str()?, bridge)?; - Ok(( - output.sql, - output.spans.into_iter().map(span_row).collect(), - output.argument_references, - output.dependencies, - output.usages, - )) + Ok(ExpansionRow { + sql: output.sql, + spans: output.spans.into_iter().map(span_row).collect(), + argument_references: output.argument_references, + dependencies: output.dependencies, + usages: output.usages, + }) } /// `render_relation`: an approved call-site reference's placeholder, else its reference call. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs index 5bdfb03605..56347e672a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs @@ -19,9 +19,9 @@ use crate::bindings::_helpers::records::construct::{ use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::declaration_values::resolved_declaration_context; -use crate::bindings::_helpers::render::macro_call_rows::span_row; +use crate::bindings::_helpers::render::macro_call_rows::{SpanRow, span_row}; use crate::bindings::_helpers::render::macro_expansion::{ - ExpansionOutput, NativeMacroExpansion, expanded, + ExpansionOutput, NativeMacroExpansion, StateInput, expanded, }; use crate::bindings::_helpers::sqltext::sql_references::extract_references; @@ -189,16 +189,16 @@ pub(crate) fn macro_output(request: ExpansionRequest<'_>) -> PyResult(request: ExpansionRequest<'py>) -> PyResult< }; let mut spans: Vec> = Vec::with_capacity(output.spans.len()); for span in output.spans { - let (source_start, source_end, output_start, output_end) = span_row(span); + let SpanRow { + source_start, + source_end, + output_start, + output_end, + } = span_row(span); spans.push(new_record_with( py, RecordId::ExpansionSpan, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs index d19447b762..215f7a4a2d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_modules.rs @@ -14,20 +14,44 @@ use sqlbuild_pysource::macro_modules::types::MacroScopeKind; use crate::bindings::_helpers::boundary::panics::{NativeCompilerError, compiler_error}; use crate::bindings::types::CompilerDetach; -/// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. -type SourceRow = ( - String, - String, - Vec, - Option>, - Option>, - String, - String, - Option, -); - -/// `(name, file index, parameters, line, source digest, dependency macro names)`. -type ExportRow = (String, usize, Vec, usize, String, Vec); +positional_record! { + extract; + /// Path as `str`, `as_posix` and parts, root and owner parts, scope, source and CPython error. + pub(crate) struct SourceRow { + display: String, + posix: String, + parts: Vec, + declaration_root: Option>, + owning_path: Option>, + scope_kind: String, + contents: String, + syntax_error: Option, + } +} + +positional_record! { + into; + /// `(name, file index, parameters, line, source digest, dependency macro names)`. + pub(crate) struct ExportRow { + name: String, + file_index: usize, + parameters: Vec, + line: usize, + source_digest: String, + dependencies: Vec, + } +} + +positional_record! { + extract; + /// `(name, path, contents, dependencies)` of one macro module to load. + struct ModuleSpecRow { + name: String, + file_path: String, + contents: String, + dependencies: Vec, + } +} /// `(file index, message)`. type FaultRow = (usize, String); @@ -49,8 +73,16 @@ fn scope_kind_of(value: &str) -> MacroScopeKind { } fn source(row: SourceRow) -> MacroModuleSource { - let (display, posix, parts, declaration_root, owning_path, scope_kind, contents, syntax_error) = - row; + let SourceRow { + display, + posix, + parts, + declaration_root, + owning_path, + scope_kind, + contents, + syntax_error, + } = row; MacroModuleSource { relative_path: MacroModulePath { display, @@ -66,14 +98,14 @@ fn source(row: SourceRow) -> MacroModuleSource { } fn export_row(export: MacroExport) -> ExportRow { - ( - export.name, - export.file_index, - export.parameters, - export.line, - export.source_digest, - export.dependencies, - ) + ExportRow { + name: export.name, + file_index: export.file_index, + parameters: export.parameters, + line: export.line, + source_digest: export.source_digest, + dependencies: export.dependencies, + } } fn fault_row(fault: MacroFault) -> FaultRow { @@ -112,13 +144,16 @@ pub(crate) fn inventory_macro_modules( /// `(synthetic root, packages shallowest first, module execution order)` of `(name, path, contents, dependencies)` rows. #[pyfunction] -fn macro_module_load_plan( - modules: Vec<(String, String, String, Vec)>, -) -> (String, Vec, Vec) { +fn macro_module_load_plan(modules: Vec) -> (String, Vec, Vec) { let specs: Vec = modules .into_iter() .map( - |(name, file_path, contents, dependencies)| MacroModuleSpec { + |ModuleSpecRow { + name, + file_path, + contents, + dependencies, + }| MacroModuleSpec { name, file_path, contents, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs index 52734f0837..d3a30786e6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/custom_rule_evidence.rs @@ -102,13 +102,24 @@ pub(crate) fn custom_rule_implementation_fingerprint( Ok(implementation_fingerprint(&source, &closure, &root)?) } +positional_record! { + into; + /// `(path, line, column, owner)` of one public-harness case. + struct EvidenceRow { + path: String, + line: usize, + column: usize, + owner: String, + } +} + /// `(path, line, column, owner)` of each public-harness case that statically evaluates the rule. #[pyfunction] #[pyo3(signature = (*, rule, project_dir))] fn custom_rule_test_evidence<'py>( rule: &Bound<'py, PyAny>, project_dir: &Bound<'py, PyAny>, -) -> PyResult> { +) -> PyResult> { let py: Python<'py> = rule.py(); let Some(source) = resolved_source(rule)? else { return Ok(Vec::new()); @@ -128,7 +139,12 @@ fn custom_rule_test_evidence<'py>( }; Ok(evidence .into_iter() - .map(|item: RuleTestEvidence| (item.path, item.line, item.column, item.owner)) + .map(|item: RuleTestEvidence| EvidenceRow { + path: item.path, + line: item.line, + column: item.column, + owner: item.owner, + }) .collect()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs index 03d70f111e..932c2a997c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_session.rs @@ -461,12 +461,12 @@ impl<'py> FixSession<'py> { .map(|(path, reason)| { violation( &path, - ( - "format-fix-skipped", - &format!("Rule fixes skipped this file: {}", reason.str()?), - "warning", - "Fix the reported problem, then rerun `sqb format --fix`.", - ), + ViolationDetails { + code: "format-fix-skipped", + message: &format!("Rule fixes skipped this file: {}", reason.str()?), + severity: "warning", + remediation: "Fix the reported problem, then rerun `sqb format --fix`.", + }, ) }) .collect() @@ -681,9 +681,17 @@ fn refused<'py>(report: &Bound<'py, PyAny>, reason: &str) -> PyResult { + code: &'a str, + message: &'a str, + severity: &'a str, + remediation: &'a str, +} + fn violation<'py>( path: &Bound<'py, PyAny>, - details: (&str, &str, &str, &str), + details: ViolationDetails<'_>, ) -> PyResult> { let py = path.py(); new_record_with( @@ -693,11 +701,14 @@ fn violation<'py>( ("file_path", path.clone()), ("line", 1_i64.into_pyobject(py)?.into_any()), ("column", 1_i64.into_pyobject(py)?.into_any()), - ("code", details.0.into_pyobject(py)?.into_any()), - ("message", details.1.into_pyobject(py)?.into_any()), - ("severity", details.2.into_pyobject(py)?.into_any()), + ("code", details.code.into_pyobject(py)?.into_any()), + ("message", details.message.into_pyobject(py)?.into_any()), + ("severity", details.severity.into_pyobject(py)?.into_any()), ("engine", "native".into_pyobject(py)?.into_any()), - ("remediation", details.3.into_pyobject(py)?.into_any()), + ( + "remediation", + details.remediation.into_pyobject(py)?.into_any(), + ), ], ) } @@ -705,12 +716,12 @@ fn violation<'py>( fn verification_fault<'py>(path: &Bound<'py, PyAny>, reason: &str) -> PyResult> { violation( path, - ( - "format-fix-verification-failed", - reason, - "fault", - "Restructure the SQL manually and recompile; the original file was retained.", - ), + ViolationDetails { + code: "format-fix-verification-failed", + message: reason, + severity: "fault", + remediation: "Restructure the SQL manually and recompile; the original file was retained.", + }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs index d58ccb5322..f1449ca5cd 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/fix_verdict.rs @@ -6,7 +6,9 @@ use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyDict, PyDictMethods, PyList, PyListMethods, PyTuple}; use sqlbuild_rules::fix_verification::main::compare::compare; -use sqlbuild_rules::fix_verification::models::{DiagnosticComparison, ModelComparison}; +use sqlbuild_rules::fix_verification::models::{ + DiagnosticComparison, DiagnosticIdentity, ModelComparison, +}; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; @@ -107,8 +109,39 @@ fn lineage<'py>(model: &Bound<'py, PyAny>) -> PyResult> { } } -fn counts( - value: Option>, -) -> PyResult> { - value.map_or_else(|| Ok(BTreeMap::new()), |value| value.extract()) +positional_record! { + extract; + /// The `(code, resource type, resource name, message)` key `compile_facts` counts. + #[derive(PartialEq, Eq, PartialOrd, Ord)] + struct IdentityKey { + code: String, + resource_type: String, + resource_name: String, + message: String, + } +} + +fn counts(value: Option>) -> PyResult> { + let Some(value) = value else { + return Ok(BTreeMap::new()); + }; + let counted: BTreeMap = value.extract()?; + Ok(counted + .into_iter() + .map(|(key, count)| { + let IdentityKey { + code, + resource_type, + resource_name, + message, + } = key; + let identity = DiagnosticIdentity { + code, + resource_type, + resource_name, + message, + }; + (identity, count) + }) + .collect()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs index 54e7413b52..c28d595f73 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint.rs @@ -2,9 +2,12 @@ use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::{FromPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_rules::lint_text::models::InterpolationSite; +use sqlbuild_rules::sql_lint::types::PreparedSql; use crate::bindings::_helpers::boundary::panics::{compiler_guard, value_error}; use crate::bindings::_helpers::rules::relation_keys; +use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; #[pyfunction] @@ -23,20 +26,31 @@ struct LintPreparationRequest { dialect: String, } +/// [`PreparedSql`] as Python reads it, each site a positional tuple. +type PreparedLintSql = (String, Vec>, Vec); + #[pyfunction] fn prepare_lint_sql( py: Python<'_>, request: LintPreparationRequest, -) -> PyResult> { - py.compiler_detach(|| { - sqlbuild_rules::sql_lint::main::preparation::prepare( - &request.expanded, - &request.before_expansion, - &request.prior_sites, - &request.dialect, +) -> PyResult> { + let prepared: Option = py + .compiler_detach(|| { + sqlbuild_rules::sql_lint::main::preparation::prepare( + &request.expanded, + &request.before_expansion, + &request.prior_sites, + &request.dialect, + ) + }) + .map_err(value_error)?; + Ok(prepared.map(|(text, sites, referenced)| { + ( + text, + sites.into_iter().map(Positional).collect(), + referenced, ) - }) - .map_err(value_error) + })) } #[pyfunction] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs index 7e6a2607ee..164895dc88 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_inputs.rs @@ -12,6 +12,7 @@ use crate::bindings::_helpers::compiled_project::model_input_view::{ }; use crate::bindings::_helpers::compiled_project::project_view::NativeProjectView; use crate::bindings::_helpers::records::fields::field as native_field; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::lint_pipeline::{ CompiledExpansion, CompiledSource, span_pass, }; @@ -39,10 +40,7 @@ fn record_path(record: &ModelInputRecord) -> PathBuf { } /// The record's shared texts with the given passes, read only when their file is linted. -fn expansion( - expansion: &ExpansionRecord, - passes: &[&Vec<(usize, usize, usize, usize)>], -) -> CompiledSource { +fn expansion(expansion: &ExpansionRecord, passes: &[&Vec]) -> CompiledSource { CompiledSource::Shared { authored_sql: Arc::clone(&expansion.authored_sql), expanded_sql: Arc::clone(&expansion.expanded_sql), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs index 5c06143522..934ad63333 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_pipeline.rs @@ -43,7 +43,7 @@ use crate::bindings::_helpers::records::construct::{new_record_with, record_clas use crate::bindings::_helpers::records::fields::field as record_field; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::render::authored_sql::{AuthoredExpansion, expand_with_inputs}; -use crate::bindings::_helpers::render::declaration_expansion::SpanRow; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; use crate::bindings::_helpers::rules::dsl_headers::header_violations; use crate::bindings::_helpers::rules::native_header_parser::{ HeaderFailure, NativeHeaderParser, external_identifiers, @@ -213,14 +213,12 @@ impl CompiledSource { /// Expansion span rows as lint spans. pub(crate) fn span_pass(rows: &[SpanRow]) -> Vec { rows.iter() - .map( - |&(source_start, source_end, output_start, output_end)| ExpansionSpan { - source_start: offset(source_start), - source_end: offset(source_end), - output_start: offset(output_start), - output_end: offset(output_end), - }, - ) + .map(|row| ExpansionSpan { + source_start: offset(row.source_start), + source_end: offset(row.source_end), + output_start: offset(row.output_start), + output_end: offset(row.output_end), + }) .collect() } @@ -809,12 +807,10 @@ pub(crate) fn prepare_single_body(request: &SingleBodyRequest<'_, '_>) -> PyResu role, dialect, } = request; - let (context, file_path, range, dialect): ( - &SqlExpansionInputs, - &Bound<'_, PyAny>, - (usize, usize), - &str, - ) = (context, file_path, *range, dialect); + let context: &SqlExpansionInputs = context; + let file_path: &Bound<'_, PyAny> = file_path; + let range: (usize, usize) = *range; + let dialect: &str = dialect; let py: Python<'_> = file_path.py(); let text: Vec = contents.chars().collect(); let planned: PlannedBody = plan_body(&text, range, neutralizer(role), None); @@ -1071,28 +1067,6 @@ fn lint_body( body: PendingBody, (lint_text, prepared_sites, externally_referenced_ctes): PreparedSql, ) -> PyResult<(LintSqlBody, Vec<(String, String)>)> { - let prepared_sites: Vec = prepared_sites - .into_iter() - .map( - |( - sentinel, - neutralized_start, - neutralized_end, - original_start, - original_end, - original_text, - )| { - InterpolationSite { - sentinel, - neutralized_start, - neutralized_end, - original_start, - original_end, - original_text, - } - }, - ) - .collect(); let dependencies = dependency_sites(&prepared_sites).map_err(value_error)?; let mut passes: Vec> = Vec::with_capacity(body.passes.len() + 2); passes.push(sentinel_spans(&body.sites)); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs index 7efa8a6682..9f6a9d83b6 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/lint_text.rs @@ -30,30 +30,14 @@ use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::rules::lint_pipeline::{ LintViolationRow, NATIVE_ENGINE, ViolationClasses, }; +use crate::bindings::models::Positional; use crate::bindings::types::CompilerDetach; const FORMAT_UNSAFE_CODE: &str = "format-unsafe"; const FORMAT_UNSAFE_REMEDIATION: &str = "Rewrite the unsupported SQL shape or report a minimal formatter reproduction. No header, SQL body, or fixture changes were written to this file."; -/// One interpolation site as `InterpolationSite` fields. -type SiteTuple = (String, usize, usize, usize, usize, String); - -fn site_tuples(neutralized: Neutralized) -> (String, Vec) { - let sites: Vec = neutralized - .sites - .into_iter() - .map(|site| { - ( - site.sentinel, - site.neutralized_start, - site.neutralized_end, - site.original_start, - site.original_end, - site.original_text, - ) - }) - .collect(); - (neutralized.text, sites) +fn site_tuples(neutralized: Neutralized) -> (String, Vec>) { + (neutralized.text, Positional::wrap_all(neutralized.sites)) } #[pyfunction] @@ -62,7 +46,7 @@ fn neutralize_lint_interpolation( body: &str, dialect: &str, for_formatting: bool, -) -> PyResult<(String, Vec)> { +) -> PyResult<(String, Vec>)> { compiler_guard(|| { neutralize_interpolation(body, dialect, for_formatting) .map(site_tuples) @@ -71,29 +55,26 @@ fn neutralize_lint_interpolation( } #[pyfunction] -fn neutralize_lint_context_interpolation(body: &str) -> PyResult<(String, Vec)> { +fn neutralize_lint_context_interpolation( + body: &str, +) -> PyResult<(String, Vec>)> { compiler_guard(|| Ok(site_tuples(neutralize_context_interpolation(body)))) } #[pyfunction] -fn neutralize_lint_audit_parameters(body: &str) -> PyResult<(String, Vec)> { +fn neutralize_lint_audit_parameters( + body: &str, +) -> PyResult<(String, Vec>)> { compiler_guard(|| Ok(site_tuples(neutralize_audit_parameters(body)))) } /// Restored text, or `(False, reason)` when a sentinel is missing or repeated. #[pyfunction] -fn restore_lint_interpolation(fixed: &str, sites: Vec) -> PyResult<(bool, String)> { - let sites: Vec = sites - .into_iter() - .map(|site| InterpolationSite { - sentinel: site.0, - neutralized_start: site.1, - neutralized_end: site.2, - original_start: site.3, - original_end: site.4, - original_text: site.5, - }) - .collect(); +fn restore_lint_interpolation( + fixed: &str, + sites: Vec>, +) -> PyResult<(bool, String)> { + let sites: Vec = Positional::unwrap_all(sites); match compiler_guard(|| Ok(restore_interpolation(fixed, &sites)))? { Ok(restored) => Ok((true, restored)), Err(LintTextError::Restoration(message)) => Ok((false, message)), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs index 989f88060c..406440f71f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/relation_keys.rs @@ -23,18 +23,21 @@ const UNIQUE_AUDIT_COLUMN_ARGUMENT: &str = "column"; const RELATION_KIND_SEED: &str = "seed"; const RELATION_KIND_SOURCE: &str = "source"; -/// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. -pub(crate) type RelationKeyRow = ( - String, - String, - String, - String, - String, - bool, - Option>, - Vec<(String, Vec, bool)>, - Vec<(String, Option)>, -); +positional_record! { + extract, into; + /// `(kind, name, materialized, mode, strategy, cursor, unique key, columns, audits)`. + pub(crate) struct RelationKeyRow { + kind: String, + name: String, + materialized: String, + mode: String, + strategy: String, + has_cursor: bool, + unique_key: Option>, + columns: Vec<(String, Vec, bool)>, + audits: Vec<(String, Option)>, + } +} /// The key row of one relation from its config values and declared column and audit specs. pub(crate) fn relation_key_row( @@ -55,19 +58,19 @@ pub(crate) fn relation_key_row( } audits = audit_key_rows(declared_audits)?; } - Ok(( - kind.to_owned(), + Ok(RelationKeyRow { + kind: kind.to_owned(), name, - config_text(values, MATERIALIZED_CONFIG)?, - config_text(values, INCREMENTAL_MODE_CONFIG)?, - config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, - values + materialized: config_text(values, MATERIALIZED_CONFIG)?, + mode: config_text(values, INCREMENTAL_MODE_CONFIG)?, + strategy: config_text(values, INCREMENTAL_STRATEGY_CONFIG)?, + has_cursor: values .get_item(CURSOR_CONFIG)? .is_some_and(|value| value.is_truthy().unwrap_or(false)), - unique_key(values)?, + unique_key: unique_key(values)?, columns, audits, - )) + }) } /// Each audit spec's `(definition name, column argument when it is text)`. @@ -93,15 +96,15 @@ pub(crate) fn audit_key_rows(audits: &Bound<'_, PyAny>) -> PyResult) -> Option> { read_source_view(view, |source, py| { let facts: &SourceKeyFacts = source.key_facts(py)?; - Ok(( - RELATION_KIND_SOURCE.to_owned(), - source.attached().name.clone(), - String::new(), - String::new(), - String::new(), - false, - None, - facts + Ok(RelationKeyRow { + kind: RELATION_KIND_SOURCE.to_owned(), + name: source.attached().name.clone(), + materialized: String::new(), + mode: String::new(), + strategy: String::new(), + has_cursor: false, + unique_key: None, + columns: facts .columns .iter() .map(|column| { @@ -112,8 +115,8 @@ pub(crate) fn source_view_key_row(view: &Bound<'_, PyAny>) -> Option(project: &Bound<'py, PyAny>) -> PyResult DeclaredRelation { - let (kind, name, materialized, mode, strategy, has_cursor, unique_key, columns, audits) = row; + let RelationKeyRow { + kind, + name, + materialized, + mode, + strategy, + has_cursor, + unique_key, + columns, + audits, + } = row; DeclaredRelation { kind, name, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs index 94c3848ffd..e90de02503 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs @@ -63,6 +63,16 @@ struct DeclarationRowPy<'py>( Option, ); +/// The code, path, line, column and message that identify one fault. +#[derive(Hash, PartialEq, Eq)] +struct FaultIdentity { + code: String, + path: String, + line: u64, + column: u64, + message: String, +} + /// The request's project facts, which models, tests and scenarios are built beside. struct ProjectRows<'py> { memo: Option<(String, String)>, @@ -410,7 +420,7 @@ fn finalized<'py>( .str()? .to_str()? .to_owned(); - let mut fixable: HashSet<(String, String, u64, u64, String)> = HashSet::new(); + let mut fixable: HashSet = HashSet::new(); let mut faults: Vec = Vec::with_capacity(findings.len()); for finding in &findings { let fault: Fault = finding_fault(finding)?; @@ -518,14 +528,14 @@ fn declaration_member((name, value): (String, Bound<'_, PyAny>)) -> PyResult (String, String, u64, u64, String) { - ( - fault.code.clone(), - fault.path.clone(), - fault.line, - fault.column, - fault.message.clone(), - ) +fn fault_identity(fault: &Fault) -> FaultIdentity { + FaultIdentity { + code: fault.code.clone(), + path: fault.path.clone(), + line: fault.line, + column: fault.column, + message: fault.message.clone(), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs index 7334eb244e..f9e58d2580 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/rule_cache.rs @@ -52,6 +52,17 @@ fn sql_rule_lint_key(config: &Bound<'_, PyAny>) -> PyResult { Ok(settings_lint_key(&LintSettings::from_record(config)?)) } +positional_record! { + extract; + /// One model's path, authored SQL, query SQL and contract. + pub(crate) struct ModelRuleRow { + pub(crate) path: String, + pub(crate) authored_sql: String, + pub(crate) query_sql: String, + pub(crate) contract: Option, + } +} + /// Each model path's SQL rule cache key from `(path, authored_sql, query_sql, contract)`. #[pyfunction] #[pyo3(signature = (*, lint_key, codes, models))] @@ -59,7 +70,7 @@ fn sql_model_rule_keys( py: Python<'_>, lint_key: &str, codes: Vec, - models: Vec<(String, String, String, Option)>, + models: Vec, ) -> BTreeMap { model_rule_keys(py, lint_key, &codes, &models) } @@ -69,20 +80,20 @@ pub(crate) fn model_rule_keys( py: Python<'_>, lint_key: &str, codes: &[String], - models: &[(String, String, String, Option)], + models: &[ModelRuleRow], ) -> BTreeMap { let inputs: Vec> = models .iter() - .map(|(_, authored_sql, query_sql, contract)| ModelKeyInput { - authored_sql, - query_sql, - contract: contract.as_deref(), + .map(|model| ModelKeyInput { + authored_sql: &model.authored_sql, + query_sql: &model.query_sql, + contract: model.contract.as_deref(), }) .collect(); let keys: Vec = py.detach(|| parallel_model_rule_keys(lint_key, codes, &inputs)); models .iter() - .map(|(path, ..)| path.clone()) + .map(|model| model.path.clone()) .zip(keys) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs index ed839bc92a..6330b98eac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs @@ -43,8 +43,8 @@ use crate::bindings::_helpers::rules::lint_pipeline::{ }; use crate::bindings::_helpers::rules::relation_keys::project_relation_keys; use crate::bindings::_helpers::rules::rule_cache::{ - StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, settings_lint_key, - stored_entries, write_rule_cache, + ModelRuleRow, StoredEntries, cached_rule_findings, encoded_findings, model_rule_keys, + settings_lint_key, stored_entries, write_rule_cache, }; const DATACLASSES_MODULE: &str = "dataclasses"; @@ -293,17 +293,22 @@ fn model_rule_identities(run: &SqlRun<'_, '_>, lint_key: &str) -> PyResult)> = Vec::new(); + let mut models: Vec = Vec::new(); for model in sql_rule_models(&run.project, true)? { if let Some(key) = model.key { - models.push((model.path, key.authored_sql, key.query_sql, key.contract)); + models.push(ModelRuleRow { + path: model.path, + authored_sql: key.authored_sql, + query_sql: key.query_sql, + contract: key.contract, + }); } } let keys: BTreeMap = model_rule_keys(run.project.py(), lint_key, run.codes, &models); let fresh: Vec<(String, String)> = models .into_iter() - .map(|(path, ..)| { + .map(|ModelRuleRow { path, .. }| { let key: String = keys[&path].clone(); (path, key) }) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs index 6cc919eaf7..b3ad6ee36e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs @@ -15,8 +15,8 @@ use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, - write_completion, + CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, + progress_relay, write_completion, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, ScenarioProject, capture_limits, capture_settings, cli_user_error, compile, @@ -101,12 +101,12 @@ pub(crate) fn run_scenario_capture_command( let relay = progress_relay( &stream, §ion, - ( - &project.project_dir, - &adapter, - CompletionKind::Capture, - style, - ), + RelayContext { + project_dir: &project.project_dir, + adapter: &adapter, + kind: CompletionKind::Capture, + style: style, + }, )?; let results: Vec> = call_with( &host::entry(py, ExecutorHost::RunScenarioCapturePipeline)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs index 1d40c492fb..0762d7b65e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs @@ -233,16 +233,24 @@ pub(crate) fn begin_section<'py>( }) } +/// The project directory, adapter, completion kind and style a progress relay reports with. +pub(crate) struct RelayContext<'a, 'py> { + pub(crate) project_dir: &'a Bound<'py, PyAny>, + pub(crate) adapter: &'a Bound<'py, PyAny>, + pub(crate) kind: CompletionKind, + pub(crate) style: CliStyle, +} + /// The relay reporting one section's scenarios. pub(crate) fn progress_relay<'py>( stream: &Bound<'py, PyAny>, section: &SectionStatus<'py>, - (project_dir, adapter, kind, style): ( - &Bound<'py, PyAny>, - &Bound<'py, PyAny>, - CompletionKind, - CliStyle, - ), + RelayContext { + project_dir, + adapter, + kind, + style, + }: RelayContext<'_, 'py>, ) -> PyResult> { Bound::new( stream.py(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs index 69f78e3f35..d125ca59c2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs @@ -17,7 +17,7 @@ use crate::bindings::_helpers::result_output::documents::format_scenario_snapsho use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, begin_section, connection_hooks, progress_relay, + CompletionKind, RelayContext, SectionStatus, begin_section, connection_hooks, progress_relay, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, boolean, capture_limits, capture_settings, compile, flush, optional_text, @@ -161,12 +161,12 @@ pub(crate) fn sync_local_snapshots<'py>( let relay = progress_relay( &run.stream, §ion, - ( - &run.project.project_dir, - &run.project_adapter, - CompletionKind::Sync { refresh }, - run.style, - ), + RelayContext { + project_dir: &run.project.project_dir, + adapter: &run.project_adapter, + kind: CompletionKind::Sync { refresh }, + style: run.style, + }, )?; let settings: Bound<'py, PyAny> = capture_settings( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs index 14018b8ddf..adadca3f31 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/test_command.rs @@ -23,8 +23,8 @@ use crate::bindings::_helpers::result_output::documents::format_scenario_executi use crate::bindings::_helpers::result_output::documents::write_execution_json_output; use crate::bindings::_helpers::scenario_command::conversions::succeeded; use crate::bindings::_helpers::scenario_command::presentation::{ - CompletionKind, SectionStatus, activity, begin_section, connection_hooks, progress_relay, - write_completion, + CompletionKind, RelayContext, SectionStatus, activity, begin_section, connection_hooks, + progress_relay, write_completion, }; use crate::bindings::_helpers::scenario_command::setup::{ CompileInputs, ScenarioProject, boolean, cli_user_error, compile, discover_project, flush, int, @@ -213,12 +213,12 @@ fn run_scenarios( let relay = progress_relay( &run.stream, §ion, - ( - &run.project.project_dir, - &run.adapter, - CompletionKind::Run { target_dir }, - run.style, - ), + RelayContext { + project_dir: &run.project.project_dir, + adapter: &run.adapter, + kind: CompletionKind::Run { target_dir }, + style: run.style, + }, )?; let project_name: Bound<'_, PyAny> = native_field( &native_field(&run.project.discovered, "project_config")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs index 46e1d892fa..58e7eddeb9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/compile_usages.rs @@ -33,21 +33,30 @@ type IdentityRow = (String, String); /// A declaration's kind, name and private owner; a resource's kind and name with no owner. type QualifiedRow = (String, String, Option); /// Consumer, declaration, usage kind, what it is used through, and the enum member. -pub(crate) type UsageRow = ( - QualifiedRow, - QualifiedRow, - String, - Option, - Option, -); +#[derive(Clone, Hash, PartialEq, Eq)] +pub(crate) struct UsageRow { + pub(crate) consumer: QualifiedRow, + pub(crate) declaration: QualifiedRow, + pub(crate) kind: String, + pub(crate) through: Option, + pub(crate) enum_member: Option, +} /// Path, scope, private owner, owning path and ownership root path of a named record. -type RecordRow = (String, String, Option, Option, String); +struct RecordRow { + path: String, + scope: String, + owner: Option, + owning_path: Option, + ownership_root: String, +} /// Name, the schema it extends, and each column's name and type. type SchemaRow = (String, Option, Vec<(String, Option)>); -/// Report key, message, path and help of one S006. -type DiagnosticRow = (Vec, String, String, String); /// Usages as `(consumer, declaration)` record positions, S006s, else a scope path error. -type SchemaUsageRows = (Vec<(usize, usize)>, Vec, Option); +type SchemaUsageRows = ( + Vec<(usize, usize)>, + Vec, + Option, +); /// `(own, position)`: an own dependency's position, else a distinct usage's. pub(crate) type SourceRow = (bool, usize); /// Distinct collected positions, merged sources per declaration, and final usage positions. @@ -58,7 +67,13 @@ fn invalid(what: &str) -> pyo3::PyErr { } fn named_record(handle: usize, row: RecordRow) -> PyResult { - let (path, scope, owner, owning_path, ownership_root) = row; + let RecordRow { + path, + scope, + owner, + owning_path, + ownership_root, + } = row; let owner: Option = match owner { None => None, Some((kind, name)) => Some(ResourceIdentity { @@ -78,15 +93,6 @@ fn named_record(handle: usize, row: RecordRow) -> PyResult { }) } -fn diagnostic_row(diagnostic: InaccessibleDiagnostic) -> DiagnosticRow { - ( - diagnostic.key, - diagnostic.message, - diagnostic.path, - diagnostic.help, - ) -} - /// `model_schema_scope_usages`: each schema's parent and enum `UsageRecord`s; S006s are reported. #[pyfunction] pub(crate) fn model_schema_scope_usages<'py>( @@ -163,10 +169,10 @@ pub(crate) fn model_schema_scope_usages<'py>( fn record_row(record: &Bound<'_, PyAny>) -> PyResult { let identity: Bound<'_, PyAny> = native_field(record, "identity")?; let owner: Bound<'_, PyAny> = native_field(&identity, "owner")?; - Ok(( - native_field(record, "path")?.extract()?, - native_field(&native_field(record, "scope")?, "value")?.extract()?, - if owner.is_none() { + Ok(RecordRow { + path: native_field(record, "path")?.extract()?, + scope: native_field(&native_field(record, "scope")?, "value")?.extract()?, + owner: if owner.is_none() { None } else { Some(( @@ -174,18 +180,25 @@ fn record_row(record: &Bound<'_, PyAny>) -> PyResult { native_field(&owner, "name")?.extract()?, )) }, - native_field(record, "owning_path")?.extract()?, - native_field(&native_field(record, "ownership_root")?, "path")?.extract()?, - )) + owning_path: native_field(record, "owning_path")?.extract()?, + ownership_root: native_field(&native_field(record, "ownership_root")?, "path")? + .extract()?, + }) } /// Report each S006 through the active compile diagnostic collector. -fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { +fn report_inaccessible(py: Python<'_>, diagnostics: Vec) -> PyResult<()> { if diagnostics.is_empty() { return Ok(()); } let path_class = py.import("pathlib")?.getattr("Path")?; - for (key, message, path, help) in diagnostics { + for InaccessibleDiagnostic { + key, + message, + path, + help, + } in diagnostics + { let fields: Bound<'_, PyDict> = PyDict::new(py); fields.set_item("phase", "compile")?; fields.set_item("severity", "error")?; @@ -230,11 +243,7 @@ fn scope_schema_usages( Ok(SchemaUsages { usages, diagnostics, - }) => ( - usages, - diagnostics.into_iter().map(diagnostic_row).collect(), - None, - ), + }) => (usages, diagnostics, None), Err(rejected) => ( Vec::new(), Vec::new(), @@ -265,7 +274,11 @@ pub(crate) fn merged_usage_rows( let mut consumed: Vec<(Option, QualifiedRow)> = Vec::with_capacity(distinct.len()); for position in &distinct { - let (consumer, declaration, _, _, _) = &collected[*position]; + let UsageRow { + consumer, + declaration, + .. + } = &collected[*position]; let by_declaration: Option = DeclarationKind::parse(&consumer.0) .is_some() .then(|| consumer.clone()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs index 2644315669..8f9d194b3c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/placement.rs @@ -13,15 +13,15 @@ pub(crate) type QualifiedRow = (String, String, Option); /// An ownership root's path, kind and resource kind. pub(crate) type RootRow = (String, String, Option); /// Identity, path, line, column, scope, ownership root and owning path. -pub(crate) type DeclarationRow = ( - QualifiedRow, - String, - i64, - i64, - String, - RootRow, - Option, -); +pub(crate) struct DeclarationRow { + pub(crate) identity: QualifiedRow, + pub(crate) path: String, + pub(crate) line: i64, + pub(crate) column: i64, + pub(crate) scope: String, + pub(crate) root: RootRow, + pub(crate) owning_path: Option, +} fn invalid(what: &str) -> pyo3::PyErr { PyValueError::new_err(format!("invalid scope placement {what}")) @@ -73,7 +73,15 @@ pub(crate) fn scope_kind(value: &str) -> PyResult { /// A declaration row as its record, without macro dependencies. pub(crate) fn declaration_record(row: DeclarationRow) -> PyResult { - let (identity, path, line, column, scope, root, owning_path) = row; + let DeclarationRow { + identity, + path, + line, + column, + scope, + root, + owning_path, + } = row; Ok(DeclarationRecord { identity: declaration_identity(identity)?, path, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs index e0e672910c..564051531b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_command.rs @@ -24,8 +24,8 @@ use crate::bindings::_helpers::scopes::scope_lookup::build_scope_lookup; use crate::bindings::_helpers::scopes::scope_objects::ScopeClasses; use crate::bindings::_helpers::scopes::scope_report_objects::read_result; use crate::bindings::_helpers::scopes::scope_reports::{ - scope_browse, scope_explain, scope_list, scope_move_preview, scope_report, - serialize_scope_report, + ExplainArguments, ListArguments, ReportArguments, scope_browse, scope_explain, scope_list, + scope_move_preview, scope_report, serialize_scope_report, }; const TARGET_ERROR_CODE: &str = "C960"; @@ -249,12 +249,12 @@ fn query_result<'py>( return Ok(scope_list( py, lookup, - ( + ListArguments { folder, - Some(filters), - optional_text(py, &target), - prospective, - ), + filters: Some(filters), + target: optional_text(py, &target), + target_is_prospective: prospective, + }, )? .into_bound(py)); } @@ -264,12 +264,12 @@ fn query_result<'py>( let mut report: Bound<'_, PyAny> = scope_report( py, lookup, - ( - optional_text(py, &target), - optional_text(py, &at), + ReportArguments { + target: optional_text(py, &target), + at: optional_text(py, &at), directory, - Some(filters), - ), + filters: Some(filters), + }, )? .into_bound(py); let explain: Option = native_field(request, "explain")?.extract()?; @@ -277,12 +277,12 @@ fn query_result<'py>( let (explanation, diagnostics): (Bound<'_, PyAny>, Bound<'_, PyAny>) = scope_explain( py, lookup, - ( - PyString::new(py, &declaration).into_any(), - optional_text(py, &target), - optional_text(py, &at), + ExplainArguments { + declaration: PyString::new(py, &declaration).into_any(), + target: optional_text(py, &target), + at: optional_text(py, &at), directory, - ), + }, ) .map(|(explanation, diagnostics)| { (explanation.into_bound(py), diagnostics.into_bound(py)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs index 0cc73f0663..3ee4cd952a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_lookup.rs @@ -29,7 +29,12 @@ use crate::bindings::types::CompilerDetach; const TYPES_MODULE: &str = "types"; /// `(declarations, canonical positions)` matched, else identity and path error messages. -pub(crate) type TargetRows = (bool, Vec, Option, Option); +pub(crate) struct TargetRows { + pub(crate) declarations: bool, + pub(crate) positions: Vec, + pub(crate) identity_error: Option, + pub(crate) path_error: Option, +} /// Visible `(position, reason, grant)`, inaccessible `(position, reason)`, else the path error. pub(crate) type ResolvedRows = ( Vec<(usize, &'static str, Option)>, @@ -119,27 +124,30 @@ impl NativeScopeLookup { None => ReportTarget::Text(text.to_owned()), }; Ok(match report_target(&self.records, &self.lookup, &target) { - Ok(TargetMatch::Resources(positions)) => ( - false, - canonical(&self.canonical.resources, positions), - None, - None, - ), - Ok(TargetMatch::Declarations(positions)) => ( - true, - canonical(&self.canonical.declarations, positions), - None, - None, - ), - Err(TargetFailure::Identity(failure)) => ( - false, - Vec::new(), - Some(identity_message(&failure, text)), - None, - ), - Err(TargetFailure::Path(rejected)) => { - (false, Vec::new(), None, Some(path_message(&rejected))) - } + Ok(TargetMatch::Resources(positions)) => TargetRows { + declarations: false, + positions: canonical(&self.canonical.resources, positions), + identity_error: None, + path_error: None, + }, + Ok(TargetMatch::Declarations(positions)) => TargetRows { + declarations: true, + positions: canonical(&self.canonical.declarations, positions), + identity_error: None, + path_error: None, + }, + Err(TargetFailure::Identity(failure)) => TargetRows { + declarations: false, + positions: Vec::new(), + identity_error: Some(identity_message(&failure, text)), + path_error: None, + }, + Err(TargetFailure::Path(rejected)) => TargetRows { + declarations: false, + positions: Vec::new(), + identity_error: None, + path_error: Some(path_message(&rejected)), + }, }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs index ab0f6eb8d3..8f9205a9c8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_objects.rs @@ -21,8 +21,8 @@ use crate::bindings::_helpers::records::construct::{new_record, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::scopes::placement::{ - IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, ownership_root, - qualified_identity, resource_identity, + DeclarationRow, IdentityRow, QualifiedRow, RootRow, declaration_identity, declaration_record, + ownership_root, qualified_identity, resource_identity, }; use crate::bindings::_helpers::scopes::scope_lookup::grant_kind; @@ -179,15 +179,15 @@ fn read_constant(facts: &Bound<'_, PyAny>) -> PyResult { /// One `DeclarationRecord` with its value metadata. pub(crate) fn read_declaration(record: &Bound<'_, PyAny>) -> PyResult { Ok(CachedDeclaration { - record: declaration_record(( - qualified_row(&native_field(record, "identity")?)?, - field(record, "path")?, - field(record, "line")?, - field(record, "column")?, - value_of(record, "scope")?, - root_row(&native_field(record, "ownership_root")?)?, - field(record, "owning_path")?, - ))?, + record: declaration_record(DeclarationRow { + identity: qualified_row(&native_field(record, "identity")?)?, + path: field(record, "path")?, + line: field(record, "line")?, + column: field(record, "column")?, + scope: value_of(record, "scope")?, + root: root_row(&native_field(record, "ownership_root")?)?, + owning_path: field(record, "owning_path")?, + })?, macro_facts: optional_item(record, "macro")? .map(|facts| read_macro(&facts)) .transpose()?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs index 9583d93bff..bb699daa8c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs @@ -40,26 +40,26 @@ use crate::bindings::_helpers::scopes::scope_report_objects::{ type PyObject = Py; /// Target, location, whether it is a directory, and the filters. -pub(crate) type ReportArguments<'py> = ( - Option>, - Option>, - bool, - Option>, -); +pub(crate) struct ReportArguments<'py> { + pub(crate) target: Option>, + pub(crate) at: Option>, + pub(crate) directory: bool, + pub(crate) filters: Option>, +} /// Declaration, target, location and whether it is a directory. -pub(crate) type ExplainArguments<'py> = ( - Bound<'py, PyAny>, - Option>, - Option>, - bool, -); +pub(crate) struct ExplainArguments<'py> { + pub(crate) declaration: Bound<'py, PyAny>, + pub(crate) target: Option>, + pub(crate) at: Option>, + pub(crate) directory: bool, +} /// Folder, filters, target and whether the target is prospective. -pub(crate) type ListArguments<'py> = ( - String, - Option>, - Option>, - bool, -); +pub(crate) struct ListArguments<'py> { + pub(crate) folder: String, + pub(crate) filters: Option>, + pub(crate) target: Option>, + pub(crate) target_is_prospective: bool, +} const CURRENT_FOLDER: &str = "."; @@ -148,7 +148,12 @@ pub(crate) fn scope_report( lookup: &Bound<'_, PyAny>, request: ReportArguments<'_>, ) -> PyResult { - let (target, at, directory, filters) = request; + let ReportArguments { + target, + at, + directory, + filters, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let selected: Bound<'_, PyAny> = match filters.filter(|filters| !filters.is_none()) { Some(filters) => filters, @@ -178,7 +183,12 @@ pub(crate) fn scope_explain( lookup: &Bound<'_, PyAny>, request: ExplainArguments<'_>, ) -> PyResult<(PyObject, PyObject)> { - let (declaration, target, at, directory) = request; + let ExplainArguments { + declaration, + target, + at, + directory, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let reports: LookupReports = lookup_reports(lookup)?; let request: ExplainRequest = ExplainRequest { @@ -240,7 +250,12 @@ pub(crate) fn scope_list( lookup: &Bound<'_, PyAny>, request: ListArguments<'_>, ) -> PyResult { - let (folder, filters, target, target_is_prospective) = request; + let ListArguments { + folder, + filters, + target, + target_is_prospective, + } = request; let classes: ScopeClasses<'_> = ScopeClasses::new(py)?; let reports: LookupReports = lookup_reports(lookup)?; let request: FolderRequest = @@ -354,12 +369,12 @@ fn query_scope_report_binding<'py>( scope_report( py, &arguments.value("lookup")?, - ( - optional(&arguments, "target")?, - optional(&arguments, "at")?, - arguments.flag("directory")?, - optional(&arguments, "filters")?, - ), + ReportArguments { + target: optional(&arguments, "target")?, + at: optional(&arguments, "at")?, + directory: arguments.flag("directory")?, + filters: optional(&arguments, "filters")?, + }, ) } @@ -379,12 +394,12 @@ fn explain_scope_declaration_binding<'py>( scope_explain( py, &arguments.value("lookup")?, - ( - arguments.value("declaration")?, - optional(&arguments, "target")?, - optional(&arguments, "at")?, - arguments.flag("directory")?, - ), + ExplainArguments { + declaration: arguments.value("declaration")?, + target: optional(&arguments, "target")?, + at: optional(&arguments, "at")?, + directory: arguments.flag("directory")?, + }, ) } @@ -435,12 +450,12 @@ fn list_scope_declarations_binding<'py>( scope_list( py, &arguments.value("lookup")?, - ( - arguments.value("folder")?.extract()?, - optional(&arguments, "filters")?, - optional(&arguments, "target")?, - arguments.flag("target_is_prospective")?, - ), + ListArguments { + folder: arguments.value("folder")?.extract()?, + filters: optional(&arguments, "filters")?, + target: optional(&arguments, "target")?, + target_is_prospective: arguments.flag("target_is_prospective")?, + }, ) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs index 8338ebd9bd..8ddc2ef87f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_validation.rs @@ -57,15 +57,15 @@ fn tuple_from<'py>(py: Python<'py>, items: Vec>) -> PyResult) -> PyResult { - Ok(( - qualified_row(&native_field(usage, "consumer")?)?, - qualified_row(&native_field(usage, "declaration")?)?, - value_of(usage, "kind")?, - optional_item(usage, "through")? + Ok(UsageRow { + consumer: qualified_row(&native_field(usage, "consumer")?)?, + declaration: qualified_row(&native_field(usage, "declaration")?)?, + kind: value_of(usage, "kind")?, + through: optional_item(usage, "through")? .map(|through| qualified_row(&through)) .transpose()?, - field(usage, "enum_member")?, - )) + enum_member: field(usage, "enum_member")?, + }) } fn dependency_row( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs index 40d9e18fbf..46b48fc41a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_visibility.rs @@ -47,8 +47,12 @@ fn target_query<'py>( } else { None }; - let (declarations, positions, identity_error, path_error): TargetRows = - native_lookup(lookup)?.get().target(row, &raw)?; + let TargetRows { + declarations, + positions, + identity_error, + path_error, + }: TargetRows = native_lookup(lookup)?.get().target(row, &raw)?; if let Some(message) = identity_error { return Err(PyErr::from_value( error_class(py, ErrorId::InvalidQualifiedIdentity)?.call1((message,))?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs index 4be1bbc2fa..02e58040ce 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs @@ -16,7 +16,9 @@ use sqlbuild_analysis::semantic_validation::models as validation; use crate::bindings::_helpers::analysis::model_reads::binding_validated; use crate::bindings::_helpers::boundary::panics::compiler_error; -use crate::bindings::_helpers::compiled_project::compiled_model_view::NativeCompiledModel; +use crate::bindings::_helpers::compiled_project::compiled_model_view::{ + ModelMetadataTexts, NativeCompiledModel, +}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get; use crate::bindings::_helpers::records::fields::mapping_get_or; @@ -282,7 +284,12 @@ fn model_diagnostics<'py>( } else { &classes.warning }; - let row: FindingRow<'_> = (&audit.code, &audit.message, audit.line, audit.column); + let row: FindingRow<'_> = FindingRow { + code: &audit.code, + message: &audit.message, + line: audit.line, + column: audit.column, + }; diagnostics.push(model_error(classes, model, &row, severity)?); } for finding in findings.references { @@ -340,17 +347,32 @@ fn pairs(mapping: &Bound<'_, PyAny>) -> PyResult> { .collect() } -/// `(code, message, line, column)`. -type FindingRow<'a> = (&'a str, &'a str, i64, i64); +/// The code, message, line and column of one finding. +struct FindingRow<'a> { + code: &'a str, + message: &'a str, + line: i64, + column: i64, +} fn finding_row(finding: &MetadataFinding) -> FindingRow<'_> { - (finding.code, &finding.message, finding.line, finding.column) + FindingRow { + code: finding.code, + message: &finding.message, + line: finding.line, + column: finding.column, + } } fn model_error<'py>( classes: &StageClasses<'py>, model: &Bound<'py, PyAny>, - &(code, message, line, column): &FindingRow<'_>, + &FindingRow { + code, + message, + line, + column, + }: &FindingRow<'_>, severity: &Bound<'py, PyAny>, ) -> PyResult> { let py: Python<'py> = classes.py(); @@ -446,8 +468,12 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes upstreams.push((upstream.str()?.extract()?, column_names(&columns)?)); } } - let (name, query_sql, authored_sql, calls_functions) = match model.cast::() - { + let ModelMetadataTexts { + name, + query_sql, + authored_sql, + calls_functions, + } = match model.cast::() { Ok(native) => native .get() .metadata_texts(model.py(), &FUNCTION_REFERENCE_KINDS)?, @@ -460,12 +486,12 @@ fn metadata_model(classes: &StageClasses<'_>, model: &Bound<'_, PyAny>) -> PyRes break; } } - ( - native_field(model, "name")?.extract()?, - native_field(model, "query_sql")?.extract()?, - native_field(model, "authored_sql")?.extract()?, + ModelMetadataTexts { + name: native_field(model, "name")?.extract()?, + query_sql: native_field(model, "query_sql")?.extract()?, + authored_sql: native_field(model, "authored_sql")?.extract()?, calls_functions, - ) + } } }; Ok(MetadataModel { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs index 42644f97bf..cdf45c3cc2 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/resource_sql.rs @@ -9,8 +9,8 @@ use pyo3::types::{PyBool, PyDict, PyDictMethods, PyList, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::semantic_checks::main::resource_sql_diagnostics::resource_sql_diagnostics; use sqlbuild_analysis::semantic_checks::models::{ - AuditSqlInput, HookModelInput, HookSqlInput, ResourceSqlDiagnostic, ResourceSqlFailure, - ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, + AuditSqlInput, HookModelInput, HookSqlInput, ResourceSpan, ResourceSqlDiagnostic, + ResourceSqlFailure, ResourceSqlKind, ResourceSqlRequest, SqlTestSqlInput, }; use crate::bindings::_helpers::boundary::panics::compiler_error; @@ -272,7 +272,7 @@ fn opts_out(values: &Bound<'_, PyAny>) -> PyResult { fn row_relations(rows: Vec) -> Vec { let mut names: Vec = rows .into_iter() - .filter_map(|(kind, name, _, _)| relation_name(&kind, name)) + .filter_map(|ReferenceRow { kind, name, .. }| relation_name(&kind, name)) .collect(); names.sort_unstable(); names.dedup(); @@ -318,7 +318,12 @@ fn python_diagnostic<'py>( classes: &DiagnosticClasses<'py>, diagnostic: ResourceSqlDiagnostic, ) -> PyResult> { - let (line, column, end_line, end_column) = diagnostic.span; + let ResourceSpan { + line, + column, + end_line, + end_column, + } = diagnostic.span; let location: Bound<'py, PyDict> = PyDict::new(py); location.set_item("path", classes.path.call1((&diagnostic.path,))?)?; location.set_item("line", line)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs index e5dfd2cf62..04336f22b9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/semantic_shapes/shapes.rs @@ -5,7 +5,7 @@ use pyo3::types::{PyDict, PyDictMethods, PyFrozenSet, PyString, PyTuple}; use pyo3::{pyfunction, wrap_pyfunction}; use crate::bindings::_helpers::analysis::binding_catalog::{ - BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, + BindingCatalog, CASE_SENSITIVE_BINDING_DIALECTS, CatalogKey, }; use crate::bindings::_helpers::analysis::model_reads::{ declared_rows, dynamic_columns_declared, inferred_rows, star_flags, unchecked_columns, @@ -74,7 +74,7 @@ fn model_relation_names(model: &Bound<'_, PyAny>) -> PyResult<(Vec, bool let referenced: bool = !rows.is_empty(); let names: Vec = relation_names( rows.into_iter() - .map(|(kind, name, _, _)| (kind, name)) + .map(|ReferenceRow { kind, name, .. }| (kind, name)) .collect(), ); return Ok((names, referenced)); @@ -155,12 +155,12 @@ fn binding_shape<'py>( if !catalog.is_truthy()? { let created: BindingCatalog = BindingCatalog::create( py, - ( - or_generic(&dialect)?.extract()?, - false, - native_field(profile, "semantic_known_functions")?.extract()?, - native_field(profile, "semantic_known_types")?.extract()?, - ), + CatalogKey { + dialect: or_generic(&dialect)?.extract()?, + quoted_ignore_case: false, + known_functions: native_field(profile, "semantic_known_functions")?.extract()?, + known_types: native_field(profile, "semantic_known_types")?.extract()?, + }, inputs.as_any(), )?; catalog = Bound::new(py, created)?.into_any(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs index 9dcd6a4f87..e35b5a5709 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/assembly.rs @@ -14,6 +14,7 @@ use sqlbuild_core::panics::main::native_failure::native_failure; use crate::bindings::_helpers::boundary::panics::value_error; use crate::bindings::_helpers::compiled_project::model_input_view::NativeModelInput; +use crate::bindings::_helpers::compiled_project::model_loop_host::ReferenceRow; use crate::bindings::_helpers::records::fields::{ field as native_field, path_posix_text, path_stem, }; @@ -26,19 +27,11 @@ const INTERNAL_FAILURE: &str = "internal"; const DECIMAL_OVERFLOW: &str = "decimal_overflow"; const SQL_TEST_ASSEMBLY_CONTEXT: &str = "native SQL test assembly"; -/// `(line, column, end line, end column, message, help)` of one P013 diagnostic. -pub(crate) type DiagnosticRow = (usize, usize, usize, usize, String, String); -/// `(name, scope deps, target models, case fingerprint, diagnostic resource, diagnostics)`. -pub(crate) type FactsRow = ( - String, - Vec<(&'static str, String)>, - Vec, - Option, - String, - Vec, -); /// The test's facts, and the `(kind, message)` of the error its assembly raises, if any. -pub(crate) type AssemblyRow = (Option, Option<(&'static str, String)>); +pub(crate) type AssemblyRow = ( + Option, + Option<(&'static str, String)>, +); /// The compile inputs assembly reads: model inputs, SQL test inputs and lexical rules. pub(crate) struct AssemblyRequest<'py> { @@ -86,11 +79,18 @@ fn assembly_model(model: &Bound<'_, PyAny>) -> PyResult { references: record .references .iter() - .map(|(kind, name, package, _)| SqlTestAssemblyReference { - kind: kind.clone(), - name: name.clone(), - package: package.clone(), - }) + .map( + |ReferenceRow { + kind, + name, + package, + .. + }| SqlTestAssemblyReference { + kind: kind.clone(), + name: name.clone(), + package: package.clone(), + }, + ) .collect(), }) }); @@ -240,11 +240,10 @@ fn parameter_values(items: &Bound<'_, PyAny>) -> PyResult AssemblyRow { match outcome { - SqlTestAssemblyOutcome::Assembled(facts) => (Some(facts_row(facts)), None), - SqlTestAssemblyOutcome::FingerprintOverflow(facts) => ( - Some(facts_row(facts)), - Some((DECIMAL_OVERFLOW, String::new())), - ), + SqlTestAssemblyOutcome::Assembled(facts) => (Some(facts), None), + SqlTestAssemblyOutcome::FingerprintOverflow(facts) => { + (Some(facts), Some((DECIMAL_OVERFLOW, String::new()))) + } SqlTestAssemblyOutcome::Failed(SqlTestAssemblyFailure::Input(message)) => { (None, Some((INPUT_FAILURE, message))) } @@ -257,27 +256,3 @@ fn assembly_row(outcome: SqlTestAssemblyOutcome) -> AssemblyRow { ), } } - -fn facts_row(facts: AssembledSqlTestFacts) -> FactsRow { - ( - facts.name, - facts.scope_deps, - facts.target_model_names, - facts.case_fingerprint, - facts.diagnostic_resource_name, - facts - .diagnostics - .into_iter() - .map(|diagnostic| { - ( - diagnostic.line, - diagnostic.column, - diagnostic.end_line, - diagnostic.end_column, - diagnostic.message, - diagnostic.help, - ) - }) - .collect(), - ) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs index 029f6f42a9..4cec1db815 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/compiled_tests.rs @@ -3,13 +3,14 @@ use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use pyo3::{IntoPyObject, pyfunction, wrap_pyfunction}; +use sqlbuild_analysis::compiler::models::{AssembledSqlTestFacts, SqlTestHelperDiagnostic}; use crate::bindings::_helpers::boundary::panics::compiler_error; use crate::bindings::_helpers::records::construct::{enum_member, record_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::RecordId; use crate::bindings::_helpers::sql_test_glue::assembly::{ - AssemblyRequest, AssemblyRow, DiagnosticRow, FactsRow, assemble_compiled_sql_tests, + AssemblyRequest, AssemblyRow, assemble_compiled_sql_tests, }; /// Fields a direct-logic compiled payload copies from its input payload, in order. @@ -159,7 +160,14 @@ fn compiled_payload<'py>( fn keyed_diagnostic<'py>( classes: &Classes<'py>, (test_input, resource_name): (&Bound<'py, PyAny>, &str), - (line, column, end_line, end_column, message, help): DiagnosticRow, + SqlTestHelperDiagnostic { + line, + column, + end_line, + end_column, + message, + help, + }: SqlTestHelperDiagnostic, ) -> PyResult> { let py: Python<'py> = classes.py(); let path: Bound<'py, PyAny> = @@ -203,11 +211,17 @@ fn keyed_diagnostic<'py>( fn compiled_test<'py>( classes: &Classes<'py>, test_input: &Bound<'py, PyAny>, - facts: FactsRow, + facts: AssembledSqlTestFacts, ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyTuple>)> { let py: Python<'py> = classes.py(); - let (name, scope_rows, target_model_names, case_fingerprint, resource_name, diagnostics) = - facts; + let AssembledSqlTestFacts { + name, + scope_deps: scope_rows, + target_model_names, + case_fingerprint, + diagnostic_resource_name: resource_name, + diagnostics, + } = facts; let scope_deps: Vec> = scope_rows .iter() .map(|(resource_type, dep)| classes.object_key(resource_type, dep)) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs index 51dac5c47a..45d8a0755b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/plan_rows.rs @@ -2,13 +2,24 @@ use crate::bindings::_helpers::records::construct::{enum_from_value, new_record_with}; use crate::bindings::_helpers::records::ids::RecordId; -use crate::bindings::_helpers::sql_test_glue::planning::{AssertionRow, ChainRow, PlanRow}; +use crate::bindings::_helpers::sql_test_glue::planning::PlanRow; use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; use pyo3::types::PyTuple; +use sqlbuild_analysis::compiler::models::{ + SqlTestAssertionStep, SqlTestChainStep, SqlTestPlan, SqlTestPlanWarning, +}; -fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult> { - let (model, sql, expected_sql, lifted, body, expected_columns, expected_lifted) = row; +fn chain_step<'py>(py: Python<'py>, step: SqlTestChainStep) -> PyResult> { + let SqlTestChainStep { + model_name: model, + resolved_sql: sql, + expected_cte_sql: expected_sql, + expected_lifted_ctes: expected_lifted, + lifted_ctes: lifted, + comparison_body_sql: body, + expected_columns, + } = step; new_record_with( py, RecordId::ChainStep, @@ -33,8 +44,13 @@ fn chain_step<'py>(py: Python<'py>, row: ChainRow) -> PyResult ) } -fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult> { - let (name, sql, lifted, body) = row; +fn assertion_step<'py>(py: Python<'py>, step: SqlTestAssertionStep) -> PyResult> { + let SqlTestAssertionStep { + name, + resolved_sql: sql, + lifted_ctes: lifted, + comparison_body_sql: body, + } = step; new_record_with( py, RecordId::SqlTestAssertionStep, @@ -48,7 +64,17 @@ fn assertion_step<'py>(py: Python<'py>, row: AssertionRow) -> PyResult, row: PlanRow) -> PyResult> { - let (sql, chain_rows, assertion_rows, model_names, warning_rows, error_messages) = row; + let PlanRow { + plan: + SqlTestPlan { + sql, + chain: chain_rows, + assertions: assertion_rows, + model_names, + warnings: warning_rows, + }, + error_messages, + } = row; let mut chain: Vec> = Vec::new(); for value in chain_rows { chain.push(chain_step(py, value)?); @@ -58,7 +84,12 @@ pub(super) fn plan(py: Python<'_>, row: PlanRow) -> PyResult> { assertions.push(assertion_step(py, value)?); } let mut warnings: Vec> = Vec::new(); - for (model_name, level, message) in warning_rows { + for SqlTestPlanWarning { + model_name, + severity: level, + message, + } in warning_rows + { warnings.push(new_record_with( py, RecordId::PlanWarning, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs index df61cf369a..fa0241d2d1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sql_test_glue/planning.rs @@ -7,9 +7,8 @@ use sqlbuild_analysis::compiler::main::sql_test_chains::resolve_sql_test_chains; use sqlbuild_analysis::compiler::main::sql_test_glue::plan_sql_test_batch; use sqlbuild_analysis::compiler::main::sql_test_plan_errors::sql_test_plan_error_messages; use sqlbuild_analysis::compiler::models::{ - SqlTestAssertionStep, SqlTestChainBatch, SqlTestChainModel, SqlTestChainStep, SqlTestCte, - SqlTestPlan, SqlTestPlanBatch, SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, - SqlTestPlanTest, + SqlTestChainBatch, SqlTestChainModel, SqlTestCte, SqlTestPlan, SqlTestPlanBatch, + SqlTestPlanFunction, SqlTestPlanModel, SqlTestPlanPayload, SqlTestPlanTest, }; use crate::bindings::_helpers::analysis::model_reads; @@ -21,37 +20,30 @@ use crate::bindings::types::CompilerDetach; const MODEL_RESOURCE_TYPE: &str = "model"; const PLANNING_WORKERS: usize = 4; -pub(crate) type CtePair = (String, String); -/// `(model, SQL, expected SQL, lifted CTEs, body, expected columns, expected lifted CTEs)`. -pub(crate) type ChainRow = ( - String, - String, - Option, - Vec, - Option, - Option>, - Vec, -); -/// `(name, resolved SQL, lifted CTEs, comparison body)`. -pub(crate) type AssertionRow = (String, String, Vec, Option); -/// `(model, severity, message)`. -pub(crate) type WarningRow = (Option, &'static str, String); -/// `(SQL, chain, assertions, model names, warnings, distinct error messages)`. -pub(crate) type PlanRow = ( - Option, - Vec, - Vec, - Vec, - Vec, - Vec, -); +/// A native SQL-test plan with its distinct error messages. +pub(crate) struct PlanRow { + pub(crate) plan: SqlTestPlan, + pub(crate) error_messages: Vec, +} + +positional_record! { + extract; + /// An SQL function's name with its rendered UDF and table-function call affixes. + pub(crate) struct FunctionRow { + pub(crate) name: String, + pub(crate) udf_prefix: String, + pub(crate) udf_suffix: String, + pub(crate) table_function_prefix: String, + pub(crate) table_function_suffix: String, + } +} /// The planning request: compiled models and tests, with the adapter's facts. pub(crate) struct PlanningRequest<'py> { pub(crate) models: Vec>, /// Cursor-rendered SQL by model index, for models whose SQL uses cursor intrinsics. pub(crate) rendered_model_sql: HashMap, - pub(crate) functions: Vec<(String, String, String, String, String)>, + pub(crate) functions: Vec, pub(crate) tests: Vec>, /// Cursor-rendered model query overrides by test index, replacing the test's own. pub(crate) rendered_test_overrides: HashMap>, @@ -171,13 +163,13 @@ fn model_dependencies(model: &Bound<'_, PyAny>) -> PyResult> { } fn plan_function( - (name, udf_prefix, udf_suffix, table_function_prefix, table_function_suffix): ( - String, - String, - String, - String, - String, - ), + FunctionRow { + name, + udf_prefix, + udf_suffix, + table_function_prefix, + table_function_suffix, + }: FunctionRow, ) -> SqlTestPlanFunction { SqlTestPlanFunction { name, @@ -242,36 +234,8 @@ fn cte(value: &Bound<'_, PyAny>) -> PyResult { fn plan_row(plan: SqlTestPlan) -> PlanRow { let error_messages: Vec = sql_test_plan_error_messages(&plan); - ( - plan.sql, - plan.chain.into_iter().map(chain_row).collect(), - plan.assertions.into_iter().map(assertion_row).collect(), - plan.model_names, - plan.warnings - .into_iter() - .map(|warning| (warning.model_name, warning.severity, warning.message)) - .collect(), + PlanRow { + plan, error_messages, - ) -} - -fn chain_row(step: SqlTestChainStep) -> ChainRow { - ( - step.model_name, - step.resolved_sql, - step.expected_cte_sql, - step.lifted_ctes, - step.comparison_body_sql, - step.expected_columns, - step.expected_lifted_ctes, - ) -} - -fn assertion_row(step: SqlTestAssertionStep) -> AssertionRow { - ( - step.name, - step.resolved_sql, - step.lifted_ctes, - step.comparison_body_sql, - ) + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs index f4eed0434b..c1ca49eb60 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/declaration_references.rs @@ -6,10 +6,17 @@ use sqlbuild_sqltext::compiler::models::{ DeclarationReferenceStop, }; -/// `(kind code, name, member, start, end)`: kind 0 is an enum member, 1 a constant. -type ReferenceRow = (u8, String, Option, usize, usize); +/// One natively scanned reference: kind 0 is an enum member, 1 a constant. +#[derive(Clone)] +pub(crate) struct ScannedReference { + pub(crate) kind: u8, + pub(crate) name: String, + pub(crate) member: Option, + pub(crate) start: usize, + pub(crate) end: usize, +} /// References and stop code: unclosed quote 0, block comment 1, invalid enum 2, constant 3. -type ScanRow = (Vec, Option); +type ScanRow = (Vec, Option); /// One SQL string's references and stop code, scanned without walking past malformed ones. pub(crate) fn declaration_scan(sql: &str) -> ScanRow { @@ -30,16 +37,16 @@ fn scan_row(scan: DeclarationReferenceScan) -> ScanRow { ) } -fn reference_row(reference: DeclarationReference) -> ReferenceRow { +fn reference_row(reference: DeclarationReference) -> ScannedReference { let kind: u8 = match reference.kind { DeclarationReferenceKind::Enum => 0, DeclarationReferenceKind::Constant => 1, }; - ( + ScannedReference { kind, - reference.name, - reference.member, - reference.start, - reference.end, - ) + name: reference.name, + member: reference.member, + start: reference.start, + end: reference.end, + } } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs index 8d2aab2a9f..881982dc82 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/model_headers.rs @@ -54,21 +54,29 @@ fn tokenize_model_header(header: &str) -> PyResult> { }) } -pub(crate) type ColumnLocation = (String, usize, usize, usize, usize); +positional_record! { + into; + /// A column name and its authored line, column, end line and end column. + pub(crate) struct ColumnLocation { + pub(crate) name: String, + pub(crate) line: usize, + pub(crate) column: usize, + pub(crate) end_line: usize, + pub(crate) end_column: usize, + } +} pub(crate) fn location_rows( locations: Vec<(String, sqlbuild_sqltext::compiler::models::TextLocation)>, ) -> Vec { locations .into_iter() - .map(|(name, location)| { - ( - name, - location.line, - location.column, - location.end_line, - location.end_column, - ) + .map(|(name, location)| ColumnLocation { + name, + line: location.line, + column: location.column, + end_line: location.end_line, + end_column: location.end_column, }) .collect() } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs index a365dde1bc..ffd2158bac 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/sql_references.rs @@ -23,10 +23,29 @@ use crate::bindings::_helpers::sqltext::lexical_syntax::LexicalSyntaxInput; const REFERENCE_CALL_SYNTAX_CODE: &str = "P012"; -/// `(kind, name, package, call_argument_count)` for one reference. -type ReferenceRow = (&'static str, String, Option, Option); -/// `(kind, call, start, message, help, corrected_call)` for one rejected call. -type InvalidCallRow = (&'static str, String, usize, String, String, String); +positional_record! { + into; + /// `(kind, name, package, call_argument_count)` for one reference. + struct ReferenceRow { + kind: &'static str, + name: String, + package: Option, + call_argument_count: Option, + } +} + +positional_record! { + into; + /// `(kind, call, start, message, help, corrected_call)` for one rejected call. + struct InvalidCallRow { + kind: &'static str, + call: String, + start: usize, + message: String, + help: String, + corrected_call: String, + } +} /// Python references and rejected calls of one scan. type ScannedObjects<'py> = (Vec>, Vec>); /// The references and rejected calls, or the scan error and its start. @@ -73,23 +92,23 @@ impl SqlReferenceScanner { } fn reference_row(reference: SqlReference) -> ReferenceRow { - ( - reference.kind, - reference.name, - reference.package, - reference.call_argument_count, - ) + ReferenceRow { + kind: reference.kind, + name: reference.name, + package: reference.package, + call_argument_count: reference.call_argument_count, + } } fn invalid_call_row(call: InvalidReferenceCall) -> InvalidCallRow { - ( - call.kind, - call.call, - call.start, - call.message, - call.help, - call.corrected_call, - ) + InvalidCallRow { + kind: call.kind, + call: call.call, + start: call.start, + message: call.message, + help: call.help, + corrected_call: call.corrected_call, + } } fn text<'py>(py: Python<'py>, value: impl IntoPyObject<'py>) -> PyResult> { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs index 415807dfa7..687b20670c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/sqltext/static_sql.rs @@ -10,16 +10,19 @@ use sqlbuild_sqltext::compiler::main::sql_interpolation::interpolate_sql; use sqlbuild_sqltext::compiler::models::InterpolationRead; use sqlbuild_sqltext::compiler::types::InterpolationHost; +use crate::bindings::_helpers::render::macro_call_rows::SpanRow; + const ENVIRONMENT_READ: &str = "env"; const CONTEXT_READ: &str = "ctx"; -/// `(sql or None when unchanged, spans, reads, error message or None)` for one SQL text. -pub(crate) type InterpolationRow = ( - Option, - Vec<(usize, usize, usize, usize)>, - Vec<(&'static str, String)>, - Option, -); +/// One SQL text interpolated: its SQL, substitution spans, reads and error message. +pub(crate) struct InterpolationRow { + /// The interpolated SQL, or `None` when unchanged. + pub(crate) sql: Option, + pub(crate) spans: Vec, + pub(crate) reads: Vec<(&'static str, String)>, + pub(crate) error: Option, +} /// Project variables, the environment, context values (or `None`) and the variable renderer. #[derive(FromPyObject)] @@ -154,13 +157,18 @@ pub(crate) fn interpolation_batch( let mut rows = Vec::with_capacity(sqls.len()); for (sql, file_path) in sqls { let row = match interpolate_sql(&host, sql, file_path) { - Ok(result) => (result.sql, result.spans, read_rows(result.reads), None), - Err(failure) => ( - None, - Vec::new(), - read_rows(failure.reads), - Some(failure.message), - ), + Ok(result) => InterpolationRow { + sql: result.sql, + spans: result.spans.into_iter().map(SpanRow::from).collect(), + reads: read_rows(result.reads), + error: None, + }, + Err(failure) => InterpolationRow { + sql: None, + spans: Vec::new(), + reads: read_rows(failure.reads), + error: Some(failure.message), + }, }; if let Some(error) = host.raised.borrow_mut().take() { return Err(error); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs index 6007f4e19d..74fe591603 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/type_system/normalization.rs @@ -10,14 +10,17 @@ use sqlbuild_analysis::type_system::models::{NormalizedType, TypeNormalization}; use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::records::fields::field as native_field; -/// `(name, family, precision, scale, length)` of one normalized type. -type NormalizedRow = ( - String, - &'static str, - Option, - Option, - Option, -); +positional_record! { + into; + /// `(name, family, precision, scale, length)` of one normalized type. + struct NormalizedRow { + name: String, + family: &'static str, + precision: Option, + scale: Option, + length: Option, + } +} /// Normalize one type: `(normalized row, Polyglot parse error)`, or Python's exception. #[pyfunction] @@ -51,13 +54,13 @@ pub(crate) fn normalization_error(python: Python<'_>, error: &TypeNormalizationE } fn normalized_row(normalized: NormalizedType) -> NormalizedRow { - ( - normalized.normalized_name, - normalized.family.as_str(), - normalized.precision.map(|value| value.as_str().to_owned()), - normalized.scale.map(|value| value.as_str().to_owned()), - normalized.length.map(|value| value.as_str().to_owned()), - ) + NormalizedRow { + name: normalized.normalized_name, + family: normalized.family.as_str(), + precision: normalized.precision.map(|value| value.as_str().to_owned()), + scale: normalized.scale.map(|value| value.as_str().to_owned()), + length: normalized.length.map(|value| value.as_str().to_owned()), + } } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/crates/sqlbuild-python/src/bindings/models.rs b/crates/sqlbuild-python/src/bindings/models.rs index 08e37dd711..cf4f54599c 100644 --- a/crates/sqlbuild-python/src/bindings/models.rs +++ b/crates/sqlbuild-python/src/bindings/models.rs @@ -1,8 +1,13 @@ //! Python-facing classes and mapping requests, converted to native models at the boundary. +use pyo3::prelude::{Bound, IntoPyObject, PyAny, PyErr, PyResult, Python}; +use pyo3::types::PyTuple; use pyo3::{FromPyObject, pyclass}; use sqlbuild_analysis::semantic_validation::models as validation; -use sqlbuild_analysis::semantic_validation::types::{Expansion, Relations}; +use sqlbuild_analysis::semantic_validation::models::Expansion; +use sqlbuild_analysis::semantic_validation::types::Relations; + +use crate::bindings::types::{FromPositional, IntoPositional}; use std::collections::HashMap; #[derive(FromPyObject, Debug)] @@ -32,7 +37,7 @@ pub(crate) struct PositionInput { pub(crate) query: String, pub(crate) expanded: String, pub(crate) cleaned: String, - pub(crate) passes: Vec>, + pub(crate) passes: Vec>>, } #[pyclass(module = "sqlbuild._native")] @@ -83,7 +88,11 @@ impl From for validation::PositionInput { query: input.query, expanded: input.expanded, cleaned: input.cleaned, - passes: input.passes, + passes: input + .passes + .into_iter() + .map(Positional::unwrap_all) + .collect(), } } } @@ -148,3 +157,35 @@ pub(crate) struct DeclarationTextRequest { pub(crate) audit_keys: Vec, pub(crate) hook_keys: Vec, } + +/// A record of another crate exchanged with Python as its positional tuple. +#[derive(Clone, Debug)] +pub(crate) struct Positional(pub(crate) T); + +impl Positional { + /// Each record of `items`, for Python. + pub(crate) fn wrap_all(items: Vec) -> Vec { + items.into_iter().map(Self).collect() + } + + /// Each record Python sent in `items`. + pub(crate) fn unwrap_all(items: Vec) -> Vec { + items.into_iter().map(|item| item.0).collect() + } +} + +impl<'py, T: FromPositional> FromPyObject<'py> for Positional { + fn extract_bound(value: &Bound<'py, PyAny>) -> PyResult { + T::from_positional(value).map(Self) + } +} + +impl<'py, T: IntoPositional> IntoPyObject<'py> for Positional { + type Target = PyTuple; + type Output = Bound<'py, PyTuple>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.0.into_positional(py) + } +} diff --git a/crates/sqlbuild-python/src/bindings/types.rs b/crates/sqlbuild-python/src/bindings/types.rs index f0038d008c..367b8e9c5e 100644 --- a/crates/sqlbuild-python/src/bindings/types.rs +++ b/crates/sqlbuild-python/src/bindings/types.rs @@ -1,6 +1,19 @@ +use pyo3::prelude::{Bound, PyAny, PyResult, Python}; +use pyo3::types::PyTuple; + pub(crate) trait CompilerDetach { fn compiler_detach Result + Send>( self, operation: F, ) -> Result; } + +/// A named record read from the positional tuple Python sends. +pub(crate) trait FromPositional: Sized { + fn from_positional(value: &Bound<'_, PyAny>) -> PyResult; +} + +/// A named record converted into the positional tuple Python receives. +pub(crate) trait IntoPositional { + fn into_positional<'py>(self, py: Python<'py>) -> PyResult>; +} diff --git a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs index 9a3becdfd7..fd68155fc5 100644 --- a/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs +++ b/crates/sqlbuild-refactor/src/refactoring/_helpers/scanning/sql_sites.rs @@ -46,8 +46,16 @@ pub(crate) struct ModelBody { pub(crate) passes: Vec>, } -/// `RESOURCE_CALL_PATTERN` on a site's text: `(kind, name, name_start, name_end)`. -fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { +/// A resource call a site's text holds, with its name span within that text. +struct ResourceCall { + kind: String, + name: String, + name_start: usize, + name_end: usize, +} + +/// `RESOURCE_CALL_PATTERN` on a site's text. +fn resource_call(text: &[char]) -> Option { if !starts_with(text, 0, "__") { return None; } @@ -82,12 +90,12 @@ fn resource_call(text: &[char]) -> Option<(String, String, usize, usize)> { if !(rest.is_empty() || rest == ['\n']) { return None; } - Some(( - kind.to_owned(), - slice(text, name_start, name_end), + Some(ResourceCall { + kind: kind.to_owned(), + name: slice(text, name_start, name_end), name_start, name_end, - )) + }) } fn skip_space(text: &[char], mut index: usize) -> usize { @@ -102,7 +110,12 @@ pub(crate) fn resource_sites(text: &[char], context: &ScanContext) -> Vec AnalysisSql let call = resource_call(&chars(&site.text)); let kind: char = call .as_ref() - .and_then(|(kind, ..)| kind.chars().next()) + .and_then(|call| call.kind.chars().next()) .unwrap_or(GENERIC_PLACEHOLDER_KIND); let placeholder = placeholder(kind, index, site.end - site.start); - if let Some((kind, name, ..)) = call { + if let Some(ResourceCall { kind, name, .. }) = call { let key = (kind, name); match tables.iter_mut().find(|(item, _)| *item == key) { Some((_, placeholders)) => { diff --git a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs index fc694bf75b..eda014fbd7 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/rows.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/rows.rs @@ -68,9 +68,14 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result schema_facts(&row, schema, inputs)?, - None => (Vec::new(), Vec::new(), false, 0), + None => SchemaFacts::default(), }; Ok(Model { sql_analysis_disabled, @@ -97,7 +102,14 @@ fn build_model(row: ModelRow, inputs: &ModelInputs<'_, '_>) -> Result, Vec, bool, u32); +/// A model's declared columns and dynamic families with their proofs and audit count. +#[derive(Default)] +struct SchemaFacts { + columns: Vec, + dynamic_columns: Vec, + dynamic_columns_proven: bool, + audit_count: u32, +} fn schema_facts( row: &ModelRow, @@ -141,7 +153,12 @@ fn schema_facts( .map(|column| column.audit_count) .sum::(); if schema.dynamic_columns.is_empty() { - return Ok((columns, Vec::new(), false, audit_count)); + return Ok(SchemaFacts { + columns, + dynamic_columns: Vec::new(), + dynamic_columns_proven: false, + audit_count, + }); } let proof_types: HashMap<&str, Option<&str>> = row .dynamic_proof @@ -172,7 +189,12 @@ fn schema_facts( .dynamic_proof .as_ref() .is_some_and(|proof| proof.output_proven); - Ok((columns, dynamic_columns, proven, audit_count)) + Ok(SchemaFacts { + columns, + dynamic_columns, + dynamic_columns_proven: proven, + audit_count, + }) } /// Python's `types_equal`, normalizing each type once; Python raises what normalization rejects. diff --git a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs index 92bc83d6e9..b39a5d3784 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_sql_scanning.rs @@ -208,7 +208,7 @@ fn given_quoted_commented_and_malformed_fragments_when_scanning_then_every_scann prepared .1 .first() - .map(|site| site.5.clone()) + .map(|site| site.original_text.clone()) .unwrap_or_default() }); assert_eq!( @@ -277,7 +277,7 @@ fn given_lint_dialect_when_preparing_macro_sites_then_backticks_follow_rules_quo for test_case in test_cases { let lint_sql = format!("SELECT @m({} x) AS y", test_case.fragment); let lint_site = prepare(&lint_sql, &lint_sql, &[], test_case.dialect)? - .and_then(|prepared| prepared.1.first().map(|site| site.5.clone())) + .and_then(|prepared| prepared.1.first().map(|site| site.original_text.clone())) .unwrap_or_default(); assert_eq!( lint_site, test_case.expected_lint_site, @@ -311,7 +311,13 @@ fn given_dollar_quoted_apostrophe_when_preparing_lint_sites_then_later_reference let sql = "SELECT $$Customer's order$$ AS note, $tag$it's$tag$ AS tag FROM __ref(\"orders\") o"; for test_case in test_cases { let sites: Vec = prepare(sql, sql, &[], test_case.dialect)? - .map(|prepared| prepared.1.into_iter().map(|site| site.5).collect()) + .map(|prepared| { + prepared + .1 + .into_iter() + .map(|site| site.original_text) + .collect() + }) .unwrap_or_default(); assert_eq!(sites, test_case.expected_sites, "{}", test_case.description); } diff --git a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs index 5e6fa94418..4eff333623 100644 --- a/crates/sqlbuild-rules/src/fix_verification/main/compare.rs +++ b/crates/sqlbuild-rules/src/fix_verification/main/compare.rs @@ -51,7 +51,7 @@ pub fn compare( .iter() .filter_map(|(identity, count)| { (*count > *diagnostic.before.get(identity).unwrap_or(&0)) - .then_some(identity.0.as_str()) + .then_some(identity.code.as_str()) }) .collect(); if !codes.is_empty() { diff --git a/crates/sqlbuild-rules/src/fix_verification/models.rs b/crates/sqlbuild-rules/src/fix_verification/models.rs index 3d5cc6dc99..8f0d439363 100644 --- a/crates/sqlbuild-rules/src/fix_verification/models.rs +++ b/crates/sqlbuild-rules/src/fix_verification/models.rs @@ -29,13 +29,22 @@ pub struct ModelComparison { pub lineage_equal: bool, } +/// A diagnostic's location-insensitive identity. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct DiagnosticIdentity { + pub code: String, + pub resource_type: String, + pub resource_name: String, + pub message: String, +} + /// Location-insensitive diagnostic multiplicities for one owner. #[derive(Debug)] pub struct DiagnosticComparison { pub owner: usize, pub edited: bool, - pub before: BTreeMap<(String, String, String, String), i64>, - pub after: BTreeMap<(String, String, String, String), i64>, + pub before: BTreeMap, + pub after: BTreeMap, } /// Failures attributed to edited owners and changes outside those owners. diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 9599fb1e2b..43cb230326 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -217,11 +217,20 @@ fn keepends_lengths(text: &str) -> Vec { .collect() } +/// The fields of one native diagnostic, its span in lint-text characters. +struct DiagnosticFields { + code: String, + message: String, + remediation: String, + start: i64, + end: i64, +} + /// The fields of one diagnostic: code, message, remediation, start and end. fn diagnostic_fields( diagnostic: &Map, body_length: i64, -) -> Result<(String, String, String, i64, i64), String> { +) -> Result { let text = |key: &str| { diagnostic .get(key) @@ -242,7 +251,13 @@ fn diagnostic_fields( if start < 0 || end < start || end > body_length { return Err(INVALID_DIAGNOSTIC.to_string()); } - Ok((code, message, remediation, start, end)) + Ok(DiagnosticFields { + code, + message, + remediation, + start, + end, + }) } /// Map one native diagnostic onto authored source, or drop it when it belongs to generated SQL. @@ -255,7 +270,13 @@ pub(crate) fn authored_diagnostic( return Err(NOT_AN_OBJECT.to_string()); }; let body_length: i64 = body.lint_text.chars().count() as i64; - let (code, message, remediation, start, end) = diagnostic_fields(diagnostic, body_length)?; + let DiagnosticFields { + code, + message, + remediation, + start, + end, + } = diagnostic_fields(diagnostic, body_length)?; let reason: Option = match diagnostic.get("fix_unavailable_reason") { None | Some(Value::Null) => None, Some(Value::String(reason)) => Some(reason.clone()), diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs index 36bb34d459..aa80d6d1ab 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/suppressions.rs @@ -54,6 +54,14 @@ fn skippable(line: &str) -> bool { stripped.is_empty() || stripped.starts_with("--") } +/// Where a retained violation sits, its code and whether a native rule reported it. +struct SuppressedTarget<'a> { + file: usize, + line: usize, + code: &'a str, + native: bool, +} + /// Whether a retained violation is the native finding one directive suppresses. pub(crate) fn matches_directive( item: &RetainedViolation, @@ -61,23 +69,28 @@ pub(crate) fn matches_directive( file: usize, directive: &SuppressionDirective, ) -> bool { - let (item_file, line, code, native): (usize, usize, &str, bool) = match item { + let target: SuppressedTarget<'_> = match item { RetainedViolation::Existing(index) => { let violation: &SuppressibleViolation = &violations[*index]; - ( - violation.file, - violation.line, - &violation.code, - violation.native, - ) + SuppressedTarget { + file: violation.file, + line: violation.line, + code: &violation.code, + native: violation.native, + } } - RetainedViolation::Added(finding) => (finding.file, finding.line, SUPPRESSION_CODE, true), + RetainedViolation::Added(finding) => SuppressedTarget { + file: finding.file, + line: finding.line, + code: SUPPRESSION_CODE, + native: true, + }, }; - item_file == file - && line == directive.target - && code == directive.code - && native - && code.starts_with(NATIVE_RULE_PREFIX) + target.file == file + && target.line == directive.target + && target.code == directive.code + && target.native + && target.code.starts_with(NATIVE_RULE_PREFIX) } /// Findings for every directive line when the project forbids suppressions. diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs index f514682f67..4ef2ad23dc 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/preparation.rs @@ -7,7 +7,7 @@ use regex::Regex; use crate::lint_text::main::neutralize_interpolation::neutralize_interpolation; use crate::lint_text::models::Neutralized; -use crate::sql_lint::types::{InterpolationSite, PreparedSql}; +use crate::sql_lint::types::PreparedSql; use sqlbuild_sqltext::sql_scan::main::quote_policy::quote_policy; static CTES: LazyLock> = LazyLock::new(|| { @@ -68,21 +68,7 @@ pub(crate) fn prepare( } let mut referenced: Vec = referenced.into_iter().collect(); referenced.sort(); - let sites: Vec = neutralized - .sites - .into_iter() - .map(|site| { - ( - site.sentinel, - site.neutralized_start, - site.neutralized_end, - site.original_start, - site.original_end, - site.original_text, - ) - }) - .collect(); - Ok(Some((neutralized.text, sites, referenced))) + Ok(Some((neutralized.text, neutralized.sites, referenced))) } /// The byte offset of character position `position`, or `None` past the end. diff --git a/crates/sqlbuild-rules/src/sql_lint/types.rs b/crates/sqlbuild-rules/src/sql_lint/types.rs index 1cb260e2c3..ef7601e9a1 100644 --- a/crates/sqlbuild-rules/src/sql_lint/types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/types.rs @@ -1,2 +1,4 @@ -pub(crate) type InterpolationSite = (String, usize, usize, usize, usize, String); +use crate::lint_text::models::InterpolationSite; + +/// Lint text with sentinels, its interpolation sites and the CTEs interpolation references. pub type PreparedSql = (String, Vec, Vec); diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs index cde7244b9a..3f09e406fa 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/grants.rs @@ -26,12 +26,14 @@ pub(crate) fn unique_grants(index: &ScopeIndex, grants: Vec) -> Vec .collect() } -type GrantKey<'a> = ( - &'a ResourceIdentity, - &'a DeclarationIdentity, - GrantThroughKey<'a>, - GrantKind, -); +/// What makes two grants the same: resource, declaration, what grants it and its kind. +#[derive(PartialEq, Eq, Hash)] +struct GrantKey<'a> { + resource: &'a ResourceIdentity, + declaration: &'a DeclarationIdentity, + through: GrantThroughKey<'a>, + kind: GrantKind, +} #[derive(PartialEq, Eq, Hash)] enum GrantThroughKey<'a> { @@ -40,17 +42,17 @@ enum GrantThroughKey<'a> { } fn grant_key<'a>(index: &'a ScopeIndex, grant: &'a GrantRecord) -> GrantKey<'a> { - ( - &grant.resource, - &index.declarations[grant.declaration].identity, - match &grant.through { + GrantKey { + resource: &grant.resource, + declaration: &index.declarations[grant.declaration].identity, + through: match &grant.through { GrantThrough::Model(name) => GrantThroughKey::Model(name), GrantThrough::Declaration(declaration) => { GrantThroughKey::Declaration(&index.declarations[*declaration].identity) } }, - grant.kind, - ) + kind: grant.kind, + } } /// Visibility and identity lookups over one index, as the Python grant functions use them. diff --git a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs index 0a9859764e..fb3f786713 100644 --- a/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs +++ b/crates/sqlbuild-scopes/src/scope_index/_helpers/ordering.rs @@ -11,14 +11,23 @@ pub(crate) fn resource_key(record: &ResourceRecord) -> (String, String) { (record.path.clone(), resource_text(&record.identity)) } +/// A declaration sort key: identity text, path, then line and column as `P` compares them. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct DeclarationKey

{ + identity: String, + path: String, + line: P, + column: P, +} + /// The builder's `_declaration_key`, which compares line and column as integers. -pub(crate) fn declaration_key(record: &DeclarationRecord) -> (String, String, i64, i64) { - ( - declaration_text(&record.identity), - record.path.clone(), - record.line, - record.column, - ) +pub(crate) fn declaration_key(record: &DeclarationRecord) -> DeclarationKey { + DeclarationKey { + identity: declaration_text(&record.identity), + path: record.path.clone(), + line: record.line, + column: record.column, + } } /// The lookup's `resource_sort_key`. @@ -34,13 +43,13 @@ pub(crate) fn canonical_resource_key( /// The lookup's `declaration_sort_key`, which compares line and column as text. pub(crate) fn canonical_declaration_key( record: &DeclarationRecord, -) -> Result<(String, String, String, String), ScopeDeferral> { - Ok(( - declaration_text(&record.identity), - normalize_path(&record.path)?, - record.line.to_string(), - record.column.to_string(), - )) +) -> Result, ScopeDeferral> { + Ok(DeclarationKey { + identity: declaration_text(&record.identity), + path: normalize_path(&record.path)?, + line: record.line.to_string(), + column: record.column.to_string(), + }) } /// Stable sort of `members` by a fallible key, as Python's `sorted(..., key=...)`. diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs index 50af50a8f9..9a6e8a92eb 100644 --- a/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/declarations.rs @@ -14,7 +14,12 @@ use crate::scope_report::models::{ }; /// `_declaration_container`: role, visibility label, role root and bucket path. -pub(crate) type Container = (Option, &'static str, Option, Option); +pub(crate) struct Container { + pub(crate) role: Option, + pub(crate) visibility: &'static str, + pub(crate) role_root: Option, + pub(crate) bucket: Option, +} pub(crate) fn declaration_text(record: &DeclarationRecord) -> String { format_identity(&QualifiedIdentity::Declaration(record.identity.clone())) @@ -50,7 +55,12 @@ fn visibility_label(scope: ScopeKind) -> &'static str { pub(crate) fn container(record: &DeclarationRecord) -> Container { let visibility: &'static str = visibility_label(record.scope); if record.scope == ScopeKind::Private { - return (None, visibility, None, None); + return Container { + role: None, + visibility, + role_root: None, + bucket: None, + }; } let role_parts: &[&str] = record.identity.kind.role_parts(); let safe: String = safe_scope_path(&record.path); @@ -65,12 +75,12 @@ pub(crate) fn container(record: &DeclarationRecord) -> Container { let role_end: usize = (role_index + role_parts.len()).min(parts.len()); let bucket_end: usize = parts.len().saturating_sub(1).max(role_end); let bucket: String = parts[role_end..bucket_end].join("/"); - ( - Some(role_parts.join("/")), + Container { + role: Some(role_parts.join("/")), visibility, - Some(parts[..role_end].join("/")), - (!bucket.is_empty()).then_some(bucket), - ) + role_root: Some(parts[..role_end].join("/")), + bucket: (!bucket.is_empty()).then_some(bucket), + } } /// The report metadata of one declaration's values and container. @@ -112,7 +122,12 @@ fn metadata(values: Option<&DeclarationValues>, container: Container) -> Vec<(St push("render_as", optional_text(constant.render_as.as_deref())); } } - let (role, visibility, role_root, bucket) = container; + let Container { + role, + visibility, + role_root, + bucket, + } = container; push("role", optional_text(role.as_deref())); push("visibility_label", text(visibility)); push("role_root", optional_text(role_root.as_deref())); diff --git a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs index 07e79ead32..04f4774014 100644 --- a/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs +++ b/crates/sqlbuild-scopes/src/scope_report/_helpers/projection.rs @@ -6,7 +6,9 @@ use crate::scope_index::main::format_identity::format_identity; use crate::scope_index::models::{ DeclarationIdentity, DeclarationRecord, QualifiedIdentity, ResourceIdentity, }; -use crate::scope_report::_helpers::declarations::{container, integer, optional_text, text, texts}; +use crate::scope_report::_helpers::declarations::{ + Container, container, integer, optional_text, text, texts, +}; use crate::scope_report::_helpers::json::{diagnostic_json, object}; use crate::scope_report::_helpers::paths::safe_scope_path; use crate::scope_report::constants::METADATA_SCHEMA_VERSION; @@ -91,7 +93,12 @@ fn declaration_projection( record: &DeclarationRecord, values: Option<&DeclarationValues>, ) -> JsonValue { - let (role, visibility, role_root, bucket) = container(record); + let Container { + role, + visibility, + role_root, + bucket, + } = container(record); object(vec![ ("identity", text(&declaration_key(&record.identity))), ("kind", text(record.identity.kind.as_str())), diff --git a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs index ea40a71473..ba1f16bef2 100644 --- a/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs +++ b/crates/sqlbuild-scopes/src/scope_text/_helpers/results.rs @@ -80,28 +80,47 @@ fn chain_lines( lines } +/// One availability section: its title, declarations, report section name and whether optional. +struct AvailabilityGroup<'a> { + title: &'a str, + declarations: &'a [DeclarationReport], + name: &'a str, + optional: bool, +} + fn availability_lines( (style, request): (CliStyle, &ScopeTextRequest), report: &ScopeReport, used: &HashSet, ) -> Vec { - let groups: [(&str, &[DeclarationReport], &str, bool); 3] = [ - ("Available", &report.available, "available", false), - ( - "Relationship grants", - &report.relationship_scope, - "relationship_scope", - true, - ), - ( - "Nearby unavailable", - &report.nearby_unavailable, - "nearby_unavailable", - true, - ), + let groups: [AvailabilityGroup<'_>; 3] = [ + AvailabilityGroup { + title: "Available", + declarations: &report.available, + name: "available", + optional: false, + }, + AvailabilityGroup { + title: "Relationship grants", + declarations: &report.relationship_scope, + name: "relationship_scope", + optional: true, + }, + AvailabilityGroup { + title: "Nearby unavailable", + declarations: &report.nearby_unavailable, + name: "nearby_unavailable", + optional: true, + }, ]; let mut lines: Vec = Vec::new(); - for (title, declarations, name, optional) in groups { + for AvailabilityGroup { + title, + declarations, + name, + optional, + } in groups + { let section: Option<&ScopeSection> = report_section(report, name); let collapsed_count: i64 = section.map_or(0, |section| section.collapsed_count); if optional && !request.verbose && declarations.is_empty() && collapsed_count == 0 { diff --git a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs index da4557cda7..6d583b0502 100644 --- a/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs +++ b/crates/sqlbuild-sqltext/src/compiler/_helpers/sql_interpolation/substitution.rs @@ -1,7 +1,8 @@ //! `@@name`, `@@ENV:NAME` and `@@CTX:name` interpolation of authored SQL. +use crate::compiler::models::CharSpan; use crate::compiler::models::{InterpolatedSql, InterpolationFailure, InterpolationRead}; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::types::InterpolationHost; use crate::sql_scan::main::non_code_end::non_code_end; use crate::sql_scan::models::{QuotePolicy, Unclosed}; use sqlbuild_core::text::main::is_letter::is_letter; @@ -11,8 +12,13 @@ const TOKEN: &str = "@@"; const ENVIRONMENT_PREFIX: &str = "ENV:"; const CONTEXT_PREFIX: &str = "CTX:"; -/// One substitution in byte offsets: `(source start, source end, output start, output end)`. -type ByteSpan = (usize, usize, usize, usize); +/// One substitution in bytes: source start and end, output start and end. +struct ByteSpan { + source_start: usize, + source_end: usize, + output_start: usize, + output_end: usize, +} struct Interpolation<'run, H: InterpolationHost> { host: &'run H, @@ -112,8 +118,12 @@ impl Interpolation<'_, H> { let text = &sql[..limit]; let output_start = self.output.len(); let end = self.rendered_token(text, start)?; - self.spans - .push((start, end, output_start, self.output.len())); + self.spans.push(ByteSpan { + source_start: start, + source_end: end, + output_start, + output_end: self.output.len(), + }); Ok(end) } @@ -256,13 +266,11 @@ fn character_spans(source: &str, output: &str, spans: &[ByteSpan]) -> Vec, - pub spans: Vec, + pub spans: Vec, pub reads: Vec, } diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs index a8225ae81c..0326016fed 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/helpers.rs @@ -4,12 +4,12 @@ use crate::compiler::_helpers::model_headers::tokenization::{ use crate::compiler::main::declaration_references::scan_declaration_references; use crate::compiler::main::model_header_single_parsing::parse_one; use crate::compiler::main::sql_interpolation::interpolate_sql; +use crate::compiler::models::CharSpan; use crate::compiler::models::{ AuthoredValue, DeclarationReferenceKind, DeclarationReferenceScan, DeclarationReferenceStop, InterpolationRead, }; use crate::compiler::tests::test_types::{HeaderNestingTestCase, MapHost}; -use crate::compiler::types::CharSpan; use std::collections::BTreeMap; use std::thread; @@ -251,13 +251,22 @@ pub(crate) fn interpolated(host: &MapHost, sql: &str) -> Result pub(crate) fn interpolation_facts( host: &MapHost, sql: &str, -) -> (Vec, Vec) { +) -> (Vec<[usize; 4]>, Vec) { interpolate_sql(host, sql, "models/orders.sql").map_or_else( |failure| (Vec::new(), failure.reads), - |result| (result.spans, result.reads), + |result| (result.spans.iter().map(span_parts).collect(), result.reads), ) } +fn span_parts(span: &CharSpan) -> [usize; 4] { + [ + span.source_start, + span.source_end, + span.output_start, + span.output_end, + ] +} + /// Named locations from `[line, column, end_line, end_column]` rows. pub(crate) fn locations( columns: Vec<(&str, [usize; 4])>, diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs index 7a2eafc102..308d2501e4 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_sql_interpolation.rs @@ -136,10 +136,10 @@ fn given_tokens_when_interpolating_then_spans_count_code_points_and_reads_keep_o description: "variables, environment, context and a bare @@ after non-ASCII text", sql: "SELECT 'ü', @@größe, @@ENV:REGION, @@CTX:this @@", expected_spans: vec![ - (12, 19, 12, 17), - (21, 33, 19, 21), - (35, 45, 23, 29), - (46, 48, 30, 32), + [12, 19, 12, 17], + [21, 33, 19, 21], + [35, 45, 23, 29], + [46, 48, 30, 32], ], expected_reads: vec![ InterpolationRead::Environment("REGION".to_owned()), diff --git a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs index 319db3016a..d116ba54d6 100644 --- a/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/tests/test_types.rs @@ -1,5 +1,5 @@ use crate::compiler::models::{InterpolationRead, NestingFailure}; -use crate::compiler::types::{CharSpan, InterpolationHost}; +use crate::compiler::types::InterpolationHost; use std::collections::BTreeMap; pub(crate) struct ModelHeaderTokenizationTestCase { @@ -51,7 +51,7 @@ pub(crate) struct InterpolationTestCase { pub(crate) struct InterpolationFactsTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, - pub(crate) expected_spans: Vec, + pub(crate) expected_spans: Vec<[usize; 4]>, pub(crate) expected_reads: Vec, } diff --git a/crates/sqlbuild-sqltext/src/compiler/types.rs b/crates/sqlbuild-sqltext/src/compiler/types.rs index a16c3efdff..95b3a0bf9d 100644 --- a/crates/sqlbuild-sqltext/src/compiler/types.rs +++ b/crates/sqlbuild-sqltext/src/compiler/types.rs @@ -1,6 +1,3 @@ -/// One substitution in code points: `(source start, source end, output start, output end)`. -pub type CharSpan = (usize, usize, usize, usize); - /// The project variables, process environment and context values one interpolation reads. pub trait InterpolationHost { /// The text of project variable `name`, `Err(message)` when it cannot be text, or `None`. diff --git a/fensu.toml b/fensu.toml index 9394b3890e..e762cf3943 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1151,6 +1151,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/invocation.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouse_scope.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/warehouses.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/compile_attachment/sql_expansion_context.rs=15", @@ -1177,11 +1178,11 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/python_plan_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/relation_targets.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/target_defaults.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_loop.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/audit_views.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_analysis.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/compiled_model_view.rs=16", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_loop.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/function_views.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/inputs_view.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_input_view.rs=20", @@ -1198,11 +1199,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/model_loop_visibility.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/project_view.rs=27", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/resource_diagnostics.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/scenario_loop.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/seed_views.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_loop.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/source_views.rs=9", - "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/compiled_project/sql_test_loop.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/contract_command/write.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/contracts/effective_config.rs=1", @@ -1227,6 +1229,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/log_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/diagnostics/trackers.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/discovery/declaration_records.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/faults.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/discovery/header_declarations.rs=1", @@ -1259,9 +1262,10 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_old_names.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_phases.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=4", + "crates/sqlbuild-python/src/bindings/_helpers/lineage/column_lineage_graph.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_index.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/lineage/lineage_objects.rs=16", + "crates/sqlbuild-python/src/bindings/_helpers/load/dlt_destination.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/load/load_command.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/load/load_progress.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/load/load_selection.rs=4", @@ -1275,6 +1279,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/plan_command/plan.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/plan_command/rules.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/plan_output/cursor_details.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/plan_output/plan_json.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_changes/detect.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/clone_plans.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_entries/command_plans.rs=2", @@ -1288,7 +1293,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/model_entries.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/models.rs=66", "crates/sqlbuild-python/src/bindings/_helpers/planner_migrations/state_inspection.rs=11", - "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/models.rs=41", "crates/sqlbuild-python/src/bindings/_helpers/planner_phases/retention.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/cursor_ranges.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/planner_resolution/models.rs=7", @@ -1304,10 +1309,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/fingerprint_cache.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/planner_warehouse/snapshot.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/presentation_text/functions.rs=1", + "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/declared_relations.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/models.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/project_assembly/syntax_validation.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=3", + "crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/lineage_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/records/error_classes.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/records/models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/refactor/contract_compare.rs=2", @@ -1320,7 +1327,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/render/macro_arguments.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_classes.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_evaluation.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=26", + "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion.rs=33", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_facts.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_expansion_result.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/render/macro_generated_references.rs=6", @@ -1342,12 +1349,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/rules/rows.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/rules/rule_dispatch.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/rules/sql_rules.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=6", + "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/presentation.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/setup.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/scenario_command/snapshot_sync.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_contexts.rs=12", "crates/sqlbuild-python/src/bindings/_helpers/scopes/declaration_resolution.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_index.rs=8", + "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_reports.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/scopes/scope_tolerant.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/seed_command/command.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/semantic_checks/metadata.rs=4", From 7c7bf79c39c44ad3e1f09b7a3e55025f4c0f8ca8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:10:10 +0100 Subject: [PATCH 11/17] chore: enable fensu dead-code enforcement and remove dead code --- fensu.toml | 1137 ++++++++++++++++- .../_helpers/random_dag_project.py | 153 --- scripts/cold_compile_performance/models.py | 14 - .../_helpers/analysis_corpus/failure_cases.py | 3 - .../_helpers/comparing/capture_text.py | 43 + .../_helpers/coverage/analysis.py | 2 +- .../_helpers/coverage/capture_values.py | 2 +- .../_helpers/coverage/discovery.py | 2 +- .../_helpers/coverage/render.py | 2 +- .../classes/capture_file.py | 92 -- scripts/fensu_policy/constants.py | 14 - .../_helpers/host_roots.py | 54 + scripts/native_module_paths/constants.py | 3 + .../_helpers/capturing/canonical.py | 2 +- src/sqlbuild/adapter/contract/constants.py | 16 - src/sqlbuild/adapters/bigquery/constants.py | 3 - src/sqlbuild/adapters/clickhouse/constants.py | 1 - .../adapters/duckdb/_helpers/__init__.py | 1 - src/sqlbuild/adapters/snowflake/constants.py | 8 - src/sqlbuild/adapters/sqlite/constants.py | 1 - src/sqlbuild/compiler/frontier/__init__.py | 0 src/sqlbuild/compiler/lineage/__init__.py | 0 src/sqlbuild/compiler/migrations/__init__.py | 0 src/sqlbuild/compiler/planner/__init__.py | 0 src/sqlbuild/executor/clone/types.py | 4 - src/sqlbuild/executor/python_nodes/models.py | 10 - .../dagster/_helpers/translation.py | 12 - .../integrations/dagster/constants.py | 15 - .../_helpers/_test_types.py | 7 + .../native_module_paths/_helpers/helpers.py | 3 + .../_helpers/test_native_host.py | 25 +- .../compiler/frontier/test_stage_capture.py | 2 +- 32 files changed, 1249 insertions(+), 382 deletions(-) delete mode 100644 scripts/cold_compile_performance/_helpers/random_dag_project.py create mode 100644 scripts/compiler_differential/_helpers/comparing/capture_text.py delete mode 100644 scripts/compiler_differential/classes/capture_file.py create mode 100644 scripts/native_module_paths/_helpers/host_roots.py delete mode 100644 src/sqlbuild/adapters/duckdb/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/frontier/__init__.py delete mode 100644 src/sqlbuild/compiler/lineage/__init__.py delete mode 100644 src/sqlbuild/compiler/migrations/__init__.py delete mode 100644 src/sqlbuild/compiler/planner/__init__.py diff --git a/fensu.toml b/fensu.toml index e762cf3943..4aa8804da8 100644 --- a/fensu.toml +++ b/fensu.toml @@ -21,6 +21,1097 @@ require_cacheable = true [targets.python.evaluation] exclude = ["tests/e2e/fixtures/**"] +[targets.python.dead_code] +enabled = true + +[[targets.python.dead_code.roots]] +modules = ["scripts.*"] +symbols = ["main"] +reason = "Repository tooling runs these modules with `python -m scripts.` from the Makefile, CI and pre-commit." + +[[targets.python.dead_code.roots]] +modules = ["scripts.fensu_policy.rules.*"] +symbols = ["*"] +reason = "Fensu loads these custom rule modules from rule_modules in this file." + +[[targets.python.dead_code.roots]] +modules = [ + "sqlbuild.rule_engine._helpers.host.custom_host", + "sqlbuild.rule_engine._helpers.host.custom_host_launch", +] +symbols = ["main"] +reason = "The custom-rule host processes run these modules with `python -m`; native code starts the launcher, which starts the host." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild._native_host"] +symbols = ["*"] +reason = "Native code imports this module by path and reads its attributes by name." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.classes.base_adapter"] +symbols = ["BaseAdapter"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.exceptions"] +symbols = ["AdapterUserError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.models"] +symbols = ["ColumnInfo", "ExpressionInferenceProfile", "RelationInfo", "RelationLookup", "RetentionRequest", "RowDiffSampling", "RowDiffTolerance", "RowDiffTolerances", "TableFreshnessMetadata", "TableFreshnessRequest"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract.types"] +symbols = ["BuiltinAdapter", "FrameworkType", "LifeCycleEventKind", "MigrationTransfer", "RetentionScope", "TableFreshnessStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main._session_warehouse_support"] +symbols = ["adapter_supports_session_warehouse"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.builtins"] +symbols = ["builtin_adapter_classes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.project_adapters"] +symbols = ["discover_project_adapters"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.discovery.main.resolve_adapter"] +symbols = ["resolve_adapter"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main._get_columns_for_inspection"] +symbols = ["get_columns_for_inspection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main._list_relations_for_inspection"] +symbols = ["list_relations_for_inspection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.active_inspection_catalog"] +symbols = ["active_inspection_catalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.inspection_query_sink"] +symbols = ["inspection_query_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.open_inspection_catalog"] +symbols = ["open_inspection_catalog"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.record_inspection_query"] +symbols = ["record_inspection_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.relation_lookup"] +symbols = ["build_relation_lookup"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.resolve_relation_location_qualified_name"] +symbols = ["resolve_relation_location_qualified_name"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.run_bounded_inspections"] +symbols = ["run_bounded_inspections"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.main.run_recorded_inspection_query"] +symbols = ["run_recorded_inspection_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.relations.models"] +symbols = ["InspectionQueryRecord", "StatementMetadataEffect"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.conditional_result_nullability"] +symbols = ["conditional_result_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.first_arg_nullability"] +symbols = ["first_arg_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.normalize_relation_type"] +symbols = ["normalize_relation_type"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.normalize_type"] +symbols = ["normalize_type"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.type_system.main.safe_cast_nullability"] +symbols = ["safe_cast_nullability"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"] +symbols = ["dispatch_with_compute_logs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse._helpers.provider_settings"] +symbols = ["provider_settings_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse._helpers.startup_inputs"] +symbols = ["startup_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cli.compile_reuse.models"] +symbols = ["CompileReuseRequest"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.auditing.types"] +symbols = ["AuditEvaluationMode", "AuditOutcome", "AuditSeverity", "ThresholdOperator"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.diagnostics.python_resources"] +symbols = ["python_resource_rows"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.macro_bridge.call_store"] +symbols = ["target_context_with_macro_call_store"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile._helpers.render.user_macros"] +symbols = ["inventory_project_macros", "load_project_macros", "macro_invocation_context"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile.types"] +symbols = ["AttachedAuditTargetKind", "FunctionLanguage", "SqlTestMode"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks"] +symbols = ["discover_command_output_sink_functions"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.filesystem.project_modules"] +symbols = ["import_python_root_module", "install_python_root_package", "load_materialize_function", "load_provider_module", "load_python_node_module", "load_sink_module", "module_functions", "module_provider_classes", "parameter_annotations", "provider_usages", "requires_arguments", "restore_python_root_modules", "source_line", "validate_event_exporter_signature"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery._helpers.validation.discovery"] +symbols = ["validate_discovered_inputs"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.discovery.main.runtime_extensions"] +symbols = ["discover_runtime_extensions"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.classes.macro_bridge"] +symbols = ["run_with_macro_bridge"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.main._macro_store_module_paths"] +symbols = ["macro_store_module_paths"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.macro_bridge.main.active_macro_bridge"] +symbols = ["active_macro_bridge"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.pipeline._helpers.materializations"] +symbols = ["load_custom_materializations"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes._helpers.user_inventory"] +symbols = ["python_node_dependency_key", "python_node_identity"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes.main._hook_identities"] +symbols = ["build_hook_identities"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.python_nodes.types"] +symbols = ["PythonNodeKind", "PythonNodeStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.references.models"] +symbols = ["ProjectRelation", "ProjectRelationIndex", "RelationName"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cost.models"] +symbols = ["CostResourceContext", "RunCostSummary"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cost.types"] +symbols = ["CostCapability", "CostStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cursor_algebra.exceptions"] +symbols = ["CursorAlgebraError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.cursor_algebra.types"] +symbols = ["BoundSentinel"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics._helpers.process_resources"] +symbols = ["read_process_resources"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.constants"] +symbols = ["SQL_PRIVATE_RECORD_MARKER"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.diagnostics_context"] +symbols = ["diagnostics_context"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.log_debug_event"] +symbols = ["log_debug_event"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.log_sql"] +symbols = ["log_sql"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.main.process_resource_reporting"] +symbols = ["process_resource_reporting"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.diagnostics.models"] +symbols = ["ProcessResourceUsage"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.errors.contracts.exceptions"] +symbols = ["ExecutorInputError", "SharedInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.main._current_audit_result_projection"] +symbols = ["current_audit_result_projection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.auditing.models"] +symbols = ["AuditExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.build.models"] +symbols = ["BuildCallbacks", "BuildCustomizations", "BuildExecutionResult", "BuildInitialState", "BuildRuntimeParams", "FunctionExecutionResult", "SeedExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.build.types"] +symbols = ["BuildStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._build_destination_transient_models"] +symbols = ["build_destination_transient_models"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._build_retention_requests"] +symbols = ["build_destination_retention_requests"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._fingerprinting"] +symbols = ["copy_clone_fingerprints"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._run_prephase_clone_stream"] +symbols = ["run_prephase_clone_stream"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main.execute"] +symbols = ["execute_clone"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.models"] +symbols = ["CloneExecutionInput", "CloneItemResult", "CloneSourceEntries"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.types"] +symbols = ["CloneStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.classes.query_artifact_lifecycle"] +symbols = ["QueryDiffArtifactLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.exceptions"] +symbols = ["FullDiffSizeGuardError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.main._execute_query"] +symbols = ["execute_query_diff"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.main.execute"] +symbols = ["execute_diff"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.diff.models"] +symbols = ["DiffExecutionOptions", "DiffExecutionResult", "FullDiffSizeLimits", "RowDiffSamplingOverride"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.janitor.main.execute"] +symbols = ["execute_janitor_plan"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.janitor.models"] +symbols = ["JanitorArchiveCandidate", "JanitorArchivedRelation", "JanitorBlockedSchema", "JanitorDeleteCandidate", "JanitorDirectStatePruneCandidate", "JanitorOldNameView", "JanitorOldNameViewPlanning", "JanitorPlan", "JanitorQueryDiffArtifactCandidate", "JanitorRelationKey", "JanitorSkippedRelation", "JanitorSkippedSchema"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.load.main.run"] +symbols = ["run_load_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.load.models"] +symbols = ["LoadCallbacks", "LoadExecutionResult", "LoadRuntimeParams"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.migrations._helpers.relation_for_location"] +symbols = ["migration_relation_for_location"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.old_name_views._helpers.janitor"] +symbols = ["is_compatibility_view"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.auditing"] +symbols = ["run_audit_pipeline_with_callbacks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.scenario"] +symbols = ["run_scenario_capture_pipeline", "run_scenario_local_test_pipeline", "run_scenario_test_pipeline", "select_scenario_snapshot_capture_candidates"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.seeding"] +symbols = ["run_seed_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.testing"] +symbols = ["run_test_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.exceptions"] +symbols = ["AuditExecutionError", "TemporaryTestFunctionError", "WarehouseConnectionError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure"] +symbols = ["raise_warehouse_connection_failure"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.main.run"] +symbols = ["run_build_pipeline"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline.models"] +symbols = ["AuditPipelineCallbacks", "TestPipelineCallbacks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes._helpers.ingress_execution"] +symbols = ["execute_ingress_python_loader_nodes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.classes.run_state"] +symbols = ["PythonNodeRunState"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main._checks"] +symbols = ["execute_python_checks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main._ingress"] +symbols = ["run_ingress_python_loader_nodes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.main.read_side"] +symbols = ["create_read_side_python_execution_tracker"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.python_nodes.models"] +symbols = ["IngressCallbacks", "PythonCheckCallbacks", "PythonCheckExecutionResult", "PythonNodeExecutionResult", "PythonNodeRuntime"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.run.models"] +symbols = ["ModelExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.scenario.models"] +symbols = ["ScenarioCaptureSettings", "ScenarioLocalReplaySource", "ScenarioSnapshotCaptureLimits", "ScenarioTestSettings"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.scheduling.types"] +symbols = ["ExecutionStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.testing.models"] +symbols = ["SqlTestExecutionResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.constants"] +symbols = ["ASSET_SELECTION_COMMANDS", "CHECK_METADATA_EXCLUDED_KEYS", "CLONE_COMMAND", "COMPLETED_EXECUTION_STATUSES", "DAGSTER_ASSET_ENABLED_FIELD", "DAGSTER_ASSET_NODE_KINDS", "DAGSTER_CHECK_ENABLED_FIELD", "DAGSTER_CHECK_NAME_FIELD", "DAGSTER_DIRECT_KIND_NODE_KINDS", "DEFAULT_SELECTABLE_NODE_KINDS", "EXPLICIT_SELECTION_FLAGS", "INSUFFICIENT_CHECK_STATUS", "LIVE_EVENT_COMMANDS", "LOADER_NODE_KIND", "LOAD_SELECTABLE_NODE_KINDS", "MATERIALIZABLE_NODE_KINDS", "MODEL_NODE_KIND", "SCENARIO_CHECK_KIND", "SOURCE_NODE_KIND", "STANDALONE_CHECK_COMMANDS", "SUCCESS_EXECUTION_STATUS", "VIEW_MATERIALIZATION_TYPE", "WARNING_CHECK_SEVERITY"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.exceptions"] +symbols = ["DagsterDagInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt._helpers.profile.load"] +symbols = ["load_dbt_yaml"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt._helpers.profile.render"] +symbols = ["render_selected_dbt_profile_output"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dbt.exceptions"] +symbols = ["DbtProfileError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.constants"] +symbols = ["DLT_INTEGRATION_KIND"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.exceptions"] +symbols = ["DltIntegrationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.main._loaders"] +symbols = ["dlt_loader_function"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dlt.models"] +symbols = ["DltResourceConfig", "DltSourceConfig"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.constants"] +symbols = ["INGESTR_INTEGRATION_KIND"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.exceptions"] +symbols = ["IngestrIntegrationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.main._loaders"] +symbols = ["ingestr_loader_function"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.ingestr.models"] +symbols = ["IngestrSourceConfig"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.rivers.constants"] +symbols = ["RIVERS_ASSET_NODE_KINDS"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.classes.event_codec"] +symbols = ["MicrobatchEventCodec"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.constants"] +symbols = ["MICROBATCH_COLUMNS", "MICROBATCH_COLUMN_TYPES", "MICROBATCH_DIRECT_INDEXES", "MICROBATCH_GENERATION_WILDCARD", "MICROBATCH_TABLE_NAME"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.exceptions"] +symbols = ["MicrobatchStateError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.models"] +symbols = ["MicrobatchCoverageProjection", "MicrobatchInterval", "ProjectedMicrobatchInterval", "ReplayRequirementProjection"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.microbatches.types"] +symbols = ["ReplayRequirementState"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.provider.exceptions"] +symbols = ["ProviderInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.provider.main.session"] +symbols = ["build_provider_session"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.constants"] +symbols = ["RELATION_RENDERER"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_asset_definition"] +symbols = ["read_asset_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_audit_factory_definition"] +symbols = ["read_audit_factory_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_check_definition"] +symbols = ["read_check_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_factory_definition"] +symbols = ["read_factory_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_hook_definition"] +symbols = ["read_hook_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_loader_definition"] +symbols = ["read_loader_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.main.read_task_definition"] +symbols = ["read_task_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.models"] +symbols = ["AuditCase", "SqlResourceRef"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.python_nodes.types"] +symbols = ["PythonCheckSeverity", "SqlResourceRefKind"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.engine._custom_catalogue"] +symbols = ["load_custom_rules"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.engine.custom_payloads"] +symbols = ["custom_rule_payloads"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.classes.text_tee"] +symbols = ["TextComputeLogTee"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.exceptions"] +symbols = ["CaptureAlreadyExistsError", "CaptureNotFoundError", "CaptureStateError", "ComputeLogMetadataError", "ComputeLogPathError", "InvalidCaptureIdError", "InvalidComputeLogCursorError", "InvalidComputeLogLimitError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.models"] +symbols = ["CaptureByteCounts", "CaptureInventory", "CaptureInventoryItem", "CaptureMetadata", "ComputeLogReadChunk", "FinalCaptureMetadata", "PruneResult"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.compute_logs.types"] +symbols = ["ComputeLogStream"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.contracts.models"] +symbols = ["ConnectionHooks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.contracts.types"] +symbols = ["ExecutionResourceKind"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.exceptions"] +symbols = ["EventExporterInputError", "EventExporterStateError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"] +symbols = ["invoke_lifecycle_event_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._run_command_output_sinks"] +symbols = ["run_command_output_sinks"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main.get_lifecycle_event_sink_definition"] +symbols = ["get_lifecycle_event_sink_definition"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.exceptions"] +symbols = ["ExecutionHistoryStorageError", "IntegrityConflictError", "InvalidCursorError", "InvalidEventError", "InvalidFilterError", "InvalidLimitError", "ProjectionConsistencyError", "UnsupportedSchemaVersionError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.models"] +symbols = ["EventPage", "RunPage", "RunRecord", "StoredEvent"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_history.types"] +symbols = ["RunStatus"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.exceptions"] +symbols = ["ExecutionDurationLimitError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main.execution_deadline_scope"] +symbols = ["execution_deadline_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.models"] +symbols = ["RunItemTimeouts"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.event_dispatcher"] +symbols = ["EventDispatcher"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.operation_lifecycle"] +symbols = ["OperationLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.exceptions"] +symbols = ["ObservabilityValidationError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.create_lifecycle_event"] +symbols = ["create_lifecycle_event"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.current_execution_identity"] +symbols = ["current_execution_identity"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.diagnostic_log_to_json"] +symbols = ["diagnostic_log_to_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.dispatcher_scope"] +symbols = ["dispatcher_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.invocation_external_context_scope"] +symbols = ["invocation_external_context_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.invocation_scope"] +symbols = ["invocation_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.lifecycle_event_from_json"] +symbols = ["lifecycle_event_from_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.lifecycle_event_to_json"] +symbols = ["lifecycle_event_to_json"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.main.run_scope"] +symbols = ["run_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.models"] +symbols = ["DiagnosticLog", "LifecycleEvent", "OperationAttributes"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.exceptions"] +symbols = ["CommandOutputValidationError", "OutputCaptureInputError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.main.configured_output_capture_scope"] +symbols = ["configured_output_capture_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.models"] +symbols = ["BoundCommandOutputSink", "CommandOutputCaptureSummary", "CommandOutputRecord", "OutputCaptureContext"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.output_capture.types"] +symbols = ["CommandOutputStream", "OutputRecordPriority"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.sql_values.exceptions"] +symbols = ["SqlValueRenderingError"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.bigquery.classes.bigquery_adapter"] +symbols = ["BigQueryAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.clickhouse.classes.clickhouse_adapter"] +symbols = ["ClickHouseAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.databricks.classes.databricks_adapter"] +symbols = ["DatabricksAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.duckdb.classes.duckdb_adapter"] +symbols = ["DuckDbAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.motherduck.classes.motherduck_adapter"] +symbols = ["MotherDuckAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.postgres.classes.postgres_adapter"] +symbols = ["PostgresAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.snowflake.classes.snowflake_adapter"] +symbols = ["SnowflakeAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.sqlite.classes.sqlite_adapter"] +symbols = ["SqliteAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.sqlserver.classes.sqlserver_adapter"] +symbols = ["SqlServerAdapter"] +reason = "Built-in adapter discovery imports this class by name from BUILTIN_ADAPTER_IMPORTS." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.dense_project"] +symbols = ["dense_model_path", "dense_test_path"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.diagnostic_measurement"] +symbols = ["measure_diagnostic_compile", "measure_function_name_scan", "measure_position_mapping"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.diagnostic_project"] +symbols = ["write_cascade_project", "write_diagnostic_project", "write_function_name_project", "write_single_model_diagnostic_project", "write_undescribed_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_benchmark"] +symbols = ["run_varied_cache_benchmark"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_project"] +symbols = ["write_varied_compile_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance._helpers.varied_statistics"] +symbols = ["varied_project_statistics"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.assert_required_cgroup_memory_limit"] +symbols = ["assert_required_cgroup_memory_limit"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.read_compile_measurement"] +symbols = ["read_compile_measurement"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.main.semantic_compile_fingerprint"] +symbols = ["semantic_compile_fingerprint"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.cold_compile_performance.models"] +symbols = ["VariedProjectStatistics"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.corpus.emitted_codes"] +symbols = ["analysis_error_codes", "case_emitted_codes", "discovery_error_codes", "emitted_failure_codes", "inline_compile_codes", "render_error_codes", "unaccounted_inline_codes"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.analysis"] +symbols = ["analysis_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.discovery"] +symbols = ["discovery_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential._helpers.coverage.render"] +symbols = ["render_capture_problems"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.constants"] +symbols = ["ANALYSIS_UNREACHABLE_CODES", "DISCOVERY_UNREACHABLE_CODES", "RENDER_UNREACHABLE_CODES"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.main.failure_cases"] +symbols = ["failure_cases"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.main.generate_project"] +symbols = ["generate_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.compiler_differential.models"] +symbols = ["EmittedCodes"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series._helpers.outcomes"] +symbols = ["full_compile_difference"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.constants"] +symbols = ["DELTA_COMPILE_LINE", "DOUBLE_CAST", "DOUBLE_COLUMN", "FLOAT_CAST", "FLOAT_COLUMN", "PROJECT_CONFIG"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.run_delta_compile_sequence"] +symbols = ["run_delta_compile_sequence"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.run_delta_compile_series"] +symbols = ["run_delta_compile_series"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.main.write_delta_compile_project"] +symbols = ["write_delta_compile_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.models"] +symbols = ["DeltaCompileOutcome", "DeltaCompileSequence", "DeltaCompileSeries"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.delta_compile_series.types"] +symbols = ["SequenceEdit"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.docs_skill._helpers.navigation"] +symbols = ["list_ordered_page_paths"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.format_performance._helpers.model_project"] +symbols = ["write_model_format_project"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.crates"] +symbols = ["native_module_path_errors", "scan_native_crates"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.host_roots"] +symbols = ["host_dead_code_roots", "host_targets"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths._helpers.scan"] +symbols = ["scan_native_source"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.native_module_paths.models"] +symbols = ["NativeModulePathScan"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.setting_help.main.find_settings_without_snippet"] +symbols = ["find_settings_without_snippet"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["scripts.setting_help.main.parse_setting_snippet"] +symbols = ["parse_setting_snippet"] +reason = "Tests and benchmarks run this tooling; Fensu does not count tests as roots." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.integrations.dagster.constants", "sqlbuild.integrations.rivers.constants"] +symbols = ["*_EXCLUSIONS", "*_MEMBERS"] +reason = "Exhaustiveness guards: tests prove every enum member is either a member or an exclusion of each derived set." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.compiler.compile.types"] +symbols = ["AuditInput", "TypedSqlValueRenderer"] +reason = "src/sqlbuild/_native.pyi annotates native bindings with these protocols; Fensu does not read stubs." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapter.contract._helpers.raw_statement_audit"] +symbols = ["audit_raw_statement_calls"] +reason = "A unit test runs this repository audit over the adapter sources." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.clone.main._prephase_row_from_clone_item"] +symbols = ["prephase_row_from_clone_item"] +reason = "The clone prephase-row tests use this entry point; production calls the helper directly." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.auditing"] +symbols = ["run_audit_pipeline"] +reason = "The audit pipeline tests drive this callback-free form; production calls run_audit_pipeline_with_callbacks." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.executor.pipeline._helpers.scenario"] +symbols = ["_ScenarioPipelineResult"] +reason = "PEP 695 type-parameter bounds in this module use the protocol; Fensu does not count bounds as references." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.rule_engine._helpers.host.module_state"] +symbols = ["module_state_token"] +reason = "The custom-rule host tests compute expected module-state tokens with it." + [targets.python.thresholds] max_positional_args = 1 @@ -50,11 +1141,6 @@ rule = "FFR309" path = "src/sqlbuild/cost/models.py" reason = "Native bindings own run-cost allocation, Snowflake cost collection, the statement ledger and the cost context; this package only declares the cost models and types they use." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/lint/__init__.py" -reason = "Native bindings own lint entry points, expansion preparation and compiler-verified fix planning; this package retains boundary models, custom-rule and macro callbacks." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/cli/compile_reuse/__init__.py" @@ -67,33 +1153,38 @@ reason = "Native bindings own microbatch coverage and replay projection, event I [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/cursor_algebra/__init__.py" -reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." +path = "src/sqlbuild/lint/__init__.py" +reason = "Empty package kept so the tests of the native lint, expansion and fix planning keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/spec/contracts/__init__.py" -reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." +rule = "FFL108" +path = "src/sqlbuild/lint/__init__.py" +reason = "Empty package kept so the tests of the native lint, expansion and fix planning keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/integrations/dbt/__init__.py" -reason = "Native bindings own the dbt interop commands, the dbt runner, its JSON event stream and project initialisation; this package keeps the dbt models, types and errors and the PyYAML and Jinja profile glue they call." +path = "src/sqlbuild/presentation/__init__.py" +reason = "Empty package kept so the tests of the native CLI presentation keep their mirrored location; it declares nothing." + +[[targets.python.rule_exceptions]] +rule = "FFL108" +path = "src/sqlbuild/presentation/__init__.py" +reason = "Empty package kept so the tests of the native CLI presentation keep their mirrored location; it declares nothing." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/frontier/__init__.py" -reason = "Native bindings own the compile frontier and the installed code identity; this package only declares the frontier constants, types and errors they read and raise." +path = "src/sqlbuild/cursor_algebra/__init__.py" +reason = "The native cursor algebra in sqlbuild-cursor owns cursor behavior; this package only declares the cursor value models, sentinels and grain metadata callers import." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/planner/__init__.py" -reason = "Native bindings own planning, plan entries, plan output, command plans and the planner value records; this package only declares the plan models, types, constants and errors the executor, integrations and pipeline import." +path = "src/sqlbuild/spec/contracts/__init__.py" +reason = "Native bindings own configuration resolution; this package only declares the configuration models, constants and errors callers import." [[targets.python.rule_exceptions]] rule = "FFR309" -path = "src/sqlbuild/compiler/migrations/__init__.py" -reason = "Native bindings own migration planning and state rows; this package only declares the migration models, decisions and table names the executor imports." +path = "src/sqlbuild/integrations/dbt/__init__.py" +reason = "Native bindings own the dbt interop commands, the dbt runner, its JSON event stream and project initialisation; this package keeps the dbt models, types and errors and the PyYAML and Jinja profile glue they call." [[targets.python.rule_exceptions]] rule = "FFR309" @@ -115,11 +1206,6 @@ rule = "FFR309" path = "src/sqlbuild/errors/contracts/__init__.py" reason = "Native bindings read expected-error codes, messages and help; this package only declares the shared and executor input errors callers raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/presentation/__init__.py" -reason = "Native bindings own CLI styling, terminal width and the transient line coordinator; this package only declares the display options and presentation types callers pass." - [[targets.python.rule_exceptions]] rule = "FFR309" path = "src/sqlbuild/compiler/dag/types.py" @@ -190,11 +1276,6 @@ rule = "FFR309" path = "src/sqlbuild/sql_values/exceptions.py" reason = "Typed SQL values are normalized and state literals rendered natively; the namespace package keeps the value models, types and the rendering error adapters raise." -[[targets.python.rule_exceptions]] -rule = "FFR309" -path = "src/sqlbuild/compiler/lineage/__init__.py" -reason = "Column lineage is built natively; the package keeps only the lineage models, enums and errors the native graph and its callers share." - [[targets.python.rule_exceptions]] rule = "FFR601" path = "src/sqlbuild/adapter/contract/classes/base_adapter.py" diff --git a/scripts/cold_compile_performance/_helpers/random_dag_project.py b/scripts/cold_compile_performance/_helpers/random_dag_project.py deleted file mode 100644 index 7ecb629896..0000000000 --- a/scripts/cold_compile_performance/_helpers/random_dag_project.py +++ /dev/null @@ -1,153 +0,0 @@ -"""Seeded DuckDB projects whose models bind against shapes inferred from their producers.""" - -from __future__ import annotations - -import random -from collections.abc import Callable -from pathlib import Path - -from scripts.cold_compile_performance.models import RandomDagProject - -_RANDOM_DAG_ROOTS: int = 3 -_RANDOM_DAG_ROOT_SHARE: float = 0.1 -_RANDOM_DAG_FEATURE_SHARE: float = 0.1 -_RANDOM_DAG_WINDOW: int = 12 -_RANDOM_DAG_SOURCES: tuple[tuple[str, tuple[str, ...]], ...] = ( - ("raw_orders", ("id", "quantity", "status")), - ("raw_customers", ("id", "region")), -) - - -def write_random_dag_project(*, project_dir: Path, project: RandomDagProject) -> tuple[str, ...]: - """Write seeded models over inferred shapes; return model names in topological order.""" - - rng: random.Random = random.Random(project.seed) - project_dir.mkdir(parents=True, exist_ok=True) - rules: str = ", ".join(f'"{code}"' for code in project.rules) - (project_dir / "sqlbuild_project.toml").write_text( - f'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = [{rules}]\n', encoding="utf-8" - ) - (project_dir / "sources").mkdir(exist_ok=True) - (project_dir / "sources" / "raw.yml").write_text( - "sources:\n" - " - name: raw_orders\n description: Test source raw_orders.\n" - " expression: SELECT 1 AS id, 2 AS quantity, 'open' AS status\n" - " - name: raw_customers\n description: Test source raw_customers.\n" - " expression: SELECT 1 AS id, 'north' AS region\n", - encoding="utf-8", - ) - models: Path = project_dir / "models" - models.mkdir(exist_ok=True) - columns: dict[str, tuple[str, ...]] = {} - names: list[str] = [] - for index in range(project.model_count): - name: str = f"orders_{index:03d}" - options: str = "" - body: str - output: tuple[str, ...] - if index < _RANDOM_DAG_ROOTS or rng.random() < _RANDOM_DAG_ROOT_SHARE: - source, output = rng.choice(_RANDOM_DAG_SOURCES) - body = f'SELECT * FROM __source("{source}")' - else: - body, output = _random_dag_query( - rng=rng, index=index, names=names, columns=columns, errors=project.errors - ) - if project.run_ids and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - options += ', tags ["${CTX:run.id}"]' - if project.analysis_opt_outs and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - options += ", sql_analysis false" - (models / f"{name}.sql").write_text( - f'MODEL (description "Test model {name}.", materialized view{options});\n{body}\n', - encoding="utf-8", - ) - columns[name] = output - names.append(name) - if project.missing_reference: - (models / "orders_missing.sql").write_text( - 'MODEL (description "Test model orders_missing.", materialized view);\n' - 'SELECT id FROM __ref("orders_absent")\n', - encoding="utf-8", - ) - if project.cycle: - for name, upstream in ( - ("orders_loop_a", "orders_loop_b"), - ("orders_loop_b", "orders_loop_a"), - ): - (models / f"{name}.sql").write_text( - f'MODEL (description "Test model {name}.", materialized view);\n' - f'SELECT id FROM __ref("{upstream}")\n', - encoding="utf-8", - ) - return tuple(names) - - -def _random_dag_query( - *, - rng: random.Random, - index: int, - names: list[str], - columns: dict[str, tuple[str, ...]], - errors: bool, -) -> tuple[str, tuple[str, ...]]: - upstream: str = rng.choice(names[max(0, index - _RANDOM_DAG_WINDOW) :]) - other: str = rng.choice(names) - if errors and rng.random() < _RANDOM_DAG_FEATURE_SHARE: - return ( - f'SELECT id, missing_{index} FROM __ref("{upstream}")', - ("id", f"missing_{index}"), - ) - return rng.choice(_query_builders())( - index=index, upstream=upstream, other=other, columns=columns - ) - - -def _star_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del index, other - return f'SELECT * FROM __ref("{upstream}")', columns[upstream] - - -def _projection_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del other - return ( - f"SELECT id, {columns[upstream][-1]} AS kept_{index}, " - f'CAST(id AS BIGINT) * {index} AS value_{index} FROM __ref("{upstream}")', - ("id", f"kept_{index}", f"value_{index}"), - ) - - -def _join_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - return ( - f"SELECT a.*, b.{columns[other][-1]} AS joined_{index} " - f'FROM __ref("{upstream}") AS a LEFT JOIN __ref("{other}") AS b ON a.id = b.id', - (*columns[upstream], f"joined_{index}"), - ) - - -def _union_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del index, columns - return ( - f'SELECT id FROM __ref("{upstream}") UNION ALL SELECT id FROM __ref("{other}")', - ("id",), - ) - - -def _aggregate_query( - *, index: int, upstream: str, other: str, columns: dict[str, tuple[str, ...]] -) -> tuple[str, tuple[str, ...]]: - del other, columns - return ( - f'SELECT id, COUNT(*) AS row_count_{index} FROM __ref("{upstream}") GROUP BY id', - ("id", f"row_count_{index}"), - ) - - -def _query_builders() -> tuple[Callable[..., tuple[str, tuple[str, ...]]], ...]: - return (_star_query, _projection_query, _join_query, _union_query, _aggregate_query) diff --git a/scripts/cold_compile_performance/models.py b/scripts/cold_compile_performance/models.py index 566a6ca7d8..9118eaf908 100644 --- a/scripts/cold_compile_performance/models.py +++ b/scripts/cold_compile_performance/models.py @@ -13,17 +13,3 @@ class VariedProjectStatistics: leaf_models: int array_models: int window_models: int - - -@dataclass(frozen=True) -class RandomDagProject: - """One seeded DuckDB project whose models bind against inferred upstream shapes.""" - - seed: int - model_count: int - errors: bool = False - run_ids: bool = False - analysis_opt_outs: bool = False - missing_reference: bool = False - cycle: bool = False - rules: tuple[str, ...] = () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py index 63172895f0..8f95cc214a 100644 --- a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -54,9 +54,6 @@ _EXPRESSION_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' _EXPRESSION_AUDIT_PATH: str = "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql" _TEST_PATH: str = "tests/unit/test_stg_orders.sql" -_ORDER_ROW: str = ( - " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status\n" -) def _mart(body: str) -> dict[str, str]: diff --git a/scripts/compiler_differential/_helpers/comparing/capture_text.py b/scripts/compiler_differential/_helpers/comparing/capture_text.py new file mode 100644 index 0000000000..a783261382 --- /dev/null +++ b/scripts/compiler_differential/_helpers/comparing/capture_text.py @@ -0,0 +1,43 @@ +"""Expand the shared nodes of a stage capture document.""" + +from __future__ import annotations + +import json + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest +from scripts.compiler_differential.constants import ( + STAGE_CAPTURE_ROOT_KEY, + STAGE_CAPTURE_SHARED_NODES_KEY, +) + +_SHARED_DOCUMENT_KEYS: frozenset[str] = frozenset( + {STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_ROOT_KEY} +) + + +def expand_capture_text(text: str) -> object: + """Parse a whole capture document, replacing every shared reference by its node.""" + + document: object = json.loads(text) + mapping: dict[str, object] | None = as_json_object(document) + nodes: dict[str, object] | None = as_json_object( + None if mapping is None else mapping.get(STAGE_CAPTURE_SHARED_NODES_KEY) + ) + if mapping is None or nodes is None or set(mapping) != _SHARED_DOCUMENT_KEYS: + return document + expanded: dict[str, object] = {} + for digest, node in nodes.items(): + expanded[digest] = _expanded(value=node, nodes=expanded) + return _expanded(value=mapping[STAGE_CAPTURE_ROOT_KEY], nodes=expanded) + + +def _expanded(*, value: object, nodes: dict[str, object]) -> object: + digest: str | None = shared_digest(value) + if digest is not None: + return nodes[digest] + items: dict[str, object] | None = as_json_object(value) + if items is not None: + return {key: _expanded(value=item, nodes=nodes) for key, item in items.items()} + if isinstance(value, list): + return [_expanded(value=item, nodes=nodes) for item in value] + return value diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py index 5116c308c5..d7a532970f 100644 --- a/scripts/compiler_differential/_helpers/coverage/analysis.py +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -7,6 +7,7 @@ from collections.abc import Iterator, Sequence from pathlib import Path +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -15,7 +16,6 @@ has_content, records, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( ANALYSIS_CALLABLE_CAPTURE_FIELDS, ANALYSIS_DETAIL_KINDS, diff --git a/scripts/compiler_differential/_helpers/coverage/capture_values.py b/scripts/compiler_differential/_helpers/coverage/capture_values.py index 16d29d8956..12744bf938 100644 --- a/scripts/compiler_differential/_helpers/coverage/capture_values.py +++ b/scripts/compiler_differential/_helpers/coverage/capture_values.py @@ -5,8 +5,8 @@ import re from collections.abc import Iterator +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( CALLABLE_MARKER, COLLECTION_CAPTURE_MARKERS, diff --git a/scripts/compiler_differential/_helpers/coverage/discovery.py b/scripts/compiler_differential/_helpers/coverage/discovery.py index da0e2810aa..b6823157d5 100644 --- a/scripts/compiler_differential/_helpers/coverage/discovery.py +++ b/scripts/compiler_differential/_helpers/coverage/discovery.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from pathlib import Path +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -17,7 +18,6 @@ named_values, records, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( CALLABLE_CAPTURE_FIELDS, CRLF_BYTES, diff --git a/scripts/compiler_differential/_helpers/coverage/render.py b/scripts/compiler_differential/_helpers/coverage/render.py index 8f314e0bce..ee553ba958 100644 --- a/scripts/compiler_differential/_helpers/coverage/render.py +++ b/scripts/compiler_differential/_helpers/coverage/render.py @@ -8,6 +8,7 @@ from collections.abc import Iterator from pathlib import Path, PurePosixPath +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.compiler_differential._helpers.comparing.compare import as_json_object from scripts.compiler_differential._helpers.coverage.capture_values import ( capture_problems, @@ -20,7 +21,6 @@ macro_function_source, takes_typed_reference, ) -from scripts.compiler_differential.classes.capture_file import expand_capture_text from scripts.compiler_differential.constants import ( BUILT_IN_AUDIT_NAMES, COMPILE_INPUTS_CAPTURE_TYPE, diff --git a/scripts/compiler_differential/classes/capture_file.py b/scripts/compiler_differential/classes/capture_file.py deleted file mode 100644 index 3017d571fd..0000000000 --- a/scripts/compiler_differential/classes/capture_file.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Read one stage capture from disk node by node instead of parsing it whole.""" - -from __future__ import annotations - -import json -from pathlib import Path - -from scripts.compiler_differential._helpers.comparing.compare import as_json_object, shared_digest -from scripts.compiler_differential._helpers.comparing.normalize import normalize_artifact_text -from scripts.compiler_differential.constants import ( - STAGE_CAPTURE_ROOT_KEY, - STAGE_CAPTURE_SHARED_NODES_KEY, -) - -_SHARED_HEADER: bytes = f"{{{json.dumps(STAGE_CAPTURE_SHARED_NODES_KEY)}: {{\n".encode() -_ROOT_PREFIX: bytes = f"{json.dumps(STAGE_CAPTURE_ROOT_KEY)}: ".encode() -_SHARED_DOCUMENT_KEYS: frozenset[str] = frozenset( - {STAGE_CAPTURE_SHARED_NODES_KEY, STAGE_CAPTURE_ROOT_KEY} -) -_DIGEST_CHARACTERS: int = 64 -_NODE_PREFIX_BYTES: int = _DIGEST_CHARACTERS + len(b'"": ') - - -class CaptureFile: - """A normalized stage capture whose shared nodes are parsed only when resolved.""" - - def __init__(self, path: Path) -> None: - self._path: Path = path - self._offsets: dict[str, int] = {} - self.root: object = self._index() - - def resolve(self, value: object) -> object: - """Return the shared node a reference names, or the value itself.""" - - digest: str | None = shared_digest(value) - return value if digest is None else self._node(digest) - - def _index(self) -> object: - with self._path.open("rb") as stream: - if stream.readline() != _SHARED_HEADER: - return _parse(self._path.read_bytes()) - root: object = None - offset: int = stream.tell() - for line in iter(stream.readline, b""): - if line.startswith(_ROOT_PREFIX): - root = _parse(line[len(_ROOT_PREFIX) :]) - elif line.startswith(b'"') and len(line) > _NODE_PREFIX_BYTES: - self._offsets[line[1 : _DIGEST_CHARACTERS + 1].decode("ascii")] = offset - offset += len(line) - return root - - def _node(self, digest: str) -> object: - offset: int | None = self._offsets.get(digest) - if offset is None: - raise json.JSONDecodeError(f"missing shared node {digest}", str(self._path), 0) - with self._path.open("rb") as stream: - _ = stream.seek(offset) - line: bytes = stream.readline() - return _parse(line[_NODE_PREFIX_BYTES:]) - - -def expand_capture_text(text: str) -> object: - """Parse a whole capture document, replacing every shared reference by its node.""" - - document: object = json.loads(text) - mapping: dict[str, object] | None = as_json_object(document) - nodes: dict[str, object] | None = as_json_object( - None if mapping is None else mapping.get(STAGE_CAPTURE_SHARED_NODES_KEY) - ) - if mapping is None or nodes is None or set(mapping) != _SHARED_DOCUMENT_KEYS: - return document - expanded: dict[str, object] = {} - for digest, node in nodes.items(): - expanded[digest] = _expanded(value=node, nodes=expanded) - return _expanded(value=mapping[STAGE_CAPTURE_ROOT_KEY], nodes=expanded) - - -def _expanded(*, value: object, nodes: dict[str, object]) -> object: - digest: str | None = shared_digest(value) - if digest is not None: - return nodes[digest] - items: dict[str, object] | None = as_json_object(value) - if items is not None: - return {key: _expanded(value=item, nodes=nodes) for key, item in items.items()} - if isinstance(value, list): - return [_expanded(value=item, nodes=nodes) for item in value] - return value - - -def _parse(data: bytes) -> object: - text: str = data.decode("utf-8").rstrip("\n") - return json.loads(normalize_artifact_text(text.removesuffix(","))) diff --git a/scripts/fensu_policy/constants.py b/scripts/fensu_policy/constants.py index f7d6084c08..d199a76cf2 100644 --- a/scripts/fensu_policy/constants.py +++ b/scripts/fensu_policy/constants.py @@ -125,15 +125,6 @@ ADAPTER_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild", "adapter") ADAPTERS_ROOT_PARTS: tuple[str, ...] = ("src", "sqlbuild", "adapters") ADAPTER_PACKAGE_IMPORT_PREFIX: str = "sqlbuild.adapters." -ADAPTER_CONTRACT_CLASS_IMPORT_PREFIX: tuple[str, ...] = ( - "sqlbuild", - "adapter", - "contract", - "classes", -) -ADAPTER_CONTRACT_BASE_CLASS_NAMES: frozenset[str] = frozenset( - {"BaseAdapter", "DuckDbBackedAdapter", "StrictAdapter"} -) ADAPTER_CONTRACT_BASE_SYMBOLS: frozenset[tuple[str, ...]] = frozenset( { ( @@ -185,7 +176,6 @@ PLANNER_PATH_PREFIX: str = "src/sqlbuild/compiler/planner/" POLICY_EVALUATION_SCOPES: frozenset[str] = frozenset({"root", "tooling"}) POLICY_IMPLEMENTATION_PATH_PREFIX: str = "scripts/fensu_policy/" -PROVIDER_CLASS_NAME: str = "Provider" PROVIDER_MODULE_PARTS: tuple[str, ...] = ("src", "sqlbuild", "providers.py") PUBLIC_EXPORT_ASSIGNMENT_TARGETS: tuple[str, ...] = ("__all__",) ROOT_SCOPE_NAME: str = "root" @@ -280,10 +270,6 @@ EVENT_EXPORTER_DECORATOR_NAMES: frozenset[str] = frozenset( {"lifecycle_event_sink", "command_output_sink"} ) -OBSERVABILITY_MODULE_PREFIXES: tuple[tuple[str, ...], ...] = ( - ("sqlbuild", "observability"), - ("sqlbuild", "runtime", "observability"), -) OBSERVABILITY_LIFECYCLE_EVENT_MODULES: tuple[tuple[str, ...], ...] = ( ("sqlbuild", "observability"), ("sqlbuild", "runtime", "observability", "models"), diff --git a/scripts/native_module_paths/_helpers/host_roots.py b/scripts/native_module_paths/_helpers/host_roots.py new file mode 100644 index 0000000000..7c425af671 --- /dev/null +++ b/scripts/native_module_paths/_helpers/host_roots.py @@ -0,0 +1,54 @@ +"""Fensu dead-code roots that keep the native host table's Python targets alive.""" + +import ast +import importlib.util +import tomllib +from importlib.machinery import ModuleSpec +from pathlib import Path +from typing import Any + +import sqlbuild._native_host as native_host +from scripts.native_module_paths.constants import ( + HOST_ROOT_REASON, + MODULE_FILE_ATTRIBUTE, + PYTHON_SUFFIX, +) + + +def defining_target(*, target: tuple[str, str]) -> tuple[str, str]: + """Follow `from module import name as name` re-exports to the module that defines a target.""" + module, attribute = target + spec: ModuleSpec | None = importlib.util.find_spec(module) + origin: str | None = None if spec is None else spec.origin + if origin is None or not origin.endswith(PYTHON_SUFFIX): + return target + for node in ast.parse(Path(origin).read_text(encoding="utf-8")).body: + if not isinstance(node, ast.ImportFrom) or node.level != 0 or node.module is None: + continue + for alias in node.names: + if (alias.asname or alias.name) == attribute: + return defining_target(target=(node.module, alias.name)) + return target + + +def host_targets() -> set[tuple[str, str]]: + """The defining module and symbol of every host table entry that names a symbol.""" + targets: set[tuple[str, str]] = set() + for target in native_host.host_targets().values(): + if target[1] != MODULE_FILE_ATTRIBUTE: + targets.add(defining_target(target=target)) + return targets + + +def host_dead_code_roots(*, config: Path) -> set[tuple[str, str]]: + """Every module and symbol the Fensu dead-code roots in `config` keep for the host table.""" + parsed: dict[str, Any] = tomllib.loads(config.read_text(encoding="utf-8")) + roots: list[dict[str, Any]] = parsed["targets"]["python"]["dead_code"]["roots"] + found: set[tuple[str, str]] = set() + for root in roots: + if root["reason"] != HOST_ROOT_REASON: + continue + for module in root["modules"]: + for symbol in root["symbols"]: + found.add((module, symbol)) + return found diff --git a/scripts/native_module_paths/constants.py b/scripts/native_module_paths/constants.py index 192e7b5ecf..407caf0bd7 100644 --- a/scripts/native_module_paths/constants.py +++ b/scripts/native_module_paths/constants.py @@ -34,3 +34,6 @@ ) HOST_MODULE: str = "sqlbuild._native_host" HOST_ENTRY: re.Pattern[str] = re.compile(r"\bHostEntry\s*\{\s*name:\s*" + ATTRIBUTE_NAME) +HOST_ROOT_REASON: str = "Native code resolves these through the sqlbuild._native_host name table." +MODULE_FILE_ATTRIBUTE: str = "__file__" +PYTHON_SUFFIX: str = ".py" diff --git a/scripts/planner_differential/_helpers/capturing/canonical.py b/scripts/planner_differential/_helpers/capturing/canonical.py index 927fb20359..051a6d8013 100644 --- a/scripts/planner_differential/_helpers/capturing/canonical.py +++ b/scripts/planner_differential/_helpers/capturing/canonical.py @@ -6,7 +6,7 @@ import re from datetime import date, timedelta -from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from scripts.planner_differential.constants import ( COMPACT_TIMESTAMP_PATTERN, DATE_PATTERN, diff --git a/src/sqlbuild/adapter/contract/constants.py b/src/sqlbuild/adapter/contract/constants.py index 2f323717dd..0a895325e8 100644 --- a/src/sqlbuild/adapter/contract/constants.py +++ b/src/sqlbuild/adapter/contract/constants.py @@ -1,21 +1,5 @@ """Adapter constants.""" -INTEGER_TYPE_NAMES: frozenset[str] = frozenset( - {"TINYINT", "SMALLINT", "INT", "INTEGER", "BIGINT", "LONG", "INT64"} -) -DECIMAL_TYPE_NAMES: frozenset[str] = frozenset( - {"DECIMAL", "BIGDECIMAL", "NUMERIC", "NUMBER", "BIGNUMERIC"} -) -FLOAT_TYPE_NAMES: frozenset[str] = frozenset({"DOUBLE", "FLOAT", "FLOAT64", "REAL"}) -STRING_TYPE_NAMES: frozenset[str] = frozenset({"VARCHAR", "CHAR", "CHARACTER", "TEXT", "STRING"}) -BOOLEAN_TYPE_NAMES: frozenset[str] = frozenset({"BOOLEAN", "BOOL"}) -TIMESTAMP_TYPE_NAMES: frozenset[str] = frozenset( - {"TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMPNTZ", "TIMESTAMPLTZ"} -) -DATE_TYPE_NAME: str = "DATE" -DATETIME_TYPE_NAME: str = "DATETIME" -POLYGLOT_CUSTOM_TYPE_NAME: str = "CUSTOM" -TIMESTAMP_TYPE_TOKEN: str = "TIMESTAMP" INTEGER_TYPE_TOKEN: str = "INT" TYPED_OBJECT_ENTRY_PART_COUNT: int = 2 TABLE_RELATION_TYPE_NAMES: frozenset[str] = frozenset( diff --git a/src/sqlbuild/adapters/bigquery/constants.py b/src/sqlbuild/adapters/bigquery/constants.py index d3024011fb..498d176b89 100644 --- a/src/sqlbuild/adapters/bigquery/constants.py +++ b/src/sqlbuild/adapters/bigquery/constants.py @@ -2,15 +2,12 @@ BOOLEAN_METADATA_TYPE_NAME: str = "BOOLEAN" BOOLEAN_WIRE_TYPE_NAME: str = "BOOL" -BIGNUMERIC_TYPE_NAME: str = "BIGNUMERIC" -CUSTOM_NORMALIZATION_TYPE_NAMES: frozenset[str] = frozenset({"INT64", "FLOAT64", "BIGNUMERIC"}) DATE_TYPE_NAME: str = "DATE" DATETIME_TYPE_NAME: str = "DATETIME" DECIMAL_METADATA_TYPE_NAMES: frozenset[str] = frozenset({"NUMERIC", "BIGNUMERIC"}) FLOAT_METADATA_TYPE_NAME: str = "FLOAT" FLOAT_WIRE_TYPE_NAME: str = "FLOAT64" INTEGER_METADATA_TYPE_NAME: str = "INTEGER" -INTEGER_PARSE_TYPE_NAMES: frozenset[str] = frozenset({"INT", "BIGINT"}) INTEGER_TYPE_TOKEN: str = "INT" INTEGER_WIRE_TYPE_NAME: str = "INT64" NOT_FOUND_ERROR_CLASS_NAME: str = "NotFound" diff --git a/src/sqlbuild/adapters/clickhouse/constants.py b/src/sqlbuild/adapters/clickhouse/constants.py index 7dce598a92..7b9047e237 100644 --- a/src/sqlbuild/adapters/clickhouse/constants.py +++ b/src/sqlbuild/adapters/clickhouse/constants.py @@ -40,7 +40,6 @@ ) VIEW_ENGINES: frozenset[str] = frozenset({"View", "MaterializedView", "LiveView", "WindowView"}) RELATION_NOT_FOUND_CODES: frozenset[int] = frozenset({60, 81}) -ATOMIC_DATABASE_ENGINES: frozenset[str] = frozenset({"Atomic", "Replicated"}) NULLABLE_WRAPPER: str = "Nullable" TYPE_WRAPPERS: tuple[str, ...] = ("Nullable", "LowCardinality") TYPE_ALIASES: dict[str, str] = { diff --git a/src/sqlbuild/adapters/duckdb/_helpers/__init__.py b/src/sqlbuild/adapters/duckdb/_helpers/__init__.py deleted file mode 100644 index d39eda62bb..0000000000 --- a/src/sqlbuild/adapters/duckdb/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""DuckDB adapter helpers.""" diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 3816e406b6..c816a0575e 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -10,17 +10,9 @@ SECONDARY_ROLES_ALL: str = "ALL" SECONDARY_ROLES_NONE: str = "NONE" NUMBER_TYPE_NAME: str = "NUMBER" -DEFAULT_TEXT_LENGTH: int = 16_777_216 -INTEGER_PRECISION: int = 38 -INTEGER_SCALE: int = 0 -NORMALIZED_LTZ_INPUT_TYPE_NAME: str = "TIMESTAMPLTZ" -NORMALIZED_NTZ_INPUT_TYPE_NAMES: frozenset[str] = frozenset({"TIMESTAMP", "TIMESTAMPNTZ"}) -NORMALIZED_TZ_INPUT_TYPE_NAME: str = "TIMESTAMPTZ" STATUS_COLUMN_NAME: str = "status" SUCCESS_STATUS_TOKENS: tuple[str, ...] = ("success", "created", "dropped") TEXT_TYPE_NAMES: frozenset[str] = frozenset({"TEXT", "VARCHAR", "CHAR", "CHARACTER"}) -TEXT_TYPE_NAME: str = "TEXT" -UNBOUNDED_TEXT_TYPE_NAMES: frozenset[str] = frozenset({"STRING", "TEXT", "VARCHAR"}) TRUE_METADATA_VALUE: str = "YES" SHOW_LIKE_ESCAPE_CHARACTER: str = "\\" CURRENT_DATABASE_ATTRIBUTE: str = "_sqlbuild_current_database" diff --git a/src/sqlbuild/adapters/sqlite/constants.py b/src/sqlbuild/adapters/sqlite/constants.py index dcebc49096..8b35836965 100644 --- a/src/sqlbuild/adapters/sqlite/constants.py +++ b/src/sqlbuild/adapters/sqlite/constants.py @@ -15,7 +15,6 @@ LOCKED_ERROR_FRAGMENTS: tuple[str, ...] = ("database is locked", "database table is locked") MISSING_RELATION_ERROR_FRAGMENT: str = "no such table" READ_STATEMENT_KEYWORDS: frozenset[str] = frozenset({"SELECT", "PRAGMA", "EXPLAIN", "VALUES"}) -TRANSACTION_END_KEYWORDS: frozenset[str] = frozenset({"COMMIT", "END", "ROLLBACK"}) WRITE_GATE_POLL_SECONDS: float = 0.1 TYPEOF_TYPE_NAMES: dict[str, str] = { "integer": "INTEGER", diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/lineage/__init__.py b/src/sqlbuild/compiler/lineage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/migrations/__init__.py b/src/sqlbuild/compiler/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/compiler/planner/__init__.py b/src/sqlbuild/compiler/planner/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/executor/clone/types.py b/src/sqlbuild/executor/clone/types.py index 9f30a97f63..2a9606126d 100644 --- a/src/sqlbuild/executor/clone/types.py +++ b/src/sqlbuild/executor/clone/types.py @@ -19,10 +19,6 @@ def __call__( ) -> None: ... -class CloneFingerprintProgressReporter(CloneFingerprintProgressCallback, Protocol): - def write_interrupted(self) -> None: ... - - class CloneStatus(StrEnum): SUCCESS = "success" WARNING = "warning" diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 1ff25a8a67..4237e12f99 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -112,16 +112,6 @@ class PythonNodeFanInDecision: skip_mode: SkipMode | None = None -@dataclass(frozen=True) -class CursorWindow: - """Resolved cursor bounds for one Python execution run.""" - - start_cursor_ts: datetime | None = None - end_cursor_ts: datetime | None = None - start_cursor_int: int | None = None - end_cursor_int: int | None = None - - from sqlbuild.executor.python_nodes.classes.run_state import PythonNodeRunState # noqa: E402 diff --git a/src/sqlbuild/integrations/dagster/_helpers/translation.py b/src/sqlbuild/integrations/dagster/_helpers/translation.py index 4a1997d92e..a8dff56731 100644 --- a/src/sqlbuild/integrations/dagster/_helpers/translation.py +++ b/src/sqlbuild/integrations/dagster/_helpers/translation.py @@ -41,15 +41,3 @@ def translate_sqlbuild_dag( ) return {**dag, "nodes": translated_nodes, "checks": translated_checks} - - -def is_dagster_asset_enabled(node: Mapping[str, Any]) -> bool: - """Return whether a translated DAG node is owned by the SQLBuild asset definition.""" - - return bool(node.get(DAGSTER_ASSET_ENABLED_FIELD, True)) - - -def is_dagster_check_enabled(check: Mapping[str, Any]) -> bool: - """Return whether a translated DAG check is owned by the SQLBuild asset definition.""" - - return bool(check.get(DAGSTER_CHECK_ENABLED_FIELD, True)) diff --git a/src/sqlbuild/integrations/dagster/constants.py b/src/sqlbuild/integrations/dagster/constants.py index d02a1fee47..8afa16ddce 100644 --- a/src/sqlbuild/integrations/dagster/constants.py +++ b/src/sqlbuild/integrations/dagster/constants.py @@ -49,23 +49,9 @@ CLONE_COMMAND: str = "clone" CHECK_COMMAND: str = "check" STANDALONE_CHECK_COMMANDS: frozenset[str] = frozenset({"audit", CHECK_COMMAND, "test"}) -EVENT_OUTPUT_FLAG: str = "--event-output" LIVE_EVENT_COMMANDS: frozenset[str] = ASSET_SELECTION_COMMANDS EXPLICIT_SELECTION_FLAGS: frozenset[str] = frozenset({"--select", "-s", "--select-file"}) -JSON_OUTPUT_FLAGS: frozenset[str] = frozenset({"--json", "--json-output"}) -JSON_OUTPUT_FLAG: str = "--json-output" -SELECT_FILE_FLAG: str = "--select-file" -SCENARIO_TEST_COMMAND: tuple[str, str] = ("scenario", "test") SCENARIO_CHECK_KIND: str = NodeKind.SCENARIO.value -SCENARIO_VALUE_FLAGS: frozenset[str] = frozenset( - { - "--max-snapshot-rows", - "--max-snapshot-total-rows", - "--max-snapshot-bytes", - "--max-snapshot-total-bytes", - } -) -LOAD_COMMAND: str = "load" LOAD_SELECTABLE_NODE_KIND_MEMBERS: frozenset[NodeKind] = frozenset( {NodeKind.SOURCE, NodeKind.LOADER} ) @@ -146,7 +132,6 @@ ) SUCCESS_EXECUTION_STATUS: str = ExecutionStatus.SUCCESS.value FAILED_EXECUTION_STATUS: str = ExecutionStatus.FAILED.value -CHECK_NAME_SEPARATOR_CHARACTER: str = "_" CHECK_METADATA_EXCLUDED_KEYS: frozenset[str] = frozenset( {"passed", "steps", "expected_results", "assertion_results"} ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py index 7040d2fcc8..89d2e334f2 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/_test_types.py +++ b/tests/unit/scripts/native_module_paths/_helpers/_test_types.py @@ -40,3 +40,10 @@ class HostUnknownNameCase: description: str name: str expected_error: str + + +@dataclass(frozen=True) +class HostDeadCodeRootCase: + description: str + expected_missing_targets: tuple[tuple[str, str], ...] + expected_extra_targets: tuple[tuple[str, str], ...] diff --git a/tests/unit/scripts/native_module_paths/_helpers/helpers.py b/tests/unit/scripts/native_module_paths/_helpers/helpers.py index facfc3700e..5f2fe828c8 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/helpers.py +++ b/tests/unit/scripts/native_module_paths/_helpers/helpers.py @@ -1,7 +1,10 @@ +from pathlib import Path + from scripts.native_module_paths._helpers.crates import native_module_path_errors from scripts.native_module_paths._helpers.scan import scan_native_source SOURCE_LABEL: str = "lib.rs" +FENSU_CONFIG: Path = Path(__file__).parents[5] / "fensu.toml" def native_source_errors(source: str) -> tuple[str, ...]: diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py b/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py index 5ff77bc5ee..aa49f0ad8f 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_native_host.py @@ -10,13 +10,15 @@ import sqlbuild._native as native import sqlbuild._native_host as native_host +from scripts.native_module_paths._helpers.host_roots import host_dead_code_roots, host_targets from tests.unit.scripts.native_module_paths._helpers._test_types import ( + HostDeadCodeRootCase, HostImportCase, HostKindCase, HostTableCase, HostUnknownNameCase, ) -from tests.unit.scripts.native_module_paths._helpers.helpers import resolved_kind +from tests.unit.scripts.native_module_paths._helpers.helpers import FENSU_CONFIG, resolved_kind @pytest.mark.parametrize( @@ -115,5 +117,26 @@ def test_given_unknown_name_when_reading_native_host_then_raises_attribute_error assert str(raised.value) == test_case.expected_error +@pytest.mark.parametrize( + "test_case", + [ + HostDeadCodeRootCase( + description="every host target is a dead-code root and every host root a target", + expected_missing_targets=(), + expected_extra_targets=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_native_host_table_when_reading_fensu_roots_then_roots_match_targets( + test_case: HostDeadCodeRootCase, +) -> None: + targets: set[tuple[str, str]] = host_targets() + roots: set[tuple[str, str]] = host_dead_code_roots(config=FENSU_CONFIG) + + assert tuple(sorted(targets - roots)) == test_case.expected_missing_targets + assert tuple(sorted(roots - targets)) == test_case.expected_extra_targets + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py index 86d7bd65ee..616cc76a25 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -8,7 +8,7 @@ import pytest -from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential._helpers.comparing.capture_text import expand_capture_text from sqlbuild._native import ( STAGE_CAPTURE_DIR_ENV_VAR, CompilerStage, From 738e9bfc17145dc9b096d43cd29a4845b2b0fcd0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:53:38 +0100 Subject: [PATCH 12/17] refactor: name large tuple types introduced on main --- .../scenario_command/capture_command.rs | 2 +- .../src/store/main/read_output_transcript.rs | 25 ++++++------------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs index b3ad6ee36e..470ab6f736 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/scenario_command/capture_command.rs @@ -105,7 +105,7 @@ pub(crate) fn run_scenario_capture_command( project_dir: &project.project_dir, adapter: &adapter, kind: CompletionKind::Capture, - style: style, + style, }, )?; let results: Vec> = call_with( diff --git a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs index c21864bb78..44be378a54 100644 --- a/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs +++ b/crates/sqlbuild-recall/src/store/main/read_output_transcript.rs @@ -52,7 +52,7 @@ pub fn read_output_transcript( .optional(), "read Recall invocation", )?; - let completeness: Option<(i64, i64, i64, bool, bool, Option)> = checked( + checked( store .connection .query_row( @@ -60,26 +60,17 @@ pub fn read_output_transcript( gap_reason FROM output_invocations WHERE invocation_id = ?1", [invocation_id], |row| { - Ok(( - row.get(0)?, - row.get(1)?, - row.get(2)?, - row.get(3)?, - row.get(4)?, - row.get(5)?, - )) + transcript.missing_sequences = row.get(0)?; + transcript.reported_drops = row.get(1)?; + transcript.filtered = row.get(2)?; + transcript.flush_complete = row.get(3)?; + transcript.complete = row.get(4)?; + transcript.gap_reason = row.get(5)?; + Ok(()) }, ) .optional(), "read Recall output completeness", )?; - if let Some((missing, drops, filtered, flushed, complete, reason)) = completeness { - transcript.missing_sequences = missing; - transcript.reported_drops = drops; - transcript.filtered = filtered; - transcript.flush_complete = flushed; - transcript.complete = complete; - transcript.gap_reason = reason; - } Ok(transcript) } From 2655063bb600220fccef743cbff24a5326aab1f0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:20:57 +0100 Subject: [PATCH 13/17] chore: satisfy fensu 0.27 in code added on main --- .../src/clickhouse/constants.rs | 52 ++++++++++++++----- fensu.toml | 10 ++++ 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-recall/src/clickhouse/constants.rs b/crates/sqlbuild-recall/src/clickhouse/constants.rs index 46bf41da3d..d4a540e63a 100644 --- a/crates/sqlbuild-recall/src/clickhouse/constants.rs +++ b/crates/sqlbuild-recall/src/clickhouse/constants.rs @@ -6,24 +6,50 @@ use crate::schema::models::Table; pub const BATCH_ROWS: usize = 5_000; /// Longest wait for a batch to fill, so each table gets at most about one part per second per writer. pub const BATCH_WINDOW_MS: u64 = 1_000; +/// ClickHouse `READONLY`. +pub const READONLY_CODE: u64 = 164; +/// ClickHouse `DATABASE_ACCESS_DENIED`. +pub const DATABASE_ACCESS_DENIED_CODE: u64 = 291; +/// ClickHouse `ACCESS_DENIED`. +pub const ACCESS_DENIED_CODE: u64 = 497; +/// ClickHouse `AUTHENTICATION_FAILED`. +pub const AUTHENTICATION_FAILED_CODE: u64 = 516; +/// ClickHouse `TIMEOUT_EXCEEDED`. +pub const TIMEOUT_EXCEEDED_CODE: u64 = 159; +/// ClickHouse `TOO_MANY_SIMULTANEOUS_QUERIES`. +pub const TOO_MANY_SIMULTANEOUS_QUERIES_CODE: u64 = 202; +/// ClickHouse `SOCKET_TIMEOUT`. +pub const SOCKET_TIMEOUT_CODE: u64 = 209; +/// ClickHouse `NETWORK_ERROR`. +pub const NETWORK_ERROR_CODE: u64 = 210; +/// ClickHouse `MEMORY_LIMIT_EXCEEDED`. +pub const MEMORY_LIMIT_EXCEEDED_CODE: u64 = 241; +/// ClickHouse `TABLE_IS_READ_ONLY`. +pub const TABLE_IS_READ_ONLY_CODE: u64 = 242; +/// ClickHouse `TOO_MANY_PARTS`. +pub const TOO_MANY_PARTS_CODE: u64 = 252; +/// ClickHouse `UNKNOWN_STATUS_OF_INSERT`. +pub const UNKNOWN_STATUS_OF_INSERT_CODE: u64 = 319; +/// ClickHouse `SYSTEM_ERROR`. +pub const SYSTEM_ERROR_CODE: u64 = 425; /// Server error codes meaning the user may not create or write the store. pub const PERMISSION_CODES: [u64; 4] = [ - 164, // READONLY - 291, // DATABASE_ACCESS_DENIED - 497, // ACCESS_DENIED - 516, // AUTHENTICATION_FAILED + READONLY_CODE, + DATABASE_ACCESS_DENIED_CODE, + ACCESS_DENIED_CODE, + AUTHENTICATION_FAILED_CODE, ]; /// Server error codes worth retrying with the same rows and deduplication tokens. pub const TRANSIENT_CODES: [u64; 9] = [ - 159, // TIMEOUT_EXCEEDED - 202, // TOO_MANY_SIMULTANEOUS_QUERIES - 209, // SOCKET_TIMEOUT - 210, // NETWORK_ERROR - 241, // MEMORY_LIMIT_EXCEEDED - 242, // TABLE_IS_READ_ONLY - 252, // TOO_MANY_PARTS - 319, // UNKNOWN_STATUS_OF_INSERT - 425, // SYSTEM_ERROR + TIMEOUT_EXCEEDED_CODE, + TOO_MANY_SIMULTANEOUS_QUERIES_CODE, + SOCKET_TIMEOUT_CODE, + NETWORK_ERROR_CODE, + MEMORY_LIMIT_EXCEEDED_CODE, + TABLE_IS_READ_ONLY_CODE, + TOO_MANY_PARTS_CODE, + UNKNOWN_STATUS_OF_INSERT_CODE, + SYSTEM_ERROR_CODE, ]; /// UNKNOWN_DATABASE: Recall never creates databases. pub const UNKNOWN_DATABASE_CODE: u64 = 81; diff --git a/fensu.toml b/fensu.toml index 4aa8804da8..a58a5bdf7f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -167,6 +167,11 @@ modules = ["sqlbuild.adapter.type_system.main.safe_cast_nullability"] symbols = ["safe_cast_nullability"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.adapters.clickhouse._helpers.recall_store"] +symbols = ["close_recall_store", "connect_recall_store", "insert_recall_rows", "run_recall_command", "run_recall_query"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs"] symbols = ["dispatch_with_compute_logs"] @@ -737,6 +742,11 @@ modules = ["sqlbuild.runtime.event_exporting.main._invoke_lifecycle_event_sink"] symbols = ["invoke_lifecycle_event_sink"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.event_exporting.main._invoke_recall_sink"] +symbols = ["invoke_recall_sink"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.runtime.event_exporting.main._run_command_output_sinks"] symbols = ["run_command_output_sinks"] From 2dd43629a59117122465d726799dd22b8871af9d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:55:50 +0100 Subject: [PATCH 14/17] fix(clickhouse): accept ClickHouse Cloud SharedMergeTree engines Shared* MergeTree engines now pass model validation like their MergeTree counterparts. Replicated* engines stay rejected. Support is best effort and untested on ClickHouse Cloud. --- .../_helpers/materialization.rs | 2 +- .../src/model_validation/constants.rs | 4 +- .../tests/test_physical_storage.rs | 38 ++++++++++++++++++- .../docs/concepts/adapters/clickhouse.md | 8 ++-- .../adapters/clickhouse/test_table_layout.py | 12 ++++++ .../docs/concepts/adapters/clickhouse.mdx | 8 ++-- 6 files changed, 61 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs index 0500b3c29b..46bc71a1ad 100644 --- a/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs +++ b/crates/sqlbuild-model-config/src/model_validation/_helpers/materialization.rs @@ -434,7 +434,7 @@ fn check_engine(config: &ConfigView<'_, N>) -> Check { _ => Err(ValidationStop::Error( config .config_error(format!( - "{ENGINE_KEY} '{}' must be a single-server MergeTree-family table engine", + "{ENGINE_KEY} '{}' must be a non-replicated MergeTree-family table engine", value.python_str() )) .with_help(model_header_help( diff --git a/crates/sqlbuild-model-config/src/model_validation/constants.rs b/crates/sqlbuild-model-config/src/model_validation/constants.rs index fdef85e14e..74d56179a8 100644 --- a/crates/sqlbuild-model-config/src/model_validation/constants.rs +++ b/crates/sqlbuild-model-config/src/model_validation/constants.rs @@ -300,5 +300,5 @@ pub const PHYSICAL_STORAGE_KEYS: [&str; 5] = [ pub const PHYSICAL_STORAGE_MATERIALIZATIONS: [&str; 2] = ["table", "incremental"]; /// The name suffix every supported table engine shares. pub const ENGINE_FAMILY_SUFFIX: &str = "MergeTree"; -/// Engine name prefixes of replicated and shared engines, which need a cluster. -pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 2] = ["Replicated", "Shared"]; +/// Engine name prefixes of replicated engines, which need a cluster. +pub const UNSUPPORTED_ENGINE_PREFIXES: [&str; 1] = ["Replicated"]; diff --git a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs index 06366bc6d4..c367ec9999 100644 --- a/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs +++ b/crates/sqlbuild-model-config/src/model_validation/tests/test_physical_storage.rs @@ -50,6 +50,40 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a query_sql: "select 1", expected_outcome: "model 'orders_daily': order_by is only valid for table and incremental models", }, + ValidateTestCase { + description: "a shared engine", + config: table(vec![("engine", Value::Str("SharedMergeTree"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a shared replacing engine with arguments", + config: table(vec![( + "engine", + Value::Str("SharedReplacingMergeTree(updated_at)"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "accepted", + }, + ValidateTestCase { + description: "a replicated replacing engine", + config: table(vec![( + "engine", + Value::Str("ReplicatedReplacingMergeTree(updated_at)"), + )]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'ReplicatedReplacingMergeTree(updated_at)' must be a non-replicated MergeTree-family table engine", + }, + ValidateTestCase { + description: "a shared engine outside the MergeTree family", + config: table(vec![("engine", Value::Str("SharedSet"))]), + references: &[], + query_sql: "select 1", + expected_outcome: "model 'orders_daily': engine 'SharedSet' must be a non-replicated MergeTree-family table engine", + }, ValidateTestCase { description: "a replicated engine", config: table(vec![( @@ -58,14 +92,14 @@ fn given_physical_layout_keys_when_validating_then_shapes_and_materializations_a )]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a single-server MergeTree-family table engine", + expected_outcome: "model 'orders_daily': engine 'ReplicatedMergeTree('/tables/orders', '{replica}')' must be a non-replicated MergeTree-family table engine", }, ValidateTestCase { description: "an engine outside the MergeTree family", config: table(vec![("engine", Value::Str("Log"))]), references: &[], query_sql: "select 1", - expected_outcome: "model 'orders_daily': engine 'Log' must be a single-server MergeTree-family table engine", + expected_outcome: "model 'orders_daily': engine 'Log' must be a non-replicated MergeTree-family table engine", }, ValidateTestCase { description: "a blank sorting expression", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md index b8bdd8f42d..83c01eecc8 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/adapters/clickhouse.md @@ -113,13 +113,15 @@ MODEL ( | Field | Value | Default | |-------|-------|---------| -| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `engine` | A non-replicated MergeTree-family engine, with its arguments | `MergeTree` | | `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | | `partition_by` | A partitioning expression | No partitioning | | `primary_key` | A prefix of `order_by` | The sorting key | | `table_settings` | Setting names mapped to strings, integers or booleans | None | -Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. +Quote expressions that are not plain column names. Compile rejects malformed values, `Replicated*` engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +ClickHouse Cloud's `Shared*` engines (`SharedMergeTree`, `SharedReplacingMergeTree` and the other `SharedMergeTree` variants) are accepted and handled like their `MergeTree` counterparts. Support is best effort: SQLBuild is not tested on ClickHouse Cloud. ClickHouse Cloud already creates a `MergeTree` table as `SharedMergeTree`, so naming the plain engine works there too. The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. @@ -181,7 +183,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## Limits -- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. ClickHouse Cloud and its `Shared*` engines are best effort and untested. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). - Snapshots always use an unsorted `MergeTree` table. diff --git a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py index ff381b1932..3305b4054f 100644 --- a/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py +++ b/tests/unit/src/sqlbuild/adapters/clickhouse/test_table_layout.py @@ -39,6 +39,18 @@ layout={"order_by": "order_id"}, expected_clauses="ENGINE = MergeTree ORDER BY (order_id)", ), + TableLayoutTestCase( + description="a shared engine", + layout={"engine": "SharedMergeTree", "partition_by": "toYYYYMM(ordered_at)"}, + expected_clauses=( + "ENGINE = SharedMergeTree ORDER BY tuple() PARTITION BY toYYYYMM(ordered_at)" + ), + ), + TableLayoutTestCase( + description="a shared replacing engine with its version column", + layout={"engine": "SharedReplacingMergeTree(version)", "order_by": ["customer_id"]}, + expected_clauses="ENGINE = SharedReplacingMergeTree(version) ORDER BY (customer_id)", + ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx index 5aa6becff3..3876b673df 100644 --- a/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx +++ b/website/src/content/docs/docs/concepts/adapters/clickhouse.mdx @@ -100,13 +100,15 @@ MODEL ( | Field | Value | Default | |-------|-------|---------| -| `engine` | A single-server MergeTree-family engine, with its arguments | `MergeTree` | +| `engine` | A non-replicated MergeTree-family engine, with its arguments | `MergeTree` | | `order_by` | A sorting expression, or a list of them | `tuple()` (unsorted) | | `partition_by` | A partitioning expression | No partitioning | | `primary_key` | A prefix of `order_by` | The sorting key | | `table_settings` | Setting names mapped to strings, integers or booleans | None | -Quote expressions that are not plain column names. Compile rejects malformed values, replicated and shared engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. +Quote expressions that are not plain column names. Compile rejects malformed values, `Replicated*` engines, and these fields on views and snapshots; other adapters reject them with `A011` when planning. + +ClickHouse Cloud's `Shared*` engines (`SharedMergeTree`, `SharedReplacingMergeTree` and the other `SharedMergeTree` variants) are accepted and handled like their `MergeTree` counterparts. Support is best effort: SQLBuild is not tested on ClickHouse Cloud. ClickHouse Cloud already creates a `MergeTree` table as `SharedMergeTree`, so naming the plain engine works there too. The fields are part of the model's version, so `sqb plan` reports a changed layout as a config change. A table model is rebuilt with the new layout. An incremental model keeps writing into its existing table with the old layout, and `sqb plan` warns with `PHYSICAL_LAYOUT_PENDING` until a full refresh (`sqb build --full-refresh`, with `--start-cursor-ts` and `--end-cursor-ts` for a cursor model) rebuilds the table with the new layout. @@ -168,7 +170,7 @@ ClickHouse returns a type's default value (`0`, the epoch, an empty string) from ## Limits -- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. +- Single server only. `ON CLUSTER`, `ReplicatedMergeTree` and `Distributed` tables are not managed. ClickHouse Cloud and its `Shared*` engines are best effort and untested. - No transactions. Each step above is designed so a rerun converges instead. - Concurrent `sqb` processes writing the same model can lose rows; see [One writer per model](#one-writer-per-model). - Snapshots always use an unsorted `MergeTree` table. From c438f702cf7d67d7c0d562895e1e8d7dc85e5e7d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 01:11:10 +0100 Subject: [PATCH 15/17] test: pin buffered stdout in broken-pipe parity tests and list the compute log logger --- tests/integration/src/sqlbuild/cli/entry/helpers.py | 5 ++++- .../native_module_paths/_helpers/test_module_paths.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/cli/entry/helpers.py b/tests/integration/src/sqlbuild/cli/entry/helpers.py index 15825c8079..0db3751b7d 100644 --- a/tests/integration/src/sqlbuild/cli/entry/helpers.py +++ b/tests/integration/src/sqlbuild/cli/entry/helpers.py @@ -299,13 +299,16 @@ def run_without_reader(program: Path, *, args: tuple[str, ...], cwd: Path) -> La reader, writer = os.pipe() os.close(reader) + environment: dict[str, str] = {**os.environ, "NO_COLOR": "1"} + # The broken-pipe contract follows Python's default buffered stdout. + _ = environment.pop("PYTHONUNBUFFERED", None) try: completed: subprocess.CompletedProcess[bytes] = subprocess.run( [str(program), *args], cwd=cwd, stdout=writer, stderr=subprocess.PIPE, - env={**os.environ, "NO_COLOR": "1"}, + env=environment, check=False, timeout=600, ) diff --git a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py index f8e98b8787..5fd81a5f4d 100644 --- a/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py +++ b/tests/unit/scripts/native_module_paths/_helpers/test_module_paths.py @@ -125,6 +125,7 @@ def test_given_native_source_when_checking_module_paths_then_reports_errors( "sqlbuild.cli", "sqlbuild.cli.commands._helpers.cost.collection", "sqlbuild.cli.commands.classes.prepared_compile_artifacts", + "sqlbuild.cli.compute_logs", "sqlbuild.cli.observability", "sqlbuild.compile", "sqlbuild.compile.reuse", From 8fff721a28ee3dd79a9360e0f124e6ad366b9965 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 04:08:39 +0100 Subject: [PATCH 16/17] ci: give native-building verify jobs 45 minutes --- .github/workflows/verify.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4430b01a54..9f0ea1398e 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -255,7 +255,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.checks == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -342,7 +342,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.compiler_differential == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -379,7 +379,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.compiler_differential == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -415,7 +415,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.compiler_differential == 'true' runs-on: windows-2025 - timeout-minutes: 40 + timeout-minutes: 45 defaults: run: shell: bash @@ -468,7 +468,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.format_performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -499,7 +499,7 @@ jobs: # These budgets are calibrated for the 8-vCPU Blacksmith runner. if: false runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 @@ -524,7 +524,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -572,7 +572,7 @@ jobs: if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 # The 3k and 10k cells also run the delta edit-loop guard, about five minutes at 10k. - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -717,7 +717,7 @@ jobs: needs: [changes, lockfile] if: needs.changes.outputs.performance == 'true' runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -788,7 +788,7 @@ jobs: # These budgets are calibrated for the 8-vCPU Blacksmith runner. if: false runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 45 steps: - uses: actions/checkout@v4 - uses: astral-sh/setup-uv@v5 From d03c5d03219f0818b200c3d3ef96b50b6c73f197 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 20 Oct 2026 05:03:09 +0100 Subject: [PATCH 17/17] fix(sqlite): switch a new database file to WAL one connection at a time Opening several connections to a new file at once could fail with A702 at once: SQLite reports the WAL switch as busy without waiting while another connection switches the same file. The switch now goes through the per-file writer gate. Also give wheel builds 75 minutes, since the Intel macOS build reached the 45-minute limit. --- .github/workflows/build-distributions.yml | 2 +- .../adapters/sqlite/classes/sqlite_adapter.py | 2 +- .../sqlite/classes/sqlite_connection.py | 15 ++++-- .../sqlbuild/adapters/sqlite/_test_types.py | 8 ++++ .../sqlbuild/adapters/sqlite/test_locking.py | 48 +++++++++++++++++++ 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index b27cad102c..eb008b7177 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -88,7 +88,7 @@ jobs: name: Build wheel (${{ matrix.platform.name }}) needs: source runs-on: ${{ matrix.platform.runner }} - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false matrix: diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py index 92a52a6696..1d648e01bd 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_adapter.py @@ -224,7 +224,7 @@ def connect(self, config: dict[str, Any]) -> SqliteConnection: raw_connection=raw_connection, path=path, busy_timeout_seconds=busy_timeout ) if path != MEMORY_DATABASE_PATH and not read_only: - connection.execute("PRAGMA journal_mode=WAL") + connection.execute_as_writer("PRAGMA journal_mode=WAL") connection.execute(f"PRAGMA busy_timeout={int(busy_timeout * 1000)}") connection.execute("PRAGMA foreign_keys=ON") connection.execute("PRAGMA legacy_alter_table=ON") diff --git a/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py b/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py index 7a1c833f18..7b2b582e33 100644 --- a/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py +++ b/src/sqlbuild/adapters/sqlite/classes/sqlite_connection.py @@ -40,11 +40,18 @@ def execute(self, sql: str) -> ObservedCursor: def execute_with(self, *, sql: str, parameters: Sequence[object]) -> ObservedCursor: return self._run(sql=sql, run=lambda cursor: cursor.execute(sql, parameters)) + def execute_as_writer(self, sql: str) -> ObservedCursor: + """Run a statement that needs the writer gate, such as switching a new file to WAL.""" + + return self._run(sql=sql, run=lambda cursor: cursor.execute(sql), writes=True) + def executemany(self, *, sql: str, rows: Iterable[Sequence[object]]) -> ObservedCursor: return self._run(sql=sql, run=lambda cursor: cursor.executemany(sql, rows)) - def _run(self, *, sql: str, run: Callable[[ObservedCursor], object]) -> ObservedCursor: - self._enter(sql) + def _run( + self, *, sql: str, run: Callable[[ObservedCursor], object], writes: bool = False + ) -> ObservedCursor: + self._enter(sql=sql, writes=writes) cursor: ObservedCursor = self.cursor() try: run(cursor) @@ -72,8 +79,8 @@ def close(self) -> None: finally: self._release() - def _enter(self, sql: str) -> None: - if self._holds_gate or leading_keyword(sql) in READ_STATEMENT_KEYWORDS: + def _enter(self, *, sql: str, writes: bool) -> None: + if self._holds_gate or (not writes and leading_keyword(sql) in READ_STATEMENT_KEYWORDS): return self._holds_gate = self._gate.acquire() diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py index d1d89ecfc7..803668b253 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/_test_types.py @@ -123,6 +123,14 @@ class SqliteInProcessWritersTestCase: expected_rows: tuple[tuple[object, ...], ...] +@dataclass(frozen=True) +class SqliteConcurrentConnectTestCase: + description: str + connections: int + attempts: int + expected_open_connections: int + + @dataclass(frozen=True) class SqliteKilledWriterTestCase: description: str diff --git a/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py b/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py index 31e43e0b82..8df254a290 100644 --- a/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py +++ b/tests/integration/src/sqlbuild/adapters/sqlite/test_locking.py @@ -13,6 +13,7 @@ from sqlbuild.adapters.sqlite.classes.sqlite_adapter import SqliteAdapter from tests.integration.src.sqlbuild.adapters.sqlite._test_types import ( + SqliteConcurrentConnectTestCase, SqliteInProcessWritersTestCase, SqliteKilledWriterTestCase, SqliteLockTestCase, @@ -208,3 +209,50 @@ def test_given_writer_killed_mid_transaction_when_reopening_then_target_is_uncha assert rows == test_case.expected_rows assert tables == test_case.expected_tables + + +@pytest.mark.parametrize( + "test_case", + [ + SqliteConcurrentConnectTestCase( + description="threads opening a new file together all switch it to WAL", + connections=4, + attempts=25, + expected_open_connections=100, + ), + ], + ids=lambda case: case.description, +) +def test_given_new_file_when_threads_connect_together_then_every_connection_opens( + test_case: SqliteConcurrentConnectTestCase, adapter: SqliteAdapter, tmp_path: Path +) -> None: + errors: list[BaseException] = [] + opened_count: int = 0 + for attempt in range(test_case.attempts): + config: dict[str, object] = {"path": str(tmp_path / f"shop_{attempt}.sqlite")} + barrier: threading.Barrier = threading.Barrier(test_case.connections) + opened: list[Any] = [] + + def connect( + config: dict[str, object] = config, + barrier: threading.Barrier = barrier, + opened: list[Any] = opened, + ) -> None: + barrier.wait(timeout=10) + try: + opened.append(adapter.connect(config)) + except BaseException as error: + errors.append(error) + + threads: tuple[threading.Thread, ...] = tuple( + threading.Thread(target=connect) for _ in range(test_case.connections) + ) + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + opened_count += len(opened) + for connection in opened: + adapter.close(connection) + + assert (opened_count, errors) == (test_case.expected_open_connections, [])