From 6d65c59e6100f730099cc23472bd8338195d0d8e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:23:50 +0000 Subject: [PATCH 01/30] build: require fensu 0.28.0 --- pyproject.toml | 2 +- uv.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f9f3c7a72c..34baca77ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-xdist>=3.8.0", "ruff>=0.12.0", - "fensu>=0.27.0", + "fensu>=0.28.0", "testcontainers>=4.8.1", "ty>=0.0.1a19", ] diff --git a/uv.lock b/uv.lock index bf5d70d180..b10ee0c368 100644 --- a/uv.lock +++ b/uv.lock @@ -1343,31 +1343,31 @@ wheels = [ [[package]] name = "fensu" -version = "0.27.0" +version = "0.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fensu-cli" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6a/a3/d1eea20ab654f3c4b1b170cc8eeaf27d5092095cd74e6bc819bfff8d9312/fensu-0.27.0.tar.gz", hash = "sha256:023692a73c4ac2fdadbb167c59a58dc944c7a41b0d78a40fa260f6923a8b3e7b", size = 538895, upload-time = "2026-10-17T05:38:49.174Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/69/a132ffc407be97a0fdcfa052cb8b49408091db748e6afe0f7a6c77e6fa7d/fensu-0.28.0.tar.gz", hash = "sha256:11bcea3207f7fd92babc200f40d074e3da789ee11e7fc7024575a415de7c1176", size = 538990, upload-time = "2026-10-27T00:39:16.19Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/b7/7b4053d22df1307dbaebb4ed48280bb9c3663ee649005eb6ab5e2fec28b8/fensu-0.27.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d982c4fec0d2210b78fcf67098c32ca9f8b8592f338b2a327fcd1747b59bb310", size = 4154209, upload-time = "2026-10-17T05:38:28.74Z" }, - { url = "https://files.pythonhosted.org/packages/7a/0d/b7058d187e9908ec18f245c1b168c24b746633a0b13666d69052157da3eb/fensu-0.27.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:8c3cca4a4391028d7b07bb9ead34340f6ba0662c36ea4ef31f69b44a46847289", size = 4004244, upload-time = "2026-10-17T05:38:31.356Z" }, - { url = "https://files.pythonhosted.org/packages/f8/bc/b1eee94e617e954a2188c1802d3770b27624a97f775cbed310f54389c100/fensu-0.27.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5640ffe3cf8916b1533dfe51ca27d65bcea71c9fbd62ae5cfaa2079e8007bdaf", size = 4230195, upload-time = "2026-10-17T05:38:33.057Z" }, - { url = "https://files.pythonhosted.org/packages/7c/01/9f895e6a4427c8ee80ce5d779aceab5a2b994a156a4c1f3bf664108ab489/fensu-0.27.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7d26d2713a69f2e05b07c6f778e1e2f86896e26b74247e60952018288381c04", size = 4307270, upload-time = "2026-10-17T05:38:34.827Z" }, - { url = "https://files.pythonhosted.org/packages/08/61/2a380a704a49eea91a3539935af7b2d4ad09aa1d87d787f87c1de847ae4b/fensu-0.27.0-cp312-abi3-win_amd64.whl", hash = "sha256:15a9aef82221f25812d9c4f74e782dd677c73d566980eaed60b762da13ee2b3d", size = 3977639, upload-time = "2026-10-17T05:38:36.612Z" }, + { url = "https://files.pythonhosted.org/packages/59/e9/7434bd0565d54ff58796105728b3ccc8b838c7200997184978e6ebfdec4d/fensu-0.28.0-cp312-abi3-macosx_10_12_x86_64.whl", hash = "sha256:29df31aff900d20b0c845940af0cd91eead55e158a2511e682e3e887a22dada8", size = 4154207, upload-time = "2026-10-27T00:38:52.311Z" }, + { url = "https://files.pythonhosted.org/packages/7f/d8/e0875b269515d9a75643d5488bdc6f70a4b6c346995f12ce07304a3610f7/fensu-0.28.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:33a6d8b7d2f734c48f48087462c0e5a1ff94535c0e27e6a4820bc10303ff3653", size = 4004243, upload-time = "2026-10-27T00:38:54.994Z" }, + { url = "https://files.pythonhosted.org/packages/e8/05/b87c0ca5cf3f724517d6958f1ebd68948f9399306bd6d6814783635bad9a/fensu-0.28.0-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de07b3fee5d908d2a766ca136753c8040d6fe25a1346d9b3b5a98483fc23d874", size = 4230192, upload-time = "2026-10-27T00:38:57.321Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d2/fdfc7fd322c3497c53c22754a33433e33678a6de84832fd41add09481a21/fensu-0.28.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f44293a2da6bf4558d00eb4f859b28f3fb69d396dad1c25ac466a92a7d1dd734", size = 4307267, upload-time = "2026-10-27T00:38:59.679Z" }, + { url = "https://files.pythonhosted.org/packages/56/c0/2ab2c2c1af4ddd1b45c750679bd9f0b777d7eca3c0e00a233c991aaa13e7/fensu-0.28.0-cp312-abi3-win_amd64.whl", hash = "sha256:ffa88f1d259a1830d513ae930ff856d5b5e4941ba433889dea04d3db5753f281", size = 3977828, upload-time = "2026-10-27T00:39:01.761Z" }, ] [[package]] name = "fensu-cli" -version = "0.27.0" +version = "0.28.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/97/2540b6ea4bca959ac577502fe0c96b432c3b4e694daa1ca68df3a2a06b8a/fensu_cli-0.27.0.tar.gz", hash = "sha256:4276e7fcf7dddc4c36afc347536bf1fd7114fff246ad344544991a1edf7b4ee3", size = 760869, upload-time = "2026-10-17T05:38:50.61Z" } +sdist = { url = "https://files.pythonhosted.org/packages/79/d9/f347f514fb224f6a1f4ca6e616ac49b573f9dceba4ff8d9efcd334f48bea/fensu_cli-0.28.0.tar.gz", hash = "sha256:1feb379ed92e193cbd4d28ab3d7729798b88120aaee2f59fd7f341519b6ee945", size = 765638, upload-time = "2026-10-27T00:39:17.752Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/69/9491487602985731af4a7363754970bb35baece4e27cf2a7a0a95ebe4ae7/fensu_cli-0.27.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e90c8d298904d02c639e87541ba51a31ea3912af0bc354cdfea9d95f4428fd41", size = 7935880, upload-time = "2026-10-17T05:38:38.606Z" }, - { url = "https://files.pythonhosted.org/packages/de/4e/7e84f824b3902708ce4c8399f3402fb278cba9c48ed37877d67a4bad5245/fensu_cli-0.27.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1a9f3a2fb8ec8930af9b5cd59f3f1c027cd80a1ebae1a22c7d07133b80d7c65a", size = 7457913, upload-time = "2026-10-17T05:38:40.624Z" }, - { url = "https://files.pythonhosted.org/packages/05/9f/fc20897baafdac0039faef2cc3a324b059fdfe7f4a1522bac386f1e5f8ae/fensu_cli-0.27.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cf73108b3778d0d74e11857e743042f4241174ec8f16aeee7e5c8c4769e41e7b", size = 7537447, upload-time = "2026-10-17T05:38:42.834Z" }, - { url = "https://files.pythonhosted.org/packages/46/be/1fc7c87cbf320ff215dcb204df808c592fefec6a0892a1103564a1c69b76/fensu_cli-0.27.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:435dad914ec0d061d5fceb10d8b049f75cdb5c031a156efe2f8309647c744b8c", size = 8035160, upload-time = "2026-10-17T05:38:45.074Z" }, - { url = "https://files.pythonhosted.org/packages/ef/5b/53050980866370c7a41dc7e3e61de86cf67e5848d8560e6a9d166040679a/fensu_cli-0.27.0-py3-none-win_amd64.whl", hash = "sha256:68e91a38dab7e081c4ba504e22a7709169d8d6c470137834adcad116b52fa280", size = 7652792, upload-time = "2026-10-17T05:38:47.26Z" }, + { url = "https://files.pythonhosted.org/packages/3b/b3/57f1de89e1cb7534b0ef6415822c8982a247b9d2efe8d2fddd85e35389c8/fensu_cli-0.28.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13ecc7a224a4b32eef19456f21659e682e636ef53f305919aa57cae27c8e3fec", size = 7978247, upload-time = "2026-10-27T00:39:04.644Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ff/2e887d727f82c61ea7cb6773429b61d4711b93f7f94814eb9a7e891384dc/fensu_cli-0.28.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:149e5e22c8895c0a3ed3956b30e693e78df7dfb67ce8e0c5682b84c77e811e7e", size = 7490546, upload-time = "2026-10-27T00:39:07.384Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/daa4c2d70e5b0398382f84c846aacc4a937c4d00b3938d00a67351b0f212/fensu_cli-0.28.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:99932d17956bb10da1ac677867567b504a961d0565fcda730c1e682434244113", size = 7570220, upload-time = "2026-10-27T00:39:09.852Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f9/c9e1095c507da3a53b3e7bf3f7c95f75824429ea45a72cb2c7af67a605f7/fensu_cli-0.28.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee51c5d6d0d5fa0b36c0f70d3d6fde429793b643f0075cdfdf3fc8b8d73e3ccb", size = 8086364, upload-time = "2026-10-27T00:39:12.052Z" }, + { url = "https://files.pythonhosted.org/packages/9c/89/ece32e452d2b4ade243dd66a2612619be7400f6acf59f854ce674bb78e30/fensu_cli-0.28.0-py3-none-win_amd64.whl", hash = "sha256:acca6ac39cbe0614d3a8e4ce7c2c12cc5fd3c800fc4c66b5484c54dc19ef4a66", size = 7700477, upload-time = "2026-10-27T00:39:14.15Z" }, ] [[package]] @@ -4346,7 +4346,7 @@ dev = [ { name = "dbt-snowflake", specifier = "==1.12.0b1" }, { name = "dbt-sqlserver", specifier = ">=1.9.0,<1.11" }, { name = "dlt", extras = ["duckdb", "filesystem", "sql-database"], specifier = ">=1.28.0,<2" }, - { name = "fensu", specifier = ">=0.27.0" }, + { name = "fensu", specifier = ">=0.28.0" }, { name = "ingestr", specifier = ">=1.1.73" }, { name = "jsonschema", specifier = ">=4.23.0" }, { name = "maturin", specifier = ">=1.9,<2" }, From dc484bb984a9ce487e65547cf638014325d3470b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:34 +0000 Subject: [PATCH 02/30] fix(audit): store standalone audit history beside the models on preserve targets A target whose schema or database is `preserve` names no location, but a standalone `sqb audit` created its result history table in a literal `preserve` schema. It now stores history where a build stores it, and a history write that fails prints a warning with the warehouse error before the audit summary. --- .../_helpers/audit_command/command.rs | 33 +++++++- .../auditing/_helpers/result_projection.py | 15 +++- src/sqlbuild/executor/auditing/models.py | 1 + .../sqlbuild/cli/commands/main/_test_types.py | 7 ++ .../main/test_audit_command_storage.py | 77 +++++++++++++++++++ 5 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index 620d94adcf..ade1c89087 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -5,6 +5,7 @@ use std::collections::BTreeSet; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyDict, PyDictMethods, PyInt, PyList, PyListMethods, PyString, PyTuple}; use pyo3::{pyclass, pyfunction, pymethods, wrap_pyfunction}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_core::audit_command::main::audit_exit_code::audit_exit_code; use sqlbuild_core::audit_command::main::render_audit_header::render_audit_header; use sqlbuild_core::audit_command::main::render_audit_summary::render_audit_summary; @@ -43,6 +44,15 @@ fn text<'py>(py: Python<'py>, value: &str) -> Bound<'py, PyAny> { PyString::new(py, value).into_any() } +/// A target database or schema as a storage location: `preserve` names no location, so audit history is stored beside the models, as a build stores it. +fn literal_location<'py>(value: &Bound<'py, PyAny>) -> PyResult> { + let py: Python<'py> = value.py(); + if !value.is_none() && value.extract::()? == PRESERVE_TARGET_VALUE { + return Ok(py.None().into_bound(py)); + } + Ok(value.clone()) +} + /// Forward audit results to the event writer and connection timing to its reporter. #[pyclass(module = "sqlbuild._native", frozen)] struct AuditCallbackRelay { @@ -388,11 +398,11 @@ fn execute<'py>( ("run_id", run_id.clone()), ( "storage_database", - native_field(&project, "effective_target_database")?, + literal_location(&native_field(&project, "effective_target_database")?)?, ), ( "storage_schema", - native_field(&project, "effective_target_schema")?, + literal_location(&native_field(&project, "effective_target_schema")?)?, ), ], )? @@ -452,6 +462,22 @@ fn audit_payload<'py>( Ok(PyString::new(py, &format!("{dumped}\n")).into_any()) } +/// The warning line for audit results that could not be stored in the warehouse history table; audit outcomes are unaffected. +fn projection_warning(py: Python<'_>, style: CliStyle) -> PyResult> { + let projection: Bound<'_, PyAny> = + host::entry(py, ExecutorHost::CurrentAuditResultProjection)?.call0()?; + if projection.is_none() || !native_field(&projection, "degraded")?.is_truthy()? { + return Ok(None); + } + let failed: i64 = native_field(&projection, "failed_count")?.extract()?; + let reason: Option = native_field(&projection, "failure_reason")?.extract()?; + let reason: String = reason.map_or_else(String::new, |reason| format!(": {reason}")); + Ok(Some(format!( + "{} {failed} audit result(s) were not written to the audit history table{reason}\n", + style.warning("warning:") + ))) +} + /// Write the summary footer and JSON output; returns the exit code. fn complete( request: &Bound<'_, PyAny>, @@ -481,6 +507,9 @@ fn complete( let style: CliStyle = CliStyle { use_color: native_field(invocation, "use_color")?.is_truthy()?, }; + if let Some(warning) = projection_warning(py, style)? { + stream.call_method1("write", (warning,))?; + } stream.call_method1("write", (render_audit_summary(counts, style),))?; stream.call_method0("flush")?; let payload: Bound<'_, PyAny> = audit_payload( diff --git a/src/sqlbuild/executor/auditing/_helpers/result_projection.py b/src/sqlbuild/executor/auditing/_helpers/result_projection.py index 0c350d4400..a6a8f91cb6 100644 --- a/src/sqlbuild/executor/auditing/_helpers/result_projection.py +++ b/src/sqlbuild/executor/auditing/_helpers/result_projection.py @@ -40,6 +40,7 @@ from sqlbuild.runtime.observability.types import JSONValue _LOGGER: logging.Logger = logging.getLogger("sqlbuild.execution") +_MAX_FAILURE_REASON_LENGTH: int = 500 _LAST_AUDIT_RESULT_PROJECTION: ContextVar[AuditResultProjection | None] = ContextVar( "sqlbuild_last_audit_result_projection", default=None ) @@ -267,14 +268,15 @@ def project_audit_result_batch_impl( ), ) except AuditResultStorageError as error: + reason: str = _storage_failure_reason(error) _LOGGER.warning( "Audit result persistence degraded: attempted=%d written=0 failed=%d (%s)", len(executed), len(executed), - error, + reason, ) projection = AuditResultProjection( - attempted_count=len(executed), failed_count=len(executed) + attempted_count=len(executed), failed_count=len(executed), failure_reason=reason ) _LAST_AUDIT_RESULT_PROJECTION.set(projection) return projection @@ -437,6 +439,15 @@ def _render_bound(bound: MeasurementThresholdBound) -> dict[str, object]: return {"operator": bound.operator.value, "limit": bound.limit} +def _storage_failure_reason(error: AuditResultStorageError) -> str: + """The storage error with the warehouse error that caused it, on one bounded line.""" + + cause: BaseException | None = error.__cause__ + text: str = str(error) if cause is None else f"{error}: {cause}" + line: str = " ".join(text.split()) + return line[:_MAX_FAILURE_REASON_LENGTH] + + def _json_or_none(value: object | None) -> str | None: return None if value is None else json.dumps(value, sort_keys=True, separators=(",", ":")) diff --git a/src/sqlbuild/executor/auditing/models.py b/src/sqlbuild/executor/auditing/models.py index 39458e8b8b..eaf0f4dd21 100644 --- a/src/sqlbuild/executor/auditing/models.py +++ b/src/sqlbuild/executor/auditing/models.py @@ -38,6 +38,7 @@ class AuditResultProjection: attempted_count: int = 0 written_count: int = 0 failed_count: int = 0 + failure_reason: str | None = None @property def degraded(self) -> bool: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 84f3707c59..3dcd09aa4d 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1310,3 +1310,10 @@ class EnumLiteralCompileTestCase: expected_exit_code: int expected_diagnostics: tuple[tuple[str, str, int | None], ...] contract: str = "enforced" + + +@dataclass(frozen=True) +class PreservedTargetAuditTestCase: + description: str + target_toml: str + expected_history_schema: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py b/tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py new file mode 100644 index 0000000000..4f9736e76c --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_audit_command_storage.py @@ -0,0 +1,77 @@ +"""Real CLI coverage for where a standalone audit stores its history.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + PreservedTargetAuditTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import query_duckdb_rows + +_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "shop.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' +) +_ORDERS_MODEL: str = ( + 'MODEL (\n description "One row per order",\n materialized table,\n schema sales,\n' + " columns (order_id (audits [not_null])),\n);\n\n" + "SELECT 1 AS order_id\n" +) + + +@pytest.mark.parametrize( + "test_case", + [ + PreservedTargetAuditTestCase( + description="a preserved schema stores history beside the models", + target_toml='schema = "preserve"\n', + expected_history_schema="sales", + ), + PreservedTargetAuditTestCase( + description="a named schema stores history in it", + target_toml='schema = "dev"\n', + expected_history_schema="dev", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_target_schema_when_auditing_alone_then_history_is_stored_where_a_build_stores_it( + test_case: PreservedTargetAuditTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + _ = (tmp_path / "sqlbuild_project.toml").write_text( + _PROJECT + test_case.target_toml, encoding="utf-8" + ) + (tmp_path / "models").mkdir() + _ = (tmp_path / "models" / "orders.sql").write_text(_ORDERS_MODEL, encoding="utf-8") + assert main(["--no-color", "--project-dir", str(tmp_path), "build"]) == 0 + + exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "audit"]) + + output: str = "".join(capsys.readouterr()) + assert exit_code == 0, output + assert "warning:" not in output + db_path: Path = tmp_path / "shop.duckdb" + assert query_duckdb_rows( + db_path=db_path, + sql="SELECT table_schema FROM information_schema.tables " + "WHERE table_name = '_sqlbuild_audit_results'", + ) == ((test_case.expected_history_schema,),) + assert query_duckdb_rows( + db_path=db_path, + sql=f"SELECT COUNT(*) FROM {test_case.expected_history_schema}._sqlbuild_audit_results", + ) == ((2,),) + assert query_duckdb_rows( + db_path=db_path, + sql="SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = 'preserve'", + ) == ((0,),) From 84521a02724da3c8d7c0dc28395e3aca9d22579d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:35 +0000 Subject: [PATCH 03/30] feat(query)!: print query results as a table when they fit `sqb query` and `sqb recall query` share one text renderer. By default it prints a table with each column's warehouse type when the result fits the terminal (or 120 characters when redirected) and one record per row otherwise, so no column or value is cut. The footer says how to see more rows and, above 50 rows, suggests aggregating or limiting the query. BREAKING CHANGE: `--format table` and `--format csv` are removed from `sqb query` and `sqb recall query`/`schema`; they fail with a message naming what to use instead. Use the default output, `--format long`, or `--json`. --- Cargo.toml | 2 +- .../sqlbuild-cli/src/argv/_helpers/table.rs | 9 ++ .../sqlbuild-cli/src/argv/_helpers/values.rs | 10 ++ .../sqlbuild-cli/src/argv/command_table.json | 54 +++++--- crates/sqlbuild-cli/src/argv/constants.rs | 4 +- crates/sqlbuild-cli/src/argv/models.rs | 2 + .../sqlbuild-core/src/progress/constants.rs | 1 - .../src/progress/main/machine_output_mode.rs | 8 +- .../src/progress/tests/test_progress_text.rs | 4 +- .../src/query_command/_helpers/boxed.rs | 109 +++++++++++++++ .../src/query_command/_helpers/footer.rs | 26 ++-- .../src/query_command/_helpers/mod.rs | 1 + .../src/query_command/constants.rs | 11 +- .../src/query_command/main/mod.rs | 3 +- .../query_command/main/render_query_csv.rs | 34 ----- .../query_command/main/render_query_table.rs | 53 -------- .../query_command/main/render_query_text.rs | 19 +++ .../sqlbuild-core/src/query_command/models.rs | 15 +- .../query_command/tests/test_query_command.rs | 128 +++++++++++------- .../src/query_command/tests/test_types.rs | 13 +- .../src/bindings/_helpers/cli/entry.rs | 5 +- .../bindings/_helpers/query_command/mod.rs | 1 + .../bindings/_helpers/query_command/query.rs | 114 ++++++++++++---- .../_helpers/query_command/query_values.rs | 13 +- .../_helpers/query_command/text_output.rs | 42 ++++++ .../_helpers/recall/clickhouse_calls.rs | 6 +- .../src/bindings/_helpers/recall/command.rs | 29 ++-- .../src/bindings/_helpers/recall/constants.rs | 7 +- .../src/bindings/_helpers/recall/rendering.rs | 70 ++++++---- .../src/store/main/query_store.rs | 32 +++++ crates/sqlbuild-recall/src/store/models.rs | 2 + .../sqlbuild/references/docs/cli/query.md | 68 +++++----- .../sqlbuild/references/docs/cli/recall.md | 34 +++-- src/sqlbuild/_native.pyi | 6 +- .../contract/classes/duckdb_backed_adapter.py | 3 + src/sqlbuild/adapter/contract/models.py | 1 + .../clickhouse/_helpers/recall_store.py | 12 +- .../clickhouse/classes/clickhouse_adapter.py | 10 +- .../postgres/classes/postgres_adapter.py | 7 +- .../adapters/test_project_local_adapters.py | 2 +- .../commands/main/bigquery/test_bigquery.py | 8 +- .../main/databricks/test_databricks.py | 8 +- .../cli/commands/main/query/test_query.py | 18 ++- .../main/query/test_query_project_inputs.py | 8 +- .../commands/main/snowflake/test_snowflake.py | 8 +- .../cli/commands/main/sqlite/test_sqlite.py | 8 +- .../src/sqlbuild/cli/recall/_test_types.py | 12 +- .../sqlbuild/cli/recall/test_local_recall.py | 2 +- .../sqlbuild/cli/recall/test_recall_cli.py | 106 ++++++++++++--- .../sqlbuild/adapters/snowflake/helpers.py | 20 ++- .../commands/_helpers/entry/test_parsing.py | 4 +- .../commands/_helpers/query/_test_types.py | 2 +- .../commands/_helpers/query/test_output.py | 33 ++++- website/src/content/docs/docs/cli/query.mdx | 66 ++++----- website/src/content/docs/docs/cli/recall.mdx | 34 +++-- .../docs/concepts/observability/recall.mdx | 11 +- 56 files changed, 862 insertions(+), 456 deletions(-) create mode 100644 crates/sqlbuild-core/src/query_command/_helpers/boxed.rs delete mode 100644 crates/sqlbuild-core/src/query_command/main/render_query_csv.rs delete mode 100644 crates/sqlbuild-core/src/query_command/main/render_query_table.rs create mode 100644 crates/sqlbuild-core/src/query_command/main/render_query_text.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs diff --git a/Cargo.toml b/Cargo.toml index 129467fd7f..e5ac97c0b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,7 +106,7 @@ ruff_text_size = "=0.0.4" # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output. rustc-hash = "2" # Recall's local store: SQLite compiled in, so every build has the same engine version. -rusqlite = { version = "0.37", features = ["bundled"] } +rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } saphyr-parser = "0.1" regex = "1" diff --git a/crates/sqlbuild-cli/src/argv/_helpers/table.rs b/crates/sqlbuild-cli/src/argv/_helpers/table.rs index 94b61666df..12bfc5b45b 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/table.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/table.rs @@ -131,6 +131,15 @@ impl TableReader { help: help(choice.get("help").unwrap_or(&Json::Null)), }) .collect(), + removed_choices: items(json, "removed_choices") + .iter() + .map(|removed| { + ( + text(removed, "value").unwrap_or_default(), + text(removed, "instead").unwrap_or_default(), + ) + }) + .collect(), } } } diff --git a/crates/sqlbuild-cli/src/argv/_helpers/values.rs b/crates/sqlbuild-cli/src/argv/_helpers/values.rs index df7da8834b..79602bc483 100644 --- a/crates/sqlbuild-cli/src/argv/_helpers/values.rs +++ b/crates/sqlbuild-cli/src/argv/_helpers/values.rs @@ -114,6 +114,16 @@ impl Run<'_> { if choices.iter().any(|choice| choice == text) { return Ok(()); } + if let Some((_, instead)) = action + .removed_choices + .iter() + .find(|(removed, _)| removed == text) + { + return Err(argument_error( + action, + &format!("{} was removed; {instead}", self.host.repr(text)), + )); + } let listed: Vec = choices .iter() .map(|choice| python_str_repr(choice)) diff --git a/crates/sqlbuild-cli/src/argv/command_table.json b/crates/sqlbuild-cli/src/argv/command_table.json index 9947a4dcdf..d2b8b61be1 100644 --- a/crates/sqlbuild-cli/src/argv/command_table.json +++ b/crates/sqlbuild-cli/src/argv/command_table.json @@ -4397,13 +4397,21 @@ "--format" ], "dest": "query_format", - "default": "long", "choices": [ "long", - "table", - "json", - "csv" + "json" + ], + "removed_choices": [ + { + "value": "table", + "instead": "the default output prints a table when the result fits; use --format long for one record per row" + }, + { + "value": "csv", + "instead": "use --json for programs, or the warehouse's own export for bulk data" + } ], + "help": "by default a table when the result fits, else one record per row; long always prints records; json for programs", "mutex_group": 0, "group": 1 }, @@ -7183,14 +7191,21 @@ "--format" ], "dest": "recall_format", - "default": "long", "choices": [ "long", - "table", - "json", - "csv" - ], - "help": "long (default) for reading, json for programs, csv for export; table is deprecated", + "json" + ], + "removed_choices": [ + { + "value": "table", + "instead": "the default output prints a table when the result fits; use --format long for one record per row" + }, + { + "value": "csv", + "instead": "use --json for programs" + } + ], + "help": "by default a table when the result fits, else one record per row; long always prints records; json for programs", "mutex_group": 0, "group": 1 }, @@ -7374,14 +7389,21 @@ "--format" ], "dest": "recall_format", - "default": "long", "choices": [ "long", - "table", - "json", - "csv" - ], - "help": "long (default) for reading, json for programs, csv for export; table is deprecated", + "json" + ], + "removed_choices": [ + { + "value": "table", + "instead": "use the default long output" + }, + { + "value": "csv", + "instead": "use --json for programs" + } + ], + "help": "long (default) for reading, json for programs", "mutex_group": 0, "group": 1 }, diff --git a/crates/sqlbuild-cli/src/argv/constants.rs b/crates/sqlbuild-cli/src/argv/constants.rs index 29d4726dc1..ef1b833fbb 100644 --- a/crates/sqlbuild-cli/src/argv/constants.rs +++ b/crates/sqlbuild-cli/src/argv/constants.rs @@ -159,7 +159,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ ("max_columns", NamespaceDefault::None), ("query_sql", NamespaceDefault::None), ("query_file", NamespaceDefault::None), - ("query_format", NamespaceDefault::Text("long")), + ("query_format", NamespaceDefault::None), ("query_limit", NamespaceDefault::Int(20)), ("query_no_limit", NamespaceDefault::Bool(false)), ("cost_selector", NamespaceDefault::Text("latest")), @@ -228,7 +228,7 @@ pub(crate) const CLI_NAMESPACE_DEFAULTS: [(&str, NamespaceDefault); 184] = [ ("recall_command", NamespaceDefault::None), ("recall_sql", NamespaceDefault::None), ("recall_backend", NamespaceDefault::Text("logs")), - ("recall_format", NamespaceDefault::Text("long")), + ("recall_format", NamespaceDefault::None), ("recall_limit", NamespaceDefault::Int(100)), ("recall_no_limit", NamespaceDefault::Bool(false)), ("recall_max_cell_bytes", NamespaceDefault::None), diff --git a/crates/sqlbuild-cli/src/argv/models.rs b/crates/sqlbuild-cli/src/argv/models.rs index 037208ddb0..44d88ac46a 100644 --- a/crates/sqlbuild-cli/src/argv/models.rs +++ b/crates/sqlbuild-cli/src/argv/models.rs @@ -98,6 +98,8 @@ pub struct ActionSpec { /// Sub-command names and their parsers, in order. pub parsers: Vec<(String, ParserSpec)>, pub choices_help: Vec, + /// Values no longer accepted, each with what to use instead; never listed in help. + pub removed_choices: Vec<(String, String)>, } /// One argument group's title and description. diff --git a/crates/sqlbuild-core/src/progress/constants.rs b/crates/sqlbuild-core/src/progress/constants.rs index 96c15345fc..36ee9d8e7e 100644 --- a/crates/sqlbuild-core/src/progress/constants.rs +++ b/crates/sqlbuild-core/src/progress/constants.rs @@ -113,7 +113,6 @@ pub(crate) const QUERY_COMMAND: &str = "query"; pub(crate) const DBT_COMMAND: &str = "dbt"; pub(crate) const DBT_JSON_OPTION: &str = "--json"; pub(crate) const JSON_FORMAT: &str = "json"; -pub(crate) const CSV_FORMAT: &str = "csv"; /// What replaces the cut tail of a transient status message. pub(crate) const STATUS_ELLIPSIS: &str = "..."; /// The resource kind of audits and the prefix of their resource identifiers. diff --git a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs index e483f76b46..7a6c3bc531 100644 --- a/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs +++ b/crates/sqlbuild-core/src/progress/main/machine_output_mode.rs @@ -1,11 +1,11 @@ //! Whether a command's stdout carries machine output, so human progress must use stderr. use crate::progress::constants::{ - CSV_FORMAT, DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, + DBT_COMMAND, DBT_JSON_OPTION, JSON_FORMAT, LINEAGE_COMMAND, QUERY_COMMAND, }; use crate::progress::models::OutputModeFacts; -/// `--json`, `sqb dbt ... --json`, `lineage --format json` or `query --format csv|json`. +/// `--json`, `sqb dbt ... --json`, `lineage --format json` or `query --format json`. #[must_use] pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { let format = |value: &Option| value.as_deref().unwrap_or_default().to_owned(); @@ -13,9 +13,7 @@ pub fn machine_output_mode(facts: &OutputModeFacts) -> bool { _ if facts.json => true, Some(DBT_COMMAND) => facts.dbt_args.iter().any(|arg| arg == DBT_JSON_OPTION), Some(LINEAGE_COMMAND) => format(&facts.lineage_format) == JSON_FORMAT, - Some(QUERY_COMMAND) => { - [CSV_FORMAT, JSON_FORMAT].contains(&format(&facts.query_format).as_str()) - } + Some(QUERY_COMMAND) => format(&facts.query_format) == JSON_FORMAT, _ => false, } } diff --git a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs index 4bee5218ae..b1e56752e3 100644 --- a/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs +++ b/crates/sqlbuild-core/src/progress/tests/test_progress_text.rs @@ -269,9 +269,9 @@ fn given_parsed_options_when_choosing_output_mode_then_machine_output_is_detecte expected_machine_output: true, }, OutputModeTestCase { - description: "query csv", + description: "query json", facts: OutputModeFacts { - query_format: Some("csv".to_owned()), + query_format: Some("json".to_owned()), ..facts("query", &[]) }, expected_machine_output: true, diff --git a/crates/sqlbuild-core/src/query_command/_helpers/boxed.rs b/crates/sqlbuild-core/src/query_command/_helpers/boxed.rs new file mode 100644 index 0000000000..dfed793a8d --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/_helpers/boxed.rs @@ -0,0 +1,109 @@ +//! Query rows in a box: names and types centered over aligned columns. + +use crate::query_command::constants::{ + BOX_BOTTOM, BOX_HORIZONTAL, BOX_RULE, BOX_TOP, BOX_VERTICAL, +}; +use crate::query_command::models::QueryDisplay; +use crate::text::main::cell_len::cell_len; + +/// Whether `value` prints on one line, so a box can hold it. +fn single_line(value: &str) -> bool { + !value.chars().any(char::is_control) +} + +fn rule(widths: &[usize], [left, middle, right]: [char; 3]) -> String { + let segments: Vec = widths + .iter() + .map(|width| BOX_HORIZONTAL.to_string().repeat(width + 2)) + .collect(); + format!("{left}{}{right}", segments.join(&middle.to_string())) +} + +fn centered(text: &str, width: usize) -> String { + let spare: usize = width.saturating_sub(cell_len(text)); + let left: usize = spare / 2; + format!("{}{text}{}", " ".repeat(left), " ".repeat(spare - left)) +} + +fn aligned(text: &str, width: usize, right: bool) -> String { + let spare: String = " ".repeat(width.saturating_sub(cell_len(text))); + if right { + format!("{spare}{text}") + } else { + format!("{text}{spare}") + } +} + +fn line(cells: Vec) -> String { + let joined: String = cells.join(&format!(" {BOX_VERTICAL} ")); + format!("{BOX_VERTICAL} {joined} {BOX_VERTICAL}") +} + +/// The box's lines when it is at most `limit` cells wide and every value is one line; `None` otherwise, so nothing is cut to fit. +pub(crate) fn boxed_lines(display: &QueryDisplay, limit: usize) -> Option> { + let types: Vec<&str> = (0..display.columns.len()) + .map(|index| { + display + .types + .get(index) + .and_then(Option::as_deref) + .unwrap_or_default() + }) + .collect(); + let mut widths: Vec = display + .columns + .iter() + .zip(&types) + .map(|(column, kind)| cell_len(column).max(cell_len(kind))) + .collect(); + for row in &display.rows { + for (index, value) in row.iter().enumerate() { + if !single_line(value) { + return None; + } + if let Some(width) = widths.get_mut(index) { + *width = (*width).max(cell_len(value)); + } + } + } + let total: usize = widths.iter().map(|width| width + 3).sum::() + 1; + if total > limit || display.columns.iter().any(|column| !single_line(column)) { + return None; + } + let mut lines: Vec = vec![ + rule(&widths, BOX_TOP), + line( + display + .columns + .iter() + .zip(&widths) + .map(|(column, width)| centered(column, *width)) + .collect(), + ), + ]; + if types.iter().any(|kind| !kind.is_empty()) { + lines.push(line( + types + .iter() + .zip(&widths) + .map(|(kind, width)| centered(kind, *width)) + .collect(), + )); + } + lines.push(rule(&widths, BOX_RULE)); + for row in &display.rows { + lines.push(line( + widths + .iter() + .enumerate() + .map(|(index, width)| { + let value: &str = row.get(index).map_or("", String::as_str); + let right: bool = display.numeric.get(index).copied().unwrap_or(false); + aligned(value, *width, right) + }) + .collect(), + )); + } + lines.push(rule(&widths, BOX_BOTTOM)); + Some(lines) +} diff --git a/crates/sqlbuild-core/src/query_command/_helpers/footer.rs b/crates/sqlbuild-core/src/query_command/_helpers/footer.rs index e145c57f3f..d6697b1545 100644 --- a/crates/sqlbuild-core/src/query_command/_helpers/footer.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/footer.rs @@ -1,20 +1,18 @@ -//! The row count and truncation lines under long and table output. +//! The row count line under every text layout. +use crate::query_command::constants::MANY_ROWS; use crate::query_command::models::QueryDisplay; -/// A blank line, ` row(s)`, and the truncation hint when the limit cut rows off. +/// A blank line and the row count: when the limit cut rows off, the limit and how to change it; above [`MANY_ROWS`], a hint to aggregate or limit the query. pub(crate) fn footer(display: &QueryDisplay) -> Vec { let count: usize = display.rows.len(); - let mut lines: Vec = vec![ - String::new(), - format!("{count} {}", if count == 1 { "row" } else { "rows" }), - ]; - if display.truncated { - lines.push(display.limit.map_or_else(String::new, |limit| { - format!( - "Showing {limit} rows. Use --limit to show more or --no-limit to disable the limit." - ) - })); - } - lines + let noun: &str = if count == 1 { "row" } else { "rows" }; + let line: String = match display.limit.filter(|_| display.truncated) { + Some(limit) => format!("{count} {noun} shown (limit {limit}; use --limit or --no-limit)"), + None if count > MANY_ROWS => { + format!("{count} {noun}; to read fewer, aggregate with GROUP BY or add a LIMIT") + } + None => format!("{count} {noun}"), + }; + vec![String::new(), line] } diff --git a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs index c8f10e5269..af7531f4f2 100644 --- a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs @@ -1 +1,2 @@ +pub(crate) mod boxed; pub(crate) mod footer; diff --git a/crates/sqlbuild-core/src/query_command/constants.rs b/crates/sqlbuild-core/src/query_command/constants.rs index 7d9aac4965..a13e21d4b5 100644 --- a/crates/sqlbuild-core/src/query_command/constants.rs +++ b/crates/sqlbuild-core/src/query_command/constants.rs @@ -1,6 +1,11 @@ //! Query output layout. pub(crate) const RECORD_RULE_WIDTH: usize = 40; -pub(crate) const TABLE_SEPARATOR: &str = " | "; -pub(crate) const CSV_DELIMITER: char = ','; -pub(crate) const CSV_QUOTE: char = '"'; +/// Above this many rows the footer suggests aggregating or limiting the query. +pub(crate) const MANY_ROWS: usize = 50; +/// The box layout's borders: corners, tees, crosses and lines. +pub(crate) const BOX_TOP: [char; 3] = ['┌', '┬', '┐']; +pub(crate) const BOX_RULE: [char; 3] = ['├', '┼', '┤']; +pub(crate) const BOX_BOTTOM: [char; 3] = ['└', '┴', '┘']; +pub(crate) const BOX_HORIZONTAL: char = '─'; +pub(crate) const BOX_VERTICAL: char = '│'; diff --git a/crates/sqlbuild-core/src/query_command/main/mod.rs b/crates/sqlbuild-core/src/query_command/main/mod.rs index 7496e3a434..b4a9c3648e 100644 --- a/crates/sqlbuild-core/src/query_command/main/mod.rs +++ b/crates/sqlbuild-core/src/query_command/main/mod.rs @@ -1,5 +1,4 @@ pub mod render_dag_summary; -pub mod render_query_csv; pub mod render_query_json; pub mod render_query_long; -pub mod render_query_table; +pub mod render_query_text; diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs b/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs deleted file mode 100644 index 1247dfa793..0000000000 --- a/crates/sqlbuild-core/src/query_command/main/render_query_csv.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! Query rows as CSV, as Python's default `csv.writer` writes them. - -use crate::query_command::constants::{CSV_DELIMITER, CSV_QUOTE}; - -fn field(value: &str) -> String { - if value - .chars() - .any(|character| matches!(character, CSV_DELIMITER | CSV_QUOTE | '\n' | '\r')) - { - format!("\"{}\"", value.replace('"', "\"\"")) - } else { - value.to_owned() - } -} - -fn record(values: &[String]) -> String { - if let [only] = values - && only.is_empty() - { - return "\"\"\n".to_owned(); - } - let fields: Vec = values.iter().map(|value| field(value)).collect(); - format!("{}\n", fields.join(",")) -} - -/// The header record then one record per row; fields quoted only when they must be. -#[must_use] -pub fn render_query_csv(columns: &[String], rows: &[Vec]) -> String { - let mut text: String = record(columns); - for row in rows { - text.push_str(&record(row)); - } - text -} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_table.rs b/crates/sqlbuild-core/src/query_command/main/render_query_table.rs deleted file mode 100644 index 1ad9803744..0000000000 --- a/crates/sqlbuild-core/src/query_command/main/render_query_table.rs +++ /dev/null @@ -1,53 +0,0 @@ -//! Query rows as an aligned table. - -use crate::query_command::_helpers::footer::footer; -use crate::query_command::constants::TABLE_SEPARATOR; -use crate::query_command::models::QueryDisplay; -use crate::text::main::python_isspace::python_isspace; - -fn padded(value: &str, width: usize) -> String { - format!("{value: String { - let mut widths: Vec = display - .columns - .iter() - .map(|column| column.chars().count()) - .collect(); - for row in &display.rows { - for (index, value) in row.iter().enumerate() { - if let Some(width) = widths.get_mut(index) { - *width = (*width).max(value.chars().count()); - } - } - } - let width_at = |index: usize| widths.get(index).copied().unwrap_or(0); - let mut lines: Vec = vec![ - display - .columns - .iter() - .enumerate() - .map(|(index, column)| padded(column, width_at(index))) - .collect::>() - .join(TABLE_SEPARATOR), - widths - .iter() - .map(|width| "-".repeat(*width)) - .collect::>() - .join(TABLE_SEPARATOR), - ]; - for row in &display.rows { - let line: String = row - .iter() - .enumerate() - .map(|(index, value)| padded(value, width_at(index))) - .collect::>() - .join(TABLE_SEPARATOR); - lines.push(line.trim_end_matches(python_isspace).to_owned()); - } - lines.extend(footer(display)); - format!("{}\n", lines.join("\n")) -} diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_text.rs b/crates/sqlbuild-core/src/query_command/main/render_query_text.rs new file mode 100644 index 0000000000..252ba6d141 --- /dev/null +++ b/crates/sqlbuild-core/src/query_command/main/render_query_text.rs @@ -0,0 +1,19 @@ +//! Query rows as text: a box when they fit, one record per row otherwise. + +use crate::query_command::_helpers::boxed::boxed_lines; +use crate::query_command::_helpers::footer::footer; +use crate::query_command::main::render_query_long::render_query_long; +use crate::query_command::models::{QueryDisplay, QueryLayout}; + +/// The rows in `layout`, then the row count. A fitted layout never drops columns or rows: a result wider than the width, or with a value spanning lines, prints one record per row. +#[must_use] +pub fn render_query_text(display: &QueryDisplay, layout: QueryLayout) -> String { + let QueryLayout::Fitted { width } = layout else { + return render_query_long(display); + }; + let Some(mut lines) = boxed_lines(display, width) else { + return render_query_long(display); + }; + lines.extend(footer(display)); + format!("{}\n", lines.join("\n")) +} diff --git a/crates/sqlbuild-core/src/query_command/models.rs b/crates/sqlbuild-core/src/query_command/models.rs index 724b1349f8..324f3a9309 100644 --- a/crates/sqlbuild-core/src/query_command/models.rs +++ b/crates/sqlbuild-core/src/query_command/models.rs @@ -1,10 +1,23 @@ //! A query result as its text formats display it. -/// Column names, each row's cells as display text, and whether the limit cut rows off. +/// Column names and types, each row's cells as display text, and whether the limit cut rows off. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct QueryDisplay { pub columns: Vec, + /// Each column's type as the warehouse names it, where known; empty when no type is known. + pub types: Vec>, + /// Columns whose values are numbers, right-aligned in the box layout. + pub numeric: Vec, pub rows: Vec>, pub truncated: bool, pub limit: Option, } + +/// How a result is laid out as text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QueryLayout { + /// A box when every row fits `width` terminal cells, else one record per row. + Fitted { width: usize }, + /// One record per row. + Long, +} diff --git a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs index 0cdd792d93..5d471dbe02 100644 --- a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs +++ b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs @@ -1,52 +1,91 @@ use crate::json::models::JsonValue; use crate::query_command::main::render_dag_summary::render_dag_summary; -use crate::query_command::main::render_query_csv::render_query_csv; use crate::query_command::main::render_query_json::render_query_json; -use crate::query_command::main::render_query_long::render_query_long; -use crate::query_command::main::render_query_table::render_query_table; -use crate::query_command::models::QueryDisplay; +use crate::query_command::main::render_query_text::render_query_text; +use crate::query_command::models::{QueryDisplay, QueryLayout}; use crate::query_command::tests::helpers::texts; -use crate::query_command::tests::test_types::{ - CsvTestCase, DagTestCase, DisplayTestCase, JsonTestCase, -}; +use crate::query_command::tests::test_types::{DagTestCase, DisplayTestCase, JsonTestCase}; + +fn invocations() -> QueryDisplay { + QueryDisplay { + columns: texts(&["status", "origin", "invocations"]), + types: vec![ + Some("varchar".to_owned()), + Some("varchar".to_owned()), + Some("int32".to_owned()), + ], + numeric: vec![false, false, true], + rows: vec![ + texts(&["completed", "integration", "133"]), + texts(&["cancelled", "local", "NULL"]), + ], + truncated: false, + limit: None, + } +} #[test] -fn given_rows_when_rendering_text_then_aligns_and_counts() { +fn given_rows_when_rendering_text_then_a_fitting_result_is_boxed_and_others_are_records() { let test_cases = [ DisplayTestCase { - description: "two rows with unicode and a truncation hint", + description: "a narrow result is boxed with types and right-aligned numbers", + display: invocations(), + layout: QueryLayout::Fitted { width: 120 }, + expected_text: "┌───────────┬─────────────┬─────────────┐\n\ + │ status │ origin │ invocations │\n\ + │ varchar │ varchar │ int32 │\n\ + ├───────────┼─────────────┼─────────────┤\n\ + │ completed │ integration │ 133 │\n\ + │ cancelled │ local │ NULL │\n\ + └───────────┴─────────────┴─────────────┘\n\n2 rows\n", + }, + DisplayTestCase { + description: "a result wider than the limit prints every value as records", + display: invocations(), + layout: QueryLayout::Fitted { width: 40 }, + expected_text: "-[ RECORD 1 ]---------------------------+\nstatus | completed\n\ + origin | integration\ninvocations | 133\n\n\ + -[ RECORD 2 ]---------------------------+\nstatus | cancelled\n\ + origin | local\ninvocations | NULL\n\n2 rows\n", + }, + DisplayTestCase { + description: "a value spanning lines prints as records, and the limit is named", display: QueryDisplay { - columns: texts(&["id", "name"]), - rows: vec![texts(&["1", "héllo"]), texts(&["22", ""])], + columns: texts(&["sql"]), + rows: vec![texts(&["SELECT 1\nFROM orders"])], truncated: true, - limit: Some(2), + limit: Some(1), + ..QueryDisplay::default() }, - expected_long: "-[ RECORD 1 ]---------------------------+\nid | 1\nname | héllo\n\n-[ RECORD 2 ]---------------------------+\nid | 22\nname | \n\n2 rows\nShowing 2 rows. Use --limit to show more or --no-limit to disable the limit.\n", - expected_table: "id | name \n-- | -----\n1 | héllo\n22 |\n\n2 rows\nShowing 2 rows. Use --limit to show more or --no-limit to disable the limit.\n", + layout: QueryLayout::Fitted { width: 120 }, + expected_text: "-[ RECORD 1 ]---------------------------+\nsql | SELECT 1\nFROM orders\n\n\ + 1 row shown (limit 1; use --limit or --no-limit)\n", }, DisplayTestCase { - description: "no rows and an unlimited truncation leaves an empty line", + description: "no rows still show the columns, without a type line when none is known", display: QueryDisplay { - columns: texts(&["x"]), - rows: Vec::new(), - truncated: true, - limit: None, + columns: texts(&["order_id"]), + ..QueryDisplay::default() + }, + layout: QueryLayout::Fitted { width: 120 }, + expected_text: "┌──────────┐\n│ order_id │\n├──────────┤\n└──────────┘\n\n0 rows\n", + }, + DisplayTestCase { + description: "the long layout is kept when forced", + display: QueryDisplay { + columns: texts(&["id"]), + rows: vec![texts(&["1"])], + ..QueryDisplay::default() }, - expected_long: "\n0 rows\n\n", - expected_table: "x\n-\n\n0 rows\n\n", + layout: QueryLayout::Long, + expected_text: "-[ RECORD 1 ]---------------------------+\nid | 1\n\n1 row\n", }, ]; for test_case in test_cases { assert_eq!( - render_query_long(&test_case.display), - test_case.expected_long, - "{}", - test_case.description - ); - assert_eq!( - render_query_table(&test_case.display), - test_case.expected_table, + render_query_text(&test_case.display, test_case.layout), + test_case.expected_text, "{}", test_case.description ); @@ -54,27 +93,18 @@ fn given_rows_when_rendering_text_then_aligns_and_counts() { } #[test] -fn given_rows_when_rendering_csv_then_quotes_minimally() { - let test_cases = [CsvTestCase { - description: "delimiters, quotes, newlines and a lone empty field", - columns: texts(&["a", "b"]), - rows: vec![ - texts(&["x,y", "q\"z"]), - texts(&["l\nm", ""]), - texts(&[""]), - Vec::new(), - ], - expected_csv: "a,b\n\"x,y\",\"q\"\"z\"\n\"l\nm\",\n\"\"\n\n", - }]; +fn given_more_than_fifty_rows_when_rendering_text_then_the_footer_suggests_aggregating() { + let display = QueryDisplay { + columns: texts(&["n"]), + rows: (0..51).map(|_| texts(&["7"])).collect(), + ..QueryDisplay::default() + }; - for test_case in test_cases { - assert_eq!( - render_query_csv(&test_case.columns, &test_case.rows), - test_case.expected_csv, - "{}", - test_case.description - ); - } + let rendered: String = render_query_text(&display, QueryLayout::Fitted { width: 120 }); + + assert!( + rendered.ends_with("\n51 rows; to read fewer, aggregate with GROUP BY or add a LIMIT\n") + ); } #[test] diff --git a/crates/sqlbuild-core/src/query_command/tests/test_types.rs b/crates/sqlbuild-core/src/query_command/tests/test_types.rs index 99d7d0b43a..e7930431d9 100644 --- a/crates/sqlbuild-core/src/query_command/tests/test_types.rs +++ b/crates/sqlbuild-core/src/query_command/tests/test_types.rs @@ -1,18 +1,11 @@ use crate::json::models::JsonValue; -use crate::query_command::models::QueryDisplay; +use crate::query_command::models::{QueryDisplay, QueryLayout}; pub(super) struct DisplayTestCase { pub(super) description: &'static str, pub(super) display: QueryDisplay, - pub(super) expected_long: &'static str, - pub(super) expected_table: &'static str, -} - -pub(super) struct CsvTestCase { - pub(super) description: &'static str, - pub(super) columns: Vec, - pub(super) rows: Vec>, - pub(super) expected_csv: &'static str, + pub(super) layout: QueryLayout, + pub(super) expected_text: &'static str, } pub(super) struct JsonTestCase { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index c2496e31fd..21411f7628 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -218,8 +218,9 @@ fn write_machine_error( let format: Option = invocation .args .getattr_opt("recall_format")? - .map(|format| format.extract()) - .transpose()?; + .map(|format| format.extract::>()) + .transpose()? + .flatten(); format.as_deref() == Some(JSON_FORMAT) } else if invocation.command_is("janitor") { invocation.flag("json")? && !take_machine_output_written() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs index 7b60b819a0..61669cbe11 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/mod.rs @@ -3,3 +3,4 @@ pub(crate) mod project; pub(crate) mod query; pub(crate) mod query_values; pub(crate) mod registration; +pub(crate) mod text_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index a77de9d6a1..4e7c18c3e1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -4,18 +4,15 @@ use pyo3::exceptions::PyException; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyDictMethods, PyInt, PyString, PyStringMethods}; +use pyo3::types::{PyBool, PyDict, PyDictMethods, PyFloat, PyInt, PyString, PyStringMethods}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_core::json::models::JsonValue; -use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; use sqlbuild_core::query_command::main::render_query_json::render_query_json; -use sqlbuild_core::query_command::main::render_query_long::render_query_long; -use sqlbuild_core::query_command::main::render_query_table::render_query_table; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::keywords; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -23,16 +20,14 @@ use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use crate::bindings::_helpers::query_command::query_values::{ - csv_text, display_text, json_cell, python_json_rows, strict_width, + display_text, json_cell, python_json_rows, strict_width, }; +use crate::bindings::_helpers::query_command::text_output::{LONG_FORMAT, render_text}; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::ids::ErrorId; -const LONG_FORMAT: &str = "long"; -const TABLE_FORMAT: &str = "table"; const JSON_FORMAT: &str = "json"; -const CSV_FORMAT: &str = "csv"; const STATEMENT_OK: &str = "OK\n"; const DEFAULT_LIMIT: i64 = 20; @@ -248,8 +243,66 @@ fn render_json( } } +/// Whether every value of the column at `index` is a number or NULL, and at least one is a number. +fn numeric_column( + rows: &[Vec>], + index: usize, + decimal: &Bound<'_, PyAny>, +) -> PyResult { + let mut seen: bool = false; + for row in rows { + let Some(value) = row.get(index) else { + continue; + }; + if value.is_none() { + continue; + } + if !number(value, decimal)? { + return Ok(false); + } + seen = true; + } + Ok(seen) +} + +fn number(value: &Bound<'_, PyAny>, decimal: &Bound<'_, PyAny>) -> PyResult { + if value.is_instance_of::() { + return Ok(false); + } + Ok(value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance(decimal)?) +} + +/// The column types the adapter reported, where it knows them. +fn column_types(result: &Bound<'_, PyAny>, width: usize) -> PyResult>> { + let Some(types) = result.getattr_opt("column_types")? else { + return Ok(Vec::new()); + }; + let types: Vec> = types + .try_iter()? + .map(|kind| { + let kind: Bound<'_, PyAny> = kind?; + if kind.is_none() { + Ok(None) + } else { + Ok(Some(kind.str()?.to_string_lossy().into_owned())) + } + }) + .collect::>()?; + Ok(if types.len() == width { + types + } else { + Vec::new() + }) +} + /// The result in the requested format; a statement without columns prints `OK`. -fn render(result: &Bound<'_, PyAny>, output_format: &str, limit: Option) -> PyResult { +fn render( + result: &Bound<'_, PyAny>, + output_format: Option<&str>, + limit: Option, +) -> PyResult { let py: Python<'_> = result.py(); let columns: Vec = native_field(result, "columns")? .try_iter()? @@ -259,27 +312,28 @@ fn render(result: &Bound<'_, PyAny>, output_format: &str, limit: Option) -> return Ok(STATEMENT_OK.to_owned()); } let rows: Vec>> = rows(result)?; - let display = |strict: bool| -> PyResult { - if strict { + match output_format { + Some(JSON_FORMAT) => render_json(result, &columns, &rows), + None | Some(LONG_FORMAT) => { for row in &rows { strict_width(columns.len(), row.len())?; } + let decimal: Bound<'_, PyAny> = imported(py, "decimal", "Decimal")?; + let display = QueryDisplay { + types: column_types(result, columns.len())?, + numeric: (0..columns.len()) + .map(|index| numeric_column(&rows, index, &decimal)) + .collect::>()?, + columns, + rows: texts(&rows, display_text)?, + truncated: native_field(result, "truncated")?.is_truthy()?, + limit, + }; + render_text(py, &display, output_format) } - Ok(QueryDisplay { - columns: columns.clone(), - rows: texts(&rows, display_text)?, - truncated: native_field(result, "truncated")?.is_truthy()?, - limit, - }) - }; - match output_format { - LONG_FORMAT => Ok(render_query_long(&display(true)?)), - TABLE_FORMAT => Ok(render_query_table(&display(false)?)), - JSON_FORMAT => render_json(result, &columns, &rows), - CSV_FORMAT => Ok(render_query_csv(&columns, &texts(&rows, csv_text)?)), - _ => Err(user_error( + Some(other) => Err(user_error( py, - &format!("unsupported query output format '{output_format}'"), + &format!("unsupported query output format '{other}'"), None, None, )), @@ -302,10 +356,10 @@ pub(crate) fn run_query_command( &["query_file", "selected_target", "output_format", "limit"], ), )?; - let output_format: String = if arguments.passed("output_format")? { + let output_format: Option = if arguments.passed("output_format")? { arguments.value("output_format")?.extract()? } else { - LONG_FORMAT.to_owned() + None }; let limit: Option = if arguments.passed("limit")? { arguments.value("limit")?.extract()? @@ -336,7 +390,7 @@ pub(crate) fn run_query_command( sql, limit, )?; - let rendered: String = render(&result, &output_format, limit)?; + let rendered: String = render(&result, output_format.as_deref(), limit)?; let stdout: Bound<'_, PyAny> = py.import("sys")?.getattr("stdout")?; stdout.call_method1("write", (PyString::new(py, &rendered),))?; stdout.call_method0("flush")?; @@ -348,7 +402,7 @@ pub(crate) fn run_query_command( #[pyo3(signature = (*, result, output_format, limit))] fn render_query_result( result: &Bound<'_, PyAny>, - output_format: &str, + output_format: Option<&str>, limit: Option, ) -> PyResult { render(result, output_format, limit) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs index 65f0682dc9..811f812b3e 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs @@ -1,4 +1,4 @@ -//! Query cell values as display text, CSV fields and JSON values. +//! Query cell values as display text and JSON values. use pyo3::exceptions::{PyUnicodeEncodeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; @@ -40,17 +40,6 @@ pub(crate) fn display_text(value: &Bound<'_, PyAny>) -> PyResult { python_text(value) } -/// The text `csv.writer` writes: empty for `None`, `repr` for floats, else `str`. -pub(crate) fn csv_text(value: &Bound<'_, PyAny>) -> PyResult { - if value.is_none() { - return Ok(String::new()); - } - if value.is_instance_of::() { - return Ok(value.repr()?.to_string_lossy().into_owned()); - } - python_text(value) -} - /// The JSON `default` hook: Decimal in fixed-point notation, anything else as its `str`. #[pyfunction] pub(crate) fn query_json_default(value: &Bound<'_, PyAny>) -> PyResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs new file mode 100644 index 0000000000..e388e9b04c --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs @@ -0,0 +1,42 @@ +//! The text layout `sqb query` and `sqb recall query` share: a box when the result fits the output width, records otherwise. + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use sqlbuild_core::query_command::main::render_query_text::render_query_text; +use sqlbuild_core::query_command::models::{QueryDisplay, QueryLayout}; + +use crate::bindings::_helpers::boundary::python_calls::imported; + +/// The format that always prints one record per row. +pub(crate) const LONG_FORMAT: &str = "long"; +/// The width a box may take when stdout is not a terminal. +const PIPED_WIDTH: usize = 120; + +/// The terminal's width when stdout is one, else [`PIPED_WIDTH`]. +fn output_width(py: Python<'_>) -> PyResult { + let stdout: Bound<'_, PyAny> = imported(py, "sys", "stdout")?; + let interactive: bool = stdout + .call_method0("isatty") + .and_then(|answer| answer.is_truthy()) + .unwrap_or(false); + if !interactive { + return Ok(PIPED_WIDTH); + } + let size: Bound<'_, PyAny> = imported(py, "shutil", "get_terminal_size")?.call0()?; + size.getattr("columns")?.extract() +} + +/// `display` as text: records for `--format long`, otherwise a box when it fits the output. +pub(crate) fn render_text( + py: Python<'_>, + display: &QueryDisplay, + format: Option<&str>, +) -> PyResult { + let layout: QueryLayout = if format == Some(LONG_FORMAT) { + QueryLayout::Long + } else { + QueryLayout::Fitted { + width: output_width(py)?, + } + }; + Ok(render_query_text(display, layout)) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs index 3278baa93c..d9281ed54f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs @@ -31,9 +31,10 @@ enum Cell { Bytes(Vec), } -/// A query reply: column names, rows of cells, and rows read, bytes read and query ID. +/// A query reply: column names and types, rows of cells, and rows read, bytes read and query ID. type QueryReply = ( Vec, + Vec>, Vec>>, (Option, Option, Option), ); @@ -175,11 +176,12 @@ pub(crate) fn query( ], action, )?; - let (columns, rows, (rows_read, bytes_read, query_id)): QueryReply = + let (columns, types, rows, (rows_read, bytes_read, query_id)): QueryReply = value.extract().map_err(|error| raised(action, &error))?; Ok(( QueryResult { columns, + types, rows: rows .into_iter() .map(|row| row.into_iter().map(cell_value).collect()) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 2e4b45435b..2d57fba2e4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -30,10 +30,9 @@ use crate::bindings::_helpers::query_command::query::run_query_command; use crate::bindings::_helpers::recall::clickhouse_store::{ClickHouseLocation, ClickHouseReader}; use crate::bindings::_helpers::recall::console::{write_stderr, write_stdout_bytes}; use crate::bindings::_helpers::recall::constants::{ - DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, LONG_FORMAT, + DEFAULT_MAX_CELL_BYTES, DEFAULT_QUERY_LIMIT, JSON_FORMAT, LAST_INVOCATION, RECALL_AMBIGUOUS_CODE, RECALL_DISABLED_CODE, RECALL_NOT_FOUND_CODE, RECALL_OPTIONS, - RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, TABLE_FORMAT, - TABLE_FORMAT_DEPRECATION, WAREHOUSE_BACKEND, + RECALL_QUERY_CODE, RECALL_SETTINGS_CODE, RECALL_STORE_CODE, WAREHOUSE_BACKEND, }; use crate::bindings::_helpers::recall::rendering::{QueryContext, render_result, render_schema}; use crate::bindings::_helpers::records::construct::new_error; @@ -60,7 +59,8 @@ struct RecallRequest { selected_target: Option, sql: Option, backend: Option, - format: String, + /// `long`, `json`, or `None` for the default layout. + format: Option, limit: Option, no_limit: bool, max_cell_bytes: Option, @@ -89,7 +89,7 @@ fn recall_request(py: Python<'_>, options: Option>) -> PyResul selected_target: text("selected_target")?, sql: text("sql")?, backend: text("backend")?, - format: text("output_format")?.unwrap_or_else(|| LONG_FORMAT.to_owned()), + format: text("output_format")?, limit: number("limit")?, no_limit: arguments.value("no_limit")?.is_truthy()?, max_cell_bytes: number("max_cell_bytes")?, @@ -101,14 +101,6 @@ fn recall_request(py: Python<'_>, options: Option>) -> PyResul }) } -/// Warn once that `--format table` goes away in a later release; it still works in this one. -fn warn_table_format(py: Python<'_>, recall: &RecallRequest) -> PyResult<()> { - if recall.format == TABLE_FORMAT { - write_stderr(py, TABLE_FORMAT_DEPRECATION)?; - } - Ok(()) -} - /// The project directory, target and settings, or the coded failure. fn settings(py: Python<'_>, recall: &RecallRequest) -> PyResult<(PathBuf, RecallSettings)> { let raw: PathBuf = match &recall.project_dir { @@ -280,7 +272,6 @@ fn limit(recall: &RecallRequest) -> Option { } fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { - warn_table_format(py, recall)?; let (project_dir, settings) = settings(py, recall)?; let sql: String = recall.sql.clone().unwrap_or_default(); if sql.trim().is_empty() { @@ -315,14 +306,15 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { let (adapter, label) = (store.adapter(), store.label()); store.close(); let (result, statistics) = queried.map_err(|error| store_failure(py, &error))?; - let format: &str = &recall.format; + let format: Option<&str> = recall.format.as_deref(); let max_bytes: Option = match recall.max_cell_bytes { Some(bytes) if bytes <= 0 => None, Some(bytes) => Some(usize::try_from(bytes).unwrap_or(usize::MAX)), - None if format == JSON_FORMAT => None, + None if format == Some(JSON_FORMAT) => None, None => Some(DEFAULT_MAX_CELL_BYTES), }; let rendered = render_result( + py, &result, format, max_bytes, @@ -334,7 +326,7 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { elapsed_ms: started.elapsed().as_secs_f64() * 1000.0, statistics: statistics.as_ref(), }, - ); + )?; write_stdout_bytes(py, rendered.as_bytes())?; Ok(0) } @@ -390,7 +382,6 @@ fn run_schema_ddl(py: Python<'_>, settings: &RecallSettings) -> PyResult { } fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { - warn_table_format(py, recall)?; let (project_dir, settings) = settings(py, recall)?; if recall.ddl { if recall.backend.as_deref() == Some(WAREHOUSE_BACKEND) { @@ -426,7 +417,7 @@ fn run_schema(py: Python<'_>, recall: &RecallRequest) -> PyResult { let (dialect, label) = (store.dialect(), store.label()); store.close(); let relations = described.map_err(|error| store_failure(py, &error))?; - let json_output: bool = recall.format == JSON_FORMAT; + let json_output: bool = recall.format.as_deref() == Some(JSON_FORMAT); write_stdout_bytes( py, render_schema( diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 004722f9f0..57c2229562 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -28,12 +28,9 @@ pub(crate) const RECALL_SETTINGS_CODE: &str = "C975"; pub(crate) const WAREHOUSE_BACKEND: &str = "warehouse"; /// Default cell width in text formats, in bytes. pub(crate) const DEFAULT_MAX_CELL_BYTES: usize = 4096; -pub(crate) const LONG_FORMAT: &str = "long"; -pub(crate) const TABLE_FORMAT: &str = "table"; -/// What `--format table` prints in the release before it is removed. -pub(crate) const TABLE_FORMAT_DEPRECATION: &str = "warning: --format table is deprecated and will be removed; use the default long format, or --json/--format csv\n"; -pub(crate) const CSV_FORMAT: &str = "csv"; pub(crate) const JSON_FORMAT: &str = "json"; +/// How text output shows SQL NULL. +pub(crate) const NULL_TEXT: &str = "NULL"; /// The standard base64 alphabet binary JSON cells are encoded with. pub(crate) const BASE64_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index 84b0e96a13..b2917bada0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -1,17 +1,16 @@ -//! `sqb recall query` and `schema` output: text cells, CSV and the JSON document. +//! `sqb recall query` and `schema` output: text cells, the shared text layout and the JSON document. +use pyo3::prelude::{PyResult, Python}; use serde_json::{Map, Value, json}; -use sqlbuild_core::query_command::main::render_query_csv::render_query_csv; -use sqlbuild_core::query_command::main::render_query_long::render_query_long; -use sqlbuild_core::query_command::main::render_query_table::render_query_table; use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_recall::records::models::RecallValue; use sqlbuild_recall::schema::models::Dialect; use sqlbuild_recall::store::models::{DescribedColumn, DescribedRelation, QueryResult}; +use crate::bindings::_helpers::query_command::text_output::render_text; use crate::bindings::_helpers::recall::clickhouse_calls::QueryStatistics; use crate::bindings::_helpers::recall::constants::{ - BASE64_ALPHABET, CSV_FORMAT, JOIN_KEYS, JSON_FORMAT, TABLE_FORMAT, + BASE64_ALPHABET, JOIN_KEYS, JSON_FORMAT, NULL_TEXT, }; fn base64(data: &[u8]) -> String { @@ -102,14 +101,28 @@ pub(crate) struct QueryContext<'a> { pub(crate) statistics: Option<&'a QueryStatistics>, } -/// The rendered result in `format`: `long`, `table`, `csv` or `json`. +/// Whether every non-NULL value of the column at `index` is a number, and at least one is. +fn numeric_column(result: &QueryResult, index: usize) -> bool { + let mut seen: bool = false; + for row in &result.rows { + match row.get(index) { + Some(RecallValue::Integer(_) | RecallValue::Real(_)) => seen = true, + Some(RecallValue::Null) | None => {} + Some(_) => return false, + } + } + seen +} + +/// The rendered result in `format`: `json`, `long`, or `None` for the default layout. pub(crate) fn render_result( + py: Python<'_>, result: &QueryResult, - format: &str, + format: Option<&str>, max_bytes: Option, context: &QueryContext<'_>, -) -> String { - if format == JSON_FORMAT { +) -> PyResult { + if format == Some(JSON_FORMAT) { let rows: Vec = result .rows .iter() @@ -131,42 +144,39 @@ pub(crate) fn render_result( "query_id": context.statistics.and_then(|statistics| statistics.query_id.clone()), }, }); - return format!("{document}\n"); - } - let csv: bool = format == CSV_FORMAT; - let null: &str = if csv { "" } else { "NULL" }; - let cells: Vec> = result - .rows - .iter() - .map(|row| display_row(row, null, max_bytes)) - .collect(); - if csv { - return render_query_csv(&result.columns, &cells); + return Ok(format!("{document}\n")); } let display = QueryDisplay { columns: result.columns.clone(), - rows: cells, + types: result.types.clone(), + numeric: (0..result.columns.len()) + .map(|index| numeric_column(result, index)) + .collect(), + rows: result + .rows + .iter() + .map(|row| display_row(row, NULL_TEXT, max_bytes)) + .collect(), truncated: result.truncated, limit: context .limit .map(|limit| i64::try_from(limit).unwrap_or(i64::MAX)), }; - if format == TABLE_FORMAT { - render_query_table(&display) - } else { - render_query_long(&display) - } + render_text(py, &display, format) } fn relation_note(name: &str) -> &'static str { match name { "invocations" => { - "one row per invocation; status is completed, failed, running, abandoned or unknown; \ - origin is integration, ci or local; integration, integration_job, \ + "one row per invocation; status is completed, failed, cancelled, running, abandoned \ + or unknown; origin is integration, ci or local; integration, integration_job, \ integration_run_id and integration_retry come from external_context" } - "statements" => "one row per statement; outcome no_terminal means no terminal fact arrived", - "audit_results" => "one row per evaluated audit", + "statements" => { + "one row per statement; outcome handled means SQLBuild recovered from the failure; \ + no_terminal means no terminal fact arrived" + } + "audit_results" => "one row per evaluated audit; its evidence rows are in artifacts", "output_invocations" => { "output completeness per invocation: gaps, drops, filtering, truncation, flush" } diff --git a/crates/sqlbuild-recall/src/store/main/query_store.rs b/crates/sqlbuild-recall/src/store/main/query_store.rs index 26461a3ac5..edf8e8eb8e 100644 --- a/crates/sqlbuild-recall/src/store/main/query_store.rs +++ b/crates/sqlbuild-recall/src/store/main/query_store.rs @@ -1,5 +1,6 @@ //! `query_store`: run one read-only query against a store. +use crate::records::models::RecallValue; use crate::store::_helpers::sqlite::{checked, from_sql}; use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::models::{QueryResult, SqliteStore}; @@ -27,10 +28,16 @@ pub fn query_store( .into_iter() .map(str::to_owned) .collect(); + let declared: Vec> = statement + .columns() + .iter() + .map(|column| column.decl_type().map(str::to_lowercase)) + .collect(); let width: usize = columns.len(); let mut rows = checked(statement.query([]), "run Recall query")?; let mut result = QueryResult { columns, + types: Vec::new(), rows: Vec::new(), truncated: false, }; @@ -50,5 +57,30 @@ pub fn query_store( } result.rows.push(values); } + result.types = declared + .into_iter() + .enumerate() + .map(|(index, declared)| declared.or_else(|| stored_type(&result.rows, index))) + .collect(); Ok(result) } + +/// The storage class every non-NULL value of a computed column holds, as SQLite names it; `None` when they differ or all are NULL. +fn stored_type(rows: &[Vec], index: usize) -> Option { + let mut found: Option<&'static str> = None; + for value in rows.iter().filter_map(|row| row.get(index)) { + let class: &'static str = match value { + RecallValue::Null => continue, + RecallValue::Integer(_) => "integer", + RecallValue::Real(_) => "real", + RecallValue::Text(_) => "text", + RecallValue::Blob(_) => "blob", + }; + match found { + None => found = Some(class), + Some(previous) if previous == class => {} + Some(_) => return None, + } + } + found.map(str::to_owned) +} diff --git a/crates/sqlbuild-recall/src/store/models.rs b/crates/sqlbuild-recall/src/store/models.rs index 3e3d6ec05d..a92d4ead30 100644 --- a/crates/sqlbuild-recall/src/store/models.rs +++ b/crates/sqlbuild-recall/src/store/models.rs @@ -91,6 +91,8 @@ pub struct MaintenanceOutcome { #[derive(Clone, Debug, Default, PartialEq)] pub struct QueryResult { pub columns: Vec, + /// Each column's type as the store names it, where known. + pub types: Vec>, pub rows: Vec>, /// More rows existed beyond the limit. pub truncated: bool, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md index c2dbb5f2fa..994aa9bbff 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md @@ -10,7 +10,7 @@ Online: https://sqlbuild.com/docs/cli/query/ - Usage - Flags -- Output formats +- Output - Errors - Examples @@ -38,68 +38,70 @@ File paths are resolved from the current working directory. |------|-------------| | `sql` | SQL to execute (positional argument) | | `--file` | Read SQL from a file instead of the command line | -| `--format` | Output format: `long` (default), `table`, `json`, or `csv` | +| `--format` | `long` prints one record per row; `json` prints a JSON array. Without it, the [default output](#output) applies | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 20) | | `--no-limit` | Disable the row limit | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](../concepts/project-configuration.md#command-group-warehouses) and the connection warehouse | -## Output formats +## Output -### long (default) +By default `sqb query` prints a table with each column's name and warehouse type: -Vertical record format, one field per line: +``` +┌──────────┬─────────────┬───────────┐ +│ order_id │ customer_id │ status │ +│ INTEGER │ INTEGER │ VARCHAR │ +├──────────┼─────────────┼───────────┤ +│ 1 │ 100 │ completed │ +│ 2 │ 200 │ completed │ +└──────────┴─────────────┴───────────┘ +2 rows ``` --[ RECORD 1 ]---------------------------+ -order_id | 1 -customer_id | 100 -status | completed --[ RECORD 2 ]---------------------------+ -order_id | 2 -customer_id | 200 -status | completed +The table must fit the terminal width, or 120 characters when output is redirected. When it is +wider, or a value spans several lines, `sqb query` prints one record per row instead, so no column +or value is ever cut. Numbers are right-aligned. Types show where the adapter reports them. -2 rows +The last line counts the rows. When the limit cut rows off, it says so and how to see more: + +``` +20 rows shown (limit 20; use --limit or --no-limit) ``` -### table +Above 50 rows it suggests reading fewer: `60 rows; to read fewer, aggregate with GROUP BY or add a +LIMIT`. -Horizontal table format: +### long + +`--format long` always prints one record per row: ``` -order_id | customer_id | status --------- | ----------- | --------- -1 | 100 | completed -2 | 200 | completed +-[ RECORD 1 ]---------------------------+ +order_id | 1 +customer_id | 100 +status | completed 2 rows ``` ### json -JSON array of objects: +`--format json`, or `--json`, prints a JSON array of objects for programs: ```json [{"order_id": 1, "customer_id": 100, "status": "completed"}] ``` -### csv - -Standard CSV with headers: - -```csv -order_id,customer_id,status -1,100,completed -2,200,completed -``` +`--format table` and `--format csv` are not accepted. The default output already prints a table +when the result fits; use `--json` for programs. ## Errors When the warehouse rejects the SQL, `sqb query` prints the warehouse message as `error[C108]: ...` on stderr and exits with code `2`. Nothing is written to stdout, so -redirected `json` or `csv` output is never partial. Invalid command input, such as a missing +redirected `json` output is never partial. Invalid command input, such as a missing query file, exits with code `1`. ## Examples @@ -108,8 +110,8 @@ query file, exits with code `1`. # Quick inspection with default format sqb query "SELECT * FROM dev.fact_orders" -# Table format with higher limit -sqb query "SELECT * FROM dev.dim_customers" --format table --limit 50 +# One record per row with a higher limit +sqb query "SELECT * FROM dev.dim_customers" --format long --limit 50 # Export to JSON sqb query "SELECT * FROM dev.daily_revenue" --json --no-limit diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md index dfb94e424f..ee013c108d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/recall.md @@ -35,7 +35,7 @@ Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| | `sql` | The query (positional argument) | -| `--format` | `long` (default) for reading, `json` for programs, or `csv` for export. `table` still works in this release but prints a deprecation warning and will be removed | +| `--format` | `long` prints one record per row; `json` is for programs. Without it, a table when the result fits, as [`sqb query`](query.md#output) prints it | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 100) | | `--no-limit` | Disable the row limit | @@ -52,15 +52,21 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -command | build -status | completed -exit_code | 0 -duration_ms | 752.0 +┌─────────┬───────────┬───────────┬─────────────┐ +│ command │ status │ exit_code │ duration_ms │ +│ text │ text │ integer │ real │ +├─────────┼───────────┼───────────┼─────────────┤ +│ build │ completed │ 0 │ 752.0 │ +└─────────┴───────────┴───────────┴─────────────┘ 1 row ``` +The second header line is each column's type in the store. A result wider than the terminal (120 +characters when output is redirected), or with a value spanning lines, prints one record per row +instead, so nothing is cut. `--format table` and `--format csv` are not accepted: the default +output already prints a table when it fits, and `--json` is for programs. + The slowest statements of past builds, with their SQL: ```bash @@ -71,15 +77,13 @@ sqb recall query --max-cell-bytes 60 \ ``` ``` --[ RECORD 1 ]---------------------------+ -resource_name | orders -ms | 3.6 -sql | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] - --[ RECORD 2 ]---------------------------+ -resource_name | NULL -ms | 3.2 -sql | CREATE SCHEMA IF NOT EXISTS dev +┌───────────────┬──────┬──────────────────────────────────────────────────────────────────────────┐ +│ resource_name │ ms │ sql │ +│ text │ real │ text │ +├───────────────┼──────┼──────────────────────────────────────────────────────────────────────────┤ +│ orders │ 5.1 │ CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 AS o…[+81 bytes] │ +│ NULL │ 3.2 │ CREATE SCHEMA IF NOT EXISTS dev │ +└───────────────┴──────┴──────────────────────────────────────────────────────────────────────────┘ 2 rows ``` diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 8c941e3403..0a0c616cbb 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -3145,7 +3145,7 @@ class CliNamespace: max_columns: int | None query_sql: str | None query_file: str | None - query_format: str + query_format: str | None query_limit: int | None query_no_limit: bool cost_selector: str @@ -4818,7 +4818,9 @@ def run_query_command( output_format: str = "long", limit: int | None = 20, ) -> int: ... -def render_query_result(*, result: QueryResult, output_format: str, limit: int | None) -> str: ... +def render_query_result( + *, result: QueryResult, output_format: str | None, limit: int | None +) -> str: ... def run_dag_command( *, project_dir: Path | None, diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 2a0c9d48a7..3b8c3660e8 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1127,16 +1127,19 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + column_types: tuple[str | None, ...] = tuple(str(column[1]) for column in description) if limit is None: return QueryResult( columns=columns, rows=tuple(tuple(row) for row in cursor.fetchall()), + column_types=column_types, ) fetched_rows: list[tuple[object, ...]] = [tuple(row) for row in cursor.fetchmany(limit + 1)] return QueryResult( columns=columns, rows=tuple(fetched_rows[:limit]), truncated=len(fetched_rows) > limit, + column_types=column_types, ) def describe_relation(self, *, connection: Any, relation: str) -> tuple[ColumnInfo, ...]: diff --git a/src/sqlbuild/adapter/contract/models.py b/src/sqlbuild/adapter/contract/models.py index a66147d3ad..5c0a97c92a 100644 --- a/src/sqlbuild/adapter/contract/models.py +++ b/src/sqlbuild/adapter/contract/models.py @@ -335,6 +335,7 @@ class QueryResult: columns: tuple[str, ...] = field(default_factory=tuple) rows: tuple[tuple[object, ...], ...] = field(default_factory=tuple) truncated: bool = False + column_types: tuple[str | None, ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py index ae367239ac..9f3d55ec7c 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -116,8 +116,11 @@ def _query_settings( def run_recall_query( *, handle: int, sql: str, settings: dict[str, int], row_cap: int | None -) -> tuple[RecallFailure, tuple[list[str], list[list[RecallCell]], RecallStatistics] | None]: - """Columns, at most `row_cap` rows of plain values, and server statistics.""" +) -> tuple[ + RecallFailure, + tuple[list[str], list[str | None], list[list[RecallCell]], RecallStatistics] | None, +]: + """Columns and their types, at most `row_cap` rows of plain values, and server statistics.""" try: client: Any = SharedRecallConnections.connections.get(handle=handle).client @@ -134,6 +137,9 @@ def run_recall_query( if row_cap is not None and len(rows) >= row_cap: break columns: list[str] = list(result.column_names) + types: list[str | None] = [ + getattr(column_type, "name", None) for column_type in result.column_types + ] summary: dict[str, Any] = dict(result.summary or {}) except Exception as error: return _failure(error), None @@ -142,7 +148,7 @@ def run_recall_query( _number(summary=summary, key="read_bytes"), summary.get("query_id"), ) - return None, (columns, rows[:row_cap] if row_cap is not None else rows, statistics) + return None, (columns, types, rows[:row_cap] if row_cap is not None else rows, statistics) def run_recall_command(*, handle: int, sql: str) -> tuple[RecallFailure, None]: diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index a92876ca3d..65f5850936 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -230,11 +230,17 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + column_types: tuple[str | None, ...] = tuple(str(column[1]) for column in description) if limit is None: - return QueryResult(columns=columns, rows=tuple(cursor.fetchall())) + return QueryResult( + columns=columns, rows=tuple(cursor.fetchall()), column_types=column_types + ) fetched: list[tuple[object, ...]] = cursor.fetchmany(limit + 1) return QueryResult( - columns=columns, rows=tuple(fetched[:limit]), truncated=len(fetched) > limit + columns=columns, + rows=tuple(fetched[:limit]), + truncated=len(fetched) > limit, + column_types=column_types, ) def supports_zero_copy_clone(self) -> bool: diff --git a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py index 5dd568d615..3232567671 100644 --- a/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py +++ b/src/sqlbuild/adapters/postgres/classes/postgres_adapter.py @@ -360,7 +360,12 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: ] truncated = len(fetched_rows) > limit rows = tuple(fetched_rows[:limit]) - return QueryResult(columns=columns, rows=rows, truncated=truncated) + column_types: tuple[str | None, ...] = tuple( + getattr(column, "type_display", None) for column in description + ) + return QueryResult( + columns=columns, rows=rows, truncated=truncated, column_types=column_types + ) def relation_exists( self, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py b/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py index 14c807538c..7d1d0b2a5b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/adapters/test_project_local_adapters.py @@ -21,7 +21,7 @@ [ ProjectLocalAdapterCliTestCase( description="query uses nested project local adapter", - command=("query", "SELECT 'ignored' AS value", "--format", "csv"), + command=("query", "SELECT 'ignored' AS value", "--format", "long"), expected_stdout_fragment="duckdb_plus", ) ], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index ede85b5f93..d2b3d50036 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -289,7 +289,7 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u BigQueryCliTestCase( description="query command uses bigquery local override", command=("query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID"), - expected_stdout_fragments=("ID | 1", "NAME | alice", "ID | 2", "NAME | bob"), + expected_stdout_fragments=("┌", "│ alice │", "│ bob │", "2 rows"), ), BigQueryCliTestCase( description="query command renders json output", @@ -302,14 +302,14 @@ def test_given_physical_source_without_freshness_when_running_on_bigquery_then_u expected_stdout_fragments=('"ID": 1', '"NAME": "alice"'), ), BigQueryCliTestCase( - description="query command renders csv output", + description="query command renders long output", command=( "query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID LIMIT 1", "--format", - "csv", + "long", ), - expected_stdout_fragments=("ID,NAME", "1,alice"), + expected_stdout_fragments=("ID | 1", "NAME | alice"), ), BigQueryCliTestCase( description="query command prints ok for ddl statements", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 074f14f4a7..c8ca0e1e46 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -460,7 +460,7 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_databricks_th DatabricksCliTestCase( description="query command uses databricks local override", command=("query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID"), - expected_stdout_fragments=("ID | 1", "NAME | alice", "ID | 2", "NAME | bob"), + expected_stdout_fragments=("┌", "│ alice │", "│ bob │", "2 rows"), ), DatabricksCliTestCase( description="query command renders json output", @@ -473,14 +473,14 @@ def test_given_intermediate_strategy_project_when_loading_twice_on_databricks_th expected_stdout_fragments=('"ID": 1', '"NAME": "alice"'), ), DatabricksCliTestCase( - description="query command renders csv output", + description="query command renders long output", command=( "query", "SELECT id AS ID, name AS NAME FROM {source} ORDER BY ID LIMIT 1", "--format", - "csv", + "long", ), - expected_stdout_fragments=("ID,NAME", "1,alice"), + expected_stdout_fragments=("ID | 1", "NAME | alice"), ), DatabricksCliTestCase( description="query command prints ok for ddl statements", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py index b87431f535..72b8c5ac14 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py @@ -21,16 +21,24 @@ "test_case", [ QueryCliTestCase( - description="renders long output with default limit", + description="renders a table with column types when the result fits", command=("query", "SELECT 1 AS id, 'alice' AS name"), expected_stdout_fragment=( - "-[ RECORD 1 ]---------------------------+\nid | 1\nname | alice\n\n1 row\n" + "┌─────────┬─────────┐\n" + "│ id │ name │\n" + "│ INTEGER │ VARCHAR │\n" + "├─────────┼─────────┤\n" + "│ 1 │ alice │\n" + "└─────────┴─────────┘\n\n" + "1 row\n" ), ), QueryCliTestCase( - description="renders table output when requested", - command=("query", "SELECT 1 AS id", "--format", "table"), - expected_stdout_fragment="id\n--\n1\n\n1 row\n", + description="renders one record per row when requested", + command=("query", "SELECT 1 AS id, 'alice' AS name", "--format", "long"), + expected_stdout_fragment=( + "-[ RECORD 1 ]---------------------------+\nid | 1\nname | alice\n\n1 row\n" + ), ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py index dcac52db7c..a2efbe07fc 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py @@ -149,8 +149,8 @@ def test_given_project_when_running_command_then_project_discovery_runs_only_whe ( QueryTargetConnectionTestCase( description="default target connection", - command=("query", "SELECT label FROM destination", "--format", "csv"), - expected_stdout_fragment="label\ndev\n", + command=("query", "SELECT label FROM destination", "--format", "long"), + expected_stdout_fragment="label | dev\n", ), QueryTargetConnectionTestCase( description="explicit target connection", @@ -158,11 +158,11 @@ def test_given_project_when_running_command_then_project_discovery_runs_only_whe "query", "SELECT label FROM destination", "--format", - "csv", + "long", "--target", "prod", ), - expected_stdout_fragment="label\nprod\n", + expected_stdout_fragment="label | prod\n", ), ), ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index 63f13f6223..3a5a7c0790 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -300,6 +300,8 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ command=( "query", "SELECT CURRENT_DATABASE() AS database_name, CURRENT_SCHEMA() AS schema_name", + "--format", + "long", ), expected_stdout_fragments=("DATABASE_NAME | SQB_DB", "SCHEMA_NAME |"), expected_schema_fragment="SQLBUILD_E2E_", @@ -310,9 +312,9 @@ def test_given_physical_source_without_freshness_when_running_on_snowflake_then_ expected_stdout_fragments=('"ID": 1', '"NAME": "alice"'), ), SnowflakeCliTestCase( - description="query command renders csv output", - command=("query", "SELECT 1 AS id, 'alice' AS name", "--format", "csv"), - expected_stdout_fragments=("ID,NAME", "1,alice"), + description="query command renders a table with Snowflake types by default", + command=("query", "SELECT 1 AS id, 'alice' AS name"), + expected_stdout_fragments=("┌", "NUMBER(1,0)", "TEXT", "alice", "1 row"), ), SnowflakeCliTestCase( description="query command prints ok for ddl statements", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py index 26bdae45dc..0c5409bfe8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlite/test_sqlite.py @@ -99,10 +99,14 @@ def test_given_orders_project_when_building_twice_then_sqlite_matches_duckdb( "--no-color", "query", "--format", - "csv", + "long", "SELECT customer_id, total_cents FROM analytics__customer_totals ORDER BY 1", ), - expected_fragments=("customer_id,total_cents", "1,1975", "2,3000", "3,500"), + expected_fragments=( + "customer_id | 1\ntotal_cents | 1975", + "customer_id | 2\ntotal_cents | 3000", + "customer_id | 3\ntotal_cents | 500", + ), ), SqliteCommandE2ETestCase( description="debug connects and runs a query", diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 3852fd9470..d03c984f4b 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -233,10 +233,18 @@ class JsonErrorTestCase: @dataclass(frozen=True) -class DeprecatedFormatTestCase: +class RemovedFormatTestCase: description: str args: tuple[str, ...] - expected_warning: bool + expected_message: str + + +@dataclass(frozen=True) +class QueryLayoutTestCase: + description: str + args: tuple[str, ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 802145907f..f8163563fc 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -245,7 +245,7 @@ def test_given_a_recall_query_when_rendered_as_json_then_it_matches_the_schema( "8", "SELECT 'first line' || char(10) || 'second line' AS note", ), - expected_fragments=("note | first li…[+14 bytes]", "1 row"), + expected_fragments=("│ first li…[+14 bytes] │", "1 row"), ), TextOutputTestCase( description="the schema lists views and join keys", diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py index 4834953e71..d2def05b76 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_cli.py @@ -5,11 +5,12 @@ import pytest from tests.integration.src.sqlbuild.cli.recall._test_types import ( - DeprecatedFormatTestCase, JsonErrorTestCase, LeanRecallTestCase, + QueryLayoutTestCase, RecallDiagnosticsTestCase, RecallUsageErrorTestCase, + RemovedFormatTestCase, StatementStartTestCase, StoreBeingCreatedTestCase, ) @@ -22,8 +23,6 @@ write_sinks, ) -TABLE_DEPRECATION: str = "--format table is deprecated and will be removed" - @pytest.mark.parametrize( "test_case", @@ -81,35 +80,106 @@ def test_given_a_failing_recall_command_with_json_when_run_then_stdout_is_a_json @pytest.mark.parametrize( "test_case", [ - DeprecatedFormatTestCase( - description="table still renders and warns", + RemovedFormatTestCase( + description="recall query rejects table and points at the default", args=("recall", "query", "--format", "table", "SELECT 1 AS answer"), - expected_warning=True, + expected_message=( + "'table' was removed; the default output prints a table when the result fits" + ), ), - DeprecatedFormatTestCase( - description="the schema table warns too", + RemovedFormatTestCase( + description="recall schema rejects table too", args=("recall", "schema", "--format", "table"), - expected_warning=True, + expected_message="'table' was removed", + ), + RemovedFormatTestCase( + description="recall query rejects csv and points at json", + args=("recall", "query", "--format", "csv", "SELECT 1 AS answer"), + expected_message="'csv' was removed; use --json for programs", + ), + RemovedFormatTestCase( + description="query rejects table", + args=("query", "--format", "table", "SELECT 1 AS answer"), + expected_message="'table' was removed", + ), + RemovedFormatTestCase( + description="query rejects csv", + args=("query", "--format", "csv", "SELECT 1 AS answer"), + expected_message="'csv' was removed; use --json for programs", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_removed_format_when_querying_then_it_is_a_usage_error_naming_the_replacement( + test_case: RemovedFormatTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + + rejected: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) + + assert rejected.returncode == 2 + assert rejected.stdout == b"" + assert test_case.expected_message in rejected.stderr.decode() + + +@pytest.mark.parametrize( + "test_case", + [ + QueryLayoutTestCase( + description="a narrow Recall result prints a box with column types", + args=("recall", "query", "SELECT 7 AS order_count, 'open' AS status"), + expected_fragments=("│ order_count │ status │", "│ integer │ text │", "1 row"), + unexpected_fragments=("RECORD",), + ), + QueryLayoutTestCase( + description="a Recall result wider than the output prints whole records", + args=("recall", "query", "SELECT replace(hex(zeroblob(70)), '0', 'x') AS note"), + expected_fragments=("-[ RECORD 1 ]", "note | " + "x" * 140, "1 row"), + unexpected_fragments=("┌",), + ), + QueryLayoutTestCase( + description="long always prints records", + args=("recall", "query", "--format", "long", "SELECT 7 AS order_count"), + expected_fragments=("-[ RECORD 1 ]", "order_count | 7"), + unexpected_fragments=("┌",), + ), + QueryLayoutTestCase( + description="a warehouse result shows the warehouse type names", + args=("query", "SELECT 7 AS order_id, CAST(12.5 AS DECIMAL(10,2)) AS amount"), + expected_fragments=("│ order_id │ amount │", "INTEGER", "DECIMAL(10,2)"), + unexpected_fragments=("RECORD",), + ), + QueryLayoutTestCase( + description="a limited result says how to see more", + args=("query", "SELECT range AS order_id FROM range(30)"), + expected_fragments=("20 rows shown (limit 20; use --limit or --no-limit)",), + unexpected_fragments=("GROUP BY",), ), - DeprecatedFormatTestCase( - description="the default long format does not warn", - args=("recall", "query", "SELECT 1 AS answer"), - expected_warning=False, + QueryLayoutTestCase( + description="many rows suggest reading fewer", + args=("query", "--no-limit", "SELECT range AS order_id FROM range(60)"), + expected_fragments=("60 rows; to read fewer, aggregate with GROUP BY or add a LIMIT",), + unexpected_fragments=(), ), ], ids=lambda case: case.description, ) -def test_given_the_table_format_when_querying_then_it_works_with_a_deprecation_warning( - test_case: DeprecatedFormatTestCase, tmp_path: Path +def test_given_a_query_result_when_printed_by_default_then_it_fits_the_output_without_hiding_data( + test_case: QueryLayoutTestCase, tmp_path: Path ) -> None: project_dir: Path = write_project(tmp_path / "orders_shop") assert run(project_dir, "build").returncode == 0 rendered: subprocess.CompletedProcess[bytes] = run(project_dir, *test_case.args) - assert rendered.returncode == 0 - assert rendered.stdout - assert (TABLE_DEPRECATION in rendered.stderr.decode()) is test_case.expected_warning + stdout: str = rendered.stdout.decode() + assert rendered.returncode == 0, rendered.stderr.decode(errors="replace") + assert [fragment in stdout for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ], stdout + assert [fragment in stdout for fragment in test_case.unexpected_fragments] == [ + False for _ in test_case.unexpected_fragments + ], stdout @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py index dd72a9d08d..f3424538b0 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/helpers.py @@ -29,8 +29,8 @@ class FakeSnowflakeDescribeCursor: """Cursor double exposing Snowflake-style description metadata.""" - def __init__(self, description: tuple[tuple[str], ...]) -> None: - self.description: tuple[tuple[str], ...] = description + def __init__(self, description: tuple[tuple[object, ...], ...]) -> None: + self.description: tuple[tuple[object, ...], ...] = description self.executed_sql: str | None = None self.executemany_sql: str | None = None self.executemany_rows: list[tuple[object, ...]] = [] @@ -47,6 +47,22 @@ def close(self) -> None: self.closed = True +class FakeSnowflakeQueryCursor(FakeSnowflakeDescribeCursor): + """Cursor double returning rows under Snowflake result metadata.""" + + def __init__( + self, description: tuple[tuple[object, ...], ...], rows: list[tuple[object, ...]] + ) -> None: + super().__init__(description=description) + self._rows: list[tuple[object, ...]] = rows + + def fetchall(self) -> list[tuple[object, ...]]: + return self._rows + + def fetchmany(self, size: int) -> list[tuple[object, ...]]: + return self._rows[:size] + + class FakeSnowflakeDescribeConnection: """Connection double returning a preconfigured describe cursor.""" diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py index 0c6990f34e..97f857f613 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_parsing.py @@ -384,9 +384,9 @@ def test_given_positional_selectors_when_parsing_then_they_join_the_selection( ), OutputFormatAliasTestCase( description="query explicit format", - argv=("query", "SELECT 1", "--format", "csv"), + argv=("query", "SELECT 1", "--format", "long"), format_attribute="query_format", - expected_format="csv", + expected_format="long", ), ], ids=lambda case: case.description, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py index 065d9b916a..a8dc4ef844 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/_test_types.py @@ -7,6 +7,6 @@ class QueryOutputTestCase: description: str result: QueryResult - output_format: str + output_format: str | None limit: int | None expected_output: str diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py index 74548abcd1..8fbe090eb0 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py @@ -48,8 +48,7 @@ "-[ RECORD 1 ]---------------------------+\n" "id | 1\n" "\n" - "1 row\n" - "Showing 1 rows. Use --limit to show more or --no-limit to disable the limit.\n" + "1 row shown (limit 1; use --limit or --no-limit)\n" ), ), QueryOutputTestCase( @@ -60,11 +59,33 @@ expected_output='[{"id": 1, "name": "alice"}]\n', ), QueryOutputTestCase( - description="renders csv rows", - result=QueryResult(columns=("id", "name"), rows=((1, "alice"),)), - output_format="csv", + description="renders a box with the warehouse types when the result fits", + result=QueryResult( + columns=("order_id", "amount"), + rows=((7, "12.50"),), + column_types=("INTEGER", "DECIMAL(10,2)"), + ), + output_format=None, limit=20, - expected_output="id,name\n1,alice\n", + expected_output=( + "┌──────────┬───────────────┐\n" + "│ order_id │ amount │\n" + "│ INTEGER │ DECIMAL(10,2) │\n" + "├──────────┼───────────────┤\n" + "│ 7 │ 12.50 │\n" + "└──────────┴───────────────┘\n" + "\n" + "1 row\n" + ), + ), + QueryOutputTestCase( + description="renders records when a value spans lines, so nothing is cut", + result=QueryResult(columns=("note",), rows=(("first\nsecond",),)), + output_format=None, + limit=20, + expected_output=( + "-[ RECORD 1 ]---------------------------+\nnote | first\nsecond\n\n1 row\n" + ), ), ], ids=lambda case: case.description, diff --git a/website/src/content/docs/docs/cli/query.mdx b/website/src/content/docs/docs/cli/query.mdx index 94128a39b7..b5805ed023 100644 --- a/website/src/content/docs/docs/cli/query.mdx +++ b/website/src/content/docs/docs/cli/query.mdx @@ -29,68 +29,70 @@ File paths are resolved from the current working directory. |------|-------------| | `sql` | SQL to execute (positional argument) | | `--file` | Read SQL from a file instead of the command line | -| `--format` | Output format: `long` (default), `table`, `json`, or `csv` | +| `--format` | `long` prints one record per row; `json` prints a JSON array. Without it, the [default output](#output) applies | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 20) | | `--no-limit` | Disable the row limit | | `--warehouse ` | Snowflake warehouse for this invocation; overrides the target's `query` [warehouse group](/docs/concepts/project-configuration#command-group-warehouses) and the connection warehouse | -## Output formats +## Output -### long (default) +By default `sqb query` prints a table with each column's name and warehouse type: -Vertical record format, one field per line: +``` +┌──────────┬─────────────┬───────────┐ +│ order_id │ customer_id │ status │ +│ INTEGER │ INTEGER │ VARCHAR │ +├──────────┼─────────────┼───────────┤ +│ 1 │ 100 │ completed │ +│ 2 │ 200 │ completed │ +└──────────┴─────────────┴───────────┘ +2 rows ``` --[ RECORD 1 ]---------------------------+ -order_id | 1 -customer_id | 100 -status | completed --[ RECORD 2 ]---------------------------+ -order_id | 2 -customer_id | 200 -status | completed +The table must fit the terminal width, or 120 characters when output is redirected. When it is +wider, or a value spans several lines, `sqb query` prints one record per row instead, so no column +or value is ever cut. Numbers are right-aligned. Types show where the adapter reports them. -2 rows +The last line counts the rows. When the limit cut rows off, it says so and how to see more: + +``` +20 rows shown (limit 20; use --limit or --no-limit) ``` -### table +Above 50 rows it suggests reading fewer: `60 rows; to read fewer, aggregate with GROUP BY or add a +LIMIT`. -Horizontal table format: +### long + +`--format long` always prints one record per row: ``` -order_id | customer_id | status --------- | ----------- | --------- -1 | 100 | completed -2 | 200 | completed +-[ RECORD 1 ]---------------------------+ +order_id | 1 +customer_id | 100 +status | completed 2 rows ``` ### json -JSON array of objects: +`--format json`, or `--json`, prints a JSON array of objects for programs: ```json [{"order_id": 1, "customer_id": 100, "status": "completed"}] ``` -### csv - -Standard CSV with headers: - -```csv -order_id,customer_id,status -1,100,completed -2,200,completed -``` +`--format table` and `--format csv` are not accepted. The default output already prints a table +when the result fits; use `--json` for programs. ## Errors When the warehouse rejects the SQL, `sqb query` prints the warehouse message as `error[C108]: ...` on stderr and exits with code `2`. Nothing is written to stdout, so -redirected `json` or `csv` output is never partial. Invalid command input, such as a missing +redirected `json` output is never partial. Invalid command input, such as a missing query file, exits with code `1`. ## Examples @@ -99,8 +101,8 @@ query file, exits with code `1`. # Quick inspection with default format sqb query "SELECT * FROM dev.fact_orders" -# Table format with higher limit -sqb query "SELECT * FROM dev.dim_customers" --format table --limit 50 +# One record per row with a higher limit +sqb query "SELECT * FROM dev.dim_customers" --format long --limit 50 # Export to JSON sqb query "SELECT * FROM dev.daily_revenue" --json --no-limit diff --git a/website/src/content/docs/docs/cli/recall.mdx b/website/src/content/docs/docs/cli/recall.mdx index b7d15976ec..d219edfe46 100644 --- a/website/src/content/docs/docs/cli/recall.mdx +++ b/website/src/content/docs/docs/cli/recall.mdx @@ -26,7 +26,7 @@ Statements that write, and multiple statements, are rejected. | Flag | Description | |------|-------------| | `sql` | The query (positional argument) | -| `--format` | `long` (default) for reading, `json` for programs, or `csv` for export. `table` still works in this release but prints a deprecation warning and will be removed | +| `--format` | `long` prints one record per row; `json` is for programs. Without it, a table when the result fits, as [`sqb query`](/docs/cli/query#output) prints it | | `--json` | Shorthand for `--format json` | | `--limit` | Maximum rows to return (default: 100) | | `--no-limit` | Disable the row limit | @@ -43,15 +43,21 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -command | build -status | completed -exit_code | 0 -duration_ms | 752.0 +┌─────────┬───────────┬───────────┬─────────────┐ +│ command │ status │ exit_code │ duration_ms │ +│ text │ text │ integer │ real │ +├─────────┼───────────┼───────────┼─────────────┤ +│ build │ completed │ 0 │ 752.0 │ +└─────────┴───────────┴───────────┴─────────────┘ 1 row ``` +The second header line is each column's type in the store. A result wider than the terminal (120 +characters when output is redirected), or with a value spanning lines, prints one record per row +instead, so nothing is cut. `--format table` and `--format csv` are not accepted: the default +output already prints a table when it fits, and `--json` is for programs. + The slowest statements of past builds, with their SQL: ```bash @@ -62,15 +68,13 @@ sqb recall query --max-cell-bytes 60 \ ``` ``` --[ RECORD 1 ]---------------------------+ -resource_name | orders -ms | 3.6 -sql | CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 …[+74 bytes] - --[ RECORD 2 ]---------------------------+ -resource_name | NULL -ms | 3.2 -sql | CREATE SCHEMA IF NOT EXISTS dev +┌───────────────┬──────┬──────────────────────────────────────────────────────────────────────────┐ +│ resource_name │ ms │ sql │ +│ text │ real │ text │ +├───────────────┼──────┼──────────────────────────────────────────────────────────────────────────┤ +│ orders │ 5.1 │ CREATE OR REPLACE TABLE dev.orders__staging AS SELECT 1 AS o…[+81 bytes] │ +│ NULL │ 3.2 │ CREATE SCHEMA IF NOT EXISTS dev │ +└───────────────┴──────┴──────────────────────────────────────────────────────────────────────────┘ 2 rows ``` diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 401c4f772a..b13eb8717c 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -70,11 +70,12 @@ sqb recall query --backend warehouse \ ``` ``` --[ RECORD 1 ]---------------------------+ -payment_id | 9 -order_id | 8 -amount_cents | -2200 -paid_at | 2026-04-03 15:42:00 +┌────────────┬──────────┬──────────────┬─────────────────────┐ +│ payment_id │ order_id │ amount_cents │ paid_at │ +│ INTEGER │ INTEGER │ INTEGER │ TIMESTAMP │ +├────────────┼──────────┼──────────────┼─────────────────────┤ +│ 9 │ 8 │ -2200 │ 2026-04-03 15:42:00 │ +└────────────┴──────────┴──────────────┴─────────────────────┘ 1 row ``` From 9741519e6e75a3e753af87640668d66da8e2d63e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:55 +0000 Subject: [PATCH 04/30] docs(skill): keep the default query format and reshape the question instead The bundled skill tells agents to keep the default output for anything a person reads, to aggregate, select fewer columns or add a LIMIT when a result is too long, never to switch format to save space, and to say why when they do change it. --- src/sqlbuild/.agents/skills/sqlbuild/SKILL.md | 10 ++++++---- .../sqlbuild/references/investigating-runs.md | 16 +++++++++------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md index db4f9017dc..960d4d018d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/SKILL.md @@ -94,10 +94,12 @@ Check that target's `read_connection` in `[recall]` or `[targets..recall]` ClickHouse SQL, and `--json` reports `rows_read` and `bytes_read`; filter on `invocation_id` or `invocation_started_at` and keep a `LIMIT` on large stores. -The default `long` format is fine in most cases: read it as it is. If it is too verbose, select -fewer columns or add a `LIMIT` rather than changing the format or reshaping the output. Use -`--json` only when a program does something meaningful with the result, such as feeding it into -further processing, not just to print rows more compactly. +Keep the default output format for anything a person reads: it prints a table when the result fits +and one record per row otherwise, so nothing is cut. If a result is too long, reshape the question, +not the output: aggregate with `GROUP BY`, select only the columns you need, or add a `LIMIT`. A +long listing usually means the query is not precise enough yet. Never switch format to shorten +output or save context; `--json` is for programs that process the result. If you change the +format, say why. Every command except `sqb recall` is recorded in the store of the target it runs against, including `sqb debug` and failed commands. To check a shared target, prefer `sqb recall` reads: diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 6e1d004192..cd7ba3fb21 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -88,13 +88,15 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of ## Output format -- The default `long` format is fine in most cases: read it as it is, every value in full, one - record at a time. -- If it is too verbose, select fewer columns or add a `LIMIT` (or aggregate) rather than changing - the format or reshaping the output. -- Use `--json` only when a program does something meaningful with the result, for example to chain - an `invocation_id` into the next query, or when you need `statistics` (`rows_read`, - `bytes_read`, elapsed time). Not just to print rows more compactly. +- Keep the default format for anything a person reads. It prints a table with column types when + the result fits and one record per row otherwise, every value in full. +- If a result is too long, reshape the question, not the output: aggregate with `GROUP BY`, select + only the columns you need, or add a `LIMIT`. A long listing usually means the query is not + precise enough yet. +- Never switch format to shorten output or save context. `--format long` always prints records; + `--json` is for programs, for example to chain an `invocation_id` into the next query, or when + you need `statistics` (`rows_read`, `bytes_read`, elapsed time). If you change the format, say + why. ## Worked investigations From 5e4470a6dc489d41a09ba6ff9b32bb1bd5fbff78 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:56 +0000 Subject: [PATCH 05/30] feat(recall): record the effective selection of each invocation Recall records the selectors a command ran with, including those read from each `--select-file`, whose paths are kept in `selector_files`. `selection_count` counts them and `selection_truncated` marks a list cut at 64 KiB. --- crates/sqlbuild-cli/src/dispatch/constants.rs | 2 ++ crates/sqlbuild-cli/src/dispatch/main/mod.rs | 1 + .../dispatch/main/selector_file_selectors.rs | 17 +++++++++++ .../src/bindings/_helpers/cli/dispatch.rs | 16 ++++------- .../src/bindings/_helpers/cli/entry.rs | 7 +++-- .../src/bindings/_helpers/recall/session.rs | 28 ++++++++++++++++++- .../src/records/main/bounded_selection.rs | 20 +++++++++++++ .../src/records/main/invocation_row.rs | 6 ++++ .../sqlbuild-recall/src/records/main/mod.rs | 1 + crates/sqlbuild-recall/src/records/models.rs | 7 +++++ .../src/schema/_helpers/clickhouse_views.rs | 13 ++++++++- .../src/schema/_helpers/sqlite_views.rs | 7 +++++ .../sqlbuild-recall/src/schema/constants.rs | 7 +++-- .../src/schema/tests/test_ddl.rs | 2 +- .../sqlbuild/references/investigating-runs.md | 3 ++ 15 files changed, 118 insertions(+), 19 deletions(-) create mode 100644 crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs create mode 100644 crates/sqlbuild-recall/src/records/main/bounded_selection.rs diff --git a/crates/sqlbuild-cli/src/dispatch/constants.rs b/crates/sqlbuild-cli/src/dispatch/constants.rs index 71b7c4de91..550b31ebe3 100644 --- a/crates/sqlbuild-cli/src/dispatch/constants.rs +++ b/crates/sqlbuild-cli/src/dispatch/constants.rs @@ -12,6 +12,8 @@ pub(crate) const DBT_INIT_SUBCOMMAND: &str = "init"; pub(crate) const DIFF_COMMAND_NAME: &str = "diff"; /// `sqb recall` reads history and is never recorded itself. pub(crate) const RECALL_COMMAND_NAME: &str = "recall"; +/// A `--select-file` line starting with this is a comment. +pub(crate) const SELECTOR_COMMENT_PREFIX: &str = "#"; const PROJECT_DIR: Field = field("project_dir", Source::ProjectDir); const SELECT: Field = field("select", Source::Select); diff --git a/crates/sqlbuild-cli/src/dispatch/main/mod.rs b/crates/sqlbuild-cli/src/dispatch/main/mod.rs index 7cc59e2303..594f4fab79 100644 --- a/crates/sqlbuild-cli/src/dispatch/main/mod.rs +++ b/crates/sqlbuild-cli/src/dispatch/main/mod.rs @@ -3,4 +3,5 @@ pub mod command_dispatch; pub mod event_export_warning; pub mod failure_report; pub mod invocation_policy; +pub mod selector_file_selectors; pub mod unavailable_handler; diff --git a/crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs b/crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs new file mode 100644 index 0000000000..ea47a6e995 --- /dev/null +++ b/crates/sqlbuild-cli/src/dispatch/main/selector_file_selectors.rs @@ -0,0 +1,17 @@ +//! The selectors a `--select-file` lists. + +use sqlbuild_core::text::main::python_splitlines::python_splitlines; +use sqlbuild_core::text::main::python_strip::python_strip; + +use crate::dispatch::constants::SELECTOR_COMMENT_PREFIX; + +/// One selector per stripped line of `text`, without blank lines and `#` comments. +#[must_use] +pub fn selector_file_selectors(text: &str) -> Vec { + python_splitlines(text) + .into_iter() + .map(python_strip) + .filter(|line| !line.is_empty() && !line.starts_with(SELECTOR_COMMENT_PREFIX)) + .map(str::to_owned) + .collect() +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs index f255f63599..f92b9941c0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs @@ -46,12 +46,12 @@ use pyo3::types::{PyDict, PyDictMethods, PyString, PyTuple}; use sqlbuild_cli::diff_report::errors::DiffUserError; use sqlbuild_cli::diff_report::main::parse_diff_name_range::parse_diff_name_range; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; use sqlbuild_cli::dispatch::main::unavailable_handler::unavailable_handler; use sqlbuild_cli::dispatch::models::{ ClassRef, CommandFacts, DiffSide, Dispatch, DispatchFailure, Field, Route, Source, }; use sqlbuild_core::format_command::main::format_path_selectors::format_path_selectors; -use sqlbuild_core::text::main::python_strip::python_strip; use crate::bindings::_helpers::boundary::python_calls::{imported, within}; use crate::bindings::_helpers::records::construct::error_class; @@ -60,7 +60,6 @@ use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::records::registry; const PATHLIB_MODULE: &str = "pathlib"; -const COMMENT_PREFIX: &str = "#"; const UTF8: &str = "utf-8"; /// The code of a `--select-file` that cannot be read. const SELECTOR_FILE_CODE: &str = "C114"; @@ -265,15 +264,10 @@ fn selector_inputs<'py>( Ok(text) => text, Err(error) => return Err(selector_file_error(py, &path, &error)?), }; - let lines: Vec = text.call_method0("splitlines")?.extract()?; - let mut count: usize = 0; - for line in &lines { - let stripped: &str = python_strip(line); - if stripped.is_empty() || stripped.starts_with(COMMENT_PREFIX) { - continue; - } - selectors.push(PyString::new(py, stripped).into_any()); - count += 1; + let listed: Vec = selector_file_selectors(&text.extract::()?); + let count: usize = listed.len(); + for selector in &listed { + selectors.push(PyString::new(py, selector).into_any()); } files.push(new_record_with( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 21411f7628..e0eea39e96 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -36,9 +36,9 @@ use crate::bindings::_helpers::presentation_text::terminal::stdout_supports_colo use crate::bindings::_helpers::recall::constants::{JSON_FORMAT, RECALL_SETTINGS_CODE}; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::session::{ - InvocationRecall, finish_invocation_recall, start_invocation_recall, + InvocationRecall, effective_selection, finish_invocation_recall, start_invocation_recall, }; -use crate::bindings::_helpers::recall::sinks::RecallLifecycleSink; +use crate::bindings::_helpers::recall::sinks::{RecallLifecycleSink, publish_lifecycle_event}; use crate::bindings::_helpers::records::construct::record_class; use crate::bindings::_helpers::records::construct::{error_class, new_error}; use crate::bindings::_helpers::records::fields::field as native_field; @@ -496,7 +496,8 @@ fn add_invocation_context( } let select: Bound<'_, PyAny> = invocation.args.getattr("select")?; if !select.is_none() { - payload.set_item("selection_count", select.len()?)?; + let (selectors, _) = effective_selection(&invocation.args)?; + payload.set_item("selection_count", selectors.len())?; } Ok(()) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 7ab0b0644e..85ef379906 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -8,9 +8,11 @@ use std::time::{Duration, Instant, SystemTime}; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use sqlbuild_cli::dispatch::main::closing_status::closing_status; +use sqlbuild_cli::dispatch::main::selector_file_selectors::selector_file_selectors; use sqlbuild_cli::dispatch::models::CommandFacts; use sqlbuild_core::command_output::constants::INVOCATION_CONTEXT_ENV; use sqlbuild_host::entries::models::ExecutorHost; +use sqlbuild_recall::records::main::bounded_selection::bounded_selection; use sqlbuild_recall::records::main::command_mode::command_mode; use sqlbuild_recall::records::main::format_timestamp::format_timestamp; use sqlbuild_recall::records::main::invocation_origin::invocation_origin; @@ -90,6 +92,24 @@ fn list_attribute(args: &Bound<'_, PyAny>, name: &str) -> PyResult> Ok(items) } +/// The effective selection: positional selectors and `--select`, then what every `--select-file` lists, with the files as passed. A file that cannot be read adds no selectors here; the command reports it. +pub(crate) fn effective_selection(args: &Bound<'_, PyAny>) -> PyResult<(Vec, Vec)> { + let mut selectors: Vec = list_attribute(args, "select")?; + let files: Vec = match args.getattr_opt("select_file")? { + Some(value) if !value.is_none() => value + .try_iter()? + .map(|item| escaped_text(&item?.str()?)) + .collect::>()?, + _ => Vec::new(), + }; + for file in &files { + if let Some(text) = file_text(Path::new(file)) { + selectors.extend(selector_file_selectors(&text)); + } + } + Ok((selectors, files)) +} + fn environment(name: &str) -> Option { std::env::var_os(name).map(|value| value.to_string_lossy().into_owned()) } @@ -239,6 +259,9 @@ fn invocation_facts( settings.destination_schema.as_deref().unwrap_or_default() ); let external_context: Option = external_context(py)?; + let (selectors, selector_files) = effective_selection(args)?; + let selection_count: u64 = selectors.len() as u64; + let (selection, selection_truncated) = bounded_selection(selectors); let no_connection: bool = facts.command.as_deref() == Some(DEBUG_COMMAND) && attribute(args, NO_CONNECTION_ATTRIBUTE)?.is_truthy()?; Ok(InvocationContext { @@ -246,7 +269,10 @@ fn invocation_facts( command_mode: command_mode(facts.command.as_deref(), no_connection).map(str::to_owned), command: facts.command.clone(), subcommand: facts.subcommand.clone(), - selection: list_attribute(args, "select")?, + selection, + selection_count: Some(selection_count), + selection_truncated, + selector_files, exclusion: list_attribute(args, "exclude")?, project_name: settings.project_name.clone(), project_dir_hash: Some(sha256_hex(project_dir.to_string_lossy().as_bytes())), diff --git a/crates/sqlbuild-recall/src/records/main/bounded_selection.rs b/crates/sqlbuild-recall/src/records/main/bounded_selection.rs new file mode 100644 index 0000000000..2bdfd968b9 --- /dev/null +++ b/crates/sqlbuild-recall/src/records/main/bounded_selection.rs @@ -0,0 +1,20 @@ +//! `bounded_selection`: the selectors a context row records, within its bound. + +use crate::records::constants::MAX_RECORDED_SELECTION_BYTES; + +/// The leading selectors whose text fits [`MAX_RECORDED_SELECTION_BYTES`], and whether any were left out. +#[must_use] +pub fn bounded_selection(selectors: Vec) -> (Vec, bool) { + let mut used: usize = 0; + let mut kept: Vec = Vec::with_capacity(selectors.len()); + let total: usize = selectors.len(); + for selector in selectors { + used += selector.len(); + if used > MAX_RECORDED_SELECTION_BYTES { + break; + } + kept.push(selector); + } + let truncated: bool = kept.len() < total; + (kept, truncated) +} diff --git a/crates/sqlbuild-recall/src/records/main/invocation_row.rs b/crates/sqlbuild-recall/src/records/main/invocation_row.rs index dc5d54213b..e9d533309c 100644 --- a/crates/sqlbuild-recall/src/records/main/invocation_row.rs +++ b/crates/sqlbuild-recall/src/records/main/invocation_row.rs @@ -18,6 +18,12 @@ pub fn invocation_row(envelope: &RowEnvelope, context: &InvocationContext) -> Re ("command", optional_text(context.command.as_deref())), ("subcommand", optional_text(context.subcommand.as_deref())), ("selection", json_list(&context.selection)), + ( + "selection_count", + context.selection_count.map_or(RecallValue::Null, integer), + ), + ("selection_truncated", boolean(context.selection_truncated)), + ("selector_files", json_list(&context.selector_files)), ("exclusion", json_list(&context.exclusion)), ( "project_name", diff --git a/crates/sqlbuild-recall/src/records/main/mod.rs b/crates/sqlbuild-recall/src/records/main/mod.rs index 3ccedf2f77..4c8e37f114 100644 --- a/crates/sqlbuild-recall/src/records/main/mod.rs +++ b/crates/sqlbuild-recall/src/records/main/mod.rs @@ -1,4 +1,5 @@ pub mod artifact_rows; +pub mod bounded_selection; pub mod command_mode; pub mod conflict_row; pub mod delivery_row; diff --git a/crates/sqlbuild-recall/src/records/models.rs b/crates/sqlbuild-recall/src/records/models.rs index de2e666efb..e9da820c87 100644 --- a/crates/sqlbuild-recall/src/records/models.rs +++ b/crates/sqlbuild-recall/src/records/models.rs @@ -103,7 +103,14 @@ pub struct OmittedOutput { pub struct InvocationContext { pub command: Option, pub subcommand: Option, + /// The effective selectors: positional, `--select` and the contents of every selector file, at most [`MAX_RECORDED_SELECTION_BYTES`](crate::records::constants::MAX_RECORDED_SELECTION_BYTES). pub selection: Vec, + /// How many effective selectors there were, recorded ones and truncated ones alike. + pub selection_count: Option, + /// Selectors were left out of `selection` to keep it within its bound. + pub selection_truncated: bool, + /// The selector files as passed, in order. + pub selector_files: Vec, pub exclusion: Vec, pub project_name: Option, pub project_dir_hash: Option, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index ccf1b80752..d5d4bba41d 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -6,7 +6,7 @@ use crate::schema::main::table_spec::table_spec; use crate::schema::models::{DdlOptions, Table}; /// The columns the `invocations` pass reads from its three sources, NULL (the terminal flag 0) where a source does not give them. -const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 30] = [ +const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 35] = [ ("ctx_command", "Nullable(String)", "NULL"), ("ctx_subcommand", "Nullable(String)", "NULL"), ("ctx_target", "Nullable(String)", "NULL"), @@ -23,6 +23,9 @@ const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 30] = [ ("ctx_recall_filter_active", "Nullable(UInt8)", "NULL"), ("ctx_origin", "Nullable(String)", "NULL"), ("ctx_command_mode", "Nullable(String)", "NULL"), + ("ctx_selection_count", "Nullable(Int64)", "NULL"), + ("ctx_selection_truncated", "Nullable(UInt8)", "NULL"), + ("ctx_selector_files", "Nullable(String)", "NULL"), ("heartbeat_at", "Nullable(DateTime64(6, 'UTC'))", "NULL"), ("beat_stale_at", "Nullable(DateTime64(6, 'UTC'))", "NULL"), ("beat_command", "Nullable(String)", "NULL"), @@ -176,6 +179,9 @@ fn invocations(options: &DdlOptions) -> String { ("ctx_recall_filter_active", "recall_filter_active"), ("ctx_origin", "origin"), ("ctx_command_mode", "command_mode"), + ("ctx_selection_count", "selection_count"), + ("ctx_selection_truncated", "selection_truncated"), + ("ctx_selector_files", "selector_files"), ], &context, ); @@ -280,6 +286,11 @@ SELECT context_adapter AS adapter, context_project_name AS project_name, context_selection AS selection, + coalesce(context_selection_count, + if(isValidJSON(context_selection), toInt64(JSONLength(context_selection)), NULL)) + AS selection_count, + context_selection_truncated AS selection_truncated, + context_selector_files AS selector_files, context_exclusion AS exclusion, context_sqlbuild_version AS sqlbuild_version, context_git_revision AS git_revision, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index c679a80e08..080de0f4bf 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -293,6 +293,10 @@ projected AS ( context.adapter, context.project_name, context.selection, + COALESCE(context.selection_count, json_array_length(context.selection)) + AS selection_count, + context.selection_truncated, + context.selector_files, context.exclusion, context.sqlbuild_version, context.git_revision, @@ -348,6 +352,9 @@ SELECT adapter, project_name, selection, + selection_count, + selection_truncated, + selector_files, exclusion, sqlbuild_version, git_revision, diff --git a/crates/sqlbuild-recall/src/schema/constants.rs b/crates/sqlbuild-recall/src/schema/constants.rs index 3e846c7b5d..48e5a69a36 100644 --- a/crates/sqlbuild-recall/src/schema/constants.rs +++ b/crates/sqlbuild-recall/src/schema/constants.rs @@ -3,7 +3,7 @@ use crate::schema::models::{ColumnSpec, ColumnType, Table, TableSpec}; /// The Recall schema version this build writes. Raise it when a column or view is added. -pub const RECALL_SCHEMA_VERSION: i64 = 4; +pub const RECALL_SCHEMA_VERSION: i64 = 5; /// Text that only older schema versions wrote into ClickHouse view definitions, with the last version that wrote it. Version 3 changed only view bodies, so view columns cannot tell version 2 views, which set `join_use_nulls`, from current ones. pub const SUPERSEDED_CLICKHOUSE_VIEW_MARKERS: [(&str, i64); 1] = [("join_use_nulls", 2)]; /// The oldest SQLite library the store's views run on (ordered `group_concat`). @@ -39,7 +39,7 @@ const CORRELATIONS: [ColumnSpec; 5] = [ ColumnSpec::optional("statement_id", ColumnType::Text), ]; -const INVOCATION_COLUMNS: [ColumnSpec; 21] = [ +const INVOCATION_COLUMNS: [ColumnSpec; 24] = [ ColumnSpec::optional("command", ColumnType::Text), ColumnSpec::optional("subcommand", ColumnType::Text), ColumnSpec::optional("selection", ColumnType::Json), @@ -61,6 +61,9 @@ const INVOCATION_COLUMNS: [ColumnSpec; 21] = [ ColumnSpec::optional("argv_digest", ColumnType::Text), ColumnSpec::optional("origin", ColumnType::Text), ColumnSpec::optional("command_mode", ColumnType::Text), + ColumnSpec::optional("selection_count", ColumnType::Integer), + ColumnSpec::optional("selection_truncated", ColumnType::Boolean), + ColumnSpec::optional("selector_files", ColumnType::Json), ]; const OUTPUT_COLUMNS: [ColumnSpec; 11] = [ diff --git a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs index d43869eb41..588e30a678 100644 --- a/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs +++ b/crates/sqlbuild-recall/src/schema/tests/test_ddl.rs @@ -154,7 +154,7 @@ fn given_clickhouse_statement_sets_when_rendered_then_every_statement_has_the_sh .collect(), required_fragments: &[ "INSERT INTO `observability`.`recall_meta`", - "SELECT 'schema', 'schema_version', '4'", + "SELECT 'schema', 'schema_version', '5'", ], expected_count: 1, }, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index cd7ba3fb21..41951591cd 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -317,6 +317,9 @@ invocations (view): one row per invocation; status is completed, failed, running adapter TEXT project_name TEXT selection TEXT + selection_count ANY + selection_truncated INTEGER + selector_files TEXT exclusion TEXT sqlbuild_version TEXT git_revision TEXT From 26471e38ed262b7c06219e660d0094bd04b945de Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:57 +0000 Subject: [PATCH 06/30] fix(recall): keep the SQL of failed and cancelled statements Each statement gets its own reference to its SQL text, and the SQL of a statement that fails or is cancelled is queued again at its terminal event's priority, so it survives a full queue. `statements.sql_available` checks the statement's own invocation, and `sql_unavailable_reason` says why SQL is missing. --- .../src/bindings/_helpers/cli/entry.rs | 4 +- .../src/bindings/_helpers/recall/constants.rs | 3 ++ .../src/bindings/_helpers/recall/sinks.rs | 46 ++++++++++++++++++- .../src/schema/_helpers/clickhouse_views.rs | 11 ++++- .../src/schema/_helpers/sqlite_views.rs | 22 +++++++-- .../src/writer/_helpers/session.rs | 23 +++++++++- .../sqlbuild/references/investigating-runs.md | 1 + 7 files changed, 100 insertions(+), 10 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index e0eea39e96..42c2f51455 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -477,13 +477,14 @@ fn subscribe_recall( RecallLifecycleSink { session, replacements: Arc::clone(&recall.replacements), + running_sql: Arc::default(), }, )?; dispatcher.call_method1("subscribe_history", (sink,))?; Ok(()) } -/// `target`, `adapter` and `selection_count` of the started invocation, when known. +/// `target`, `adapter` and `selection_count` of the started invocation, when known; the count includes what selector files list. fn add_invocation_context( invocation: &Invocation<'_>, payload: &Bound<'_, PyDict>, @@ -692,6 +693,7 @@ fn parsed_invocation<'py>( RecallLifecycleSink { session, replacements: Arc::clone(&recall.replacements), + running_sql: Arc::default(), }, )? .into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 57c2229562..1330869523 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -115,3 +115,6 @@ pub(crate) const UTC_OFFSET: &str = "+00:00"; pub(crate) const DEBUG_COMMAND: &str = "debug"; /// The namespace attribute of `sqb debug --no-connection`. pub(crate) const NO_CONNECTION_ATTRIBUTE: &str = "no_connection"; +/// The statement terminal events, at which the sink settles a statement's SQL. +pub(crate) const STATEMENT_COMPLETED: &str = "statement_completed"; +pub(crate) const STATEMENT_FAILED: &str = "statement_failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index 21d3360b94..363e973c0f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -1,6 +1,7 @@ //! The native subscriber that publishes every lifecycle fact and artifact into Recall. -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; use pyo3::exceptions::PyException; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python, pyclass, pymethods}; @@ -11,6 +12,7 @@ use sqlbuild_recall::records::main::parse_event::parse_event; use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::constants::{STATEMENT_COMPLETED, STATEMENT_FAILED}; use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; use crate::bindings::_helpers::recall::replacements::RecallReplacements; @@ -42,11 +44,15 @@ pub(crate) fn publish_lifecycle_event( Ok(()) } -/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. +/// The SQL and resource of a statement that has not reached its terminal event yet. +type RunningSql = HashMap)>; + +/// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. The SQL of a statement that fails or is cancelled is queued again with its terminal event's priority. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct RecallLifecycleSink { pub(crate) session: Arc, pub(crate) replacements: Arc, + pub(crate) running_sql: Arc>, } #[pymethods] @@ -77,13 +83,49 @@ impl RecallLifecycleSink { texts.map(|(sql, statement_id, resource_id)| { self.session .publish_sql(&sql, statement_id.as_deref(), resource_id.as_deref()); + if let Some(statement_id) = statement_id { + self.running().insert(statement_id, (sql, resource_id)); + } }), ) } } impl RecallLifecycleSink { + fn running(&self) -> std::sync::MutexGuard<'_, RunningSql> { + self.running_sql + .lock() + .unwrap_or_else(PoisonError::into_inner) + } + + /// Forget a statement's SQL at its terminal event; a failed one's SQL is queued again first. + fn settle_statement(&self, event: &Bound<'_, PyAny>) -> PyResult<()> { + let Some(event_type) = event.getattr_opt("event_type")? else { + return Ok(()); + }; + let event_type: String = event_type.str()?.to_string(); + if event_type != STATEMENT_COMPLETED && event_type != STATEMENT_FAILED { + return Ok(()); + } + let Some(statement_id) = event.getattr_opt("statement_id")? else { + return Ok(()); + }; + if statement_id.is_none() { + return Ok(()); + } + let statement_id: String = escaped_text(&statement_id.str()?)?; + let Some((sql, resource_id)) = self.running().remove(&statement_id) else { + return Ok(()); + }; + if event_type == STATEMENT_FAILED { + self.session + .publish_failed_sql(&sql, Some(&statement_id), resource_id.as_deref()); + } + Ok(()) + } + fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + self.settle_statement(event)?; if self .replacements .deliver_lifecycle(py, &self.session, event)? diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index d5d4bba41d..e9e6ec284e 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -407,7 +407,16 @@ SELECT if(has_terminal, last_row_count, NULL) AS row_count, if(has_terminal, last_affected_rows, NULL) AS affected_rows, first_sql_digest AS sql_digest, - concat('sha256:', first_sql_digest) IN (SELECT digest FROM {artifacts}) AS sql_available + (invocation_id, concat('sha256:', first_sql_digest)) IN ( + SELECT invocation_id, digest FROM {refs} + WHERE digest IN (SELECT digest FROM {artifacts}) + ) AS sql_available, + multiIf( + sql_available, NULL, + (invocation_id, concat('sha256:', first_sql_digest)) + IN (SELECT invocation_id, digest FROM {refs}), 'expired_or_not_captured', + 'not_captured' + ) AS sql_unavailable_reason FROM folded" ) } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 080de0f4bf..983c54631d 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -39,6 +39,17 @@ terminals AS ( FROM facts WHERE event_type IN ('statement_completed', 'statement_failed') ), +captures AS ( + SELECT firsts.statement_id, ( + SELECT MAX(artifact.digest IS NOT NULL) + FROM recall_artifact_refs AS ref + LEFT JOIN recall_artifacts AS artifact ON artifact.digest = ref.digest + WHERE ref.invocation_id = firsts.invocation_id + AND ref.digest = 'sha256:' || firsts.sql_digest + ) AS stored + FROM firsts + WHERE firsts.position = 1 +), resources AS ( SELECT invocation_id, resource_id, MIN(resource_name) AS resource_name, MIN(resource_kind) AS resource_kind @@ -74,16 +85,19 @@ SELECT terminals.row_count, terminals.affected_rows, firsts.sql_digest, - EXISTS ( - SELECT 1 FROM recall_artifacts AS artifact - WHERE artifact.digest = 'sha256:' || firsts.sql_digest - ) AS sql_available + COALESCE(captures.stored, 0) AS sql_available, + CASE captures.stored + WHEN 1 THEN NULL + WHEN 0 THEN 'expired_or_not_captured' + ELSE 'not_captured' + END AS sql_unavailable_reason FROM firsts LEFT JOIN started ON started.statement_id = firsts.statement_id LEFT JOIN submitted ON submitted.statement_id = firsts.statement_id LEFT JOIN query_ids ON query_ids.statement_id = firsts.statement_id LEFT JOIN terminals ON terminals.statement_id = firsts.statement_id AND terminals.position = 1 +LEFT JOIN captures ON captures.statement_id = firsts.statement_id LEFT JOIN resources ON resources.invocation_id = firsts.invocation_id AND resources.resource_id = firsts.resource_id diff --git a/crates/sqlbuild-recall/src/writer/_helpers/session.rs b/crates/sqlbuild-recall/src/writer/_helpers/session.rs index 240150540a..ee88de2643 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/session.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/session.rs @@ -146,8 +146,27 @@ impl RecallSession { Ok(()) } - /// Queue the SQL a statement executed, once per digest per invocation. + /// Queue the SQL a statement executed: its text once per digest per invocation, and a reference per statement. pub fn publish_sql(&self, sql: &str, statement_id: Option<&str>, resource_id: Option<&str>) { + self.queue_sql(sql, (statement_id, resource_id), Priority::Artifact); + } + + /// Queue the SQL of a statement that failed or was cancelled again, above output and diagnostics, so a full queue or a spent shutdown budget keeps it; it is stored once if it already was. + pub fn publish_failed_sql( + &self, + sql: &str, + statement_id: Option<&str>, + resource_id: Option<&str>, + ) { + self.queue_sql(sql, (statement_id, resource_id), Priority::FailedTerminal); + } + + fn queue_sql( + &self, + sql: &str, + (statement_id, resource_id): (Option<&str>, Option<&str>), + priority: Priority, + ) { if sql.len() > MAX_ARTIFACT_BYTES { let mut state = locked(&self.shared.state); let counts = state.counts_mut(DeliveryStream::Artifacts); @@ -166,7 +185,7 @@ impl RecallSession { stream: DeliveryStream::Artifacts, }; let mut state = locked(&self.shared.state); - state.push_record(record, Priority::Artifact); + state.push_record(record, priority); self.shared.wake.notify_one(); } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 41951591cd..12e65b8567 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -393,6 +393,7 @@ statements (view): one row per statement; outcome no_terminal means no terminal affected_rows INTEGER sql_digest TEXT join key sql_available ANY + sql_unavailable_reason ANY ``` From dbdfaf65be95446a5a09bdaf03b512c50a0b5b59 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:59 +0000 Subject: [PATCH 07/30] feat(recall): keep failing audits' evidence rows Violation audits that warn or fail keep a bounded sample of the rows they returned, as measurement audits already did. Recall stores the evidence as an artifact named by the audit result's `sample_digest`, with `evidence_count` and `evidence_truncated` on `audit_results`. `[recall] audit_evidence` (`warn-and-error`, `error` or `off`) chooses which outcomes keep it. --- .../sqlbuild-config/src/recall/constants.rs | 5 +- .../src/recall/main/recall_section.rs | 13 +- crates/sqlbuild-config/src/recall/models.rs | 2 + .../src/recall/tests/test_recall_section.rs | 9 ++ .../src/bindings/_helpers/recall/session.rs | 1 + .../sqlbuild-recall/src/records/constants.rs | 12 ++ .../src/schema/_helpers/clickhouse_views.rs | 8 +- .../src/schema/_helpers/sqlite_views.rs | 5 +- .../src/settings/_helpers/layers.rs | 1 + .../settings/main/resolve_recall_settings.rs | 3 +- crates/sqlbuild-recall/src/settings/models.rs | 36 +++++ .../src/settings/tests/helpers.rs | 5 +- .../src/settings/tests/test_resolve.rs | 7 +- .../src/writer/_helpers/delivery.rs | 4 +- .../src/writer/_helpers/materialize.rs | 147 +++++++++++++++--- crates/sqlbuild-recall/src/writer/models.rs | 3 + crates/sqlbuild-recall/src/writer/tests.rs | 2 + .../src/writer/tests/helpers.rs | 78 +++++++++- .../src/writer/tests/test_artifacts.rs | 74 +++++++++ .../references/docs/concepts/audits.md | 16 +- .../sqlbuild/references/investigating-runs.md | 4 +- .../executor/auditing/_helpers/execution.py | 38 +++-- .../src/content/docs/docs/concepts/audits.mdx | 15 +- .../docs/concepts/observability/recall.mdx | 40 ++--- 24 files changed, 457 insertions(+), 71 deletions(-) create mode 100644 crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs diff --git a/crates/sqlbuild-config/src/recall/constants.rs b/crates/sqlbuild-config/src/recall/constants.rs index 74a066bec1..76f2a2fab1 100644 --- a/crates/sqlbuild-config/src/recall/constants.rs +++ b/crates/sqlbuild-config/src/recall/constants.rs @@ -2,7 +2,8 @@ pub const RECALL_SECTION: &str = "recall"; /// Keys `[recall]` and `targets..recall` accept, sorted. -pub const RECALL_KEYS: [&str; 7] = [ +pub const RECALL_KEYS: [&str; 8] = [ + "audit_evidence", "diagnostics_min_severity", "enabled", "max_output_bytes", @@ -15,6 +16,8 @@ pub const RECALL_KEYS: [&str; 7] = [ pub const REMOVED_CONNECTION_KEY: &str = "connection"; /// The diagnostic severities a Recall store can keep from, least severe first. pub const RECALL_SEVERITIES: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; +/// Which audit outcomes keep their evidence rows in Recall, the default first. +pub const AUDIT_EVIDENCE_MODES: [&str; 3] = ["warn-and-error", "error", "off"]; /// The longest shutdown flush budget, in milliseconds. pub const MAX_SHUTDOWN_TIMEOUT_MS: u64 = 600_000; /// The longest retention, in days. diff --git a/crates/sqlbuild-config/src/recall/main/recall_section.rs b/crates/sqlbuild-config/src/recall/main/recall_section.rs index fce31d1f59..981b50b83a 100644 --- a/crates/sqlbuild-config/src/recall/main/recall_section.rs +++ b/crates/sqlbuild-config/src/recall/main/recall_section.rs @@ -3,7 +3,8 @@ use crate::models::ConfigValue; use crate::recall::_helpers::values::{non_empty, output_bytes, retention_days, timeout_ms}; use crate::recall::constants::{ - MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES, REMOVED_CONNECTION_KEY, + AUDIT_EVIDENCE_MODES, MAX_RETENTION_DAYS, RECALL_KEYS, RECALL_SEVERITIES, + REMOVED_CONNECTION_KEY, }; use crate::recall::models::RecallSection; @@ -65,6 +66,16 @@ pub fn recall_section(value: &ConfigValue, label: &str) -> Result { + let text = non_empty(value, label, key)?; + if !AUDIT_EVIDENCE_MODES.contains(&text) { + return Err(format!( + "{label}.audit_evidence must be one of: {}", + AUDIT_EVIDENCE_MODES.join(", ") + )); + } + section.audit_evidence = Some(text.to_owned()); + } other => { return Err(format!( "{label} contains unknown key(s): {other}. Allowed keys: {}", diff --git a/crates/sqlbuild-config/src/recall/models.rs b/crates/sqlbuild-config/src/recall/models.rs index 6b8e21e9ec..219353c31b 100644 --- a/crates/sqlbuild-config/src/recall/models.rs +++ b/crates/sqlbuild-config/src/recall/models.rs @@ -13,4 +13,6 @@ pub struct RecallSection { pub diagnostics_min_severity: Option, /// Bytes of each output stream to record; `Some(0)` records everything. pub max_output_bytes: Option, + /// Which audit outcomes keep their evidence rows: one of [`AUDIT_EVIDENCE_MODES`](crate::recall::constants::AUDIT_EVIDENCE_MODES). + pub audit_evidence: Option, } diff --git a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs index 61ca10eb0e..96e9812512 100644 --- a/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs +++ b/crates/sqlbuild-config/src/recall/tests/test_recall_section.rs @@ -17,6 +17,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { ("shutdown_timeout", text("1500ms")), ("diagnostics_min_severity", text("debug")), ("max_output_bytes", ConfigValue::Integer(4_194_304)), + ("audit_evidence", text("error")), ]), expected_section: RecallSection { enabled: Some(false), @@ -26,6 +27,7 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { shutdown_timeout_ms: Some(1500), diagnostics_min_severity: Some("debug".to_owned()), max_output_bytes: Some(4_194_304), + audit_evidence: Some("error".to_owned()), }, }, ValidSectionTestCase { @@ -64,6 +66,13 @@ fn given_valid_sections_when_validated_then_settings_are_typed() { #[test] fn given_invalid_sections_when_validated_then_the_message_names_the_key() { let test_cases = [ + InvalidSectionTestCase { + description: "audit evidence names one of its modes", + value: table(&[("audit_evidence", text("always"))]), + label: "targets.prod.recall", + expected_message_prefix: "targets.prod.recall.audit_evidence must be one of: \ + warn-and-error, error, off", + }, InvalidSectionTestCase { description: "output bytes must not be negative", value: table(&[("max_output_bytes", ConfigValue::Integer(-1))]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 85ef379906..8555774aaf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -459,6 +459,7 @@ fn started_session( setup_budget: Duration::ZERO, maintenance: true, defer_context: true, + audit_evidence: settings.audit_evidence, }); Ok(Some(Arc::new(session))) } diff --git a/crates/sqlbuild-recall/src/records/constants.rs b/crates/sqlbuild-recall/src/records/constants.rs index d30b2fe176..b4f54d98a5 100644 --- a/crates/sqlbuild-recall/src/records/constants.rs +++ b/crates/sqlbuild-recall/src/records/constants.rs @@ -10,6 +10,18 @@ pub const ARTIFACT_DIGEST_PREFIX: &str = "sha256:"; pub const SQL_ARTIFACT_KIND: &str = "sql"; pub const SQL_MEDIA_TYPE: &str = "text/x-sql"; pub const EXECUTED_SQL_ROLE: &str = "executed_sql"; +/// Evidence rows of a warning or failing audit, stored as a JSON array of row objects. +pub const AUDIT_EVIDENCE_KIND: &str = "audit_evidence"; +pub const AUDIT_EVIDENCE_MEDIA_TYPE: &str = "application/json"; +pub const AUDIT_EVIDENCE_ROLE: &str = "audit_evidence"; +/// The event whose payload may carry evidence rows. +pub const AUDIT_COMPLETED_EVENT: &str = "audit_completed"; +/// The payload field holding evidence rows, which Recall stores as an artifact instead. +pub const EVIDENCE_PAYLOAD_FIELD: &str = "evidence"; +/// The payload field naming the evidence artifact's digest, without its `sha256:` prefix. +pub const SAMPLE_DIGEST_PAYLOAD_FIELD: &str = "sample_digest"; +/// The most selector text an invocation's context row records; later selectors are counted but left out. +pub const MAX_RECORDED_SELECTION_BYTES: usize = 64 * 1024; /// Diagnostic severities, least severe first. pub const SEVERITY_ORDER: [&str; 6] = ["trace", "debug", "info", "warning", "error", "critical"]; /// Event payload fields promoted to their own `recall_events` columns. diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index e9e6ec284e..1223f0a207 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -448,6 +448,9 @@ fn audit_results(options: &DdlOptions) -> String { JSONExtract(payload, 'evaluation_mode', 'Nullable(String)') AS evaluation_mode, JSONExtract(payload, 'execution_error', 'Nullable(String)') AS execution_error, sql_digest, + JSONExtract(payload, 'evidence_count', 'Nullable(Int64)') AS evidence_count, + if(JSONHas(payload, 'evidence_truncated'), + toUInt8(JSONExtractBool(payload, 'evidence_truncated')), NULL) AS evidence_truncated, JSONExtract(payload, 'sample_digest', 'Nullable(String)') AS sample_digest FROM {events} WHERE event_type = 'audit_completed'" @@ -617,8 +620,9 @@ fn artifacts(options: &DdlOptions) -> String { artifact.stored_size AS size, artifact.stored_digest IS NOT NULL AS available, if(artifact.stored_digest IS NULL, 'expired_or_not_captured', NULL) AS unavailable_reason, - if(startsWith(artifact.stored_media_type, 'text/'), toValidUTF8(artifact.stored_content), NULL) - AS text, + if(startsWith(artifact.stored_media_type, 'text/') + OR artifact.stored_media_type = 'application/json', + toValidUTF8(artifact.stored_content), NULL) AS text, artifact.stored_content AS content FROM {refs} AS refs LEFT JOIN ( diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 983c54631d..6922202a5f 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -128,6 +128,8 @@ const AUDIT_RESULTS: &str = "SELECT json_extract(payload, '$.evaluation_mode') AS evaluation_mode, json_extract(payload, '$.execution_error') AS execution_error, sql_digest, + json_extract(payload, '$.evidence_count') AS evidence_count, + json_extract(payload, '$.evidence_truncated') AS evidence_truncated, json_extract(payload, '$.sample_digest') AS sample_digest FROM recall_events WHERE event_type = 'audit_completed'"; @@ -239,7 +241,8 @@ const ARTIFACTS: &str = "SELECT artifact.size, artifact.digest IS NOT NULL AS available, CASE WHEN artifact.digest IS NULL THEN 'expired_or_not_captured' END AS unavailable_reason, - CASE WHEN artifact.media_type LIKE 'text/%' THEN CAST(artifact.content AS TEXT) END AS text, + CASE WHEN artifact.media_type LIKE 'text/%' OR artifact.media_type = 'application/json' + THEN CAST(artifact.content AS TEXT) END AS text, artifact.content FROM recall_artifact_refs AS refs LEFT JOIN recall_artifacts AS artifact ON artifact.digest = refs.digest"; diff --git a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs index 570122df90..e20b285870 100644 --- a/crates/sqlbuild-recall/src/settings/_helpers/layers.rs +++ b/crates/sqlbuild-recall/src/settings/_helpers/layers.rs @@ -37,6 +37,7 @@ pub(crate) fn overlay(base: RecallSection, layer: RecallSection) -> RecallSectio .diagnostics_min_severity .or(base.diagnostics_min_severity), max_output_bytes: layer.max_output_bytes.or(base.max_output_bytes), + audit_evidence: layer.audit_evidence.or(base.audit_evidence), } } diff --git a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs index 41852edf74..fb04f0b3d4 100644 --- a/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs +++ b/crates/sqlbuild-recall/src/settings/main/resolve_recall_settings.rs @@ -10,7 +10,7 @@ use crate::settings::constants::{ LOCAL_FILE_NAME, PROJECT_FILE_NAME, }; use crate::settings::errors::SettingsError; -use crate::settings::models::{RecallSettings, SettingsRequest}; +use crate::settings::models::{AuditEvidence, RecallSettings, SettingsRequest}; /// The effective settings: `targets..recall` over `[recall]` over the built-in local store, local configuration over project configuration at each level. Only the connection of the request's purpose is resolved, so a machine holding only the writer's or only the reader's credentials resolves its own side. pub fn resolve_recall_settings(request: &SettingsRequest) -> Result { @@ -73,5 +73,6 @@ pub fn resolve_recall_settings(request: &SettingsRequest) -> Result, pub destination_database: Option, pub destination_schema: Option, + /// Which audit outcomes keep their evidence rows as an artifact. + pub audit_evidence: AuditEvidence, +} + +/// Which audit outcomes keep their evidence rows in Recall, as `recall.audit_evidence` sets it; evidence holds data values from the audited relation. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum AuditEvidence { + /// Warnings and errors: `warn-and-error`. + #[default] + WarnAndError, + /// Errors only: `error`. + Error, + /// None: `off`. + Off, +} + +impl AuditEvidence { + /// The mode `recall.audit_evidence` names; anything else is the default. + #[must_use] + pub fn from_setting(setting: Option<&str>) -> Self { + match setting { + Some("error") => Self::Error, + Some("off") => Self::Off, + _ => Self::WarnAndError, + } + } + + /// Whether an audit with `outcome` keeps its evidence. + #[must_use] + pub fn keeps(self, outcome: &str) -> bool { + match self { + Self::WarnAndError => outcome == "warn" || outcome == "error", + Self::Error => outcome == "error", + Self::Off => false, + } + } } /// Which connection an invocation uses, so only that one is resolved and its environment references expanded. diff --git a/crates/sqlbuild-recall/src/settings/tests/helpers.rs b/crates/sqlbuild-recall/src/settings/tests/helpers.rs index 9c20bf9474..9594516108 100644 --- a/crates/sqlbuild-recall/src/settings/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/settings/tests/helpers.rs @@ -3,7 +3,9 @@ use std::path::PathBuf; use sqlbuild_config::models::ConfigValue; use crate::settings::main::resolve_recall_settings::resolve_recall_settings; -use crate::settings::models::{RecallSettings, SettingsRequest, StoreLocation, StorePurpose}; +use crate::settings::models::{ + AuditEvidence, RecallSettings, SettingsRequest, StoreLocation, StorePurpose, +}; /// The placeholder project directory resolved paths are reported under. pub(crate) const PROJECT: &str = ""; @@ -74,6 +76,7 @@ pub(crate) fn defaults() -> RecallSettings { project_adapter: Some("duckdb".to_owned()), destination_database: None, destination_schema: None, + audit_evidence: AuditEvidence::WarnAndError, } } diff --git a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs index fea7a4fcf9..b5cd7052c4 100644 --- a/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs +++ b/crates/sqlbuild-recall/src/settings/tests/test_resolve.rs @@ -1,4 +1,4 @@ -use crate::settings::models::{RecallSettings, StoreLocation, StorePurpose}; +use crate::settings::models::{AuditEvidence, RecallSettings, StoreLocation, StorePurpose}; use crate::settings::tests::helpers::{ central_store, defaults, local_store, resolution_error, resolved, }; @@ -23,7 +23,7 @@ const LAYERED_PROJECT: &[(&str, &str)] = &[ [targets.dev]\n[targets.dev.recall]\nenabled = false\n\ [targets.prod]\n[targets.prod.recall]\nwrite_connection = \"central\"\nread_connection = \"central\"\n\ [targets.capped]\n[targets.capped.recall]\nwrite_connection = \"central\"\nread_connection = \"central\"\n\ - max_output_bytes = 4096\n\ + max_output_bytes = 4096\naudit_evidence = \"error\"\n\ [targets.qa]\n[targets.qa.recall]\nwrite_connection = \"other\"\nread_connection = \"other\"\n\ [targets.laptop]\n[targets.laptop.recall]\nread_connection = \"central\"\n", ), @@ -136,7 +136,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d }, }, ResolveTestCase { - description: "a configured output limit applies to a ClickHouse store", + description: "a configured output limit and audit evidence mode apply to one target", files: LAYERED_PROJECT, target: Some("capped"), purpose: StorePurpose::Record, @@ -148,6 +148,7 @@ fn given_project_layers_when_resolved_then_target_over_local_over_project_over_d retention_days: 10, shutdown_timeout_ms: 1000, max_output_bytes: Some(4096), + audit_evidence: AuditEvidence::Error, ..defaults() }, }, diff --git a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs index dc9f69ab2f..65baa52389 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/delivery.rs @@ -344,7 +344,7 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { }; let mut next_heartbeat: SystemTime = SystemTime::now(); let mut first_heartbeat: bool = true; - let mut materializer = Materializer::new(config.envelope.clone()); + let mut materializer = Materializer::new(config.envelope.clone(), config.audit_evidence); let batching: Batching = batching(config); loop { let (batch, closing) = next_batch(shared, next_heartbeat, batching); @@ -354,9 +354,11 @@ pub(crate) fn deliver(shared: &Arc, config: &SessionConfig) { || !built.unbuildable.is_empty() || !built.filtered.is_empty() || built.extra_events > 0 + || built.extra_artifacts > 0 { let mut state = locked(&shared.state); state.counts_mut(DeliveryStream::Events).accepted += built.extra_events; + state.counts_mut(DeliveryStream::Artifacts).accepted += built.extra_artifacts; state.account(&built.settled, Ok(&InsertOutcome::default())); state.account_unbuildable(&built.unbuildable); state.account_filtered(&built.filtered); diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index dd17ee103a..b1bb6a236e 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -2,8 +2,12 @@ use std::collections::HashSet; +use serde_json::{Map, Value}; + use crate::records::constants::{ - ARTIFACT_DIGEST_PREFIX, EXECUTED_SQL_ROLE, SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, + ARTIFACT_DIGEST_PREFIX, AUDIT_COMPLETED_EVENT, AUDIT_EVIDENCE_KIND, AUDIT_EVIDENCE_MEDIA_TYPE, + AUDIT_EVIDENCE_ROLE, EVIDENCE_PAYLOAD_FIELD, EXECUTED_SQL_ROLE, SAMPLE_DIGEST_PAYLOAD_FIELD, + SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, }; use crate::records::main::artifact_rows::artifact_rows; use crate::records::main::event_object_row::event_object_row; @@ -11,6 +15,7 @@ use crate::records::main::format_timestamp::format_timestamp; use crate::records::main::output_row::output_row; use crate::records::main::sha256_hex::sha256_hex; use crate::records::models::{ArtifactInput, OutputChunk, OutputStream, RecallRow, RowEnvelope}; +use crate::settings::models::AuditEvidence; use crate::writer::_helpers::output_chunk::output_chunk; use crate::writer::_helpers::queue::{Payload, Queued, SqlInput}; use crate::writer::models::{DeliveryStream, ReplacedEvents, ReplacedOutput}; @@ -30,22 +35,29 @@ pub(crate) struct Built { pub(crate) filtered: Vec, /// Events a lifecycle replacement published beyond the one it was handed, accepted as they are built. pub(crate) extra_events: u64, + /// Audit evidence artifacts taken out of their events, accepted as they are built. + pub(crate) extra_artifacts: u64, } -/// Builds rows for one session; remembers which SQL it has stored. +/// Builds rows for one session; remembers which SQL and references it has stored. #[derive(Debug)] pub(crate) struct Materializer { envelope: RowEnvelope, artifacts_seen: HashSet, + /// Digests and the references already written for them. + refs_seen: HashSet<(String, String)>, /// Output bytes stored per stream, for the offsets of replaced chunks. output_bytes: [u64; 2], + audit_evidence: AuditEvidence, } impl Materializer { - pub(crate) fn new(envelope: RowEnvelope) -> Self { + pub(crate) fn new(envelope: RowEnvelope, audit_evidence: AuditEvidence) -> Self { Self { + audit_evidence, envelope, artifacts_seen: HashSet::new(), + refs_seen: HashSet::new(), output_bytes: [0; 2], } } @@ -92,26 +104,98 @@ impl Materializer { Ok(stored) } + /// The content row the first time a digest is seen, and a reference row for each statement that ran it; `None` when both are already written. fn sql_rows(&mut self, input: SqlInput) -> Option> { - let digest = format!( - "{ARTIFACT_DIGEST_PREFIX}{}", - sha256_hex(input.sql.as_bytes()) + let artifact: ArtifactInput = artifact_input( + input.sql.into_bytes(), + (SQL_ARTIFACT_KIND, SQL_MEDIA_TYPE, EXECUTED_SQL_ROLE), + (input.statement_id, input.resource_id), + &format_timestamp(input.seen_at), ); - if !self.artifacts_seen.insert(digest.clone()) { + self.artifact_rows(artifact) + } + + /// `artifact`'s content row the first time its digest is seen and its reference row the first time it is named; `None` when both are written. + fn artifact_rows(&mut self, artifact: ArtifactInput) -> Option> { + if !self + .refs_seen + .insert((artifact.digest.clone(), artifact.ref_id.clone())) + { + return None; + } + let first_sighting: bool = self.artifacts_seen.insert(artifact.digest.clone()); + let [content, reference] = artifact_rows(&self.envelope, &artifact); + Some(if first_sighting { + vec![content, reference] + } else { + vec![reference] + }) + } + + /// The row of `event`; an `audit_completed` event's evidence rows leave its payload and, when the outcome keeps them, become an artifact its `sample_digest` names. + fn event_rows( + &mut self, + mut event: Map, + built: &mut Built, + ) -> Result { + if let Some(artifact) = self.take_evidence(&mut event) + && let Some(rows) = self.artifact_rows(artifact) + { + built.rows.extend(rows); + built.written.push(DeliveryStream::Artifacts); + built.extra_artifacts += 1; + } + event_object_row(&self.envelope, &event) + } + + fn take_evidence(&self, event: &mut Map) -> Option { + if event.get("event_type").and_then(Value::as_str) != Some(AUDIT_COMPLETED_EVENT) { return None; } - let artifact = ArtifactInput { - ref_id: input.statement_id.clone().unwrap_or_else(|| digest.clone()), - digest, - kind: SQL_ARTIFACT_KIND.to_owned(), - media_type: SQL_MEDIA_TYPE.to_owned(), - content: input.sql.into_bytes(), - seen_at: format_timestamp(input.seen_at), - role: EXECUTED_SQL_ROLE.to_owned(), - statement_id: input.statement_id, - resource_id: input.resource_id, - }; - Some(artifact_rows(&self.envelope, &artifact).to_vec()) + let resource_id: Option = event + .get("resource_id") + .and_then(Value::as_str) + .map(str::to_owned); + let event_id: String = event + .get("event_id") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + let seen_at: String = event + .get("occurred_at") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + let payload: &mut Map = event.get_mut("payload")?.as_object_mut()?; + let evidence: Value = payload.remove(EVIDENCE_PAYLOAD_FIELD)?; + let outcome: &str = payload + .get("outcome") + .and_then(Value::as_str) + .unwrap_or_default(); + if !self.audit_evidence.keeps(outcome) { + return None; + } + let content: Vec = serde_json::to_vec(&evidence).ok()?; + let mut artifact: ArtifactInput = artifact_input( + content, + ( + AUDIT_EVIDENCE_KIND, + AUDIT_EVIDENCE_MEDIA_TYPE, + AUDIT_EVIDENCE_ROLE, + ), + (None, resource_id), + &seen_at, + ); + artifact.ref_id = event_id; + let digest: &str = artifact + .digest + .strip_prefix(ARTIFACT_DIGEST_PREFIX) + .unwrap_or(&artifact.digest); + payload.insert( + SAMPLE_DIGEST_PAYLOAD_FIELD.to_owned(), + Value::from(digest.to_owned()), + ); + Some(artifact) } /// The rows of `batch`, with each record's stream sorted by how it is accounted; lifecycle replacements see their events in publication order. @@ -139,7 +223,7 @@ impl Materializer { built.rows.extend(rows); built.written.push(stream); } - Payload::Event(event) => match event_object_row(&self.envelope, &event) { + Payload::Event(event) => match self.event_rows(event, &mut built) { Ok(row) => { built.rows.push(row); built.written.push(stream); @@ -166,7 +250,7 @@ impl Materializer { ReplacedEvents::Published(events) => { built.extra_events += events.len() as u64 - 1; for event in events { - match event_object_row(&self.envelope, &event) { + match self.event_rows(event, &mut built) { Ok(row) => { built.rows.push(row); built.written.push(stream); @@ -181,3 +265,24 @@ impl Materializer { built } } + +/// An artifact of `content`, referenced by its statement or, without one, by its digest. +fn artifact_input( + content: Vec, + (kind, media_type, role): (&str, &str, &str), + (statement_id, resource_id): (Option, Option), + seen_at: &str, +) -> ArtifactInput { + let digest = format!("{ARTIFACT_DIGEST_PREFIX}{}", sha256_hex(&content)); + ArtifactInput { + ref_id: statement_id.clone().unwrap_or_else(|| digest.clone()), + digest, + kind: kind.to_owned(), + media_type: media_type.to_owned(), + content, + seen_at: seen_at.to_owned(), + role: role.to_owned(), + statement_id, + resource_id, + } +} diff --git a/crates/sqlbuild-recall/src/writer/models.rs b/crates/sqlbuild-recall/src/writer/models.rs index bd59ae3060..9492fc4928 100644 --- a/crates/sqlbuild-recall/src/writer/models.rs +++ b/crates/sqlbuild-recall/src/writer/models.rs @@ -6,6 +6,7 @@ use std::thread::JoinHandle; use std::time::Duration; use crate::records::models::{DeliveryCounts, InvocationContext, RowEnvelope}; +use crate::settings::models::AuditEvidence; use crate::store::types::RecallStoreOpener; use crate::writer::_helpers::delivery::Shared; @@ -115,6 +116,8 @@ pub struct SessionConfig { /// Hold the context row until `RecallSession::publish_context` says whether a replacement filters history. pub defer_context: bool, pub maintenance_owner: String, + /// Which audit outcomes keep their evidence rows as an artifact; the rest are left out of the stored event. + pub audit_evidence: AuditEvidence, } /// How table setup went. diff --git a/crates/sqlbuild-recall/src/writer/tests.rs b/crates/sqlbuild-recall/src/writer/tests.rs index 81c66254ff..119759d076 100644 --- a/crates/sqlbuild-recall/src/writer/tests.rs +++ b/crates/sqlbuild-recall/src/writer/tests.rs @@ -1,3 +1,5 @@ +#[path = "tests/test_artifacts.rs"] +mod artifacts; #[path = "tests/helpers.rs"] mod helpers; #[path = "tests/test_queue.rs"] diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index 22b2ed1912..bb26531884 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -7,6 +7,7 @@ use crate::records::models::{ DeliveryCounts, InvocationContext, OutputStream, RecallValue, RowEnvelope, }; use crate::schema::models::Table; +use crate::settings::models::AuditEvidence; use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; @@ -94,6 +95,7 @@ pub(crate) fn config(path: PathBuf, invocation_id: &str) -> SessionConfig { maintenance: true, maintenance_owner: "test:42".to_owned(), defer_context: false, + audit_evidence: AuditEvidence::WarnAndError, } } @@ -583,7 +585,7 @@ pub(crate) fn replacement_order(priorities: &[Priority]) -> Vec { ); } let batch = state.take_batch(usize::MAX); - drop(Materializer::new(envelope("ordered")).build(batch, None)); + drop(Materializer::new(envelope("ordered"), AuditEvidence::WarnAndError).build(batch, None)); std::mem::take(&mut *seen.lock().expect("the record is not poisoned")) } @@ -605,3 +607,77 @@ pub(crate) fn creation_notices( matches!(setup, Some(SetupOutcome::PermissionDenied { .. })), ) } + +/// One `audit_completed` event of `outcome` carrying one evidence row. +fn audit_event(index: usize, outcome: &str) -> serde_json::Map { + let event: serde_json::Value = serde_json::json!({ + "event_id": format!("audit-{index}"), + "event_type": "audit_completed", + "schema_version": 2, + "producer": "sqlbuild", + "producer_version": "1", + "occurred_at": "2026-10-19T00:00:01.000000Z", + "invocation_id": "evidence", + "invocation_sequence": index, + "resource_id": format!("audit:orders_{outcome}"), + "payload": { + "audit_name": format!("orders_{outcome}"), + "outcome": outcome, + "evidence_count": 1, + "evidence_truncated": false, + "evidence": [{"order_id": index}], + }, + }); + match event { + serde_json::Value::Object(event) => event, + _ => serde_json::Map::new(), + } +} + +/// Per audit outcome: whether the stored event still holds its evidence, its `sample_digest`, and the artifact text that digest names, after a session keeping `audit_evidence`. +pub(crate) fn stored_evidence(audit_evidence: AuditEvidence) -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(SessionConfig { + audit_evidence, + ..config(path.clone(), "evidence") + }); + for (index, outcome) in ["warn", "error"].iter().enumerate() { + session.publish_event_object(audit_event(index, outcome)); + } + drop(session.close()); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + query_store( + &store, + "SELECT r.outcome, json_extract(e.payload, '$.evidence') IS NOT NULL, r.sample_digest, \ + r.evidence_count, a.kind, a.text \ + FROM audit_results AS r JOIN recall_events AS e USING (event_id) \ + LEFT JOIN artifacts AS a ON a.digest = 'sha256:' || r.sample_digest \ + ORDER BY r.outcome", + None, + ) + .expect("the query is valid") + .rows +} + +/// Content rows, reference rows and conflicts after two statements ran the same SQL and the second failed. +pub(crate) fn failed_statement_sql() -> Vec> { + let directory = tempfile::tempdir().expect("a temporary directory can be created"); + let path = directory.path().join("recall.sqlite3"); + let session = start_recall_session(config(path.clone(), "failed")); + session.publish_sql("select 1", Some("s1"), Some("orders")); + session.publish_sql("select 1", Some("s2"), Some("orders")); + session.publish_failed_sql("select 1", Some("s2"), Some("orders")); + session.publish_failed_sql("select 2", Some("s3"), None); + drop(session.close()); + let store = open_sqlite_store(&path, OpenMode::ReadOnly).expect("the store opens"); + query_store( + &store, + "SELECT (SELECT COUNT(*) FROM recall_artifacts), \ + (SELECT COUNT(*) FROM recall_artifact_refs), (SELECT COUNT(*) FROM recall_conflicts), \ + (SELECT group_concat(statement_id, ',' ORDER BY statement_id) FROM artifacts)", + None, + ) + .expect("the query is valid") + .rows +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs new file mode 100644 index 0000000000..01c731d561 --- /dev/null +++ b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs @@ -0,0 +1,74 @@ +use crate::records::models::RecallValue; +use crate::settings::models::AuditEvidence; +use crate::writer::tests::helpers::{failed_statement_sql, stored_evidence}; + +fn text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} + +/// The digest of `[{"order_id":}]` without its prefix. +fn evidence_digest(index: u8) -> RecallValue { + let content: String = format!("[{{\"order_id\":{index}}}]"); + text(&crate::records::main::sha256_hex::sha256_hex( + content.as_bytes(), + )) +} + +fn kept(outcome: &str, index: u8) -> Vec { + vec![ + text(outcome), + RecallValue::Integer(0), + evidence_digest(index), + RecallValue::Integer(1), + text("audit_evidence"), + text(&format!("[{{\"order_id\":{index}}}]")), + ] +} + +fn left_out(outcome: &str) -> Vec { + vec![ + text(outcome), + RecallValue::Integer(0), + RecallValue::Null, + RecallValue::Integer(1), + RecallValue::Null, + RecallValue::Null, + ] +} + +#[test] +fn given_audit_evidence_settings_when_audits_complete_then_kept_evidence_is_an_artifact_and_none_stays_in_the_event() + { + let test_cases = [ + ( + AuditEvidence::WarnAndError, + vec![kept("error", 1), kept("warn", 0)], + ), + ( + AuditEvidence::Error, + vec![kept("error", 1), left_out("warn")], + ), + ( + AuditEvidence::Off, + vec![left_out("error"), left_out("warn")], + ), + ]; + + for (setting, expected) in test_cases { + assert_eq!(stored_evidence(setting), expected, "{setting:?}"); + } +} + +#[test] +fn given_statements_sharing_sql_and_a_failed_one_when_recorded_then_text_is_stored_once_and_every_statement_is_referenced() + { + assert_eq!( + failed_statement_sql(), + vec![vec![ + RecallValue::Integer(2), + RecallValue::Integer(3), + RecallValue::Integer(0), + text("s1,s2,s3"), + ]] + ); +} diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md index 9fe8b1386d..a1495fdc8e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/audits.md @@ -10,6 +10,7 @@ Online: https://sqlbuild.com/docs/concepts/audits/ - How audits work - Measurement audits +- Evidence - Audit factories - Audit metadata - Result history @@ -81,8 +82,19 @@ MODEL ( ``` `minimum_samples` keeps low-volume measurements distinct as `insufficient` rather than inventing a -pass or failure. Evidence is diagnostic and bounded by `evidence_limit`; the measurement and -threshold determine the outcome. +pass or failure. The measurement and threshold determine the outcome. + +## Evidence + +When an audit warns or fails, SQLBuild keeps some of the rows behind the failure as evidence: a +violation audit keeps the rows its query returned, and a measurement audit runs its `EVIDENCE` +query. `evidence_limit` caps the rows kept (default 20), and at most 64 KB of them are kept. Set it +on any attachment, for example `not_null (evidence_limit 5)`. + +Evidence is diagnostic and never changes the outcome. It holds data values from the audited +relation, and is recorded in the `evidence_json` column of `_sqlbuild_audit_results`, as +`evidence` in the `audit_completed` lifecycle event, and in +[Recall](observability/recall.md#audit-evidence), which `audit_evidence` controls. ## Audit factories diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 12e65b8567..f3b12d798b 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -203,7 +203,7 @@ artifacts (view): captured SQL and other artifacts, by reference text ANY content BLOB -audit_results (view): one row per evaluated audit +audit_results (view): one row per evaluated audit; its evidence rows are in artifacts invocation_id TEXT join key invocation_started_at TEXT event_id TEXT @@ -224,6 +224,8 @@ audit_results (view): one row per evaluated audit evaluation_mode ANY execution_error ANY sql_digest TEXT join key + evidence_count ANY + evidence_truncated ANY sample_digest ANY conflicts (view): record keys reused with different content; the first row is kept diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 5ad6cdb1ce..2d4aaf22a4 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -107,6 +107,13 @@ def execute_audit_impl( # noqa: PLR0913 executed_sql=executed_sql, run_scope_phase=run_scope_phase, ) + if outcome in {AuditOutcome.WARN, AuditOutcome.ERROR}: + evidence_rows, evidence_truncated = _bounded_evidence( + cursor=cursor, raw_rows=rows, row_limit=_evidence_row_limit(audit) + ) + result = replace( + result, evidence_rows=evidence_rows, evidence_truncated=evidence_truncated + ) lifecycle.completed() return result @@ -218,13 +225,14 @@ def _measurement_result( # noqa: PLR0913 connection=context.connection, run_scope_phase=context.run_scope_phase, ) + row_limit: int = _evidence_row_limit(audit) + fetchmany: object = getattr(evidence_cursor, "fetchmany", None) evidence_rows, evidence_truncated = _bounded_evidence( cursor=evidence_cursor, - row_limit=( - DEFAULT_EVIDENCE_ROW_LIMIT - if audit.evidence_limit is None - else audit.evidence_limit + raw_rows=list( + fetchmany(row_limit + 1) if callable(fetchmany) else evidence_cursor.fetchall() ), + row_limit=row_limit, ) except Exception as caught_error: evidence_error = f"{type(caught_error).__name__}: {caught_error}" @@ -383,27 +391,31 @@ def _sample_count(*, row: Mapping[str, object], column: str) -> tuple[int | None return int(numeric), None +def _evidence_row_limit(audit: AuditPlanEntry) -> int: + return DEFAULT_EVIDENCE_ROW_LIMIT if audit.evidence_limit is None else audit.evidence_limit + + def _bounded_evidence( - *, cursor: Any, row_limit: int + *, cursor: Any, raw_rows: list[Any], row_limit: int ) -> tuple[tuple[Mapping[str, object], ...], bool]: - fetchmany: object = getattr(cursor, "fetchmany", None) - raw_rows: list[Any] = list( - fetchmany(row_limit + 1) if callable(fetchmany) else cursor.fetchall() - ) + """The leading rows within the row limit and serialized byte cap, and whether rows were cut.""" + truncated: bool = len(raw_rows) > row_limit retained: list[Mapping[str, object]] = [] + serialized_bytes: int = len(b"[]") for raw_row in raw_rows[:row_limit]: safe_row: Mapping[str, object] = { key: _json_safe(value) for key, value in _row_mapping(cursor=cursor, row=raw_row).items() } - candidate: list[Mapping[str, object]] = [*retained, safe_row] - serialized: bytes = json.dumps(candidate, sort_keys=True, separators=(",", ":")).encode( - "utf-8" + row_bytes: int = len( + json.dumps(safe_row, sort_keys=True, separators=(",", ":")).encode("utf-8") ) - if len(serialized) > MAX_EVIDENCE_SERIALIZED_BYTES: + separator_bytes: int = len(b",") if retained else 0 + if serialized_bytes + separator_bytes + row_bytes > MAX_EVIDENCE_SERIALIZED_BYTES: truncated = True break + serialized_bytes += separator_bytes + row_bytes retained.append(safe_row) return tuple(retained), truncated diff --git a/website/src/content/docs/docs/concepts/audits.mdx b/website/src/content/docs/docs/concepts/audits.mdx index 06db321a40..5b39a4fd3d 100644 --- a/website/src/content/docs/docs/concepts/audits.mdx +++ b/website/src/content/docs/docs/concepts/audits.mdx @@ -61,8 +61,19 @@ MODEL ( ``` `minimum_samples` keeps low-volume measurements distinct as `insufficient` rather than inventing a -pass or failure. Evidence is diagnostic and bounded by `evidence_limit`; the measurement and -threshold determine the outcome. +pass or failure. The measurement and threshold determine the outcome. + +## Evidence + +When an audit warns or fails, SQLBuild keeps some of the rows behind the failure as evidence: a +violation audit keeps the rows its query returned, and a measurement audit runs its `EVIDENCE` +query. `evidence_limit` caps the rows kept (default 20), and at most 64 KB of them are kept. Set it +on any attachment, for example `not_null (evidence_limit 5)`. + +Evidence is diagnostic and never changes the outcome. It holds data values from the audited +relation, and is recorded in the `evidence_json` column of `_sqlbuild_audit_results`, as +`evidence` in the `audit_completed` lifecycle event, and in +[Recall](/docs/concepts/observability/recall#audit-evidence), which `audit_evidence` controls. ## Audit factories diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index b13eb8717c..8b2a94f5e7 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -23,32 +23,30 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -run | 2026-10-20T00:01:44.784683Z -outcome | warn -violation_count | 1 -status | completed -exit_code | 0 - --[ RECORD 2 ]---------------------------+ -run | 2026-10-20T00:00:34.652468Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 - --[ RECORD 3 ]---------------------------+ -run | 2026-10-19T23:59:23.827831Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 +┌─────────────────────────────┬─────────┬─────────────────┬───────────┬───────────┐ +│ run │ outcome │ violation_count │ status │ exit_code │ +│ text │ text │ integer │ text │ integer │ +├─────────────────────────────┼─────────┼─────────────────┼───────────┼───────────┤ +│ 2026-10-20T00:01:44.784683Z │ warn │ 1 │ completed │ 0 │ +│ 2026-10-20T00:00:34.652468Z │ pass │ 0 │ completed │ 0 │ +│ 2026-10-19T23:59:23.827831Z │ pass │ 0 │ completed │ 0 │ +└─────────────────────────────┴─────────┴─────────────────┴───────────┴───────────┘ 3 rows ``` The audit started warning on the latest build, and that build still exited 0, so nothing alerted. +See the rows the audit flagged, which Recall keeps as the audit's evidence: + +```bash +sqb recall query \ + "SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + WHERE a.audit_name = 'revenue_is_non_negative' AND a.outcome <> 'pass' + ORDER BY a.invocation_started_at DESC LIMIT 1" +``` + See what that build printed, exactly as the scheduler saw it: ```bash @@ -162,6 +160,7 @@ retention = "30d" # days of history to keep; also accepts hours, suc shutdown_timeout = "3s" # longest delay Recall may add when a command exits diagnostics_min_severity = "info" max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything +audit_evidence = "warn-and-error" [targets.ci.recall] enabled = false # do not record runs against the ci target @@ -176,6 +175,7 @@ enabled = false # do not record runs against the ci target | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | | `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | +| `audit_evidence` | `"warn-and-error"` | Which failing audits keep the rows they flagged: `warn-and-error`, `error`, or `off`. See [Audit evidence](#audit-evidence) | To turn Recall off for the whole project: From 3a788d854afc44b2898f3d4e87d7d1d777381ef5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:58:00 +0000 Subject: [PATCH 08/30] feat(recall): mark cost telemetry statements and record cost attribution status Snowflake cost history reads have `purpose = 'cost_telemetry'`, a read that timed out and was retried in smaller windows ends with outcome `handled`, and each invocation records how far cost attribution got in `cost_status`. --- .../src/lifecycle_events/constants.rs | 6 +- .../src/entries/_helpers/executor.rs | 5 ++ crates/sqlbuild-host/src/entries/models.rs | 1 + .../_helpers/cost_report/attribution.rs | 22 ++++++ .../_helpers/cost_report/degradation.rs | 3 + .../src/bindings/_helpers/cost_report/mod.rs | 1 + .../_helpers/cost_telemetry/snowflake.rs | 72 ++++++++++++++----- .../src/schema/_helpers/clickhouse_views.rs | 9 ++- .../src/schema/_helpers/sqlite_views.rs | 5 +- .../sqlbuild/references/investigating-runs.md | 2 +- src/sqlbuild/_native_host.py | 4 ++ .../snowflake/classes/snowflake_adapter.py | 50 ++++++++++++- .../classes/statement_lifecycle.py | 27 +++++++ .../runtime/observability/constants.py | 1 + .../adapters/snowflake/_test_types.py | 13 ++++ .../adapters/snowflake/test_client.py | 40 +++++++++++ .../snowflake/test_cost_collection.py | 66 +++++++++++++++++ tests/unit/src/sqlbuild/cost/_test_types.py | 7 ++ .../src/sqlbuild/cost/test_snowflake_scale.py | 68 ++++++++++++++++++ .../runtime/observability/_test_types.py | 7 ++ .../observability/test_statement_lifecycle.py | 42 +++++++++++ .../docs/concepts/observability/recall.mdx | 15 +++- 22 files changed, 439 insertions(+), 27 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index cd6572eb5b..42435b65bc 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -79,7 +79,7 @@ pub(crate) const FORBIDDEN_STATEMENT_PAYLOAD_FIELDS: [&str; 8] = [ "sql", "statement_sql", ]; -pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [ +pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 56] = [ "adapter", "attached_column_name", "attached_target_kind", @@ -92,6 +92,7 @@ pub(crate) const STRING_PAYLOAD_FIELDS: [&str; 55] = [ "binding_key", "command", "configured_batch_size", + "cost_status", "cursor_end_exclusive", "cursor_start", "definition_fingerprint", @@ -653,7 +654,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ event_type: "invocation_completed", required_correlations: &[], required_payload: &[], - allowed_payload: &["command", "duration_ms", "exit_code"], + allowed_payload: &["command", "cost_status", "duration_ms", "exit_code"], terminal: true, }, EventDefinition { @@ -1010,6 +1011,7 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ "intent", "job_id", "metadata", + "outcome", "purpose", "query_id", "row_count", diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 637dbe5d5f..27e14bdf88 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -719,6 +719,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Class, edge: HostEdge::Executor, }, + HostEntry { + name: "StatementLifecycle", + kind: HostKind::Class, + edge: HostEdge::Executor, + }, HostEntry { name: "ObservabilityValidationError", kind: HostKind::Exception, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 44cb7aa246..e173ce1abb 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -344,6 +344,7 @@ pub enum ExecutorHost { ExecutionDeadlineScope, EventDispatcher, OperationLifecycle, + StatementLifecycle, ObservabilityValidationError, CreateLifecycleEvent, CurrentExecutionIdentity, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs new file mode 100644 index 0000000000..7feb2891a0 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/attribution.rs @@ -0,0 +1,22 @@ +//! How far the running command attributed warehouse cost, for the invocation's terminal event. + +use std::cell::RefCell; + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; + +thread_local! { + /// Per thread, since an invocation finalizes cost and reports on the thread that runs it. + static COST_STATUS: RefCell> = const { RefCell::new(None) }; +} + +/// Record the cost status of the latest finalization; a later one replaces it, so a run that stops before collecting keeps `pending`. +pub(crate) fn report_cost_status(status: &Bound<'_, PyAny>) -> PyResult<()> { + let status: String = status.str()?.to_string(); + COST_STATUS.with(|slot| *slot.borrow_mut() = Some(status)); + Ok(()) +} + +/// The cost status the command reported, cleared so the next invocation starts without one. +pub(crate) fn take_cost_status() -> Option { + COST_STATUS.with(|slot| slot.borrow_mut().take()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs index bdd4ba21ec..d41ffba179 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs @@ -13,6 +13,7 @@ use sqlbuild_core::cost_report::models::BreakdownOptions; use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::call_with; +use crate::bindings::_helpers::cost_report::attribution::report_cost_status; use crate::bindings::_helpers::cost_report::records::cost_run; use crate::bindings::_helpers::cost_report::store; use crate::bindings::_helpers::cost_telemetry::values::cost_aware; @@ -219,6 +220,7 @@ fn collection_failure<'py>( )?; let status: Bound<'py, PyAny> = host::entry(py, ExecutorHost::CostStatus)?.getattr("COLLECTION_FAILED")?; + report_cost_status(&status)?; let message: Bound<'py, PyAny> = PyString::new( py, &format!("Cost telemetry failed during {stage} ({error_type})."), @@ -307,6 +309,7 @@ pub(crate) fn finalize_build_cost<'py>( native_field(finalization, "cost_message")?, )? }; + report_cost_status(&native_field(&cost, "status")?)?; let record: Bound<'py, PyAny> = cost_record(finalization, cost)?; persist(finalization, &record, "artifact persistence")?; if !native_field(finalization, "render")?.is_truthy()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs index f734d355cf..854d8e9fb1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_report/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod attribution; pub(crate) mod degradation; pub(crate) mod exception_types; pub(crate) mod functions; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs index 80fbc37771..3eb528de1b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs @@ -6,7 +6,7 @@ use pyo3::exceptions::{PyTimeoutError, PyValueError}; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; +use pyo3::types::{PyCFunction, PyDict, PyList, PyListMethods, PyString, PyStringMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_analysis::snowflake_cost::main::metadata_only_statement::metadata_only_statement; use sqlbuild_analysis::snowflake_cost::main::sqlbuild_query_tag::sqlbuild_query_tag; @@ -14,14 +14,16 @@ use sqlbuild_analysis::snowflake_cost::models::QueryTag; use sqlbuild_core::cost_allocation::constants::QUERY_HISTORY_RESULT_LIMIT; use sqlbuild_core::cost_allocation::main::query_history_sql::query_history_sql; use sqlbuild_core::cost_allocation::models::{QueryHistoryWindow, RunAllocation}; +use sqlbuild_host::entries::models::ExecutorHost; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, imported, within}; use crate::bindings::_helpers::cost_telemetry::allocation::{ SummaryState, allocate_observations, priced, summary_object, }; use crate::bindings::_helpers::cost_telemetry::health::consume_ledger_failure; use crate::bindings::_helpers::cost_telemetry::ledger::read_ledger; use crate::bindings::_helpers::cursor::values::DATETIME_MODULE; +use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::replaced_keywords; @@ -85,9 +87,43 @@ fn check_deadline(history: &History<'_>, py: Python<'_>) -> PyResult<()> { Ok(()) } +/// Whether a read's statement timeout is recovered from by reading smaller windows, so its failed statement is recorded as handled. +#[derive(Clone, Copy, PartialEq, Eq)] +enum TimeoutRetry { + Split, + None, +} + +/// Run `execute`, marking a statement timeout inside it as handled when the caller retries it. +fn with_timeout_retry<'py, T>( + py: Python<'py>, + retry: TimeoutRetry, + execute: impl FnOnce() -> PyResult, +) -> PyResult { + if retry == TimeoutRetry::None { + return execute(); + } + let is_timeout = PyCFunction::new_closure( + py, + None, + None, + |arguments: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult { + Ok(arguments + .get_item(0)? + .str()? + .to_string_lossy() + .contains(STATEMENT_TIMEOUT_CODE)) + }, + )?; + let scope: Bound<'py, PyAny> = host::entry(py, ExecutorHost::StatementLifecycle)? + .call_method1("handled_failure_scope", (is_timeout,))?; + within(&scope, |_| execute()) +} + fn history_rows<'py>( history: &History<'py>, (start, end): (&Bound<'py, PyAny>, Option<&Bound<'py, PyAny>>), + retry: TimeoutRetry, ) -> PyResult>> { let py: Python<'py> = start.py(); check_deadline(history, py)?; @@ -107,10 +143,11 @@ fn history_rows<'py>( parameters.set_item("STATEMENT_TIMEOUT_IN_SECONDS", QUERY_TIMEOUT_SECONDS)?; let options: Bound<'py, PyDict> = PyDict::new(py); options.set_item("statement_params", parameters)?; - let cursor: Bound<'py, PyAny> = + let cursor: Bound<'py, PyAny> = with_timeout_retry(py, retry, || { history .connection - .call_method("execute", (sql,), Some(&options))?; + .call_method("execute", (sql,), Some(&options)) + })?; check_deadline(history, py)?; let rows: Vec> = cursor .call_method0("fetchall")? @@ -157,12 +194,14 @@ fn bounded_rows<'py>( ) -> PyResult<(Vec>, bool)> { let py: Python<'py> = start.py(); let width: Bound<'py, PyAny> = end.sub(start)?; - let rows: Vec> = match history_rows(history, (start, Some(end))) { + let retry: TimeoutRetry = if width.gt(timedelta(py, ("seconds", MIN_TIMEOUT_SPLIT_SECONDS))?)? { + TimeoutRetry::Split + } else { + TimeoutRetry::None + }; + let rows: Vec> = match history_rows(history, (start, Some(end)), retry) { Ok(rows) => rows, - Err(error) - if statement_timeout(py, &error) - && width.gt(timedelta(py, ("seconds", MIN_TIMEOUT_SPLIT_SECONDS))?)? => - { + Err(error) if retry == TimeoutRetry::Split && statement_timeout(py, &error) => { return split_rows(history, (start, end), &width); } Err(error) => return Err(error), @@ -328,13 +367,14 @@ fn open_rows<'py>( (start, window_end): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), ) -> PyResult<(Vec>, bool)> { let py: Python<'py> = start.py(); - let open_rows: Vec> = match history_rows(history, (start, None)) { - Ok(rows) => rows, - Err(error) if statement_timeout(py, &error) => { - return bounded_rows(history, (start, window_end)); - } - Err(error) => return Err(error), - }; + let open_rows: Vec> = + match history_rows(history, (start, None), TimeoutRetry::Split) { + Ok(rows) => rows, + Err(error) if statement_timeout(py, &error) => { + return bounded_rows(history, (start, window_end)); + } + Err(error) => return Err(error), + }; if open_rows.len() < history.result_limit { return Ok((open_rows, false)); } diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 1223f0a207..796400f0fc 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -43,11 +43,13 @@ const INVOCATION_SOURCE_COLUMNS: [(&str, &str, &str); 35] = [ ("event_error_type", "Nullable(String)", "NULL"), ("event_error_code", "Nullable(String)", "NULL"), ("event_error_message", "Nullable(String)", "NULL"), + ("event_signal_number", "Nullable(Int64)", "NULL"), + ("event_cost_status", "Nullable(String)", "NULL"), ("event_is_terminal", "UInt8", "0"), ]; /// The statement columns each statement event carries into the second fold, in tuple order. -const STATEMENT_FACT_COLUMNS: [&str; 17] = [ +const STATEMENT_FACT_COLUMNS: [&str; 18] = [ "statement_id", "event_type", "occurred_at", @@ -65,6 +67,7 @@ const STATEMENT_FACT_COLUMNS: [&str; 17] = [ "duration_ms", "row_count", "affected_rows", + "outcome", ]; /// The columns the `output_invocations` pass reads: the chunk's own values default to what adds nothing to a sum, and the delivery and context values to NULL. @@ -376,7 +379,8 @@ folded AS ( argMaxIf(error_code, {order}, {terminal}) AS last_error_code, argMaxIf(duration_ms, {order}, {terminal}) AS last_duration_ms, argMaxIf(row_count, {order}, {terminal}) AS last_row_count, - argMaxIf(affected_rows, {order}, {terminal}) AS last_affected_rows + argMaxIf(affected_rows, {order}, {terminal}) AS last_affected_rows, + argMaxIf(outcome, {order}, {terminal}) AS last_outcome FROM statement_events GROUP BY invocation_id, invocation_started_at, statement_id ) SELECT @@ -399,6 +403,7 @@ SELECT multiIf( NOT has_terminal, 'no_terminal', last_event_type = 'statement_completed', 'completed', + last_outcome = 'handled', 'handled', 'failed' ) AS outcome, if(has_terminal, last_error_type, NULL) AS error_type, diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 6922202a5f..3152c66763 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -31,7 +31,7 @@ query_ids AS ( ), terminals AS ( SELECT statement_id, event_type, occurred_at, query_id, error_type, error_code, duration_ms, - row_count, affected_rows, + row_count, affected_rows, outcome, ROW_NUMBER() OVER ( PARTITION BY statement_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -76,7 +76,8 @@ SELECT terminals.occurred_at AS ended_at, CASE terminals.event_type WHEN 'statement_completed' THEN 'completed' - WHEN 'statement_failed' THEN 'failed' + WHEN 'statement_failed' THEN + CASE WHEN terminals.outcome = 'handled' THEN 'handled' ELSE 'failed' END ELSE 'no_terminal' END AS outcome, terminals.error_type, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index f3b12d798b..67d7a48cfd 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -370,7 +370,7 @@ output_transcripts (view): exact stdout, stderr and combined bytes per invocatio stderr BLOB combined BLOB -statements (view): one row per statement; outcome no_terminal means no terminal fact arrived +statements (view): one row per statement; outcome handled means SQLBuild recovered from the failure; no_terminal means no terminal fact arrived invocation_id TEXT join key invocation_started_at TEXT statement_id TEXT join key diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 93ffba817f..8becdaf2da 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -581,6 +581,10 @@ "sqlbuild.runtime.observability.classes.operation_lifecycle", "OperationLifecycle", ), + "StatementLifecycle": ( + "sqlbuild.runtime.observability.classes.statement_lifecycle", + "StatementLifecycle", + ), "ObservabilityValidationError": ( "sqlbuild.runtime.observability.exceptions", "ObservabilityValidationError", diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 5122de425b..19bf8a78bb 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -166,6 +166,7 @@ from sqlbuild.cost.types import CostCapability, CostStatus from sqlbuild.diagnostics.main.log_sql import log_sql from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +from sqlbuild.runtime.observability.constants import COST_TELEMETRY_PURPOSE from sqlbuild.spec.contracts.constants import DEFAULT_SEED_CSV_SETTINGS _EXACT_COLUMN_INSPECTION_LIMIT: int = 32 @@ -363,6 +364,32 @@ def collect_run_cost( usd_per_credit: Decimal, rate_source: str, budget_seconds: float | None = None, + ) -> RunCostSummary: + with StatementLifecycle.purpose_scope(COST_TELEMETRY_PURPOSE): + return self._collect_run_cost( + connection_config=connection_config, + target_database=target_database, + run_id=run_id, + started_at=started_at, + completed_at=completed_at, + statement_ledger_path=statement_ledger_path, + usd_per_credit=usd_per_credit, + rate_source=rate_source, + budget_seconds=budget_seconds, + ) + + def _collect_run_cost( + self, + *, + connection_config: dict[str, object], + target_database: str | None, + run_id: str, + started_at: datetime, + completed_at: datetime, + statement_ledger_path: Path, + usd_per_credit: Decimal, + rate_source: str, + budget_seconds: float | None = None, ) -> RunCostSummary: telemetry_connection_config: dict[str, object] = dict(connection_config) telemetry_connection_config["login_timeout"] = _COST_LOGIN_TIMEOUT_SECONDS @@ -1863,13 +1890,16 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) + column_types: tuple[str | None, ...] = tuple( + self._result_column_type(column) for column in description + ) if limit is None: rows: tuple[tuple[object, ...], ...] = tuple( tuple(row) for row in cursor.fetchall() ) if self._is_status_only_non_row_result(columns=columns, rows=rows): return QueryResult() - return QueryResult(columns=columns, rows=rows) + return QueryResult(columns=columns, rows=rows, column_types=column_types) fetched_rows: list[tuple[object, ...]] = [ tuple(row) for row in cursor.fetchmany(limit + 1) ] @@ -1880,10 +1910,28 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: columns=columns, rows=rows, truncated=len(fetched_rows) > limit, + column_types=column_types, ) finally: cursor.close() + @staticmethod + def _result_column_type(column: Any) -> str | None: + """Name a result column's Snowflake type from its cursor description entry.""" + + from snowflake.connector.constants import FIELD_ID_TO_NAME + + type_name: str | None = FIELD_ID_TO_NAME.get(column[1]) + if type_name is None: + return None + if type_name != "FIXED": + return type_name + precision: int | None = column[4] + scale: int | None = column[5] + if precision is None: + return "NUMBER" + return f"NUMBER({precision},{scale or 0})" + @staticmethod def _is_status_only_non_row_result( *, columns: tuple[str, ...], rows: tuple[tuple[object, ...], ...] diff --git a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py index 85d0901508..9ddadf930e 100644 --- a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py @@ -44,6 +44,10 @@ "sqlbuild_statement_lifecycle_owner", default=None ) _STATEMENT_PURPOSE: ContextVar[str | None] = ContextVar("sqlbuild_statement_purpose", default=None) +_HANDLED_FAILURE: ContextVar[Callable[[BaseException], bool] | None] = ContextVar( + "sqlbuild_statement_handled_failure", default=None +) +_HANDLED_OUTCOME: str = "handled" type _ExecutionOwnerKey = tuple[int, int | None] @@ -86,6 +90,7 @@ def __init__( self._pending_failure: tuple[BaseException, str | None, str | None] | None = None self._statement_facts: tuple[str, str] | None = None self._purpose: str | None = _STATEMENT_PURPOSE.get() + self._is_handled: Callable[[BaseException], bool] | None = _HANDLED_FAILURE.get() self._item_deadline: ItemDeadline | None = None self._terminal_query_id: str | None = None @@ -108,6 +113,17 @@ def purpose_scope(purpose: str) -> Iterator[None]: finally: _STATEMENT_PURPOSE.reset(token) + @staticmethod + @contextmanager + def handled_failure_scope(is_handled: Callable[[BaseException], bool]) -> Iterator[None]: + """Mark failures the caller recovers from, such as a timeout it retries in parts.""" + + token: Token[Callable[[BaseException], bool] | None] = _HANDLED_FAILURE.set(is_handled) + try: + yield + finally: + _HANDLED_FAILURE.reset(token) + @staticmethod def discard_failed_attempt() -> None: """Forget a failed attempt deferred to the active statement before an adapter retry.""" @@ -271,6 +287,8 @@ def failed( payload["query_id"] = query_id if job_id is not None: payload["job_id"] = job_id + if self._failure_handled(error=error): + payload["outcome"] = _HANDLED_OUTCOME self._publish_terminal(event_type="statement_failed", payload=payload) def __exit__( @@ -329,6 +347,15 @@ def _terminal_payload(self) -> dict[str, JSONValue]: payload["duration_ms"] = max(0.0, (self._clock() - started) * 1000.0) return payload + def _failure_handled(self, *, error: BaseException) -> bool: + is_handled: Callable[[BaseException], bool] | None = self._is_handled + if is_handled is None: + return False + try: + return bool(is_handled(error)) + except Exception: + return False + def _stop_monitor(self) -> str | None: monitor: StatementMonitor | None = self._monitor if monitor is None: diff --git a/src/sqlbuild/runtime/observability/constants.py b/src/sqlbuild/runtime/observability/constants.py index df9f0d299d..46a00a0f79 100644 --- a/src/sqlbuild/runtime/observability/constants.py +++ b/src/sqlbuild/runtime/observability/constants.py @@ -10,6 +10,7 @@ STATEMENT_HEARTBEAT_THRESHOLD_SECONDS: float = 30.0 MODEL_BUILD_PURPOSE: str = "model_build" AUDIT_PURPOSE: str = "audit" +COST_TELEMETRY_PURPOSE: str = "cost_telemetry" LIFECYCLE_ENVELOPE_FIELDS_V1: frozenset[str] = frozenset( { "event_id", diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py index be6cfb3c96..35713fd376 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/_test_types.py @@ -37,6 +37,12 @@ class SnowflakeCostCollectionTestCase: expected_limitation_fragment: str | None = None +@dataclass(frozen=True) +class SnowflakeCostPurposeTestCase: + description: str + expected_purposes: list[str | None] + + @dataclass(frozen=True) class SnowflakeMergeExclusionTestCase: description: str @@ -105,6 +111,13 @@ class SnowflakeRenderTableFunctionTestCase: expected_sql: str +@dataclass(frozen=True) +class SnowflakeQueryColumnTypesTestCase: + description: str + cursor_description: tuple[tuple[object, ...], ...] + expected_types: tuple[str | None, ...] + + @dataclass(frozen=True) class SnowflakeQueryColumnNamesTestCase: description: str diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py index 80b645cce2..57f8ea7ea7 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_client.py @@ -13,6 +13,7 @@ from sqlbuild.adapter.contract.models import ( ColumnInfo, ExpressionInferenceProfile, + QueryResult, RelationInfo, SchemaDiffResult, TableFreshnessMetadata, @@ -36,6 +37,7 @@ SnowflakePruneSqlTestCase, SnowflakeQualifiedColumnInspectionTestCase, SnowflakeQueryColumnNamesTestCase, + SnowflakeQueryColumnTypesTestCase, SnowflakeRelationAgeMetadataTestCase, SnowflakeRenderCloneTestCase, SnowflakeRenderCursorBoundLiteralTestCase, @@ -55,6 +57,7 @@ FakeSnowflakeMetadataConnection, FakeSnowflakeMetadataCursor, FakeSnowflakeMetadataSequenceConnection, + FakeSnowflakeQueryCursor, FakeSnowflakeRawConnection, build_bulk_columns_rows, build_bulk_sequence_connection, @@ -1155,6 +1158,43 @@ def test_given_equivalent_types_when_diffing_schema_then_snowflake_ignores_alias assert result == test_case.expected_result +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeQueryColumnTypesTestCase( + description="numbers carry precision and scale, other types their name", + cursor_description=( + ("ORDER_ID", 0, None, None, 38, 0, False), + ("AMOUNT", 0, None, None, 10, 2, True), + ("STATUS", 2, None, 16777216, None, None, True), + ("ORDERED_AT", 8, None, None, 0, 9, True), + ), + expected_types=("NUMBER(38,0)", "NUMBER(10,2)", "TEXT", "TIMESTAMP_NTZ"), + ), + SnowflakeQueryColumnTypesTestCase( + description="an unknown type code has no name", + cursor_description=(("PAYLOAD", 99, None, None, None, None, True),), + expected_types=(None,), + ), + ], + ids=lambda case: case.description, +) +def test_given_snowflake_result_metadata_when_querying_then_column_types_are_named( + test_case: SnowflakeQueryColumnTypesTestCase, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + cursor: FakeSnowflakeQueryCursor = FakeSnowflakeQueryCursor( + description=test_case.cursor_description, + rows=[tuple(None for _ in test_case.cursor_description)], + ) + + result: QueryResult = adapter.query( + connection=FakeSnowflakeDescribeConnection(cursor), sql="SELECT 1", limit=20 + ) + + assert result.column_types == test_case.expected_types + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py index afbc4b0c70..aa63188772 100644 --- a/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py +++ b/tests/unit/src/sqlbuild/adapters/snowflake/test_cost_collection.py @@ -10,8 +10,16 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus +from sqlbuild.observability import ( + EventDispatcher, + LifecycleEvent, + dispatcher_scope, + invocation_scope, +) +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.adapters.snowflake._test_types import ( SnowflakeCostCollectionTestCase, + SnowflakeCostPurposeTestCase, ) @@ -37,6 +45,21 @@ def __call__(self, config: dict[str, object]) -> _CostConnection: return self.connection +class _QueryHistoryCollect: + """Runs one query-history statement, as the real collector does.""" + + def __call__(self, **kwargs: Any) -> RunCostSummary: + with StatementLifecycle( + adapter="snowflake", sql="SELECT 1 FROM query_history", intent="query" + ) as lifecycle: + lifecycle.completed(row_count=0) + return RunCostSummary( + status=CostStatus.COMPLETE, + usd_per_credit=kwargs["usd_per_credit"], + rate_source=kwargs["rate_source"], + ) + + class _Collect: def __init__(self, *, error: bool) -> None: self.error: bool = error @@ -116,3 +139,46 @@ def test_given_telemetry_stage_failure_when_collecting_then_safe_status_is_retur assert connect.config["network_timeout"] == 15 assert (test_case.expected_message_fragment or "") in (summary.message or "") assert (test_case.expected_limitation_fragment or "") in " ".join(summary.limitations) + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeCostPurposeTestCase( + description="query-history statements are recorded as cost telemetry", + expected_purposes=["cost_telemetry", "cost_telemetry"], + ), + ], + ids=lambda case: case.description, +) +def test_given_cost_collection_when_it_queries_history_then_its_statements_name_their_purpose( + test_case: SnowflakeCostPurposeTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + adapter: SnowflakeAdapter = SnowflakeAdapter() + monkeypatch.setattr( + adapter, "connect", _Connect(connection=_CostConnection(close_error=False), error=False) + ) + monkeypatch.setattr( + "sqlbuild.adapters.snowflake.classes.snowflake_adapter.collect_snowflake_cost", + _QueryHistoryCollect(), + ) + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + now: datetime = datetime(2026, 8, 23, tzinfo=UTC) + + with invocation_scope("inv-cost-purpose"), dispatcher_scope(dispatcher): + _ = adapter.collect_run_cost( + connection_config={}, + target_database="ANALYTICS", + run_id="run-1", + started_at=now, + completed_at=now, + statement_ledger_path=tmp_path / "statements.jsonl", + usd_per_credit=Decimal("3.00"), + rate_source="default", + ) + + assert [event.payload.get("purpose") for event in events] == test_case.expected_purposes diff --git a/tests/unit/src/sqlbuild/cost/_test_types.py b/tests/unit/src/sqlbuild/cost/_test_types.py index 158999812c..145f865885 100644 --- a/tests/unit/src/sqlbuild/cost/_test_types.py +++ b/tests/unit/src/sqlbuild/cost/_test_types.py @@ -104,6 +104,13 @@ class CostResourceScopeTestCase: expected_attempt: int +@dataclass(frozen=True) +class SnowflakeTimeoutOutcomeTestCase: + description: str + timeout_window: timedelta + expected_failure_outcomes: set[str | None] + + @dataclass(frozen=True) class SnowflakeScaleTestCase: description: str diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py index 7664da0998..7c471f3ca7 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py @@ -22,12 +22,20 @@ from sqlbuild.adapters.snowflake.classes.snowflake_adapter import SnowflakeAdapter from sqlbuild.cost.models import RunCostSummary from sqlbuild.cost.types import CostStatus +from sqlbuild.observability import ( + EventDispatcher, + LifecycleEvent, + dispatcher_scope, + invocation_scope, +) +from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.cli.commands._helpers.cost.helpers import write_snowflake_project from tests.unit.src.sqlbuild.cost._test_types import ( SnowflakeCostBudgetTestCase, SnowflakeCostFailureMessageTestCase, SnowflakeLargeRunCostTestCase, SnowflakeScaleTestCase, + SnowflakeTimeoutOutcomeTestCase, ) from tests.unit.src.sqlbuild.cost.helpers import ( build_cost_run_record, @@ -215,6 +223,66 @@ def test_given_large_run_when_collecting_snowflake_cost_then_sql_stays_bounded( assert len(history.reads) <= test_case.expected_maximum_reads +class _ObservedHistory(_FakeQueryHistory): + """Query history read through statement lifecycles, as the Snowflake connection records them.""" + + def execute(self, sql: str, **kwargs: Any) -> _Cursor: + with StatementLifecycle(adapter="snowflake", sql=sql, intent="query") as lifecycle: + try: + cursor: _Cursor = super().execute(sql, **kwargs) + except Exception as error: + lifecycle.failed(error=error) + raise + lifecycle.completed(row_count=len(cursor.rows)) + return cursor + + +@pytest.mark.parametrize( + "test_case", + [ + SnowflakeTimeoutOutcomeTestCase( + description="a timed-out read retried in smaller windows is handled", + timeout_window=timedelta(minutes=8), + expected_failure_outcomes={"handled"}, + ), + SnowflakeTimeoutOutcomeTestCase( + description="a timed-out read too narrow to split stays failed", + timeout_window=timedelta(0), + expected_failure_outcomes={"handled", None}, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_history_read_timeout_when_collecting_then_only_retried_reads_are_handled( + test_case: SnowflakeTimeoutOutcomeTestCase, tmp_path: Path +) -> None: + duration: timedelta = timedelta(minutes=42) + rows: list[tuple[Any, ...]] = run_history_rows( + origin=_ORIGIN, statement_count=6_500, duration=duration + ) + ledger_path: Path = tmp_path / "statements.jsonl" + write_ledger(path=ledger_path, run_id=_RUN_ID, rows=rows) + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + + with invocation_scope("inv-history-timeout"), dispatcher_scope(dispatcher): + _ = collect_snowflake_cost( + connection=_ObservedHistory(rows=rows, timeout_window=test_case.timeout_window), + run_id=_RUN_ID, + started_at=_ORIGIN, + completed_at=_ORIGIN + duration, + statement_ledger_path=ledger_path, + usd_per_credit=Decimal("3.00"), + rate_source="default", + attempts=1, + ) + + assert { + event.payload.get("outcome") for event in events if event.event_type == "statement_failed" + } == test_case.expected_failure_outcomes + + class _ClosingHistory(_FakeQueryHistory): def close(self) -> None: return None diff --git a/tests/unit/src/sqlbuild/runtime/observability/_test_types.py b/tests/unit/src/sqlbuild/runtime/observability/_test_types.py index 5105567e38..64a82652bc 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/_test_types.py +++ b/tests/unit/src/sqlbuild/runtime/observability/_test_types.py @@ -374,3 +374,10 @@ class FailedMonitorStartCase: description: str threshold_seconds: float expected_event_types: tuple[str, ...] + + +@dataclass(frozen=True) +class HandledFailureCase: + description: str + error: BaseException + expected_outcome: str | None diff --git a/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py b/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py index 4c6692bd76..257f95b52a 100644 --- a/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py +++ b/tests/unit/src/sqlbuild/runtime/observability/test_statement_lifecycle.py @@ -22,6 +22,7 @@ from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle from tests.unit.src.sqlbuild.runtime.observability._test_types import ( ErrorCodePrivacyCase, + HandledFailureCase, StatementKindPrivacyCase, StatementLifecycleCase, ) @@ -659,3 +660,44 @@ def test_given_driver_error_code_when_failing_statement_then_only_safe_identifie assert events[-1].payload.get("error_code") == test_case.expected_error_code encoded_events: str = repr(events) assert all(fragment not in encoded_events for fragment in test_case.private_fragments) + + +class _StatementTimeoutError(Exception): + pass + + +@pytest.mark.parametrize( + "test_case", + [ + HandledFailureCase( + description="a failure the caller recovers from is marked handled", + error=_StatementTimeoutError("statement timed out"), + expected_outcome="handled", + ), + HandledFailureCase( + description="any other failure stays a failure", + error=ValueError("division by zero"), + expected_outcome=None, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_handled_failure_scope_when_a_statement_fails_then_only_recovered_failures_are_handled( + test_case: HandledFailureCase, +) -> None: + events: list[LifecycleEvent] = [] + dispatcher: EventDispatcher = EventDispatcher() + dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) + + with ( + invocation_scope("inv-handled-failure"), + dispatcher_scope(dispatcher), + StatementLifecycle.handled_failure_scope( + lambda error: isinstance(error, _StatementTimeoutError) + ), + StatementLifecycle(adapter="snowflake", sql="SELECT 1", intent="execute") as lifecycle, + ): + lifecycle.failed(error=test_case.error) + + assert events[-1].event_type == "statement_failed" + assert events[-1].payload.get("outcome") == test_case.expected_outcome diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 8b2a94f5e7..4bf5e6432e 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -382,8 +382,8 @@ in the store's SQL dialect, against these views: | View | One row per | |------|-------------| | `invocations` | Invocation, with status, exit code, and timing | -| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | -| `audit_results` | Evaluated audit | +| `statements` | Executed statement, with resource, kind, purpose, outcome, duration, and row counts | +| `audit_results` | Evaluated audit, with its evidence count | | `events` | Lifecycle event | | `diagnostics` | Retained diagnostic | | `artifacts` | Reference from an invocation to captured SQL, with its text | @@ -400,7 +400,16 @@ invocation fails and lists the matches. In SQL, match a prefix with Run `sqb recall schema` for every column, and check the values a column holds before filtering on it, for example `SELECT DISTINCT resource_kind, purpose FROM statements`: a model's `resource_kind` is its materialization, such as `table` or `view`, and its build statements have -`purpose = 'model_build'`. +`purpose = 'model_build'`. On Snowflake, the statements that read query history for cost +attribution have `purpose = 'cost_telemetry'`. + +`statements.outcome` is `completed`, `failed`, `handled` or `no_terminal`. A `handled` statement +failed, and SQLBuild recovered from it: for example, a cost history read that timed out and was +read again in smaller windows. `no_terminal` means the statement never reported an end, usually +because the process died. `sql_available` is `1` when the statement's SQL is in `artifacts`; when it +is `0`, `sql_unavailable_reason` says why: `not_captured` when Recall never received the SQL, or +`expired_or_not_captured` when the statement references SQL text that is no longer stored, because +[retention](#retention) removed it or it was dropped before it was written. Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or From 63bb682057a5b3b5292776b80c9bde704f4d50a3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:58:01 +0000 Subject: [PATCH 09/30] fix(recall): record SIGTERM as a cancelled run with its real end SIGTERM stops a command the way Ctrl-C does: Recall records it as `cancelled` with `signal_number`, its real end time and its flushed output, also when it arrives during project discovery, and `sqb` exits with code 143. Runs that died without an end have `ended_at_estimated = 1`. --- .../src/lifecycle_events/constants.rs | 2 + .../src/bindings/_helpers/cli/entry.rs | 115 ++++++-- .../src/bindings/_helpers/cli/mod.rs | 1 + .../src/bindings/_helpers/cli/registration.rs | 5 +- .../src/bindings/_helpers/cli/termination.rs | 96 +++++++ .../src/bindings/_helpers/recall/command.rs | 2 +- .../src/schema/_helpers/clickhouse_views.rs | 20 +- .../src/schema/_helpers/sqlite_views.rs | 12 + .../docs/concepts/observability/recall.md | 134 +++++++--- .../sqlbuild/references/investigating-runs.md | 26 +- src/sqlbuild/_native.pyi | 1 + .../main/clickhouse/test_clickhouse_recall.py | 2 +- .../src/sqlbuild/cli/recall/_test_types.py | 38 +++ .../src/sqlbuild/cli/recall/helpers.py | 40 ++- .../sqlbuild/cli/recall/test_local_recall.py | 2 +- .../cli/recall/test_recall_run_facts.py | 252 ++++++++++++++++++ .../docs/concepts/observability/recall.mdx | 65 ++++- 17 files changed, 736 insertions(+), 77 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs create mode 100644 tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py diff --git a/crates/sqlbuild-core/src/lifecycle_events/constants.rs b/crates/sqlbuild-core/src/lifecycle_events/constants.rs index 42435b65bc..a56ae32466 100644 --- a/crates/sqlbuild-core/src/lifecycle_events/constants.rs +++ b/crates/sqlbuild-core/src/lifecycle_events/constants.rs @@ -663,11 +663,13 @@ pub(crate) const LIFECYCLE_EVENT_CATALOG_V2: [EventDefinition; 23] = [ required_payload: &[], allowed_payload: &[ "command", + "cost_status", "duration_ms", "error_code", "error_message", "error_type", "exit_code", + "signal_number", ], terminal: true, }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 42c2f51455..e007ed68f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -28,7 +28,12 @@ use crate::bindings::_helpers::cli::machine_output::take_machine_output_written; use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; +use crate::bindings::_helpers::cli::termination::{ + CommandTerminated, TERMINATED_EXIT_CODE, TerminationHandler, handle_sigterm, + interrupting_signal, report_termination, +}; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; +use crate::bindings::_helpers::cost_report::attribution::take_cost_status; use crate::bindings::_helpers::diff_report::output::render_diff_error_json; use crate::bindings::_helpers::native_host::host; use crate::bindings::_helpers::output_capture::records::invocation_context; @@ -545,7 +550,10 @@ fn observed_dispatch<'py>( } let command: String = invocation.command_name(); drop(take_failure_cause()); + drop(take_cost_status()); let started: Instant = Instant::now(); + let started_at: Bound<'py, PyAny> = imported(py, "datetime", "datetime")? + .call_method1("now", (imported(py, "datetime", "UTC")?,))?; let elapsed = |started: Instant| -> PyResult> { u64::try_from(started.elapsed().as_millis()) .unwrap_or(u64::MAX) @@ -558,23 +566,53 @@ fn observed_dispatch<'py>( invocation.project_dir()?.unbind(), ), )?; + let started_payload = || -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("command", &command)?; + add_invocation_context(invocation, &payload)?; + Ok(payload) + }; + let failed_payload = |error: &PyErr| -> PyResult> { + let payload: Bound<'py, PyDict> = PyDict::new(py); + payload.set_item("command", &command)?; + payload.set_item("duration_ms", elapsed(started)?)?; + let (error_type, code, message): (String, String, String) = raised_cause(py, error); + payload.set_item("error_type", error_type)?; + if !code.is_empty() { + payload.set_item("error_code", code)?; + } + payload.set_item("error_message", message)?; + if let Some(signal) = interrupting_signal(py, error) { + payload.set_item("signal_number", signal)?; + } + if let Some(status) = take_cost_status() { + payload.set_item("cost_status", status)?; + } + Ok(payload) + }; + let announced: std::cell::Cell = std::cell::Cell::new(false); let scope: Bound<'py, PyAny> = observability.clone().into_any(); - within(&scope, |dispatcher| { + let observed: PyResult> = within(&scope, |dispatcher| { let publish = |event: Bound<'py, PyAny>| -> PyResult<()> { dispatcher.call_method1("publish_lifecycle", (event,))?; Ok(()) }; subscribe_recall((&observability, dispatcher), &invocation.recall)?; - let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item("command", &command)?; - add_invocation_context(invocation, &payload)?; - publish(lifecycle_event(py, "invocation_started", payload)?)?; + publish(lifecycle_event( + py, + "invocation_started", + started_payload()?, + )?)?; + announced.set(true); match dispatch_command(&invocation.args, invocation.handlers.clone()) { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; payload.set_item("duration_ms", elapsed(started)?)?; payload.set_item("exit_code", &exit_code)?; + if let Some(status) = take_cost_status() { + payload.set_item("cost_status", status)?; + } let terminal: &str = if exit_code.eq(0)? { "invocation_completed" } else { @@ -589,21 +627,55 @@ fn observed_dispatch<'py>( Ok(exit_code) } Err(error) => { - let payload: Bound<'py, PyDict> = PyDict::new(py); - payload.set_item("command", &command)?; - payload.set_item("duration_ms", elapsed(started)?)?; - let (error_type, code, message): (String, String, String) = - raised_cause(py, &error); - payload.set_item("error_type", error_type)?; - if !code.is_empty() { - payload.set_item("error_code", code)?; - } - payload.set_item("error_message", message)?; - publish(lifecycle_event(py, "invocation_failed", payload)?)?; + publish(lifecycle_event( + py, + "invocation_failed", + failed_payload(&error)?, + )?)?; Err(error) } } - }) + }); + match (&observed, &invocation.recall.session) { + (Err(error), Some(session)) + if !announced.get() && interrupting_signal(py, error).is_some() => + { + record_interrupted_setup( + py, + session, + (&started_at, started_payload()?), + failed_payload(error)?, + )?; + } + _ => {} + } + observed +} + +/// Record an invocation interrupted while its observability was still being set up, such as during project discovery, so Recall shows it cancelled instead of running. +fn record_interrupted_setup( + py: Python<'_>, + session: &sqlbuild_recall::writer::models::RecallSession, + (started_at, started): (&Bound<'_, PyAny>, Bound<'_, PyDict>), + failed: Bound<'_, PyDict>, +) -> PyResult<()> { + let started: Bound<'_, PyAny> = call_with( + &host::entry(py, ExecutorHost::CreateLifecycleEvent)?, + &[ + ( + "event_type", + PyString::new(py, "invocation_started").into_any(), + ), + ("payload", started.into_any()), + ("occurred_at", started_at.clone()), + ], + )?; + publish_lifecycle_event(py, session, &started)?; + publish_lifecycle_event( + py, + session, + &lifecycle_event(py, "invocation_failed", failed)?, + ) } /// Dispatch once and project every expected failure to its coded error and exit code. @@ -745,7 +817,13 @@ fn parsed_invocation<'py>( Ok(_) | Err(_) => None, }; let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); - let exit_code: Bound<'py, PyAny> = dispatched?; + let exit_code: Bound<'py, PyAny> = match dispatched { + Err(error) if error.is_instance_of::(py) => { + report_termination(py); + TERMINATED_EXIT_CODE.into_bound_py_any(py)? + } + dispatched => dispatched?, + }; finished?; Ok(exit_code) } @@ -786,6 +864,7 @@ fn run_cli<'py>( .get_item(pyo3::types::PySlice::new(py, 1, isize::MAX, 1))?, }; let argv: Bound<'py, PyAny> = offered_launch_record(&argv)?; + let _termination: TerminationHandler<'py> = handle_sigterm(py)?; let external: Bound<'py, PyAny> = external_context(py)?; let invocation_scope: Bound<'py, PyAny> = host::entry(py, ExecutorHost::InvocationScope)?.call0()?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs index 17fbdc632b..c26744d81a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/mod.rs @@ -8,3 +8,4 @@ pub(crate) mod namespace; pub(crate) mod observability; pub(crate) mod registration; pub(crate) mod surrogates; +pub(crate) mod termination; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs index 90cea0cf56..7c15b912eb 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/registration.rs @@ -3,7 +3,9 @@ use pyo3::prelude::{Bound, PyModule, PyResult}; use crate::bindings::_helpers::build_progress::{progress, progress_reporters, progress_text}; -use crate::bindings::_helpers::cli::{argv, entry, launch_state, namespace, observability}; +use crate::bindings::_helpers::cli::{ + argv, entry, launch_state, namespace, observability, termination, +}; use crate::bindings::_helpers::{ audit_command, build_command, build_progress, change_selection, check, clone, command_runtime, compute_logs, contract_command, cost_command, cost_report, cost_telemetry, dbt_command, @@ -19,6 +21,7 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { launch_state::register(module)?; namespace::register(module)?; observability::register(module)?; + termination::register(module)?; progress::register(module)?; progress_reporters::register(module)?; progress_text::register(module)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs new file mode 100644 index 0000000000..e9c73a8545 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs @@ -0,0 +1,96 @@ +//! SIGTERM as an interrupt: the command stops the way Ctrl+C stops it, records its end and exits with the conventional code. + +use pyo3::create_exception; +use pyo3::exceptions::PyKeyboardInterrupt; +use pyo3::prelude::{ + Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, +}; +use pyo3::types::{PyCFunction, PyDict, PyTuple}; + +use crate::bindings::_helpers::boundary::python_calls::imported; + +create_exception!( + sqlbuild._native, + CommandTerminated, + PyKeyboardInterrupt, + "Raised in the main thread when the command receives SIGTERM; handled like Ctrl+C." +); + +/// The number of SIGTERM, as POSIX defines it. +pub(crate) const SIGTERM_NUMBER: i64 = 15; +/// The number of SIGINT, which Python raises as `KeyboardInterrupt`. +pub(crate) const SIGINT_NUMBER: i64 = 2; +/// The exit code of a command stopped by SIGTERM: 128 plus the signal number. +pub(crate) const TERMINATED_EXIT_CODE: i64 = 128 + SIGTERM_NUMBER; +const TERMINATED_MESSAGE: &str = "terminated by SIGTERM"; + +/// The SIGTERM handler installed for one command, and the handler it replaced. +pub(crate) struct TerminationHandler<'py> { + signal: Bound<'py, PyAny>, + previous: Option>, +} + +impl Drop for TerminationHandler<'_> { + fn drop(&mut self) { + if let Some(previous) = self.previous.take() { + let number: PyResult> = self.signal.getattr("SIGTERM"); + drop(number.and_then(|number| self.signal.call_method1("signal", (number, previous)))); + } + } +} + +/// Raise [`CommandTerminated`] on SIGTERM while the command runs: only on the main thread, and only when no other handler was installed, so an embedding application keeps its own. +pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult> { + let signal: Bound<'_, PyAny> = py.import("signal")?.into_any(); + let threading: Bound<'_, PyAny> = py.import("threading")?.into_any(); + let main_thread: bool = threading + .call_method0("current_thread")? + .is(&threading.call_method0("main_thread")?); + let number: Bound<'_, PyAny> = signal.getattr("SIGTERM")?; + let current: Bound<'_, PyAny> = signal.call_method1("getsignal", (&number,))?; + if !main_thread || !current.eq(signal.getattr("SIG_DFL")?)? { + return Ok(TerminationHandler { + signal, + previous: None, + }); + } + let handler: Bound<'_, PyCFunction> = PyCFunction::new_closure( + py, + None, + None, + |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { + Err(CommandTerminated::new_err(TERMINATED_MESSAGE)) + }, + )?; + signal.call_method1("signal", (&number, handler))?; + Ok(TerminationHandler { + signal, + previous: Some(current), + }) +} + +/// The signal that interrupted the command: SIGTERM for [`CommandTerminated`], SIGINT for any other `KeyboardInterrupt`. +pub(crate) fn interrupting_signal(py: Python<'_>, error: &PyErr) -> Option { + if error.is_instance_of::(py) { + return Some(SIGTERM_NUMBER); + } + error + .is_instance_of::(py) + .then_some(SIGINT_NUMBER) +} + +/// Write the termination notice to stderr; a stream that cannot be written changes nothing. +pub(crate) fn report_termination(py: Python<'_>) { + let written = imported(py, "sys", "stderr").and_then(|stderr| { + stderr.call_method1("write", (format!("error: {TERMINATED_MESSAGE}\n"),))?; + stderr.call_method0("flush") + }); + drop(written); +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add( + "CommandTerminated", + module.py().get_type::(), + ) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index 2d57fba2e4..be7ef480ea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -487,7 +487,7 @@ fn run_output(py: Python<'_>, recall: &RecallRequest) -> PyResult { problems.push("delivery did not report a complete flush".to_owned()); } match transcript.status.as_deref() { - Some("completed" | "failed") | None => {} + Some("completed" | "failed" | "cancelled") | None => {} Some(status) => problems.push(format!("the invocation is {status}")), } if !problems.is_empty() { diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 796400f0fc..6a55d15a51 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -215,6 +215,14 @@ fn invocations(options: &DdlOptions) -> String { "event_error_message", "JSONExtract(payload, 'error_message', 'Nullable(String)')", ), + ( + "event_signal_number", + "JSONExtract(payload, 'signal_number', 'Nullable(Int64)')", + ), + ( + "event_cost_status", + "JSONExtract(payload, 'cost_status', 'Nullable(String)')", + ), ( "event_is_terminal", "event_type IN ('invocation_completed', 'invocation_failed')", @@ -253,6 +261,8 @@ folded AS ( {terminal_error_type} {terminal_error_code} {terminal_error_message} +{terminal_signal_number} +{terminal_cost_status} maxIf(heartbeat_at, source = 'beat') AS last_heartbeat_at, maxIf(beat_stale_at, source = 'beat') AS stale_at, minIf(beat_command, source = 'beat') AS first_beat_command, @@ -266,6 +276,7 @@ SELECT coalesce(started_occurred_at, invocation_started_at) AS started_at, multiIf( terminal_type = 'invocation_completed', 'completed', + terminal_type = 'invocation_failed' AND terminal_signal_number IS NOT NULL, 'cancelled', terminal_type = 'invocation_failed', 'failed', stale_at IS NULL, 'unknown', stale_at >= now64(6), 'running', @@ -277,10 +288,14 @@ SELECT if(stale_at < now64(6), toFloat64(dateDiff('millisecond', started_at, last_heartbeat_at)), NULL)) AS duration_ms, + multiIf(terminal_type IS NOT NULL, toUInt8(0), stale_at < now64(6), toUInt8(1), NULL) + AS ended_at_estimated, terminal_exit_code AS exit_code, terminal_error_type AS error_type, terminal_error_code AS error_code, terminal_error_message AS error_message, + terminal_signal_number AS signal_number, + terminal_cost_status AS cost_status, coalesce(context_command, started_command, first_beat_command) AS command, coalesce(context_subcommand, first_beat_subcommand) AS subcommand, context_origin AS origin, @@ -323,13 +338,16 @@ FROM folded", terminal_error_type = terminal("event_error_type", "terminal_error_type"), terminal_error_code = terminal("event_error_code", "terminal_error_code"), terminal_error_message = terminal("event_error_message", "terminal_error_message"), + terminal_signal_number = terminal("event_signal_number", "terminal_signal_number"), + terminal_cost_status = terminal("event_cost_status", "terminal_cost_status"), ) } -/// Two folds over one read of the events: by resource, which names the resource and gathers its statement events, then by statement. Both group by `invocation_id` and `invocation_started_at`, so a filter on either reaches the read; `sql_available` looks digests up in the stored SQL, which is shared across invocations. +/// Two folds over one read of the events: by resource, which names the resource and gathers its statement events, then by statement. Both group by `invocation_id` and `invocation_started_at`, so a filter on either reaches the read; `sql_available` needs a reference to the digest in the statement's own invocation and the stored SQL, which is shared across invocations. fn statements(options: &DdlOptions) -> String { let events = retained(Table::Events, options, ""); let artifacts = clickhouse_table(table_spec(Table::Artifacts).name, options); + let refs = retained(Table::ArtifactRefs, options, ""); let order = "tuple(invocation_sequence, occurred_at)"; let terminal = "event_type IN ('statement_completed', 'statement_failed')"; let statement_event = "statement_id IS NOT NULL AND startsWith(event_type, 'statement_')"; diff --git a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs index 3152c66763..8d4e1c2f81 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/sqlite_views.rs @@ -271,6 +271,8 @@ started AS ( terminals AS ( SELECT invocation_id, event_type, occurred_at, exit_code, duration_ms, error_type, error_code, json_extract(payload, '$.error_message') AS error_message, + json_extract(payload, '$.signal_number') AS signal_number, + json_extract(payload, '$.cost_status') AS cost_status, ROW_NUMBER() OVER ( PARTITION BY invocation_id ORDER BY invocation_sequence DESC, occurred_at DESC, event_id DESC @@ -291,6 +293,8 @@ projected AS ( COALESCE(started.started_at, ids.invocation_started_at) AS started_at, CASE WHEN terminals.event_type = 'invocation_completed' THEN 'completed' + WHEN terminals.event_type = 'invocation_failed' + AND terminals.signal_number IS NOT NULL THEN 'cancelled' WHEN terminals.event_type = 'invocation_failed' THEN 'failed' WHEN beats.stale_at IS NULL THEN 'unknown' WHEN beats.stale_at >= {NOW} THEN 'running' @@ -303,6 +307,8 @@ projected AS ( terminals.error_type, terminals.error_code, terminals.error_message, + terminals.signal_number, + terminals.cost_status, COALESCE(context.command, started.command, beats.command) AS command, COALESCE(context.subcommand, beats.subcommand) AS subcommand, context.origin, @@ -358,10 +364,16 @@ SELECT (unixepoch(last_heartbeat_at, 'subsec') - unixepoch(started_at, 'subsec')) * 1000.0, 3 ) END AS duration_ms, + CASE + WHEN terminal_event IS NOT NULL THEN 0 + WHEN status = 'abandoned' THEN 1 + END AS ended_at_estimated, exit_code, error_type, error_code, error_message, + signal_number, + cost_status, command, subcommand, origin, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index d4ee10a7ed..d020fc3125 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -10,6 +10,9 @@ Online: https://sqlbuild.com/docs/concepts/observability/recall/ - An investigation, start to finish - What is recorded +- How a run ended +- Selection +- Audit evidence - Git revision - Where a run came from - Settings @@ -41,32 +44,30 @@ sqb recall query \ ``` ``` --[ RECORD 1 ]---------------------------+ -run | 2026-10-20T00:01:44.784683Z -outcome | warn -violation_count | 1 -status | completed -exit_code | 0 - --[ RECORD 2 ]---------------------------+ -run | 2026-10-20T00:00:34.652468Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 - --[ RECORD 3 ]---------------------------+ -run | 2026-10-19T23:59:23.827831Z -outcome | pass -violation_count | 0 -status | completed -exit_code | 0 +┌─────────────────────────────┬─────────┬─────────────────┬───────────┬───────────┐ +│ run │ outcome │ violation_count │ status │ exit_code │ +│ text │ text │ integer │ text │ integer │ +├─────────────────────────────┼─────────┼─────────────────┼───────────┼───────────┤ +│ 2026-10-20T00:01:44.784683Z │ warn │ 1 │ completed │ 0 │ +│ 2026-10-20T00:00:34.652468Z │ pass │ 0 │ completed │ 0 │ +│ 2026-10-19T23:59:23.827831Z │ pass │ 0 │ completed │ 0 │ +└─────────────────────────────┴─────────┴─────────────────┴───────────┴───────────┘ 3 rows ``` The audit started warning on the latest build, and that build still exited 0, so nothing alerted. +See the rows the audit flagged, which Recall keeps as the audit's evidence: + +```bash +sqb recall query \ + "SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id + WHERE a.audit_name = 'revenue_is_non_negative' AND a.outcome <> 'pass' + ORDER BY a.invocation_started_at DESC LIMIT 1" +``` + See what that build printed, exactly as the scheduler saw it: ```bash @@ -88,11 +89,12 @@ sqb recall query --backend warehouse \ ``` ``` --[ RECORD 1 ]---------------------------+ -payment_id | 9 -order_id | 8 -amount_cents | -2200 -paid_at | 2026-04-03 15:42:00 +┌────────────┬──────────┬──────────────┬─────────────────────┐ +│ payment_id │ order_id │ amount_cents │ paid_at │ +│ INTEGER │ INTEGER │ INTEGER │ TIMESTAMP │ +├────────────┼──────────┼──────────────┼─────────────────────┤ +│ 9 │ 8 │ -2200 │ 2026-04-03 15:42:00 │ +└────────────┴──────────┴──────────────┴─────────────────────┘ 1 row ``` @@ -109,21 +111,25 @@ For every recorded invocation, Recall keeps: for a failed invocation its cause as `error_type`, `error_code` and a one-line `error_message`, also when the command reported the failure only through its exit code (for example `sqb debug` failing to connect records `ConnectionError` and `C980`); +- the selectors it ran with, see [Selection](#selection); +- whether it finished attributing warehouse cost, as `cost_status`: `complete`, `partial`, + `pending`, `unavailable` or `collection_failed`, or `NULL` for an adapter without cost + attribution; - the commit it ran from (`git_revision`), see [Git revision](#git-revision); - where it came from (`origin`) and, for an orchestrated run, the job and run that started it, see [Where a run came from](#where-a-run-came-from); - its [lifecycle events](../observability.md): runs, resource attempts, statements, retries, and audit results; -- the SQL text of every executed statement, stored once per distinct statement and shared across - invocations; +- the SQL text of every executed statement, including statements that failed or were cancelled, + stored once per distinct statement and shared across invocations; +- the rows a failing audit flagged, see [Audit evidence](#audit-evidence); - its stdout and stderr, byte for byte, in the order they were written, up to [`max_output_bytes`](#output-limit) of each stream; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all of its history still succeeds, prints a warning followed by its own final status, and -`output_invocations` reports the gap. An invocation whose process died without finishing has -status `abandoned`. +`output_invocations` reports the gap. `sqb recall` only reads history: it publishes no lifecycle events, runs no project sinks, and keeps no compute logs, so a query starts quickly. Every command is recorded except `sqb recall` itself, commands that create a project (such as @@ -132,7 +138,58 @@ recorded either: a `sqb compile` answered from the launcher's warm cache, and `s `sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store -can contain sensitive SQL and output. +can contain sensitive SQL, output and audit evidence. + +## How a run ended + +`invocations.status` is one of: + +| Status | Meaning | +|--------|---------| +| `completed` | The command exited 0 | +| `failed` | The command exited with an error | +| `cancelled` | The command was stopped by SIGTERM or Ctrl-C; `signal_number` is `15` or `2` | +| `running` | The command is still running: its heartbeats are recent | +| `abandoned` | Its heartbeats stopped without an end: the process died, for example after SIGKILL or running out of memory | +| `unknown` | Recall has neither an end nor a heartbeat for it | + +A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it +records its real end time and its output, prints `error: terminated by SIGTERM`, and exits with +code 143. Recall keeps heartbeating while the command discovers and compiles the project, so a run +stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. + +## Selection + +`invocations.selection` is the JSON array of selectors the command ran with: positional selectors, +`--select`, and the selectors read from each `--select-file`, whose paths are in `selector_files`. +`selection_count` counts them. At most 64 KiB of selectors are kept; beyond that the list is cut +and `selection_truncated` is `1`, while `selection_count` still counts all of them. + +## Audit evidence + +When an audit warns or fails, Recall keeps its [evidence](../audits.md#evidence): up to +`evidence_limit` of the rows it flagged. `audit_results` counts them in `evidence_count`, sets +`evidence_truncated` to `1` when the limit cut rows off, and names the stored rows by +`sample_digest`. Read them from `artifacts`, joined on the audit's `event_id`: + +```sql +SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id +WHERE a.outcome <> 'pass' +``` + +The evidence holds data values from your warehouse. Choose which failures keep it with +`audit_evidence`: + +| Value | Keeps evidence for | +|-------|--------------------| +| `warn-and-error` (default) | Audits that warn or fail | +| `error` | Audits that fail at error severity | +| `off` | No audit | + +`evidence_count` always counts the rows the audit kept as evidence; `sample_digest` is `NULL` when +Recall did not store them. ## Git revision @@ -179,6 +236,7 @@ retention = "30d" # days of history to keep; also accepts hours, suc shutdown_timeout = "3s" # longest delay Recall may add when a command exits diagnostics_min_severity = "info" max_output_bytes = 2097152 # bytes of each output stream to keep; 0 keeps everything +audit_evidence = "warn-and-error" [targets.ci.recall] enabled = false # do not record runs against the ci target @@ -193,6 +251,7 @@ enabled = false # do not record runs against the ci target | `shutdown_timeout` | `"3s"` | The cap on time spent flushing history at exit, in `s` or `ms`, up to 10 minutes | | `diagnostics_min_severity` | `"info"` | `trace`, `debug`, `info`, `warning`, `error`, or `critical` | | `max_output_bytes` | `2097152` for the local store | Bytes of stdout and of stderr kept per invocation, an integer; `0` keeps everything. See [Output limit](#output-limit) | +| `audit_evidence` | `"warn-and-error"` | Which failing audits keep the rows they flagged: `warn-and-error`, `error`, or `off`. See [Audit evidence](#audit-evidence) | To turn Recall off for the whole project: @@ -398,8 +457,8 @@ in the store's SQL dialect, against these views: | View | One row per | |------|-------------| | `invocations` | Invocation, with status, exit code, and timing | -| `statements` | Executed statement, with resource, kind, purpose, duration, and row counts | -| `audit_results` | Evaluated audit | +| `statements` | Executed statement, with resource, kind, purpose, outcome, duration, and row counts | +| `audit_results` | Evaluated audit, with its evidence count | | `events` | Lifecycle event | | `diagnostics` | Retained diagnostic | | `artifacts` | Reference from an invocation to captured SQL, with its text | @@ -416,7 +475,16 @@ invocation fails and lists the matches. In SQL, match a prefix with Run `sqb recall schema` for every column, and check the values a column holds before filtering on it, for example `SELECT DISTINCT resource_kind, purpose FROM statements`: a model's `resource_kind` is its materialization, such as `table` or `view`, and its build statements have -`purpose = 'model_build'`. +`purpose = 'model_build'`. On Snowflake, the statements that read query history for cost +attribution have `purpose = 'cost_telemetry'`. + +`statements.outcome` is `completed`, `failed`, `handled` or `no_terminal`. A `handled` statement +failed, and SQLBuild recovered from it: for example, a cost history read that timed out and was +read again in smaller windows. `no_terminal` means the statement never reported an end, usually +because the process died. `sql_available` is `1` when the statement's SQL is in `artifacts`; when it +is `0`, `sql_unavailable_reason` says why: `not_captured` when Recall never received the SQL, or +`expired_or_not_captured` when the statement references SQL text that is no longer stored, because +[retention](#retention) removed it or it was dropped before it was written. Every view has `invocation_id` and `invocation_started_at`. Filter on them to keep a query cheap: on a ClickHouse store, a filter on either reaches the table reads, so a query about one invocation or diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md index 67d7a48cfd..6c7648ec1e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/investigating-runs.md @@ -31,11 +31,17 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of `invocation_started_at` and the filter reaches the table reads; without one, a view folds the whole retention window. - **Writing history never fails the command,** so a missing row means "not recorded", not "did not - happen": `invocations.status = 'abandoned'` (the process died), `statements.outcome = - 'no_terminal'` (no terminal fact arrived), and `output_invocations.complete = 0` with a - `gap_reason` when output is incomplete. + happen": `invocations.status = 'abandoned'` (the process died; `ended_at` is its last heartbeat + and `ended_at_estimated = 1`), `statements.outcome = 'no_terminal'` (no terminal fact arrived), + and `output_invocations.complete = 0` with a `gap_reason` when output is incomplete. A command + stopped by SIGTERM or Ctrl-C is `cancelled`, with `signal_number` 15 or 2 and its real end time. - **SQL is stored once by digest.** `statements.sql_digest` joins `artifacts` on - `(invocation_id, statement_id)` for the SQL text (`artifacts.text`). + `(invocation_id, statement_id)` for the SQL text (`artifacts.text`), also for failed and + cancelled statements. `statements.sql_available = 0` means the text is missing, and + `sql_unavailable_reason` says why. +- **Audit evidence is an artifact.** A warning or failing audit's flagged rows (up to + `evidence_limit`) are in `artifacts` with `ref_id = audit_results.event_id`; `sample_digest` is + `NULL` when `[recall] audit_evidence` kept none. Evidence holds warehouse data values. - **Output is a byte-for-byte copy** of stdout and stderr, up to `max_output_bytes` per stream (the local store keeps the first and last MiB). Use `sqb recall output` rather than querying `output_transcripts` for long output. @@ -50,12 +56,15 @@ what they printed. `sqb recall query` runs one read-only `SELECT` over a set of | Column | Values | |--------|--------| -| `invocations.status` | `completed`, `failed`, `running`, `abandoned`, `unknown` | +| `invocations.status` | `completed`, `failed`, `cancelled`, `running`, `abandoned`, `unknown` | +| `invocations.selection`, `selection_count`, `selector_files` | The selectors the command ran with, including those read from `--select-file` (paths in `selector_files`); `selection_truncated = 1` when the list exceeded 64 KiB | +| `invocations.cost_status` | `complete`, `partial`, `pending`, `unavailable`, `collection_failed`; `NULL` when the adapter attributes no cost | | `invocations.origin` | `integration` (an orchestrator such as Dagster started it), `ci` (CI environment variables were set), `local`; `NULL` on runs older SQLBuild versions recorded | | `invocations.command_mode` | What a diagnostic checked: `sqb debug` records `config_only` (`--no-connection`) or `config_and_connection`; `NULL` for other commands | | `invocations.integration`, `integration_job`, `integration_run_id`, `integration_retry` | The orchestrator, job, run and retry from `external_context`, for example `dagster` and its job name; `NULL` when there is none or an older version stored it as non-JSON | | `invocations.error_type`, `error_code`, `error_message` | The cause of a failed invocation, also when it failed only through its exit code (`sqb debug`: `C980` connection, `C981` query, `C982` other check) | -| `statements.outcome` | `completed`, `failed`, `no_terminal` | +| `statements.outcome` | `completed`, `failed`, `handled` (failed, and SQLBuild recovered, such as a timed-out cost history read retried in smaller windows), `no_terminal` | +| `statements.purpose` | `model_build` for a model's build statements, `cost_telemetry` for cost history reads, and others; check with `SELECT DISTINCT purpose` | | `audit_results.outcome` | `pass`, `warn`, `error`, `insufficient`; `execution_error` is set when the audit query itself failed | | `output_invocations.gap_reason` | `NULL` (complete), `truncated`, `filtered`, `dropped`, `unexplained` | | `events.event_type` | `invocation_started`, `invocation_completed`, `invocation_failed`, `run_started`, `run_completed`, `run_failed`, `resource_attempt_started`, `resource_attempt_completed`, `resource_attempt_failed`, `resource_attempt_skipped`, `operation_*`, `statement_started`, `statement_submitted`, `statement_completed`, `statement_failed`, `statement_heartbeat`, `microbatch_*`, `retry_scheduled`, `audit_completed` | @@ -300,17 +309,20 @@ events (view): every lifecycle fact ingested_at TEXT content_hash TEXT -invocations (view): one row per invocation; status is completed, failed, running, abandoned or unknown; origin is integration, ci or local; integration, integration_job, integration_run_id and integration_retry come from external_context +invocations (view): one row per invocation; status is completed, failed, cancelled, running, abandoned or unknown; origin is integration, ci or local; integration, integration_job, integration_run_id and integration_retry come from external_context invocation_id TEXT join key invocation_started_at ANY started_at ANY status ANY ended_at ANY duration_ms ANY + ended_at_estimated ANY exit_code INTEGER error_type TEXT error_code TEXT error_message ANY + signal_number ANY + cost_status ANY command ANY subcommand ANY origin TEXT diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 0a0c616cbb..af1025705b 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -6239,6 +6239,7 @@ def list_source_candidates( ) -> tuple[RelationInfo, ...]: ... class CostArtifactError(ValueError): ... +class CommandTerminated(KeyboardInterrupt): ... class SourceFreshnessInputError(ValueError): ... class SourceFreshnessObservationError(ValueError): ... diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py index a665d15cf1..20fb52540d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall.py @@ -341,7 +341,7 @@ def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abando running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) - assert recorded == [[running_id, test_case.expected_status, 1, 0]] + assert recorded == [[running_id, test_case.expected_status, 1, 1, 0]] @pytest.mark.parametrize( diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index d03c984f4b..6488e0eed8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -349,3 +349,41 @@ class InvocationPrefixTestCase: prefix_length: int expected_exit_code: int expected_error_code: str | None + + +@dataclass(frozen=True) +class RecordedSelectionTestCase: + description: str + selector_file: str + args: tuple[str, ...] + expected_selection: list[object] + + +@dataclass(frozen=True) +class TerminatedCommandTestCase: + description: str + environment: dict[str, str] + wait_sql: str + expected_output: bytes + + +@dataclass(frozen=True) +class FailedStatementSqlTestCase: + description: str + model_sql: str + expected_sql_fragment: str + + +@dataclass(frozen=True) +class AuditEvidenceTestCase: + description: str + recall_toml: str + expected_evidence: list[object] + expected_artifacts: list[list[object]] + + +@dataclass(frozen=True) +class CostAttributionTestCase: + description: str + collected: str + expected_cost_status: str diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 595fc40c59..916af0dbc5 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -213,7 +213,8 @@ def killed_build(project_dir: Path, *, heartbeat_seconds: str) -> tuple[str, lis time.sleep(5 * float(heartbeat_seconds)) recorded: list[list[object]] = recall_rows( project_dir, - "SELECT i.invocation_id, i.status, i.ended_at IS NOT NULL, o.complete " + "SELECT i.invocation_id, i.status, i.ended_at IS NOT NULL, i.ended_at_estimated, " + "o.complete " "FROM invocations i JOIN output_invocations o USING (invocation_id)", ) return str(running[0][0]), recorded @@ -224,15 +225,8 @@ def killed_during_discovery( ) -> tuple[list[list[object]], subprocess.CompletedProcess[bytes]]: """Rows left by a build SIGKILLed while importing a slow sink, and `recall output last`.""" - sink: Path = project_dir / "sinks" / "slow_import.py" - sink.parent.mkdir(parents=True, exist_ok=True) - _ = sink.write_text( - "import os\nimport time\n\n" - "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", - encoding="utf-8", - ) process: subprocess.Popen[bytes] = start( - project_dir, + write_slow_import_sink(project_dir), "build", SQLBUILD_RECALL_HEARTBEAT_SECONDS=heartbeat_seconds, ORDERS_IMPORT_SECONDS="60", @@ -251,6 +245,34 @@ def killed_during_discovery( return recorded, run(project_dir, "recall", "output", "last") +def write_slow_import_sink(project_dir: Path) -> Path: + """Add a sink module whose import sleeps for `ORDERS_IMPORT_SECONDS`, slowing discovery.""" + + sink: Path = project_dir / "sinks" / "slow_import.py" + sink.parent.mkdir(parents=True, exist_ok=True) + _ = sink.write_text( + "import os\nimport time\n\n" + "time.sleep(float(os.environ.get('ORDERS_IMPORT_SECONDS', '0')))\n", + encoding="utf-8", + ) + return project_dir + + +def terminated_build( + project_dir: Path, *, wait_sql: str, **env: str +) -> subprocess.CompletedProcess[bytes]: + """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row.""" + + process: subprocess.Popen[bytes] = start(project_dir, "build", **env) + try: + _ = wait_for(wait_sql, project_dir, timeout=120) + process.send_signal(signal.SIGTERM) + stdout, stderr = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + raise AssertionError(f"a terminated build kept running:\n{hang_dump(process)}") from None + return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + + def interrupted_build(project_dir: Path, **env: str) -> tuple[int | None, float]: """Exit code of a build sent SIGINT after its first logged event, and seconds to stop.""" diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index f8163563fc..2cb811acc8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -390,7 +390,7 @@ def test_given_a_killed_build_when_heartbeats_stop_then_the_invocation_is_abando running_id, recorded = killed_build(project_dir, heartbeat_seconds=test_case.heartbeat_seconds) assert recorded == [ - [running_id, test_case.expected_status, 1, test_case.expected_output_complete] + [running_id, test_case.expected_status, 1, 1, test_case.expected_output_complete] ] diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py new file mode 100644 index 0000000000..b640a329cf --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -0,0 +1,252 @@ +import json +import subprocess +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.recall._test_types import ( + AuditEvidenceTestCase, + CostAttributionTestCase, + FailedStatementSqlTestCase, + RecordedSelectionTestCase, + TerminatedCommandTestCase, +) +from tests.integration.src.sqlbuild.cli.recall.helpers import ( + EMITTED_TAIL, + recall_rows, + run, + terminated_build, + write_project, + write_slow_import_sink, +) + +CUSTOMERS_MODEL: str = ( + 'MODEL (\n description "One row per customer",\n materialized table,\n' + " columns (email (audits [not_null(severity warn)])),\n);\n\n" + "SELECT * FROM (VALUES (1, 'ada@example.com'), (2, NULL), (3, NULL)) " + "AS customers(customer_id, email)\n" +) + +COSTED_ADAPTER: str = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n" + "from sqlbuild.cost.models import RunCostSummary\n" + "from sqlbuild.cost.types import CostCapability, CostStatus\n\n\n" + "class DuckDbCostedAdapter(DuckDbAdapter):\n" + " adapter_name = 'duckdb_costed'\n\n" + " def cost_capability(self):\n" + " return CostCapability.SNOWFLAKE_QUERY_HISTORY\n\n" + " def collect_run_cost(self, *, usd_per_credit, rate_source, **kwargs):\n" + "{collected}" +) + + +@pytest.mark.parametrize( + "test_case", + [ + RecordedSelectionTestCase( + description="a selector file's selectors are recorded with the file", + selector_file="# nightly orders\norders\n\nemit_bytes\n", + args=("--select-file", "nightly.txt"), + expected_selection=['["orders","emit_bytes"]', 2, 0, '["nightly.txt"]'], + ), + RecordedSelectionTestCase( + description="positional, option and file selectors are recorded together", + selector_file="emit_bytes\n", + args=("orders", "--select-file", "nightly.txt"), + expected_selection=['["orders","emit_bytes"]', 2, 0, '["nightly.txt"]'], + ), + RecordedSelectionTestCase( + description="a command without selectors records none", + selector_file="", + args=(), + expected_selection=["[]", 0, 0, "[]"], + ), + ], + ids=lambda case: case.description, +) +def test_given_selectors_when_building_then_the_effective_selection_is_recorded( + test_case: RecordedSelectionTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = (project_dir / "nightly.txt").write_text(test_case.selector_file, encoding="utf-8") + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", *test_case.args) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT selection, selection_count, selection_truncated, selector_files FROM invocations", + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert recorded == [test_case.expected_selection] + + +@pytest.mark.parametrize( + "test_case", + [ + TerminatedCommandTestCase( + description="during project discovery", + environment={"ORDERS_IMPORT_SECONDS": "60"}, + wait_sql=( + "SELECT invocation_id FROM invocations WHERE status = 'running' " + "AND (julianday(last_heartbeat_at) - julianday(started_at)) * 86400 >= 2" + ), + expected_output=b"", + ), + TerminatedCommandTestCase( + description="while a task runs", + environment={"ORDERS_TASK_SLEEP_SECONDS": "60"}, + wait_sql=( + "SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'" + ), + expected_output=EMITTED_TAIL, + ), + ], + ids=lambda case: case.description, +) +def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_with_its_real_end( + test_case: TerminatedCommandTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_slow_import_sink(write_project(tmp_path / "orders_shop")) + + terminated: subprocess.CompletedProcess[bytes] = terminated_build( + project_dir, + wait_sql=test_case.wait_sql, + SQLBUILD_RECALL_HEARTBEAT_SECONDS="0.5", + **test_case.environment, + ) + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT i.status, i.signal_number, i.ended_at_estimated, i.ended_at > i.started_at, " + "o.flush_complete FROM invocations i JOIN output_invocations o USING (invocation_id)", + ) + output: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "output", "last") + + assert terminated.returncode == 143 + assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") + assert recorded == [["cancelled", 15, 0, 1, 1]] + assert test_case.expected_output in output.stdout + assert b"may be incomplete" not in output.stderr + + +@pytest.mark.parametrize( + "test_case", + [ + FailedStatementSqlTestCase( + description="a statement the warehouse rejects keeps its SQL", + model_sql="SELECT CAST('not a number' AS INTEGER) AS order_id", + expected_sql_fragment="CAST('not a number' AS INTEGER)", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failed_statement_when_recalled_then_its_sql_is_readable( + test_case: FailedStatementSqlTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + _ = (project_dir / "models" / "orders.sql").write_text( + f'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + f"{test_case.model_sql}\n", + encoding="utf-8", + ) + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "orders") + recorded: list[list[object]] = recall_rows( + project_dir, + "SELECT s.sql_available, s.sql_unavailable_reason, a.text FROM statements s " + "LEFT JOIN artifacts a USING (invocation_id, statement_id) WHERE s.outcome = 'failed'", + ) + + assert built.returncode == 1 + assert [row[:2] for row in recorded] == [[1, None]] + assert test_case.expected_sql_fragment in str(recorded[0][2]) + + +@pytest.mark.parametrize( + "test_case", + [ + AuditEvidenceTestCase( + description="a warning audit's violating rows are kept by default", + recall_toml="", + expected_evidence=["warn", 2, 2, 0, 1], + expected_artifacts=[["audit_evidence", "application/json"]], + ), + AuditEvidenceTestCase( + description="error-only evidence skips warnings", + recall_toml='\n[recall]\naudit_evidence = "error"\n', + expected_evidence=["warn", 2, 2, 0, 0], + expected_artifacts=[], + ), + AuditEvidenceTestCase( + description="evidence can be turned off", + recall_toml='\n[recall]\naudit_evidence = "off"\n', + expected_evidence=["warn", 2, 2, 0, 0], + expected_artifacts=[], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_failing_audit_when_recalled_then_its_evidence_follows_the_setting( + test_case: AuditEvidenceTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.recall_toml) + _ = (project_dir / "models" / "customers.sql").write_text(CUSTOMERS_MODEL, encoding="utf-8") + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "customers") + evidence: list[list[object]] = recall_rows( + project_dir, + "SELECT outcome, violation_count, evidence_count, evidence_truncated, " + "sample_digest IS NOT NULL FROM audit_results", + ) + artifacts: list[list[object]] = recall_rows( + project_dir, + "SELECT a.kind, a.media_type, a.text FROM artifacts a JOIN audit_results r " + "ON a.invocation_id = r.invocation_id AND a.ref_id = r.event_id", + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert evidence == [test_case.expected_evidence] + assert [row[:2] for row in artifacts] == test_case.expected_artifacts + assert [json.loads(str(row[2])) for row in artifacts] == [ + [{"email": None}, {"email": None}] for _ in test_case.expected_artifacts + ] + + +@pytest.mark.parametrize( + "test_case", + [ + CostAttributionTestCase( + description="a collected cost records its status", + collected=( + " return RunCostSummary(status=CostStatus.PARTIAL, " + "usd_per_credit=usd_per_credit, rate_source=rate_source)\n" + ), + expected_cost_status="partial", + ), + CostAttributionTestCase( + description="a failed collection is recorded as failed", + collected=" raise ConnectionError('query history unreachable')\n", + expected_cost_status="collection_failed", + ), + ], + ids=lambda case: case.description, +) +def test_given_cost_attribution_when_a_build_ends_then_the_invocation_records_how_far_it_got( + test_case: CostAttributionTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project(tmp_path / "orders_shop") + toml: Path = project_dir / "sqlbuild_project.toml" + _ = toml.write_text( + toml.read_text(encoding="utf-8").replace('"duckdb"', '"duckdb_costed"', 1), + encoding="utf-8", + ) + adapter: Path = project_dir / "adapters" / "warehouse" / "duckdb_costed.py" + adapter.parent.mkdir(parents=True) + _ = adapter.write_text(COSTED_ADAPTER.format(collected=test_case.collected), encoding="utf-8") + + built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "orders") + recorded: list[list[object]] = recall_rows( + project_dir, "SELECT status, cost_status FROM invocations" + ) + + assert built.returncode == 0, built.stderr.decode(errors="replace") + assert recorded == [["completed", test_case.expected_cost_status]] diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index 4bf5e6432e..af623d8ade 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -90,21 +90,25 @@ For every recorded invocation, Recall keeps: for a failed invocation its cause as `error_type`, `error_code` and a one-line `error_message`, also when the command reported the failure only through its exit code (for example `sqb debug` failing to connect records `ConnectionError` and `C980`); +- the selectors it ran with, see [Selection](#selection); +- whether it finished attributing warehouse cost, as `cost_status`: `complete`, `partial`, + `pending`, `unavailable` or `collection_failed`, or `NULL` for an adapter without cost + attribution; - the commit it ran from (`git_revision`), see [Git revision](#git-revision); - where it came from (`origin`) and, for an orchestrated run, the job and run that started it, see [Where a run came from](#where-a-run-came-from); - its [lifecycle events](/docs/concepts/observability): runs, resource attempts, statements, retries, and audit results; -- the SQL text of every executed statement, stored once per distinct statement and shared across - invocations; +- the SQL text of every executed statement, including statements that failed or were cancelled, + stored once per distinct statement and shared across invocations; +- the rows a failing audit flagged, see [Audit evidence](#audit-evidence); - its stdout and stderr, byte for byte, in the order they were written, up to [`max_output_bytes`](#output-limit) of each stream; - framework diagnostics at or above `diagnostics_min_severity`. Writes happen on a background thread and never fail the command. A command that cannot record all of its history still succeeds, prints a warning followed by its own final status, and -`output_invocations` reports the gap. An invocation whose process died without finishing has -status `abandoned`. +`output_invocations` reports the gap. `sqb recall` only reads history: it publishes no lifecycle events, runs no project sinks, and keeps no compute logs, so a query starts quickly. Every command is recorded except `sqb recall` itself, commands that create a project (such as @@ -113,7 +117,58 @@ recorded either: a `sqb compile` answered from the launcher's warm cache, and `s `sqb rules list` and `sqb rules show` in a project with no custom rules, rule options or sinks. The store is `logs/recall.sqlite3` in the project directory. Add `logs/` to `.gitignore`; the store -can contain sensitive SQL and output. +can contain sensitive SQL, output and audit evidence. + +## How a run ended + +`invocations.status` is one of: + +| Status | Meaning | +|--------|---------| +| `completed` | The command exited 0 | +| `failed` | The command exited with an error | +| `cancelled` | The command was stopped by SIGTERM or Ctrl-C; `signal_number` is `15` or `2` | +| `running` | The command is still running: its heartbeats are recent | +| `abandoned` | Its heartbeats stopped without an end: the process died, for example after SIGKILL or running out of memory | +| `unknown` | Recall has neither an end nor a heartbeat for it | + +A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it +records its real end time and its output, prints `error: terminated by SIGTERM`, and exits with +code 143. Recall keeps heartbeating while the command discovers and compiles the project, so a run +stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: +its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. + +## Selection + +`invocations.selection` is the JSON array of selectors the command ran with: positional selectors, +`--select`, and the selectors read from each `--select-file`, whose paths are in `selector_files`. +`selection_count` counts them. At most 64 KiB of selectors are kept; beyond that the list is cut +and `selection_truncated` is `1`, while `selection_count` still counts all of them. + +## Audit evidence + +When an audit warns or fails, Recall keeps its [evidence](/docs/concepts/audits#evidence): up to +`evidence_limit` of the rows it flagged. `audit_results` counts them in `evidence_count`, sets +`evidence_truncated` to `1` when the limit cut rows off, and names the stored rows by +`sample_digest`. Read them from `artifacts`, joined on the audit's `event_id`: + +```sql +SELECT r.text FROM audit_results a JOIN artifacts r + ON r.invocation_id = a.invocation_id AND r.ref_id = a.event_id +WHERE a.outcome <> 'pass' +``` + +The evidence holds data values from your warehouse. Choose which failures keep it with +`audit_evidence`: + +| Value | Keeps evidence for | +|-------|--------------------| +| `warn-and-error` (default) | Audits that warn or fail | +| `error` | Audits that fail at error severity | +| `off` | No audit | + +`evidence_count` always counts the rows the audit kept as evidence; `sample_digest` is `NULL` when +Recall did not store them. ## Git revision From 41bc92ab126c2885a19ad6eada6911d301c4ed64 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:07:25 +0000 Subject: [PATCH 10/30] fix(rules): judge SQL macro slot reads by how they are written SQBRSQL028 counts a plain slot read such as `@weekly.total` as unqualified, though it expands to `weekly.total`, and an aliased read as qualified. A macro's SQL is judged in its macro file rather than again at each call, so a step written only in plain reads passes, and a caller's own columns are judged without the macro output. In a macro file the help points at reading slots through an alias. Generated SQL is unchanged. --- .../src/lint_text/_helpers/sql_diagnostics.rs | 56 +++++- .../src/lint_text/_helpers/sql_requests.rs | 19 ++- .../sqlbuild-rules/src/lint_text/constants.rs | 3 + .../src/sql_lint/_helpers/additional.rs | 55 +++++- .../src/sql_lint/_helpers/engine.rs | 3 + crates/sqlbuild-rules/src/sql_lint/models.rs | 4 + .../src/sql_lint/tests/test_sql_lint.rs | 61 +++++++ .../src/sql_lint/tests/test_types.rs | 8 + .../docs/concepts/macros/sql-macros.md | 5 + .../sqlbuild/cli/commands/main/_test_types.py | 9 + .../main/test_macro_slot_qualification.py | 159 ++++++++++++++++++ .../docs/docs/concepts/macros/sql-macros.mdx | 5 + 12 files changed, 381 insertions(+), 6 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index a0c52b771b..3552e6ccf1 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -10,9 +10,10 @@ use crate::lint_text::_helpers::sites::{byte_offset, compiled, lint_policy, site use crate::lint_text::constants::{ GENERATED_FIX, GENERATED_SQL_SUFFIX, HARNESS_CTE_PREFIXES, INVALID_DIAGNOSTIC, INVALID_FIX_EDIT, INVALID_FIX_OBJECT, INVALID_FIX_REASON, LINT_API_VERSION, LONG_LITERAL_CODE, - MISSING_DIAGNOSTICS, NATIVE_LINT_ERROR_CODE, NON_CONTIGUOUS_FIX, NON_OBJECT_RESPONSE, - NOT_AN_OBJECT, PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, - UNSUPPORTED_VERSION, UNUSED_CTE_CODE, + MACRO_MIXED_QUALIFICATION_REMEDIATION, MISSING_DIAGNOSTICS, MIXED_QUALIFICATION_CODE, + NATIVE_LINT_ERROR_CODE, NON_CONTIGUOUS_FIX, NON_OBJECT_RESPONSE, NOT_AN_OBJECT, + PARSE_ERROR_POSITION_PATTERN, PARSE_FAILURE_PREFIX, PARSE_FAILURE_REMEDIATION, + SQL_MACRO_PARAMETER_SENTINEL_PREFIX, UNSUPPORTED_VERSION, UNUSED_CTE_CODE, }; use crate::lint_text::models::{AuthoredDiagnostic, AuthoredEdit, ExpansionSpan, LintSqlBody}; use crate::sql_lint::main::backtick_identifiers::backtick_identifiers; @@ -211,6 +212,48 @@ fn from_sql_macro(body: &LintSqlBody, (text, dialect): (&str, &str), offset: i64 } } +/// A pass's input offset where that pass's output holds it: inside a replaced range, the start or end of its replacement. +fn forward_pass(offset: i64, spans: &[ExpansionSpan], end: bool) -> i64 { + let mut shift: i64 = 0; + for span in spans { + if offset < span.source_start || (end && offset == span.source_start) { + break; + } + if offset < span.source_end || (end && offset == span.source_end) { + return if end { + span.output_end + } else { + span.output_start + }; + } + shift += (span.output_end - span.output_start) - (span.source_end - span.source_start); + } + offset + shift +} + +/// Character ranges of the lint text that SQL macro calls generated, so rules about the author's style judge that SQL in its macro file instead. +pub(crate) fn sql_macro_output_spans( + body: &LintSqlBody, + file: (&str, &str), +) -> Vec<(usize, usize)> { + let mut spans: Vec<(usize, usize)> = Vec::new(); + for (index, pass) in body.passes.iter().enumerate() { + for span in pass { + let (mut start, mut end): (i64, i64) = (span.output_start, span.output_end); + for later in &body.passes[index + 1..] { + start = forward_pass(start, later, false); + end = forward_pass(end, later, true); + } + if end > start && from_sql_macro(body, file, start) { + spans.push((start as usize, end as usize)); + } + } + } + spans.sort_unstable(); + spans.dedup(); + spans +} + /// Keep diagnostics starting inside a framed expression, whose frame lint generated. fn inside_frame(diagnostics: Vec, (prefix, suffix): (i64, i64), length: i64) -> Vec { let end_of_expression: i64 = length - suffix; @@ -393,6 +436,13 @@ pub(crate) fn authored_diagnostic( } else { reason }; + let remediation: String = if code == MIXED_QUALIFICATION_CODE + && body.lint_text.contains(SQL_MACRO_PARAMETER_SENTINEL_PREFIX) + { + MACRO_MIXED_QUALIFICATION_REMEDIATION.to_owned() + } else { + remediation + }; let message: String = if generated { match file.token_at(absolute) { Some(token) => format!("{message} (in SQL generated by {token})"), diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index 32f38b4e83..c9a226f485 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -7,7 +7,9 @@ use sqlbuild_attachments::ceremonial_select::main::omitted_select_offset::omitte use sqlbuild_attachments::ceremonial_select::models::OmittedSelect; use sqlbuild_sqltext::sql_scan::models::LexicalSyntax; -use crate::lint_text::_helpers::sql_diagnostics::without_generated_comments; +use crate::lint_text::_helpers::sql_diagnostics::{ + sql_macro_output_spans, without_generated_comments, +}; use crate::lint_text::constants::{ CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, @@ -48,6 +50,15 @@ pub(crate) fn lint_payload( options: &LintSqlOptions, ) -> Value { let lint_text: Cow<'_, str> = without_generated_comments(body, (file, &options.dialect)); + let inserted: usize = OMITTED_CEREMONIAL_SELECT_SQL.chars().count(); + let exempt: Vec = sql_macro_output_spans(body, (file, &options.dialect)) + .into_iter() + .map(|(start, end)| match omitted { + Some(offset) if start >= offset => (start + inserted, end + inserted), + _ => (start, end), + }) + .map(|(start, end)| Value::from(vec![start, end])) + .collect(); let sql: String = match omitted { Some(offset) => { let characters: Vec = lint_text.chars().collect(); @@ -61,6 +72,12 @@ pub(crate) fn lint_payload( let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); let _ = payload.insert("sql".to_string(), Value::from(sql)); let _ = payload.insert("dialect".to_string(), Value::from(options.dialect.clone())); + if !exempt.is_empty() { + let _ = payload.insert( + "qualification_exempt_spans".to_string(), + Value::from(exempt), + ); + } if let Some(enabled) = &options.enabled_rules { let _ = payload.insert("enabled_rules".to_string(), strings(enabled)); } diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 6bb802f943..0f11c5857c 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -93,6 +93,9 @@ pub(crate) const GROUPED_NAMED_ROLES: [(&[&str], bool); 9] = [ pub(crate) const LINT_API_VERSION: i64 = 1; pub(crate) const UNUSED_CTE_CODE: &str = "SQBRSQL005"; pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; +pub(crate) const MIXED_QUALIFICATION_CODE: &str = "SQBRSQL028"; +/// The help for mixed qualification in a SQL macro body, where slot reads are qualified through an alias. +pub(crate) const MACRO_MIXED_QUALIFICATION_REMEDIATION: &str = "Use one consistent qualification style in this query scope. A plain slot read such as `@rel.column` counts as unqualified; to qualify it, read through an alias: `FROM @rel AS s`, then `s.@rel.column`."; pub(crate) const NATIVE_LINT_ERROR_CODE: &str = "L003"; pub(crate) const PARSE_ERROR_POSITION_PATTERN: &str = r"^Parse error at line (?P\d+), column (?P\d+):"; diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs index a23e97824f..3ea2f3aacd 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs @@ -3,6 +3,7 @@ use std::collections::HashSet; use polyglot_sql::tokens::{Span, Token, TokenType}; use crate::constants::{EMPTY_FIXTURE_QUERY_TOKEN_COUNT, ENCLOSING_CTE_TOKEN_COUNT}; +use crate::lint_text::constants::SQL_MACRO_PARAMETER_SENTINEL_PREFIX; use crate::sql_lint::_helpers::engine::{ direct_indices, is_comment, is_layout, is_query_from, query_end, significant_after, significant_before, token_depths, @@ -35,6 +36,7 @@ struct ReferenceFactContext<'a> { query: &'a QuerySlice<'a>, outer_relations: &'a FxHashSet, external_identifiers: &'a HashSet, + qualification_exempt_spans: &'a [(usize, usize)], query_start: usize, query_end: usize, } @@ -50,6 +52,7 @@ struct SelectFactContext<'a> { tokens: &'a [Token], depths: &'a [usize], external_identifiers: &'a HashSet, + qualification_exempt_spans: &'a [(usize, usize)], dependency_identifiers: &'a HashSet, allows_ceremonial_select: bool, allows_dynamic_output_star: bool, @@ -171,6 +174,7 @@ pub(super) fn collect_additional_facts( tokens, depths: &depths, external_identifiers: options.external_identifiers, + qualification_exempt_spans: options.qualification_exempt_spans, dependency_identifiers: options.dependency_identifiers, allows_ceremonial_select: options.allows_ceremonial_select, allows_dynamic_output_star: options.allows_dynamic_output_star, @@ -338,6 +342,7 @@ fn collect_select_additional_facts(context: &SelectFactContext<'_>) -> Additiona query: &reference_slice, outer_relations: &outer_relations, external_identifiers: context.external_identifiers, + qualification_exempt_spans: context.qualification_exempt_spans, query_start: select_index, query_end: end, }; @@ -663,10 +668,14 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer facts .unqualified_multi_source_columns .extend(unqualified.iter().map(|&index| tokens[index].span)); - } else if !qualified.is_empty() && !unqualified.is_empty() { + } else { facts .inconsistent_single_source_qualification - .extend(unqualified.iter().map(|&index| tokens[index].span)); + .extend(inconsistent_qualification( + tokens, + (&qualified, &unqualified), + context.qualification_exempt_spans, + )); } facts.unknown_relation_qualifiers.extend( qualified @@ -685,6 +694,48 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer facts } +/// The unqualified references of a single-relation scope that also qualifies some, judged as written: a plain macro slot read such as `@orders.amount` counts as unqualified though it reads through its relation, and SQL another file authored is judged there. +fn inconsistent_qualification( + tokens: &[Token], + (qualified, unqualified): (&[usize], &[usize]), + exempt: &[(usize, usize)], +) -> Vec { + let authored = |index: &&usize| { + let start: usize = tokens[**index].span.start; + !exempt + .iter() + .any(|&(exempt_start, exempt_end)| exempt_start <= start && start < exempt_end) + }; + let (plain_reads, qualified): (Vec, Vec) = qualified + .iter() + .filter(authored) + .partition(|&&index| is_plain_slot_read(tokens, index)); + let mut unqualified: Vec = unqualified.iter().filter(authored).copied().collect(); + if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { + return Vec::new(); + } + unqualified.extend(plain_reads); + unqualified.sort_unstable(); + unqualified + .iter() + .map(|&index| tokens[index].span) + .collect() +} + +/// Whether the qualified reference at `index` is a plain SQL macro slot read, both its relation and its slot a macro parameter. +fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { + let is_parameter = |token: &Token| { + token + .text + .to_ascii_lowercase() + .starts_with(SQL_MACRO_PARAMETER_SENTINEL_PREFIX) + }; + is_parameter(&tokens[index]) + && tokens + .get(index + 2) + .is_some_and(|column| is_parameter(column)) +} + fn simple_qualifier( tokens: &[Token], references: &[usize], diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs index 34c1ce5695..ef93075a49 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs @@ -337,6 +337,7 @@ struct DiagnosticContext<'a> { struct FactBuildOptions<'a> { external_identifiers: &'a HashSet, + qualification_exempt_spans: &'a [(usize, usize)], dependency_identifiers: &'a HashSet, externally_referenced_ctes: &'a HashSet, allows_ceremonial_select: bool, @@ -406,6 +407,7 @@ pub(crate) fn lint(request: LintRequest) -> Result { .collect(); let options = FactBuildOptions { external_identifiers: &external_identifiers, + qualification_exempt_spans: &request.qualification_exempt_spans, dependency_identifiers: &dependency_identifiers, externally_referenced_ctes: &externally_referenced_ctes, allows_ceremonial_select: request.allows_ceremonial_select, @@ -516,6 +518,7 @@ fn build_facts( tokens, &AdditionalFactOptions { external_identifiers: options.external_identifiers, + qualification_exempt_spans: options.qualification_exempt_spans, dependency_identifiers: options.dependency_identifiers, allows_ceremonial_select: options.allows_ceremonial_select, allows_dynamic_output_star: options.allows_dynamic_output_star, diff --git a/crates/sqlbuild-rules/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs index ea4fbc58b2..57bf44bf3d 100644 --- a/crates/sqlbuild-rules/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules/src/sql_lint/models.rs @@ -83,12 +83,16 @@ pub(crate) struct LintRequest { pub relation_keys: HashMap>>, #[serde(default)] pub relation_names: HashMap, + /// Character ranges of SQL another file authored, such as SQL macro output, whose column qualification is judged in that file instead. + #[serde(default)] + pub qualification_exempt_spans: Vec<(usize, usize)>, } #[derive(Debug)] pub(crate) struct AdditionalFactOptions<'a> { pub external_identifiers: &'a HashSet, pub dependency_identifiers: &'a HashSet, + pub qualification_exempt_spans: &'a [(usize, usize)], pub allows_ceremonial_select: bool, pub allows_dynamic_output_star: bool, pub allows_empty_fixture_star: bool, diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs index 141ff9c8b3..e714c9db4c 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs @@ -1114,6 +1114,20 @@ fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> R expected_anchor: Some("name"), expected_replacement: None, }, + test_types::AdditionalLintRuleTestCase { + description: "plain macro slot reads count as unqualified beside a plain group read", + sql: "SELECT __sqlbuild_macro_parameter_1__, __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_2__ AS orders FROM __sqlbuild_macro_parameter_0__", + rule: "SQBRSQL028", + expected_anchor: None, + expected_replacement: None, + }, + test_types::AdditionalLintRuleTestCase { + description: "a plain macro slot read beside a hand-qualified column is mixed", + sql: "SELECT __sqlbuild_macro_parameter_0__.__sqlbuild_macro_parameter_1__ AS orders, s.refunds FROM __sqlbuild_macro_parameter_0__ AS s", + rule: "SQBRSQL028", + expected_anchor: Some("__sqlbuild_macro_parameter_0__"), + expected_replacement: None, + }, test_types::AdditionalLintRuleTestCase { description: "unknown relation qualifier", sql: "SELECT missing.id FROM items AS present", @@ -1582,6 +1596,53 @@ fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> R Ok(()) } +#[test] +fn given_exempt_macro_output_when_linting_qualification_then_only_authored_columns_are_judged() +-> Result<(), String> { + let test_cases = [ + test_types::QualificationExemptLintTestCase { + description: "a caller's bare column beside qualified macro output passes", + sql: "SELECT order_id, orders.amount - orders.discount AS net FROM orders", + exempt: "orders.amount - orders.discount", + expected_count: 0, + }, + test_types::QualificationExemptLintTestCase { + description: "the caller's own mix still fails", + sql: "SELECT order_id, orders.status, orders.amount - orders.discount AS net FROM orders", + exempt: "orders.amount - orders.discount", + expected_count: 1, + }, + ]; + for test_case in test_cases { + let start: usize = test_case + .sql + .find(test_case.exempt) + .ok_or("exempt text should be in the SQL")?; + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": ["SQBRSQL028"], + "qualification_exempt_spans": [[start, start + test_case.exempt.len()]], + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } + Ok(()) +} + #[test] fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() -> Result<(), String> { diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs index 30f7edda4a..41b6cd742f 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs @@ -55,6 +55,14 @@ pub(crate) struct PlainSqlLintTestCase { pub expected_count: usize, } +pub(crate) struct QualificationExemptLintTestCase { + pub(crate) description: &'static str, + pub(crate) sql: &'static str, + /// The text whose characters are another file's SQL. + pub(crate) exempt: &'static str, + pub(crate) expected_count: usize, +} + pub(crate) struct DynamicOutputStarLintTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index fa1c77ee06..5edc64ede2 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -316,6 +316,11 @@ fully typed. - Every read inside `USING (...)` stands unqualified, as `USING` needs, including `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. +- Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and + unqualified columns, counts every plain read as unqualified and every `alias.@weekly...` read as + qualified, and judges a macro's SQL in the macro file rather than at each call. A step written + only in plain reads, such as `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, + passes; to qualify, read every slot through an alias. A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, could bind its bare columns to the subquery's relations instead of the one the slot reads. The diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 3dcd09aa4d..b376d83e8e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1317,3 +1317,12 @@ class PreservedTargetAuditTestCase: description: str target_toml: str expected_history_schema: str + + +@dataclass(frozen=True) +class MacroSlotQualificationTestCase: + description: str + files: dict[str, str] + expected_exit_code: int + expected_fragments: tuple[str, ...] + absent_fragments: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py new file mode 100644 index 0000000000..ab7738ffad --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py @@ -0,0 +1,159 @@ +"""Real CLI coverage for how SQBRSQL028 judges the qualification of SQL macro slot reads.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + MacroSlotQualificationTestCase, +) + +_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "shop.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n\n' + "[rules]\n" + 'select = ["SQBRSQL028"]\n' +) +_WEEKLY_ORDERS: str = ( + 'MODEL (\n description "Weekly orders per product",\n materialized table,\n);\n\n' + "SELECT 1 AS product_id, CAST(10 AS BIGINT) AS orders_placed, " + "CAST(1 AS BIGINT) AS refund_count\n" +) +_REFUND_RATES: str = ( + 'MODEL (\n description "Weekly refund rate per product",\n materialized table,\n);\n\n' + 'WITH weekly AS (SELECT * FROM __ref("weekly_orders")),\n' + "@sql.weekly_rates(weekly, weekly.keys := (product_id), " + "weekly.order_count := orders_placed, __cte := by_product)\n" + "final AS (SELECT * FROM by_product)\n" + "SELECT * FROM final\n" +) + + +def _weekly_rates(first_step: str) -> str: + return ( + 'MACRO (\n description "Weekly refund rate per grain",\n' + " arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)),\n" + ");\n\n" + f"WITH counted AS (\n{first_step}\n),\n\n" + "rated AS (\n" + " SELECT\n" + " @weekly.keys,\n" + " SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n" + " FROM counted\n" + " GROUP BY @weekly.keys\n" + ")\n" + ) + + +def _rates_project(first_step: str) -> dict[str, str]: + return { + "models/weekly_orders.sql": _WEEKLY_ORDERS, + "models/refund_rates.sql": _REFUND_RATES, + "models/_macros/sql/weekly_rates.sql": _weekly_rates(first_step), + } + + +@pytest.mark.parametrize( + "test_case", + [ + MacroSlotQualificationTestCase( + description="a first step written in plain slot reads passes", + files=_rates_project( + " SELECT\n" + " @weekly.keys,\n" + " @weekly.order_count AS orders,\n" + " @weekly.refund_count AS refunds\n" + " FROM @weekly" + ), + expected_exit_code=0, + expected_fragments=("Project compiled",), + absent_fragments=("SQBRSQL028",), + ), + MacroSlotQualificationTestCase( + description="aliased slot reads beside a plain group read fail once, at the macro", + files=_rates_project( + " SELECT\n" + " @weekly.keys,\n" + " s.@weekly.order_count AS orders,\n" + " s.@weekly.refund_count AS refunds\n" + " FROM @weekly AS s" + ), + expected_exit_code=1, + expected_fragments=( + "--> models/_macros/sql/weekly_rates.sql:8:5", + "to qualify it, read through an alias: `FROM @rel AS s`, then `s.@rel.column`", + ), + absent_fragments=("--> models/refund_rates.sql",), + ), + MacroSlotQualificationTestCase( + description="an expression macro's plain reads leave the caller's style to the caller", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount, " + "CAST(1 AS DOUBLE) AS discount\n" + ), + "models/_macros/sql/net_amount.sql": ( + 'MACRO (\n description "Order amount after discount",\n' + " arguments (orders RELATION (amount DOUBLE, discount DOUBLE)),\n);\n\n" + "@orders.amount - @orders.discount\n" + ), + "models/order_totals.sql": ( + 'MODEL (\n description "Net amount per order",\n materialized table,\n);\n\n' + "SELECT order_id, @sql.net_amount(orders) AS net\n" + 'FROM __ref("orders") AS orders\n' + ), + }, + expected_exit_code=0, + expected_fragments=("Project compiled",), + absent_fragments=("SQBRSQL028",), + ), + MacroSlotQualificationTestCase( + description="a caller that mixes styles itself still fails", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" + ), + "models/open_orders.sql": ( + 'MODEL (\n description "Open orders",\n materialized table,\n);\n\n' + 'SELECT o.order_id, status FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + absent_fragments=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_macro_slot_reads_when_compiling_then_qualification_is_judged_as_written( + test_case: MacroSlotQualificationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = {"sqlbuild_project.toml": _PROJECT, **test_case.files} + for relative, contents in files.items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + exit_code: int = main(["--no-color", "--project-dir", str(tmp_path), "compile"]) + + output: str = "".join(capsys.readouterr()) + assert exit_code == test_case.expected_exit_code, output + assert [fragment in output for fragment in test_case.expected_fragments] == [ + True for _ in test_case.expected_fragments + ], output + assert [fragment in output for fragment in test_case.absent_fragments] == [ + False for _ in test_case.absent_fragments + ], output diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 7d87eff1b0..e15c793714 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -301,6 +301,11 @@ fully typed. - Every read inside `USING (...)` stands unqualified, as `USING` needs, including `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. +- Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and + unqualified columns, counts every plain read as unqualified and every `alias.@weekly...` read as + qualified, and judges a macro's SQL in the macro file rather than at each call. A step written + only in plain reads, such as `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, + passes; to qualify, read every slot through an alias. A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, could bind its bare columns to the subquery's relations instead of the one the slot reads. The From 3a1a9fb1d70176bb5756bb0bb93c89b5e8f91aec Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:44:25 +0000 Subject: [PATCH 11/30] feat(compile): write models' compiled SQL when Rules fail A compile whose Rules fail still writes each model's expanded SQL to `target/compiled/models/`, so SQL that macros generated can be read. A model with Rule errors starts with a header marking it not buildable and listing its findings. The exit code, the manifest, the DAG file and the other artifacts are unchanged. --- .../_helpers/compile_command/target_writer.rs | 96 ++++++++++++++++- .../_helpers/compile_driver/command.rs | 11 ++ .../sqlbuild/references/docs/cli/compile.md | 19 +++- .../docs/concepts/macros/sql-macros.md | 6 +- .../sqlbuild/cli/commands/main/_test_types.py | 9 ++ .../main/test_compile_rule_failures.py | 101 ++++++++++++++++++ .../sqlbuild/cli/commands/main/test_rules.py | 6 +- website/src/content/docs/docs/cli/compile.mdx | 19 +++- .../docs/docs/concepts/macros/sql-macros.mdx | 6 +- 9 files changed, 265 insertions(+), 8 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 1a10104441..dfc7283b83 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -39,8 +39,8 @@ use crate::bindings::_helpers::compile_outputs::runtime_target::{ }; use crate::bindings::_helpers::compiled_views::compiled_model_view::NativeCompiledModel; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::construct::new_record_with; +use crate::bindings::_helpers::records::construct::{enum_member, error_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; @@ -748,6 +748,100 @@ pub(crate) fn write_static_target<'py>( ) } +/// The first line of a compiled model whose Rules failed. +const RULE_FAILED_HEADER: &str = "-- sqb: compile failed for this model; not buildable"; + +/// `path` relative to `project_dir` with `/` separators, as model files name themselves. +fn project_relative(path: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>) -> PyResult { + let path: Bound<'_, PyAny> = path_class(path.py())?.call1((path,))?; + let relative: Bound<'_, PyAny> = if path.call_method0("is_absolute")?.is_truthy()? { + match path.call_method1("relative_to", (project_dir.call_method0("resolve")?,)) { + Ok(relative) => relative, + Err(_) => path, + } + } else { + path + }; + relative.call_method0("as_posix")?.extract() +} + +/// Each source file's Rule errors as compiled-file header lines: code, location and message. +fn rule_error_lines( + diagnostics: &Bound<'_, PyAny>, + project_dir: &Bound<'_, PyAny>, +) -> PyResult>> { + let py: Python<'_> = diagnostics.py(); + let rule_phase: Bound<'_, PyAny> = enum_member(py, RecordId::DiagnosticPhase, "RULE")?; + let mut lines: HashMap> = HashMap::new(); + for diagnostic in diagnostics.try_iter()? { + let diagnostic: Bound<'_, PyAny> = diagnostic?; + let path: Bound<'_, PyAny> = native_field(&diagnostic, "path")?; + if path.is_none() + || !native_field(&diagnostic, "is_error")?.is_truthy()? + || !native_field(&diagnostic, "phase")?.is(&rule_phase) + { + continue; + } + let relative: String = project_relative(&path, project_dir)?; + let position = |field: &str| -> PyResult { + let value: Bound<'_, PyAny> = native_field(&diagnostic, field)?; + Ok(if value.is_none() { + String::new() + } else { + format!(":{}", value.str()?) + }) + }; + let message: String = native_field(&diagnostic, "message")?.str()?.to_string(); + let line: String = format!( + "-- {} at {relative}{}{}: {}", + native_field(&diagnostic, "code")?.str()?, + position("line")?, + position("column")?, + message.split_whitespace().collect::>().join(" "), + ); + lines.entry(relative).or_default().push(line); + } + Ok(lines) +} + +/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. Nothing else under `target/` changes: no manifest, tests, cache or stale-file removal. +pub(crate) fn write_rule_failed_models<'py>( + target_dir: &Bound<'py, PyAny>, + (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), + diagnostics: &Bound<'py, PyAny>, +) -> PyResult<()> { + let py: Python<'py> = target_dir.py(); + let errors: HashMap> = rule_error_lines(diagnostics, project_dir)?; + let _ = prepared_target(target_dir)?; + let compiled_dir: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; + let mut batch: ArtifactBatch = ArtifactBatch::new(true, None); + for model in native_field(project, "models")?.try_iter()? { + let model: Bound<'py, PyAny> = model?; + let (relative_path, query_sql): (String, String) = + if let Ok(native) = model.cast::() { + let (relative_path, query_sql) = native.get().output_texts(py)?; + (relative_path, query_sql.to_string()) + } else { + ( + native_field(&model, "relative_path")? + .call_method0("as_posix")? + .extract()?, + native_field(&model, "query_sql")?.extract()?, + ) + }; + let sql: String = match errors.get(&relative_path) { + Some(lines) => format!("{RULE_FAILED_HEADER}\n{}\n{query_sql}", lines.join("\n")), + None => query_sql, + }; + batch.queue_sql_at( + py, + native_model_output_path(&compiled_dir, &relative_path), + &sql, + )?; + } + batch.flush(py) +} + /// Plan uncached SQL tests and report their errors without writing artifacts or cache. pub(crate) fn planning_diagnostics<'py>( target_dir: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 902d4fa14c..28a7a4ea93 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -26,6 +26,7 @@ use crate::bindings::_helpers::compile_command::reuse_recording::write_reusable_ use crate::bindings::_helpers::compile_command::reuse_recording::{ CompileArtifactWrites, RecordedCompileOutput, compile_reuse_timings, new_artifact_recorder, }; +use crate::bindings::_helpers::compile_command::target_writer::write_rule_failed_models; use crate::bindings::_helpers::compile_context::input_reads::record_compile_input_reads; use crate::bindings::_helpers::compile_driver::analysis::analyze; use crate::bindings::_helpers::compile_driver::phases::{ @@ -358,6 +359,16 @@ fn compiled_with_status<'py>( (prepared_artifacts, recorder), status, )?; + if rules_failed && !skip_write { + write_rule_failed_models( + &target_dir(&project_dir)?, + ( + &native_field(&analysis, "graph")?.getattr("project")?, + &project_dir, + ), + &native_field(&analysis, "diagnostics")?, + )?; + } let timings: Bound<'py, PyDict> = timings_ms( (&analysis, write_ms), (&recorders.detailed_timings, reuse_timings), diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 2a7c01186c..09c41541ae 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -82,7 +82,9 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb 4. **Column lineage.** Traces column-level dependencies across models. 5. **Contracts.** Checks declared columns against the inferred query output. 6. **Rules.** Runs the selected built-in Rules, then the selected custom Rules. -7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. +7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. When they fail, it + writes only the models' SQL, so you can read what the Rules checked; see + [When Rules fail](#when-rules-fail). Rules that need inferred types, such as `SQBRCONTRACT105`, are skipped when SQL analysis is off, and compile prints one note naming them. @@ -91,6 +93,21 @@ and compile prints one note naming them. findings block the artifacts. To check one Rule code or family without changing project configuration, use [`sqb rules run`](rules.md). +### When Rules fail + +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. Each +model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros +generated. A model with Rule errors starts with a header that marks it not buildable and lists +its findings: + +```sql +-- sqb: compile failed for this model; not buildable +-- SQBRSQL028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns +SELECT o.order_id, status FROM __ref("orders") AS o +``` + +The next compile that passes writes the files again without the header. + ## Compiling a selection Selectors limit analysis and reporting. Discovery still covers the whole project, so every diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index 5edc64ede2..2900c107b4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -509,8 +509,10 @@ A `* REPLACE` item takes its source column's type rather than the call's. - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. - [Rules](../rules.md) and [column lineage](../column-lineage.md) read the expanded - SQL, so they see through every call. A rule finding inside an expansion is reported at its - position in the expanded query, not in the macro file. + SQL, so they see through every call. A Rule finding inside an expansion is reported at the call, + marked `(in SQL generated by @sql.name(...))`. The model's expanded SQL is in + `target/compiled/models/`, also when the Rules fail; see + [When Rules fail](../../cli/compile.md#when-rules-fail). - Editing a macro recompiles its callers. Test SQL macros with [macro-mode unit tests](../testing.md#macro-tests), and replace one diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index b376d83e8e..438e4e9d1b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1326,3 +1326,12 @@ class MacroSlotQualificationTestCase: expected_exit_code: int expected_fragments: tuple[str, ...] absent_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class RuleFailureArtifactsTestCase: + description: str + open_orders_sql: str + expected_exit_code: int + expected_open_orders: str + expected_manifest: bool diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py new file mode 100644 index 0000000000..c971f9a58b --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -0,0 +1,101 @@ +"""Real CLI coverage for the compiled SQL a compile writes when Rules fail.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild._native import run_cli as main +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + RuleFailureArtifactsTestCase, +) + +_PROJECT: str = ( + 'name = "shop"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.local]\n" + 'database = "shop.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "local"\n' + 'schema = "dev"\n\n' + "[rules]\n" + 'select = ["SQBRSQL028"]\n' +) +_ORDERS: str = ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" +) +_HEADER: str = "-- sqb: compile failed for this model; not buildable\n" + + +def _open_orders(sql: str) -> str: + return f'MODEL (\n description "Open orders",\n materialized table,\n);\n\n{sql}\n' + + +@pytest.mark.parametrize( + "test_case", + [ + RuleFailureArtifactsTestCase( + description="a model failing a Rule is written with a header listing its findings", + open_orders_sql='SELECT o.order_id, status FROM __ref("orders") AS o', + expected_exit_code=1, + expected_open_orders=( + _HEADER + "-- SQBRSQL028 at models/open_orders.sql:6:20: " + "Single-relation query mixes qualified and unqualified columns\n" + 'SELECT o.order_id, status FROM __ref("orders") AS o\n' + ), + expected_manifest=False, + ), + RuleFailureArtifactsTestCase( + description="a passing compile writes the same model without the header", + open_orders_sql='SELECT o.order_id, o.status FROM __ref("orders") AS o', + expected_exit_code=0, + expected_open_orders='SELECT o.order_id, o.status FROM __ref("orders") AS o\n', + expected_manifest=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failure_header( + test_case: RuleFailureArtifactsTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = { + "sqlbuild_project.toml": _PROJECT, + "models/orders.sql": _ORDERS, + "models/open_orders.sql": _open_orders( + 'SELECT o.order_id, status FROM __ref("orders") AS o' + ), + } + for relative, contents in files.items(): + path: Path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + compile_args: list[str] = [ + "--no-color", + "--project-dir", + str(tmp_path), + "compile", + "--manifest", + ] + _ = main(compile_args) + _ = (tmp_path / "models" / "open_orders.sql").write_text( + _open_orders(test_case.open_orders_sql), encoding="utf-8" + ) + (tmp_path / "target" / "manifest.json").unlink(missing_ok=True) + + exit_code: int = main(compile_args) + + output: str = "".join(capsys.readouterr()) + compiled: Path = tmp_path / "target" / "compiled" / "models" + assert exit_code == test_case.expected_exit_code, output + assert (compiled / "open_orders.sql").read_text(encoding="utf-8") == ( + test_case.expected_open_orders + ) + assert (compiled / "orders.sql").read_text(encoding="utf-8") == ( + "SELECT 1 AS order_id, 'open' AS status\n" + ) + assert (tmp_path / "target" / "manifest.json").exists() is test_case.expected_manifest diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d2d435fb76..44e099df62 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -1440,7 +1440,11 @@ def final_directory(*, model: Model, ctx: RuleContext) -> list[Finding]: assert cold["diagnostics"] == warm["diagnostics"] == first["diagnostics"] assert cold["compile_timings"]["rule_cache_hits"] == 0 assert warm["compile_timings"]["rule_cache_hits"] >= 1 - assert not (tmp_path / "target" / "compiled").exists() + assert ( + (tmp_path / "target" / "compiled" / "models" / "orders.sql") + .read_text(encoding="utf-8") + .startswith("-- sqb: compile failed for this model; not buildable\n-- XSQBRARCH001 at ") + ) host_inputs: Path = tmp_path / "target" / "rules-cache" / "host-inputs" assert not tuple(host_inputs.glob("*.payload")) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 0bb775514f..1708045b28 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -66,7 +66,9 @@ Positional `SELECTOR` arguments work like `--select`, and both forms can be comb 4. **Column lineage.** Traces column-level dependencies across models. 5. **Contracts.** Checks declared columns against the inferred query output. 6. **Rules.** Runs the selected built-in Rules, then the selected custom Rules. -7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. +7. **Write.** Writes compiled SQL to `target/compiled/` when the Rules pass. When they fail, it + writes only the models' SQL, so you can read what the Rules checked; see + [When Rules fail](#when-rules-fail). Rules that need inferred types, such as `SQBRCONTRACT105`, are skipped when SQL analysis is off, and compile prints one note naming them. @@ -75,6 +77,21 @@ and compile prints one note naming them. findings block the artifacts. To check one Rule code or family without changing project configuration, use [`sqb rules run`](/docs/cli/rules). +### When Rules fail + +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. Each +model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros +generated. A model with Rule errors starts with a header that marks it not buildable and lists +its findings: + +```sql +-- sqb: compile failed for this model; not buildable +-- SQBRSQL028 at models/open_orders.sql:6:20: Single-relation query mixes qualified and unqualified columns +SELECT o.order_id, status FROM __ref("orders") AS o +``` + +The next compile that passes writes the files again without the header. + ## Compiling a selection Selectors limit analysis and reporting. Discovery still covers the whole project, so every diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index e15c793714..7fd384e3c5 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -494,8 +494,10 @@ A `* REPLACE` item takes its source column's type rather than the call's. - `sqb format` formats SQL macro files, bodies of every form and the `MACRO` header, and normalises the commas around CTE-list and column-list calls. - [Rules](/docs/concepts/rules) and [column lineage](/docs/concepts/column-lineage) read the expanded - SQL, so they see through every call. A rule finding inside an expansion is reported at its - position in the expanded query, not in the macro file. + SQL, so they see through every call. A Rule finding inside an expansion is reported at the call, + marked `(in SQL generated by @sql.name(...))`. The model's expanded SQL is in + `target/compiled/models/`, also when the Rules fail; see + [When Rules fail](/docs/cli/compile#when-rules-fail). - Editing a macro recompiles its callers. Test SQL macros with [macro-mode unit tests](/docs/concepts/testing#macro-tests), and replace one From a9847212efe42387a937423977ed9d533082da5f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:04:07 +0000 Subject: [PATCH 12/30] refactor: keep the batch within the architecture checks Move helpers and test fixtures to where the layering and test-shape rules expect them, route query output through the shared terminal helpers, and record the new native crossing budgets. --- .../render_query_long.rs => _helpers/long.rs} | 2 +- .../src/query_command/_helpers/mod.rs | 1 + .../src/query_command/main/mod.rs | 1 - .../query_command/main/render_query_text.rs | 6 +- .../src/query_command/tests/helpers.rs | 21 +++ .../query_command/tests/test_query_command.rs | 61 ++++---- .../src/query_command/tests/test_types.rs | 6 + .../_helpers/compile_command/target_writer.rs | 32 ++-- .../bindings/_helpers/query_command/query.rs | 15 +- .../_helpers/query_command/text_output.rs | 13 +- .../_helpers/recall/clickhouse_calls.rs | 5 +- .../src/bindings/_helpers/recall/command.rs | 3 +- .../src/bindings/_helpers/recall/rendering.rs | 5 +- .../sqlbuild-recall/src/settings/constants.rs | 4 + crates/sqlbuild-recall/src/settings/models.rs | 8 +- .../src/store/_helpers/column_types.rs | 51 +++++++ .../sqlbuild-recall/src/store/_helpers/mod.rs | 1 + .../src/store/main/query_store.rs | 39 +---- .../src/writer/_helpers/materialize.rs | 140 ++++++++++-------- .../src/writer/tests/helpers.rs | 41 ++++- .../src/writer/tests/test_artifacts.rs | 96 +++++------- .../src/writer/tests/test_types.rs | 12 ++ crates/sqlbuild-rules/src/sql_lint/tests.rs | 2 + .../sql_lint/tests/test_lint_exemptions.rs | 96 ++++++++++++ .../src/sql_lint/tests/test_sql_lint.rs | 90 ----------- fensu.toml | 16 +- .../clickhouse/_helpers/recall_store.py | 4 +- .../snowflake/classes/snowflake_adapter.py | 3 +- src/sqlbuild/adapters/snowflake/constants.py | 1 + .../classes/statement_lifecycle.py | 5 +- .../src/sqlbuild/cli/commands/main/helpers.py | 45 ++++++ .../main/test_compile_rule_failures.py | 9 +- .../main/test_macro_slot_qualification.py | 42 +----- .../src/sqlbuild/cost/test_snowflake_scale.py | 9 +- 34 files changed, 489 insertions(+), 396 deletions(-) rename crates/sqlbuild-core/src/query_command/{main/render_query_long.rs => _helpers/long.rs} (94%) create mode 100644 crates/sqlbuild-recall/src/store/_helpers/column_types.rs create mode 100644 crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_long.rs b/crates/sqlbuild-core/src/query_command/_helpers/long.rs similarity index 94% rename from crates/sqlbuild-core/src/query_command/main/render_query_long.rs rename to crates/sqlbuild-core/src/query_command/_helpers/long.rs index d56e24dab7..a751bbaad6 100644 --- a/crates/sqlbuild-core/src/query_command/main/render_query_long.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/long.rs @@ -6,7 +6,7 @@ use crate::query_command::models::QueryDisplay; /// `-[ RECORD n ]---+` then `column | value` lines per row, then the row count. #[must_use] -pub fn render_query_long(display: &QueryDisplay) -> String { +pub(crate) fn long_text(display: &QueryDisplay) -> String { let width: usize = display .columns .iter() diff --git a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs index af7531f4f2..32089b756c 100644 --- a/crates/sqlbuild-core/src/query_command/_helpers/mod.rs +++ b/crates/sqlbuild-core/src/query_command/_helpers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod boxed; pub(crate) mod footer; +pub(crate) mod long; diff --git a/crates/sqlbuild-core/src/query_command/main/mod.rs b/crates/sqlbuild-core/src/query_command/main/mod.rs index b4a9c3648e..5d9e192526 100644 --- a/crates/sqlbuild-core/src/query_command/main/mod.rs +++ b/crates/sqlbuild-core/src/query_command/main/mod.rs @@ -1,4 +1,3 @@ pub mod render_dag_summary; pub mod render_query_json; -pub mod render_query_long; pub mod render_query_text; diff --git a/crates/sqlbuild-core/src/query_command/main/render_query_text.rs b/crates/sqlbuild-core/src/query_command/main/render_query_text.rs index 252ba6d141..8d2b4a2323 100644 --- a/crates/sqlbuild-core/src/query_command/main/render_query_text.rs +++ b/crates/sqlbuild-core/src/query_command/main/render_query_text.rs @@ -2,17 +2,17 @@ use crate::query_command::_helpers::boxed::boxed_lines; use crate::query_command::_helpers::footer::footer; -use crate::query_command::main::render_query_long::render_query_long; +use crate::query_command::_helpers::long::long_text; use crate::query_command::models::{QueryDisplay, QueryLayout}; /// The rows in `layout`, then the row count. A fitted layout never drops columns or rows: a result wider than the width, or with a value spanning lines, prints one record per row. #[must_use] pub fn render_query_text(display: &QueryDisplay, layout: QueryLayout) -> String { let QueryLayout::Fitted { width } = layout else { - return render_query_long(display); + return long_text(display); }; let Some(mut lines) = boxed_lines(display, width) else { - return render_query_long(display); + return long_text(display); }; lines.extend(footer(display)); format!("{}\n", lines.join("\n")) diff --git a/crates/sqlbuild-core/src/query_command/tests/helpers.rs b/crates/sqlbuild-core/src/query_command/tests/helpers.rs index e611510a34..795f72a478 100644 --- a/crates/sqlbuild-core/src/query_command/tests/helpers.rs +++ b/crates/sqlbuild-core/src/query_command/tests/helpers.rs @@ -1,4 +1,25 @@ +use crate::query_command::models::QueryDisplay; + /// Owned copies of string literals. pub(super) fn texts(values: &[&str]) -> Vec { values.iter().map(|value| (*value).to_owned()).collect() } + +/// A small invocation-count result with one NULL count. +pub(super) fn invocations() -> QueryDisplay { + QueryDisplay { + columns: texts(&["status", "origin", "invocations"]), + types: vec![ + Some("varchar".to_owned()), + Some("varchar".to_owned()), + Some("int32".to_owned()), + ], + numeric: vec![false, false, true], + rows: vec![ + texts(&["completed", "integration", "133"]), + texts(&["cancelled", "local", "NULL"]), + ], + truncated: false, + limit: None, + } +} diff --git a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs index 5d471dbe02..3551975fe7 100644 --- a/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs +++ b/crates/sqlbuild-core/src/query_command/tests/test_query_command.rs @@ -3,26 +3,10 @@ use crate::query_command::main::render_dag_summary::render_dag_summary; use crate::query_command::main::render_query_json::render_query_json; use crate::query_command::main::render_query_text::render_query_text; use crate::query_command::models::{QueryDisplay, QueryLayout}; -use crate::query_command::tests::helpers::texts; -use crate::query_command::tests::test_types::{DagTestCase, DisplayTestCase, JsonTestCase}; - -fn invocations() -> QueryDisplay { - QueryDisplay { - columns: texts(&["status", "origin", "invocations"]), - types: vec![ - Some("varchar".to_owned()), - Some("varchar".to_owned()), - Some("int32".to_owned()), - ], - numeric: vec![false, false, true], - rows: vec![ - texts(&["completed", "integration", "133"]), - texts(&["cancelled", "local", "NULL"]), - ], - truncated: false, - limit: None, - } -} +use crate::query_command::tests::helpers::{invocations, texts}; +use crate::query_command::tests::test_types::{ + DagTestCase, DisplayTestCase, FooterTestCase, JsonTestCase, +}; #[test] fn given_rows_when_rendering_text_then_a_fitting_result_is_boxed_and_others_are_records() { @@ -93,18 +77,35 @@ fn given_rows_when_rendering_text_then_a_fitting_result_is_boxed_and_others_are_ } #[test] -fn given_more_than_fifty_rows_when_rendering_text_then_the_footer_suggests_aggregating() { - let display = QueryDisplay { - columns: texts(&["n"]), - rows: (0..51).map(|_| texts(&["7"])).collect(), - ..QueryDisplay::default() - }; +fn given_many_rows_when_rendering_text_then_only_more_than_fifty_suggest_aggregating() { + let test_cases = [ + FooterTestCase { + description: "fifty rows print only the count", + row_count: 50, + expected_footer: "\n50 rows\n", + }, + FooterTestCase { + description: "more than fifty rows suggest aggregating", + row_count: 51, + expected_footer: "\n51 rows; to read fewer, aggregate with GROUP BY or add a LIMIT\n", + }, + ]; - let rendered: String = render_query_text(&display, QueryLayout::Fitted { width: 120 }); + for test_case in test_cases { + let display = QueryDisplay { + columns: texts(&["n"]), + rows: (0..test_case.row_count).map(|_| texts(&["7"])).collect(), + ..QueryDisplay::default() + }; + + let rendered: String = render_query_text(&display, QueryLayout::Fitted { width: 120 }); - assert!( - rendered.ends_with("\n51 rows; to read fewer, aggregate with GROUP BY or add a LIMIT\n") - ); + assert!( + rendered.ends_with(test_case.expected_footer), + "{}", + test_case.description + ); + } } #[test] diff --git a/crates/sqlbuild-core/src/query_command/tests/test_types.rs b/crates/sqlbuild-core/src/query_command/tests/test_types.rs index e7930431d9..eddd4ac5c4 100644 --- a/crates/sqlbuild-core/src/query_command/tests/test_types.rs +++ b/crates/sqlbuild-core/src/query_command/tests/test_types.rs @@ -8,6 +8,12 @@ pub(super) struct DisplayTestCase { pub(super) expected_text: &'static str, } +pub(super) struct FooterTestCase { + pub(super) description: &'static str, + pub(super) row_count: usize, + pub(super) expected_footer: &'static str, +} + pub(super) struct JsonTestCase { pub(super) description: &'static str, pub(super) columns: Vec, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index dfc7283b83..c05f847b39 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -43,6 +43,7 @@ use crate::bindings::_helpers::records::construct::new_record_with; use crate::bindings::_helpers::records::construct::{enum_member, error_class}; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_get_or; +use crate::bindings::_helpers::records::fields::path_posix_text; use crate::bindings::_helpers::records::ids::{ErrorId, RecordId}; use crate::bindings::_helpers::sql_test_glue::plan_driver::plan_compiled_sql_test_artifacts; use crate::bindings::_helpers::sql_test_glue::plan_driver::sql_test_model_chain_names_by_key; @@ -68,6 +69,8 @@ const PUBLISHED_TREE: &str = "tree"; const TARGET_OPTIONS: (&[&str], &[&str]) = (&["target_dir", "adapter", "project"], &["manifest"]); const STAGING_CHANGED_HELP: &str = "Rerun the compile; avoid deleting target/ while a compile is running."; +/// The first line of a compiled model whose Rules failed. +const RULE_FAILED_HEADER: &str = "-- sqb: compile failed for this model; not buildable"; /// `base / part / ...`. fn joined<'py>(base: &Bound<'py, PyAny>, parts: &[&str]) -> PyResult> { @@ -748,21 +751,20 @@ pub(crate) fn write_static_target<'py>( ) } -/// The first line of a compiled model whose Rules failed. -const RULE_FAILED_HEADER: &str = "-- sqb: compile failed for this model; not buildable"; - /// `path` relative to `project_dir` with `/` separators, as model files name themselves. fn project_relative(path: &Bound<'_, PyAny>, project_dir: &Bound<'_, PyAny>) -> PyResult { - let path: Bound<'_, PyAny> = path_class(path.py())?.call1((path,))?; - let relative: Bound<'_, PyAny> = if path.call_method0("is_absolute")?.is_truthy()? { - match path.call_method1("relative_to", (project_dir.call_method0("resolve")?,)) { - Ok(relative) => relative, - Err(_) => path, - } - } else { - path - }; - relative.call_method0("as_posix")?.extract() + let text: String = path_posix_text(path)?; + if !Path::new(&text).is_absolute() { + return Ok(text); + } + let root: PathBuf = std::fs::canonicalize(path_posix_text(project_dir)?)?; + Ok(Path::new(&text) + .strip_prefix(&root) + .map_or(text.clone(), |relative| { + relative + .to_string_lossy() + .replace(std::path::MAIN_SEPARATOR, "/") + })) } /// Each source file's Rule errors as compiled-file header lines: code, location and message. @@ -823,9 +825,7 @@ pub(crate) fn write_rule_failed_models<'py>( (relative_path, query_sql.to_string()) } else { ( - native_field(&model, "relative_path")? - .call_method0("as_posix")? - .extract()?, + path_posix_text(&native_field(&model, "relative_path")?)?, native_field(&model, "query_sql")?.extract()?, ) }; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index 4e7c18c3e1..e345a934c3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -276,20 +276,7 @@ fn number(value: &Bound<'_, PyAny>, decimal: &Bound<'_, PyAny>) -> PyResult, width: usize) -> PyResult>> { - let Some(types) = result.getattr_opt("column_types")? else { - return Ok(Vec::new()); - }; - let types: Vec> = types - .try_iter()? - .map(|kind| { - let kind: Bound<'_, PyAny> = kind?; - if kind.is_none() { - Ok(None) - } else { - Ok(Some(kind.str()?.to_string_lossy().into_owned())) - } - }) - .collect::>()?; + let types: Vec> = native_field(result, "column_types")?.extract()?; Ok(if types.len() == width { types } else { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs index e388e9b04c..68055a186a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs @@ -1,10 +1,11 @@ //! The text layout `sqb query` and `sqb recall query` share: a box when the result fits the output width, records otherwise. -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{PyResult, Python}; use sqlbuild_core::query_command::main::render_query_text::render_query_text; use sqlbuild_core::query_command::models::{QueryDisplay, QueryLayout}; use crate::bindings::_helpers::boundary::python_calls::imported; +use crate::bindings::_helpers::build_progress::terminal::{terminal_columns, tty_flag}; /// The format that always prints one record per row. pub(crate) const LONG_FORMAT: &str = "long"; @@ -13,16 +14,10 @@ const PIPED_WIDTH: usize = 120; /// The terminal's width when stdout is one, else [`PIPED_WIDTH`]. fn output_width(py: Python<'_>) -> PyResult { - let stdout: Bound<'_, PyAny> = imported(py, "sys", "stdout")?; - let interactive: bool = stdout - .call_method0("isatty") - .and_then(|answer| answer.is_truthy()) - .unwrap_or(false); - if !interactive { + if !tty_flag(&imported(py, "sys", "stdout")?).unwrap_or(false) { return Ok(PIPED_WIDTH); } - let size: Bound<'_, PyAny> = imported(py, "shutil", "get_terminal_size")?.call0()?; - size.getattr("columns")?.extract() + terminal_columns(py) } /// `display` as text: records for `--format long`, otherwise a box when it fits the output. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs index d9281ed54f..d2b239e961 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/clickhouse_calls.rs @@ -33,8 +33,7 @@ enum Cell { /// A query reply: column names and types, rows of cells, and rows read, bytes read and query ID. type QueryReply = ( - Vec, - Vec>, + (Vec, Vec>), Vec>>, (Option, Option, Option), ); @@ -176,7 +175,7 @@ pub(crate) fn query( ], action, )?; - let (columns, types, rows, (rows_read, bytes_read, query_id)): QueryReply = + let ((columns, types), rows, (rows_read, bytes_read, query_id)): QueryReply = value.extract().map_err(|error| raised(action, &error))?; Ok(( QueryResult { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs index be7ef480ea..ad12342ac4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/command.rs @@ -316,8 +316,7 @@ fn run_query(py: Python<'_>, recall: &RecallRequest) -> PyResult { let rendered = render_result( py, &result, - format, - max_bytes, + (format, max_bytes), &QueryContext { target: settings.target.as_deref(), store_path: &label, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs index b2917bada0..f2a5cd4abc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/rendering.rs @@ -114,12 +114,11 @@ fn numeric_column(result: &QueryResult, index: usize) -> bool { seen } -/// The rendered result in `format`: `json`, `long`, or `None` for the default layout. +/// The rendered result in `format`: `json`, `long`, or `None` for the default layout; text cells are cut at `max_bytes`. pub(crate) fn render_result( py: Python<'_>, result: &QueryResult, - format: Option<&str>, - max_bytes: Option, + (format, max_bytes): (Option<&str>, Option), context: &QueryContext<'_>, ) -> PyResult { if format == Some(JSON_FORMAT) { diff --git a/crates/sqlbuild-recall/src/settings/constants.rs b/crates/sqlbuild-recall/src/settings/constants.rs index 97a4ea7b67..f9776e2a04 100644 --- a/crates/sqlbuild-recall/src/settings/constants.rs +++ b/crates/sqlbuild-recall/src/settings/constants.rs @@ -19,3 +19,7 @@ pub const PROJECT_FILE_NAME: &str = "sqlbuild_project.toml"; pub const LOCAL_FILE_NAME: &str = "sqlbuild_local.toml"; /// Bytes of each output stream the local SQLite store keeps per invocation: the first and last MiB. pub const DEFAULT_SQLITE_MAX_OUTPUT_BYTES: u64 = 2 * 1024 * 1024; +/// The audit outcome of a warning-severity failure, as `audit_completed` events name it. +pub const AUDIT_WARN_OUTCOME: &str = "warn"; +/// The audit outcome of an error-severity failure, as `audit_completed` events name it. +pub const AUDIT_ERROR_OUTCOME: &str = "error"; diff --git a/crates/sqlbuild-recall/src/settings/models.rs b/crates/sqlbuild-recall/src/settings/models.rs index 89546ccfb6..4e3d6a788e 100644 --- a/crates/sqlbuild-recall/src/settings/models.rs +++ b/crates/sqlbuild-recall/src/settings/models.rs @@ -4,7 +4,9 @@ use std::path::PathBuf; use sqlbuild_config::models::ConfigValue; -use crate::settings::constants::DEFAULT_SQLITE_MAX_OUTPUT_BYTES; +use crate::settings::constants::{ + AUDIT_ERROR_OUTCOME, AUDIT_WARN_OUTCOME, DEFAULT_SQLITE_MAX_OUTPUT_BYTES, +}; /// Where Recall writes or reads. #[derive(Clone, Debug, PartialEq)] @@ -85,8 +87,8 @@ impl AuditEvidence { #[must_use] pub fn keeps(self, outcome: &str) -> bool { match self { - Self::WarnAndError => outcome == "warn" || outcome == "error", - Self::Error => outcome == "error", + Self::WarnAndError => [AUDIT_WARN_OUTCOME, AUDIT_ERROR_OUTCOME].contains(&outcome), + Self::Error => outcome == AUDIT_ERROR_OUTCOME, Self::Off => false, } } diff --git a/crates/sqlbuild-recall/src/store/_helpers/column_types.rs b/crates/sqlbuild-recall/src/store/_helpers/column_types.rs new file mode 100644 index 0000000000..55211da6b7 --- /dev/null +++ b/crates/sqlbuild-recall/src/store/_helpers/column_types.rs @@ -0,0 +1,51 @@ +//! The types `query_store` reports for a result's columns. + +use crate::records::models::RecallValue; + +/// Each column's name and declared type, lowercased; `None` for a computed column. +pub(crate) fn column_facts( + statement: &rusqlite::Statement<'_>, +) -> (Vec, Vec>) { + statement + .columns() + .iter() + .map(|column| { + ( + column.name().to_owned(), + column.decl_type().map(str::to_lowercase), + ) + }) + .unzip() +} + +/// Each column's declared type, or for a computed column the storage class its values share. +pub(crate) fn column_types( + declared: Vec>, + rows: &[Vec], +) -> Vec> { + declared + .into_iter() + .enumerate() + .map(|(index, declared)| declared.or_else(|| stored_type(rows, index))) + .collect() +} + +/// The storage class every non-NULL value of a computed column holds, as SQLite names it; `None` when they differ or all are NULL. +fn stored_type(rows: &[Vec], index: usize) -> Option { + let mut found: Option<&'static str> = None; + for value in rows.iter().filter_map(|row| row.get(index)) { + let class: &'static str = match value { + RecallValue::Null => continue, + RecallValue::Integer(_) => "integer", + RecallValue::Real(_) => "real", + RecallValue::Text(_) => "text", + RecallValue::Blob(_) => "blob", + }; + match found { + None => found = Some(class), + Some(previous) if previous == class => {} + Some(_) => return None, + } + } + found.map(str::to_owned) +} diff --git a/crates/sqlbuild-recall/src/store/_helpers/mod.rs b/crates/sqlbuild-recall/src/store/_helpers/mod.rs index c1f9469a7f..3344c9a183 100644 --- a/crates/sqlbuild-recall/src/store/_helpers/mod.rs +++ b/crates/sqlbuild-recall/src/store/_helpers/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod column_types; pub(crate) mod identity; pub(crate) mod quarantine; pub(crate) mod rows; diff --git a/crates/sqlbuild-recall/src/store/main/query_store.rs b/crates/sqlbuild-recall/src/store/main/query_store.rs index edf8e8eb8e..801e4bcb10 100644 --- a/crates/sqlbuild-recall/src/store/main/query_store.rs +++ b/crates/sqlbuild-recall/src/store/main/query_store.rs @@ -1,6 +1,6 @@ //! `query_store`: run one read-only query against a store. -use crate::records::models::RecallValue; +use crate::store::_helpers::column_types::{column_facts, column_types}; use crate::store::_helpers::sqlite::{checked, from_sql}; use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::models::{QueryResult, SqliteStore}; @@ -23,16 +23,7 @@ pub fn query_store( "Recall queries must be read-only SELECT statements", )); } - let columns: Vec = statement - .column_names() - .into_iter() - .map(str::to_owned) - .collect(); - let declared: Vec> = statement - .columns() - .iter() - .map(|column| column.decl_type().map(str::to_lowercase)) - .collect(); + let (columns, declared): (Vec, Vec>) = column_facts(&statement); let width: usize = columns.len(); let mut rows = checked(statement.query([]), "run Recall query")?; let mut result = QueryResult { @@ -57,30 +48,6 @@ pub fn query_store( } result.rows.push(values); } - result.types = declared - .into_iter() - .enumerate() - .map(|(index, declared)| declared.or_else(|| stored_type(&result.rows, index))) - .collect(); + result.types = column_types(declared, &result.rows); Ok(result) } - -/// The storage class every non-NULL value of a computed column holds, as SQLite names it; `None` when they differ or all are NULL. -fn stored_type(rows: &[Vec], index: usize) -> Option { - let mut found: Option<&'static str> = None; - for value in rows.iter().filter_map(|row| row.get(index)) { - let class: &'static str = match value { - RecallValue::Null => continue, - RecallValue::Integer(_) => "integer", - RecallValue::Real(_) => "real", - RecallValue::Text(_) => "text", - RecallValue::Blob(_) => "blob", - }; - match found { - None => found = Some(class), - Some(previous) if previous == class => {} - Some(_) => return None, - } - } - found.map(str::to_owned) -} diff --git a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs index b1bb6a236e..54c22db4f2 100644 --- a/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs +++ b/crates/sqlbuild-recall/src/writer/_helpers/materialize.rs @@ -39,6 +39,31 @@ pub(crate) struct Built { pub(crate) extra_artifacts: u64, } +impl Built { + /// Account for one event's row and the artifact rows of its evidence. + fn add_event( + &mut self, + stream: DeliveryStream, + (row, evidence_rows): ( + Result, + Option>, + ), + ) { + if let Some(rows) = evidence_rows { + self.rows.extend(rows); + self.written.push(DeliveryStream::Artifacts); + self.extra_artifacts += 1; + } + match row { + Ok(row) => { + self.rows.push(row); + self.written.push(stream); + } + Err(_) => self.unbuildable.push(stream), + } + } +} + /// Builds rows for one session; remembers which SQL and references it has stored. #[derive(Debug)] pub(crate) struct Materializer { @@ -132,70 +157,67 @@ impl Materializer { }) } - /// The row of `event`; an `audit_completed` event's evidence rows leave its payload and, when the outcome keeps them, become an artifact its `sample_digest` names. + /// The row of `event`, then the artifact rows of its evidence: an `audit_completed` event's evidence rows leave its payload and, when the outcome keeps them, become an artifact its `sample_digest` names. fn event_rows( &mut self, - mut event: Map, - built: &mut Built, - ) -> Result { - if let Some(artifact) = self.take_evidence(&mut event) - && let Some(rows) = self.artifact_rows(artifact) - { - built.rows.extend(rows); - built.written.push(DeliveryStream::Artifacts); - built.extra_artifacts += 1; - } - event_object_row(&self.envelope, &event) + event: Map, + ) -> ( + Result, + Option>, + ) { + let (event, artifact): (Map, Option) = + self.take_evidence(event); + let evidence_rows: Option> = + artifact.and_then(|artifact| self.artifact_rows(artifact)); + (event_object_row(&self.envelope, &event), evidence_rows) } - fn take_evidence(&self, event: &mut Map) -> Option { - if event.get("event_type").and_then(Value::as_str) != Some(AUDIT_COMPLETED_EVENT) { - return None; - } - let resource_id: Option = event - .get("resource_id") - .and_then(Value::as_str) - .map(str::to_owned); - let event_id: String = event - .get("event_id") - .and_then(Value::as_str) - .unwrap_or_default() - .to_owned(); - let seen_at: String = event - .get("occurred_at") - .and_then(Value::as_str) - .unwrap_or_default() - .to_owned(); - let payload: &mut Map = event.get_mut("payload")?.as_object_mut()?; - let evidence: Value = payload.remove(EVIDENCE_PAYLOAD_FIELD)?; - let outcome: &str = payload - .get("outcome") - .and_then(Value::as_str) - .unwrap_or_default(); - if !self.audit_evidence.keeps(outcome) { - return None; - } - let content: Vec = serde_json::to_vec(&evidence).ok()?; + /// `event` without its evidence rows, and the artifact holding them when its outcome keeps them. + fn take_evidence( + &self, + mut event: Map, + ) -> (Map, Option) { + let evidence: Option = + if event.get("event_type").and_then(Value::as_str) == Some(AUDIT_COMPLETED_EVENT) { + event + .get_mut("payload") + .and_then(Value::as_object_mut) + .and_then(|payload| payload.remove(EVIDENCE_PAYLOAD_FIELD)) + } else { + None + }; + let text = |field: &str| -> Option { + event.get(field).and_then(Value::as_str).map(str::to_owned) + }; + let outcome: Option<&str> = event + .get("payload") + .and_then(|payload| payload.get("outcome")) + .and_then(Value::as_str); + let Some(evidence) = + evidence.filter(|_| self.audit_evidence.keeps(outcome.unwrap_or_default())) + else { + return (event, None); + }; let mut artifact: ArtifactInput = artifact_input( - content, + evidence.to_string().into_bytes(), ( AUDIT_EVIDENCE_KIND, AUDIT_EVIDENCE_MEDIA_TYPE, AUDIT_EVIDENCE_ROLE, ), - (None, resource_id), - &seen_at, + (None, text("resource_id")), + &text("occurred_at").unwrap_or_default(), ); - artifact.ref_id = event_id; - let digest: &str = artifact + artifact.ref_id = text("event_id").unwrap_or_default(); + let digest: String = artifact .digest .strip_prefix(ARTIFACT_DIGEST_PREFIX) - .unwrap_or(&artifact.digest); - payload.insert( - SAMPLE_DIGEST_PAYLOAD_FIELD.to_owned(), - Value::from(digest.to_owned()), - ); - Some(artifact) + .unwrap_or(&artifact.digest) + .to_owned(); + if let Some(payload) = event.get_mut("payload").and_then(Value::as_object_mut) { + payload.insert(SAMPLE_DIGEST_PAYLOAD_FIELD.to_owned(), Value::from(digest)); + } + (event, Some(artifact)) } /// The rows of `batch`, with each record's stream sorted by how it is accounted; lifecycle replacements see their events in publication order. @@ -223,13 +245,7 @@ impl Materializer { built.rows.extend(rows); built.written.push(stream); } - Payload::Event(event) => match self.event_rows(event, &mut built) { - Ok(row) => { - built.rows.push(row); - built.written.push(stream); - } - Err(_) => built.unbuildable.push(stream), - }, + Payload::Event(event) => built.add_event(stream, self.event_rows(event)), Payload::Pending(sequence, pending) => pending_events.push((sequence, pending)), Payload::Sql(input) => match self.sql_rows(input) { Some(rows) => { @@ -250,13 +266,7 @@ impl Materializer { ReplacedEvents::Published(events) => { built.extra_events += events.len() as u64 - 1; for event in events { - match self.event_rows(event, &mut built) { - Ok(row) => { - built.rows.push(row); - built.written.push(stream); - } - Err(_) => built.unbuildable.push(stream), - } + built.add_event(stream, self.event_rows(event)); } } ReplacedEvents::Failed => built.unbuildable.push(stream), diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index bb26531884..97a319ea81 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -628,10 +628,7 @@ fn audit_event(index: usize, outcome: &str) -> serde_json::Map event, - _ => serde_json::Map::new(), - } + serde_json::from_value(event).expect("the audit event is an object") } /// Per audit outcome: whether the stored event still holds its evidence, its `sample_digest`, and the artifact text that digest names, after a session keeping `audit_evidence`. @@ -681,3 +678,39 @@ pub(crate) fn failed_statement_sql() -> Vec> { .expect("the query is valid") .rows } + +pub(crate) fn recall_text(value: &str) -> RecallValue { + RecallValue::Text(value.to_owned()) +} + +/// The digest of `[{"order_id":}]` without its prefix. +fn evidence_digest(index: u8) -> RecallValue { + let content: String = format!("[{{\"order_id\":{index}}}]"); + recall_text(&crate::records::main::sha256_hex::sha256_hex( + content.as_bytes(), + )) +} + +/// The [`stored_evidence`] row of an audit whose one evidence row was kept as an artifact. +pub(crate) fn kept_evidence(outcome: &str, index: u8) -> Vec { + vec![ + recall_text(outcome), + RecallValue::Integer(0), + evidence_digest(index), + RecallValue::Integer(1), + recall_text("audit_evidence"), + recall_text(&format!("[{{\"order_id\":{index}}}]")), + ] +} + +/// The [`stored_evidence`] row of an audit whose evidence was not kept. +pub(crate) fn left_out_evidence(outcome: &str) -> Vec { + vec![ + recall_text(outcome), + RecallValue::Integer(0), + RecallValue::Null, + RecallValue::Integer(1), + RecallValue::Null, + RecallValue::Null, + ] +} diff --git a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs index 01c731d561..9b236a0322 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_artifacts.rs @@ -1,74 +1,60 @@ use crate::records::models::RecallValue; use crate::settings::models::AuditEvidence; -use crate::writer::tests::helpers::{failed_statement_sql, stored_evidence}; - -fn text(value: &str) -> RecallValue { - RecallValue::Text(value.to_owned()) -} - -/// The digest of `[{"order_id":}]` without its prefix. -fn evidence_digest(index: u8) -> RecallValue { - let content: String = format!("[{{\"order_id\":{index}}}]"); - text(&crate::records::main::sha256_hex::sha256_hex( - content.as_bytes(), - )) -} - -fn kept(outcome: &str, index: u8) -> Vec { - vec![ - text(outcome), - RecallValue::Integer(0), - evidence_digest(index), - RecallValue::Integer(1), - text("audit_evidence"), - text(&format!("[{{\"order_id\":{index}}}]")), - ] -} - -fn left_out(outcome: &str) -> Vec { - vec![ - text(outcome), - RecallValue::Integer(0), - RecallValue::Null, - RecallValue::Integer(1), - RecallValue::Null, - RecallValue::Null, - ] -} +use crate::writer::tests::helpers::{ + failed_statement_sql, kept_evidence, left_out_evidence, recall_text, stored_evidence, +}; +use crate::writer::tests::test_types::{AuditEvidenceTestCase, FailedStatementTestCase}; #[test] fn given_audit_evidence_settings_when_audits_complete_then_kept_evidence_is_an_artifact_and_none_stays_in_the_event() { let test_cases = [ - ( - AuditEvidence::WarnAndError, - vec![kept("error", 1), kept("warn", 0)], - ), - ( - AuditEvidence::Error, - vec![kept("error", 1), left_out("warn")], - ), - ( - AuditEvidence::Off, - vec![left_out("error"), left_out("warn")], - ), + AuditEvidenceTestCase { + description: "warn_and_error keeps both outcomes' rows", + audit_evidence: AuditEvidence::WarnAndError, + expected_rows: vec![kept_evidence("error", 1), kept_evidence("warn", 0)], + }, + AuditEvidenceTestCase { + description: "error keeps only the failing audit's rows", + audit_evidence: AuditEvidence::Error, + expected_rows: vec![kept_evidence("error", 1), left_out_evidence("warn")], + }, + AuditEvidenceTestCase { + description: "off keeps no rows", + audit_evidence: AuditEvidence::Off, + expected_rows: vec![left_out_evidence("error"), left_out_evidence("warn")], + }, ]; - for (setting, expected) in test_cases { - assert_eq!(stored_evidence(setting), expected, "{setting:?}"); + for test_case in test_cases { + assert_eq!( + stored_evidence(test_case.audit_evidence), + test_case.expected_rows, + "{}", + test_case.description + ); } } #[test] fn given_statements_sharing_sql_and_a_failed_one_when_recorded_then_text_is_stored_once_and_every_statement_is_referenced() { - assert_eq!( - failed_statement_sql(), - vec![vec![ + let test_cases = [FailedStatementTestCase { + description: "two texts, three references, no conflict", + expected_rows: vec![vec![ RecallValue::Integer(2), RecallValue::Integer(3), RecallValue::Integer(0), - text("s1,s2,s3"), - ]] - ); + recall_text("s1,s2,s3"), + ]], + }]; + + for test_case in test_cases { + assert_eq!( + failed_statement_sql(), + test_case.expected_rows, + "{}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 67ab3e837c..ad776499f5 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -3,6 +3,7 @@ use std::time::{Duration, Instant}; use crate::records::models::{OutputChunk, OutputStream, RecallRow, RecallValue, RowEnvelope}; use crate::schema::models::Table; +use crate::settings::models::AuditEvidence; use crate::store::errors::{StoreError, StoreErrorKind}; use crate::store::models::InsertOutcome; use crate::store::types::{RecallStore, RecallStoreOpener}; @@ -349,3 +350,14 @@ impl RecallStoreOpener for CreatingStore { Duration::from_millis(10) } } + +pub(crate) struct AuditEvidenceTestCase { + pub(crate) description: &'static str, + pub(crate) audit_evidence: AuditEvidence, + pub(crate) expected_rows: Vec>, +} + +pub(crate) struct FailedStatementTestCase { + pub(crate) description: &'static str, + pub(crate) expected_rows: Vec>, +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests.rs b/crates/sqlbuild-rules/src/sql_lint/tests.rs index d1ff27056d..dde04c7550 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests.rs @@ -20,6 +20,8 @@ mod idempotency_reprint; mod layout_rules; #[path = "tests/test_line_wrap.rs"] mod line_wrap; +#[path = "tests/test_lint_exemptions.rs"] +mod lint_exemptions; #[path = "tests/test_set_operation_formatter.rs"] mod set_operation_formatter; #[path = "tests/test_sql_lint.rs"] diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs new file mode 100644 index 0000000000..b058d8ee14 --- /dev/null +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs @@ -0,0 +1,96 @@ +use serde_json::{Value, json}; + +use crate::sql_lint::main::engine::lint_json; +use crate::sql_lint::tests::test_types::{ + DynamicOutputStarLintTestCase, QualificationExemptLintTestCase, +}; + +#[test] +fn given_exempt_macro_output_when_linting_qualification_then_only_authored_columns_are_judged() +-> Result<(), String> { + let test_cases = [ + QualificationExemptLintTestCase { + description: "a caller's bare column beside qualified macro output passes", + sql: "SELECT order_id, orders.amount - orders.discount AS net FROM orders", + exempt: "orders.amount - orders.discount", + expected_count: 0, + }, + QualificationExemptLintTestCase { + description: "the caller's own mix still fails", + sql: "SELECT order_id, orders.status, orders.amount - orders.discount AS net FROM orders", + exempt: "orders.amount - orders.discount", + expected_count: 1, + }, + ]; + for test_case in test_cases { + let start: usize = test_case + .sql + .find(test_case.exempt) + .ok_or("exempt text should be in the SQL")?; + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "duckdb", + "enabled_rules": ["SQBRSQL028"], + "qualification_exempt_spans": [[start, start + test_case.exempt.len()]], + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + } + Ok(()) +} + +#[test] +fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() +-> Result<(), String> { + let test_cases = [DynamicOutputStarLintTestCase { + description: "unrelated CTE star remains controlled", + sql: "WITH pivot_input AS (SELECT customer_id, category, amount FROM orders), \ +final AS (SELECT * FROM pivot_input PIVOT(MAX(amount) FOR category IN (ANY))), \ +unrelated AS (SELECT * FROM pivot_input) SELECT * FROM final", + expected_count: 1, + expected_prefix: "unrelated AS (SELECT ", + }]; + for test_case in test_cases { + let response = lint_json( + &json!({ + "version": 1, + "sql": test_case.sql, + "dialect": "snowflake", + "enabled_rules": ["SQBRSQL021"], + "allows_dynamic_output_star": true, + }) + .to_string(), + )?; + let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; + let diagnostics = payload["diagnostics"] + .as_array() + .ok_or("diagnostics should be an array")?; + + assert_eq!( + diagnostics.len(), + test_case.expected_count, + "{}", + test_case.description + ); + let start = diagnostics[0]["start"].as_u64().unwrap_or_default() as usize; + assert!( + test_case.sql[..start].ends_with(test_case.expected_prefix), + "{}", + test_case.description + ); + } + Ok(()) +} diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs index e714c9db4c..a71418df82 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_sql_lint.rs @@ -1596,96 +1596,6 @@ fn given_additional_rule_cases_when_linting_then_findings_and_fixes_match() -> R Ok(()) } -#[test] -fn given_exempt_macro_output_when_linting_qualification_then_only_authored_columns_are_judged() --> Result<(), String> { - let test_cases = [ - test_types::QualificationExemptLintTestCase { - description: "a caller's bare column beside qualified macro output passes", - sql: "SELECT order_id, orders.amount - orders.discount AS net FROM orders", - exempt: "orders.amount - orders.discount", - expected_count: 0, - }, - test_types::QualificationExemptLintTestCase { - description: "the caller's own mix still fails", - sql: "SELECT order_id, orders.status, orders.amount - orders.discount AS net FROM orders", - exempt: "orders.amount - orders.discount", - expected_count: 1, - }, - ]; - for test_case in test_cases { - let start: usize = test_case - .sql - .find(test_case.exempt) - .ok_or("exempt text should be in the SQL")?; - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "duckdb", - "enabled_rules": ["SQBRSQL028"], - "qualification_exempt_spans": [[start, start + test_case.exempt.len()]], - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or("diagnostics should be an array")?; - - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - } - Ok(()) -} - -#[test] -fn given_compiler_proven_dynamic_output_when_linting_then_only_terminal_star_is_exempt() --> Result<(), String> { - let test_cases = [test_types::DynamicOutputStarLintTestCase { - description: "unrelated CTE star remains controlled", - sql: "WITH pivot_input AS (SELECT customer_id, category, amount FROM orders), \ -final AS (SELECT * FROM pivot_input PIVOT(MAX(amount) FOR category IN (ANY))), \ -unrelated AS (SELECT * FROM pivot_input) SELECT * FROM final", - expected_count: 1, - expected_prefix: "unrelated AS (SELECT ", - }]; - for test_case in test_cases { - let response = lint_json( - &json!({ - "version": 1, - "sql": test_case.sql, - "dialect": "snowflake", - "enabled_rules": ["SQBRSQL021"], - "allows_dynamic_output_star": true, - }) - .to_string(), - )?; - let payload: Value = serde_json::from_str(&response).map_err(|error| error.to_string())?; - let diagnostics = payload["diagnostics"] - .as_array() - .ok_or("diagnostics should be an array")?; - - assert_eq!( - diagnostics.len(), - test_case.expected_count, - "{}", - test_case.description - ); - let start = diagnostics[0]["start"].as_u64().unwrap_or_default() as usize; - assert!( - test_case.sql[..start].ends_with(test_case.expected_prefix), - "{}", - test_case.description - ); - } - Ok(()) -} - #[test] fn given_ceremonial_cte_contexts_when_linting_then_only_harness_dependencies_are_reachable() -> Result<(), String> { diff --git a/fensu.toml b/fensu.toml index b554e3acbe..ce88c1a8fd 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1644,7 +1644,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=23", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=25", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1680,12 +1680,13 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/check/check_core.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", - "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=19", - "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", + "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", + "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=30", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", + "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", @@ -1724,7 +1725,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=24", + "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=25", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=3", @@ -1779,7 +1780,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/context.rs=7", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/health.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/ledger.rs=13", - "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=22", + "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/snowflake.rs=23", "crates/sqlbuild-python/src/bindings/_helpers/cost_telemetry/values.rs=9", "crates/sqlbuild-python/src/bindings/_helpers/cursor/conversion.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/cursor/durations.rs=2", @@ -2082,8 +2083,9 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", + "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", + "crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", @@ -2264,6 +2266,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=12", + "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=3", @@ -2650,6 +2653,7 @@ module_layers = [ "project_assembly=compile", "python_values=infra", "query_command=commands", + "query_command/text_output=output", "recall=output", "recall/command=commands", "record_behaviours=registry", diff --git a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py index 9f3d55ec7c..7662f86184 100644 --- a/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py +++ b/src/sqlbuild/adapters/clickhouse/_helpers/recall_store.py @@ -118,7 +118,7 @@ def run_recall_query( *, handle: int, sql: str, settings: dict[str, int], row_cap: int | None ) -> tuple[ RecallFailure, - tuple[list[str], list[str | None], list[list[RecallCell]], RecallStatistics] | None, + tuple[tuple[list[str], list[str | None]], list[list[RecallCell]], RecallStatistics] | None, ]: """Columns and their types, at most `row_cap` rows of plain values, and server statistics.""" @@ -148,7 +148,7 @@ def run_recall_query( _number(summary=summary, key="read_bytes"), summary.get("query_id"), ) - return None, (columns, types, rows[:row_cap] if row_cap is not None else rows, statistics) + return None, ((columns, types), rows[:row_cap] if row_cap is not None else rows, statistics) def run_recall_command(*, handle: int, sql: str) -> tuple[RecallFailure, None]: diff --git a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py index 19bf8a78bb..fb44ee1b78 100644 --- a/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py +++ b/src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py @@ -149,6 +149,7 @@ BASE_TABLE_METADATA_TYPE, CURRENT_DATABASE_ATTRIBUTE, EXTERNAL_BROWSER_AUTHENTICATOR, + FIXED_POINT_RESULT_TYPE, MAX_STATEMENT_TIMEOUT_SECONDS, MFA_AUTHENTICATOR, OAUTH_AUTHORIZATION_CODE_AUTHENTICATOR, @@ -1924,7 +1925,7 @@ def _result_column_type(column: Any) -> str | None: type_name: str | None = FIELD_ID_TO_NAME.get(column[1]) if type_name is None: return None - if type_name != "FIXED": + if type_name != FIXED_POINT_RESULT_TYPE: return type_name precision: int | None = column[4] scale: int | None = column[5] diff --git a/src/sqlbuild/adapters/snowflake/constants.py b/src/sqlbuild/adapters/snowflake/constants.py index 1ae48c46af..e5f457a610 100644 --- a/src/sqlbuild/adapters/snowflake/constants.py +++ b/src/sqlbuild/adapters/snowflake/constants.py @@ -46,3 +46,4 @@ DEPENDENCY_VIEW_DOMAINS: tuple[str, ...] = ("VIEW", "MATERIALIZED VIEW", "SECURE VIEW") QUOTED_IDENTIFIER_MARK: str = '"' NAME_SEPARATOR: str = "." +FIXED_POINT_RESULT_TYPE: str = "FIXED" diff --git a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py index 9ddadf930e..eb794a9f6d 100644 --- a/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py +++ b/src/sqlbuild/runtime/observability/classes/statement_lifecycle.py @@ -351,10 +351,7 @@ def _failure_handled(self, *, error: BaseException) -> bool: is_handled: Callable[[BaseException], bool] | None = self._is_handled if is_handled is None: return False - try: - return bool(is_handled(error)) - except Exception: - return False + return bool(is_handled(error)) def _stop_monitor(self) -> str | None: monitor: StatementMonitor | None = self._monitor diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index e69977b35f..d04bf7b97c 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -38,6 +38,19 @@ STAGE_CAPTURE_SUFFIX: str = ".json" E2E_FIXTURES_DIR: Path = Path(__file__).resolve().parents[6] / "e2e" / "fixtures" +_WEEKLY_ORDERS: str = ( + 'MODEL (\n description "Weekly orders per product",\n materialized table,\n);\n\n' + "SELECT 1 AS product_id, CAST(10 AS BIGINT) AS orders_placed, " + "CAST(1 AS BIGINT) AS refund_count\n" +) +_REFUND_RATES: str = ( + 'MODEL (\n description "Weekly refund rate per product",\n materialized table,\n);\n\n' + 'WITH weekly AS (SELECT * FROM __ref("weekly_orders")),\n' + "@sql.weekly_rates(weekly, weekly.keys := (product_id), " + "weekly.order_count := orders_placed, __cte := by_product)\n" + "final AS (SELECT * FROM by_product)\n" + "SELECT * FROM final\n" +) def staging_directories(target_dir: Path) -> list[str]: @@ -1214,3 +1227,35 @@ def write_enum_project(*, root: Path, model_sql: str, contract: str) -> None: " columns (\n order_id (type INTEGER),\n status (type order_status),\n ),\n);\n\n" f"{model_sql}" ) + + +def _weekly_rates(first_step: str) -> str: + return ( + 'MACRO (\n description "Weekly refund rate per grain",\n' + " arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)),\n" + ");\n\n" + f"WITH counted AS (\n{first_step}\n),\n\n" + "rated AS (\n" + " SELECT\n" + " @weekly.keys,\n" + " SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n" + " FROM counted\n" + " GROUP BY @weekly.keys\n" + ")\n" + ) + + +def refund_rates_project(first_step: str) -> dict[str, str]: + """Weekly orders, refund-rate models and the SQL macro whose first CTE is `first_step`.""" + + return { + "models/weekly_orders.sql": _WEEKLY_ORDERS, + "models/refund_rates.sql": _REFUND_RATES, + "models/_macros/sql/weekly_rates.sql": _weekly_rates(first_step), + } + + +def open_orders_model(sql: str) -> str: + """An `open_orders` table model whose query is `sql`.""" + + return f'MODEL (\n description "Open orders",\n materialized table,\n);\n\n{sql}\n' diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index c971f9a58b..f7314304ce 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -10,6 +10,7 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( RuleFailureArtifactsTestCase, ) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import open_orders_model _PROJECT: str = ( 'name = "shop"\n' @@ -30,10 +31,6 @@ _HEADER: str = "-- sqb: compile failed for this model; not buildable\n" -def _open_orders(sql: str) -> str: - return f'MODEL (\n description "Open orders",\n materialized table,\n);\n\n{sql}\n' - - @pytest.mark.parametrize( "test_case", [ @@ -66,7 +63,7 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu files: dict[str, str] = { "sqlbuild_project.toml": _PROJECT, "models/orders.sql": _ORDERS, - "models/open_orders.sql": _open_orders( + "models/open_orders.sql": open_orders_model( 'SELECT o.order_id, status FROM __ref("orders") AS o' ), } @@ -83,7 +80,7 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu ] _ = main(compile_args) _ = (tmp_path / "models" / "open_orders.sql").write_text( - _open_orders(test_case.open_orders_sql), encoding="utf-8" + open_orders_model(test_case.open_orders_sql), encoding="utf-8" ) (tmp_path / "target" / "manifest.json").unlink(missing_ok=True) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py index ab7738ffad..b7e39b835e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py @@ -10,6 +10,7 @@ from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( MacroSlotQualificationTestCase, ) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import refund_rates_project _PROJECT: str = ( 'name = "shop"\n' @@ -23,43 +24,6 @@ "[rules]\n" 'select = ["SQBRSQL028"]\n' ) -_WEEKLY_ORDERS: str = ( - 'MODEL (\n description "Weekly orders per product",\n materialized table,\n);\n\n' - "SELECT 1 AS product_id, CAST(10 AS BIGINT) AS orders_placed, " - "CAST(1 AS BIGINT) AS refund_count\n" -) -_REFUND_RATES: str = ( - 'MODEL (\n description "Weekly refund rate per product",\n materialized table,\n);\n\n' - 'WITH weekly AS (SELECT * FROM __ref("weekly_orders")),\n' - "@sql.weekly_rates(weekly, weekly.keys := (product_id), " - "weekly.order_count := orders_placed, __cte := by_product)\n" - "final AS (SELECT * FROM by_product)\n" - "SELECT * FROM final\n" -) - - -def _weekly_rates(first_step: str) -> str: - return ( - 'MACRO (\n description "Weekly refund rate per grain",\n' - " arguments (weekly RELATION (keys COLUMNS, order_count BIGINT, refund_count BIGINT)),\n" - ");\n\n" - f"WITH counted AS (\n{first_step}\n),\n\n" - "rated AS (\n" - " SELECT\n" - " @weekly.keys,\n" - " SUM(refunds) / NULLIF(SUM(orders), 0) AS refund_rate\n" - " FROM counted\n" - " GROUP BY @weekly.keys\n" - ")\n" - ) - - -def _rates_project(first_step: str) -> dict[str, str]: - return { - "models/weekly_orders.sql": _WEEKLY_ORDERS, - "models/refund_rates.sql": _REFUND_RATES, - "models/_macros/sql/weekly_rates.sql": _weekly_rates(first_step), - } @pytest.mark.parametrize( @@ -67,7 +31,7 @@ def _rates_project(first_step: str) -> dict[str, str]: [ MacroSlotQualificationTestCase( description="a first step written in plain slot reads passes", - files=_rates_project( + files=refund_rates_project( " SELECT\n" " @weekly.keys,\n" " @weekly.order_count AS orders,\n" @@ -80,7 +44,7 @@ def _rates_project(first_step: str) -> dict[str, str]: ), MacroSlotQualificationTestCase( description="aliased slot reads beside a plain group read fail once, at the macro", - files=_rates_project( + files=refund_rates_project( " SELECT\n" " @weekly.keys,\n" " s.@weekly.order_count AS orders,\n" diff --git a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py index 7c471f3ca7..04fadde417 100644 --- a/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py +++ b/tests/unit/src/sqlbuild/cost/test_snowflake_scale.py @@ -278,9 +278,12 @@ def test_given_a_history_read_timeout_when_collecting_then_only_retried_reads_ar attempts=1, ) - assert { - event.payload.get("outcome") for event in events if event.event_type == "statement_failed" - } == test_case.expected_failure_outcomes + failures: list[LifecycleEvent] = list( + filter(lambda event: event.event_type == "statement_failed", events) + ) + assert {event.payload.get("outcome") for event in failures} == ( + test_case.expected_failure_outcomes + ) class _ClosingHistory(_FakeQueryHistory): From 8f5d58584f152b8a1d1663a386e08c9c27327f58 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:18:08 +0000 Subject: [PATCH 13/30] test: follow the batch's behaviour changes in existing tests Expect DuckDB result column types, the long query format, the compiled SQL a Rules failure now writes, and keep the statement lifecycle host entry as a dead-code root. --- fensu.toml | 5 +++++ .../src/sqlbuild/adapters/duckdb/test_client.py | 5 ++++- .../cli/commands/main/test_compile_prepared_artifacts.py | 9 +++++++-- .../src/sqlbuild/cli/commands/main/entry/test_main.py | 4 ++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/fensu.toml b/fensu.toml index ce88c1a8fd..d05e32d057 100644 --- a/fensu.toml +++ b/fensu.toml @@ -812,6 +812,11 @@ modules = ["sqlbuild.runtime.observability.classes.operation_lifecycle"] symbols = ["OperationLifecycle"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.observability.classes.statement_lifecycle"] +symbols = ["StatementLifecycle"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.runtime.observability.exceptions"] symbols = ["ObservabilityValidationError"] diff --git a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py index d79ab7d5b3..42f5af9db4 100644 --- a/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/duckdb/test_client.py @@ -259,13 +259,16 @@ def test_given_config_with_settings_when_connecting_then_applies_settings( columns=("id", "name"), rows=((1, "alice"),), truncated=True, + column_types=("INTEGER", "VARCHAR"), ), ), QueryTestCase( description="returns all rows without limit", sql="SELECT * FROM (VALUES (1), (2)) AS t(id) ORDER BY id", limit=None, - expected_result=QueryResult(columns=("id",), rows=((1,), (2,))), + expected_result=QueryResult( + columns=("id",), rows=((1,), (2,)), column_types=("INTEGER",) + ), ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py index e7f1d438b3..a31ce8a271 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_prepared_artifacts.py @@ -233,7 +233,13 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu assert main(arguments) == 1 failed: dict[str, object] = json.loads(capsys.readouterr().out) assert failed["has_errors"] is True - assert {path: path.read_bytes() for path in before} == before + changed: Path = compiled / "models" / "orders_000.sql" + assert {path: path.read_bytes() for path in before} == { + **before, + changed: b"-- sqb: compile failed for this model; not buildable\n" + b"-- XSQBRARCH001 at models/orders_000.sql:1:1: Model rejected\n" + b"SELECT CAST(2 AS INTEGER) AS order_id\n", + } assert stale.exists() assert staging_directories(tmp_path / "target") == [] @@ -242,7 +248,6 @@ def test_given_existing_artifacts_when_rules_fail_then_staged_changes_are_not_pu final: dict[str, object] = json.loads(capsys.readouterr().out) assert final["has_errors"] is False assert not stale.exists() - changed: Path = compiled / "models" / "orders_000.sql" for path in modified_times.keys() - {changed}: assert path.stat().st_mtime_ns == modified_times[path] assert path.read_bytes() == before[path] diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 2db7418051..c9186d4175 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -1190,7 +1190,7 @@ def run_scenario_capture(request: ScenarioCaptureCommandRequest) -> int: "query", "select 1 as id", "--format", - "table", + "long", "--limit", "5", ], @@ -1222,7 +1222,7 @@ def run_query( ) assert exit_code == test_case.expected_exit_code - assert received_args == [(None, "select 1 as id", None, "table", 5)] + assert received_args == [(None, "select 1 as id", None, "long", 5)] @pytest.mark.parametrize( From 4516bea5bf4044a64cd0323591f1e30ef9579646 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:00:32 +0000 Subject: [PATCH 14/30] style(rules): pass the slot-read predicate directly --- crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs index 3ea2f3aacd..828448edc7 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs @@ -730,10 +730,7 @@ fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { .to_ascii_lowercase() .starts_with(SQL_MACRO_PARAMETER_SENTINEL_PREFIX) }; - is_parameter(&tokens[index]) - && tokens - .get(index + 2) - .is_some_and(|column| is_parameter(column)) + is_parameter(&tokens[index]) && tokens.get(index + 2).is_some_and(is_parameter) } fn simple_qualifier( From 97672e44a5547229b993002de77916ff80d9a9f2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:05:05 +0000 Subject: [PATCH 15/30] fix(recall): keep the interrupt as the result when recording an interrupted setup fails --- .../src/bindings/_helpers/cli/entry.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index e007ed68f3..6dc1e8e49d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -640,12 +640,15 @@ fn observed_dispatch<'py>( (Err(error), Some(session)) if !announced.get() && interrupting_signal(py, error).is_some() => { - record_interrupted_setup( - py, - session, - (&started_at, started_payload()?), - failed_payload(error)?, - )?; + let recorded: PyResult<()> = started_payload().and_then(|started| { + record_interrupted_setup( + py, + session, + (&started_at, started), + failed_payload(error)?, + ) + }); + drop(recorded); } _ => {} } From 29ad54c266cf8fdda25ef98e1e31913bac34ef35 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:47:28 +0000 Subject: [PATCH 16/30] fix(recall): stop on SIGTERM by cancelling running statements from a native handler A native SIGTERM handler sets a flag; a watcher cancels every running statement and interrupts the main thread the way Ctrl+C does. A statement error after SIGTERM becomes the termination, so the run ends cancelled with exit 143 instead of a failed model. A second SIGTERM ends the process at once. --- Cargo.lock | 23 ++ Cargo.toml | 5 + crates/sqlbuild-build-features/Cargo.toml | 7 + .../src/entries/_helpers/executor.rs | 5 + crates/sqlbuild-host/src/entries/models.rs | 1 + crates/sqlbuild-python/Cargo.toml | 1 + .../src/bindings/_helpers/cli/entry.rs | 21 +- .../src/bindings/_helpers/cli/termination.rs | 222 ++++++++++++++---- fensu.toml | 5 + .../docs/concepts/observability/recall.md | 5 +- src/sqlbuild/_native.pyi | 1 + src/sqlbuild/_native_host.py | 4 + .../adapter/contract/classes/connection.py | 5 + .../execution_limits/_helpers/running.py | 5 + .../main/_cancel_running_statements.py | 14 ++ .../main/tracked_statement_connection.py | 5 +- .../src/sqlbuild/cli/recall/_test_types.py | 8 + .../src/sqlbuild/cli/recall/helpers.py | 34 ++- .../cli/recall/test_recall_run_facts.py | 64 ++++- .../docs/concepts/observability/recall.mdx | 5 +- 20 files changed, 369 insertions(+), 71 deletions(-) create mode 100644 src/sqlbuild/runtime/execution_limits/_helpers/running.py create mode 100644 src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py diff --git a/Cargo.lock b/Cargo.lock index e76a2dcf85..25a14905cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1175,6 +1175,26 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "siphasher" version = "1.0.4" @@ -1225,12 +1245,14 @@ dependencies = [ "arrayvec", "bstr", "either", + "errno", "memchr", "once_cell", "regex-automata", "syn 2.0.119", "syn 3.0.3", "thiserror 2.0.21", + "windows-sys", ] [[package]] @@ -1350,6 +1372,7 @@ dependencies = [ "rustc-hash", "serde", "serde_json", + "signal-hook", "sqlbuild-analysis", "sqlbuild-attachments", "sqlbuild-cache", diff --git a/Cargo.toml b/Cargo.toml index e5ac97c0b0..84be0319c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,6 +105,11 @@ ruff_text_size = "=0.0.4" # Unix only: `sqb` shares the compute log prune lock with Python's `filelock` through `flock`. # Fast deterministic hasher for hot in-memory maps whose iteration order never reaches output. rustc-hash = "2" +# The native SIGTERM handler: sets a flag the CLI watches, without unsafe code in the workspace. +signal-hook = { version = "0.3", default-features = false } +# Features signal-hook's registry enables, matched in the sqb build through sqlbuild-build-features. +errno = "0.3" +windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } rustix = { version = "1.1.4", default-features = false, features = ["fs", "std", "termios"] } diff --git a/crates/sqlbuild-build-features/Cargo.toml b/crates/sqlbuild-build-features/Cargo.toml index 84a5d1ee63..949dead6d8 100644 --- a/crates/sqlbuild-build-features/Cargo.toml +++ b/crates/sqlbuild-build-features/Cargo.toml @@ -22,6 +22,13 @@ once_cell.workspace = true regex-automata.workspace = true thiserror.workspace = true +# signal-hook's registry, in the extension only, enables these on the platforms that use them. +[target.'cfg(unix)'.dependencies] +errno.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys.workspace = true + # Proc-macro dependencies resolve with build dependencies, so the syn features PyO3's macros use # are enabled here. The empty build script makes Cargo resolve them. [build-dependencies] diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index 27e14bdf88..e810c3be3a 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -709,6 +709,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Executor, }, + HostEntry { + name: "cancel_running_statements", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, HostEntry { name: "EventDispatcher", kind: HostKind::Class, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index e173ce1abb..900ee396f8 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -342,6 +342,7 @@ pub enum ExecutorHost { RunStatus, ExecutionDurationLimitError, ExecutionDeadlineScope, + CancelRunningStatements, EventDispatcher, OperationLifecycle, StatementLifecycle, diff --git a/crates/sqlbuild-python/Cargo.toml b/crates/sqlbuild-python/Cargo.toml index a0d4b49fb4..2ffda3e998 100644 --- a/crates/sqlbuild-python/Cargo.toml +++ b/crates/sqlbuild-python/Cargo.toml @@ -23,6 +23,7 @@ rayon.workspace = true rustc-hash.workspace = true serde.workspace = true serde_json.workspace = true +signal-hook.workspace = true sqlbuild-analysis.workspace = true sqlbuild-attachments.workspace = true sqlbuild-cache.workspace = true diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index 6dc1e8e49d..e6277324f4 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -1,15 +1,15 @@ //! The `sqb` entry: invocation scopes, parsing, observed dispatch and the error projection. use std::sync::Arc; -use std::time::Instant; +use std::time::{Instant, SystemTime}; use pyo3::exceptions::PySystemExit; use pyo3::prelude::{ Bound, Py, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; use pyo3::types::{ - PyBool, PyCFunction, PyDict, PyDictMethods, PyInt, PyList, PyString, PyStringMethods, PyTuple, - PyTypeMethods, + PyBool, PyCFunction, PyDateTime, PyDict, PyDictMethods, PyInt, PyList, PyString, + PyStringMethods, PyTuple, PyTypeMethods, PyTzInfo, }; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_cli::dispatch::main::failure_report::failure_report; @@ -29,8 +29,8 @@ use crate::bindings::_helpers::cli::observability::{ CliObservabilityScope, cli_observability_scope, }; use crate::bindings::_helpers::cli::termination::{ - CommandTerminated, TERMINATED_EXIT_CODE, TerminationHandler, handle_sigterm, - interrupting_signal, report_termination, + CommandTerminated, TERMINATED_MESSAGE, TerminationHandler, handle_sigterm, interrupting_signal, + signal_exit_code, terminated, terminated_exit, }; use crate::bindings::_helpers::compile_command::report::format_compile_error_json; use crate::bindings::_helpers::cost_report::attribution::take_cost_status; @@ -552,8 +552,7 @@ fn observed_dispatch<'py>( drop(take_failure_cause()); drop(take_cost_status()); let started: Instant = Instant::now(); - let started_at: Bound<'py, PyAny> = imported(py, "datetime", "datetime")? - .call_method1("now", (imported(py, "datetime", "UTC")?,))?; + let started_at: SystemTime = SystemTime::now(); let elapsed = |started: Instant| -> PyResult> { u64::try_from(started.elapsed().as_millis()) .unwrap_or(u64::MAX) @@ -603,8 +602,12 @@ fn observed_dispatch<'py>( "invocation_started", started_payload()?, )?)?; - announced.set(true); - match dispatch_command(&invocation.args, invocation.handlers.clone()) { + let dispatched: PyResult> = + match dispatch_command(&invocation.args, invocation.handlers.clone()) { + Ok(_) if terminated() => Err(CommandTerminated::new_err(TERMINATED_MESSAGE)), + dispatched => dispatched, + }; + match dispatched { Ok(exit_code) => { let payload: Bound<'py, PyDict> = PyDict::new(py); payload.set_item("command", &command)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs index e9c73a8545..3256513caa 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs @@ -1,19 +1,28 @@ -//! SIGTERM as an interrupt: the command stops the way Ctrl+C stops it, records its end and exits with the conventional code. +//! SIGTERM as an interrupt: the command cancels its running statements, stops the way Ctrl+C stops it, records its end and exits with the conventional code. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::thread::JoinHandle; +use std::time::Duration; use pyo3::create_exception; use pyo3::exceptions::PyKeyboardInterrupt; use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; -use pyo3::types::{PyCFunction, PyDict, PyTuple}; +use pyo3::{pyfunction, wrap_pyfunction}; +use sqlbuild_host::entries::models::ExecutorHost; use crate::bindings::_helpers::boundary::python_calls::imported; +use crate::bindings::_helpers::native_host::host; +use crate::bindings::_helpers::recall::console::write_stderr; +use crate::bindings::_helpers::records::fields::field as native_field; create_exception!( sqlbuild._native, CommandTerminated, PyKeyboardInterrupt, - "Raised in the main thread when the command receives SIGTERM; handled like Ctrl+C." + "Raised when the command receives SIGTERM; handled like Ctrl+C." ); /// The number of SIGTERM, as POSIX defines it. @@ -22,75 +31,190 @@ pub(crate) const SIGTERM_NUMBER: i64 = 15; pub(crate) const SIGINT_NUMBER: i64 = 2; /// The exit code of a command stopped by SIGTERM: 128 plus the signal number. pub(crate) const TERMINATED_EXIT_CODE: i64 = 128 + SIGTERM_NUMBER; -const TERMINATED_MESSAGE: &str = "terminated by SIGTERM"; +/// The exit code of a command stopped by Ctrl+C: 128 plus the signal number. +pub(crate) const INTERRUPTED_EXIT_CODE: i64 = 128 + SIGINT_NUMBER; +pub(crate) const TERMINATED_MESSAGE: &str = "terminated by SIGTERM"; +/// `signal.SIG_DFL`, as `signal.getsignal` reports a signal Python leaves to its default action. +const DEFAULT_ACTION: i64 = 0; +/// How often the watcher looks for a termination request. +const WATCH_INTERVAL: Duration = Duration::from_millis(20); + +/// The process's SIGTERM actions, registered once: take the default action while `default` is set, otherwise set `terminated`. +struct SigtermFlags { + default: Arc, + terminated: Arc, +} + +static SIGTERM_FLAGS: OnceLock> = OnceLock::new(); + +/// Register the SIGTERM actions; `None` when the platform refuses them. +fn registered_flags() -> Option<&'static SigtermFlags> { + SIGTERM_FLAGS + .get_or_init(|| { + let flags = SigtermFlags { + default: Arc::new(AtomicBool::new(true)), + terminated: Arc::new(AtomicBool::new(false)), + }; + let signal: i32 = signal_hook::consts::SIGTERM; + let registered = + signal_hook::flag::register_conditional_default(signal, Arc::clone(&flags.default)) + .and_then(|_| { + signal_hook::flag::register(signal, Arc::clone(&flags.terminated)) + }); + match registered { + Ok(_) => Some(flags), + Err(_unsupported_signal) => None, + } + }) + .as_ref() +} + +/// The SIGTERM handling of one command and the watcher that acts once SIGTERM arrives. Holds only Rust values: while it lives, SIGTERM sets a flag; once dropped, SIGTERM takes its default action again. +pub(crate) struct TerminationHandler { + flags: Option<&'static SigtermFlags>, + stop: Arc, + watcher: Option>, +} -/// The SIGTERM handler installed for one command, and the handler it replaced. -pub(crate) struct TerminationHandler<'py> { - signal: Bound<'py, PyAny>, - previous: Option>, +impl TerminationHandler { + /// Give SIGTERM its default action back and wait for the watcher to finish; whether SIGTERM arrived while the command ran. + pub(crate) fn settle(&mut self, py: Python<'_>) -> bool { + if let Some(flags) = self.flags { + flags.default.store(true, Ordering::SeqCst); + } + self.stop.store(true, Ordering::SeqCst); + if let Some(watcher) = self.watcher.take() { + py.detach(|| drop(watcher.join())); + } + terminated() + } } -impl Drop for TerminationHandler<'_> { +impl Drop for TerminationHandler { fn drop(&mut self) { - if let Some(previous) = self.previous.take() { - let number: PyResult> = self.signal.getattr("SIGTERM"); - drop(number.and_then(|number| self.signal.call_method1("signal", (number, previous)))); + self.stop.store(true, Ordering::SeqCst); + if let Some(watcher) = self.watcher.take() { + Python::attach(|py| py.detach(|| drop(watcher.join()))); + } + if let Some(flags) = self.flags { + flags.default.store(true, Ordering::SeqCst); + flags.terminated.store(false, Ordering::SeqCst); + } + } +} + +/// Where the watcher sends the interrupt: the main thread, and whether Python handles Ctrl+C there. +#[derive(Clone, Copy)] +struct MainThread { + ident: u64, + handles_interrupt: bool, +} + +/// Once SIGTERM arrives: let a second one end the process at once, cancel every running statement, and interrupt the main thread the way Ctrl+C does, where the platform can signal one thread. +fn watch(flags: &SigtermFlags, stop: &AtomicBool, main: MainThread) { + while !stop.load(Ordering::SeqCst) { + if flags.terminated.load(Ordering::SeqCst) { + flags.default.store(true, Ordering::SeqCst); + Python::attach(|py| { + let cancelled = host::entry(py, ExecutorHost::CancelRunningStatements) + .and_then(|cancel| cancel.call0()); + drop(cancelled); + if main.handles_interrupt { + let interrupted = imported(py, "signal", "pthread_kill") + .and_then(|kill| kill.call1((main.ident, SIGINT_NUMBER))); + drop(interrupted); + } + }); + return; } + std::thread::sleep(WATCH_INTERVAL); } } -/// Raise [`CommandTerminated`] on SIGTERM while the command runs: only on the main thread, and only when no other handler was installed, so an embedding application keeps its own. -pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult> { - let signal: Bound<'_, PyAny> = py.import("signal")?.into_any(); +/// Handle SIGTERM while the command runs: only on the main thread, and only when Python leaves SIGTERM to its default action, so an embedding application keeps its own handler. +pub(crate) fn handle_sigterm(py: Python<'_>) -> PyResult { let threading: Bound<'_, PyAny> = py.import("threading")?.into_any(); - let main_thread: bool = threading - .call_method0("current_thread")? - .is(&threading.call_method0("main_thread")?); - let number: Bound<'_, PyAny> = signal.getattr("SIGTERM")?; - let current: Bound<'_, PyAny> = signal.call_method1("getsignal", (&number,))?; - if !main_thread || !current.eq(signal.getattr("SIG_DFL")?)? { + let main_thread: Bound<'_, PyAny> = threading.call_method0("main_thread")?; + let on_main: bool = threading.call_method0("current_thread")?.is(&main_thread); + let action = |signal: i64| -> PyResult> { + imported(py, "signal", "getsignal")?.call1((signal,)) + }; + let default: bool = action(SIGTERM_NUMBER)? + .extract::() + .is_ok_and(|action| action == DEFAULT_ACTION); + let stop: Arc = Arc::new(AtomicBool::new(false)); + let Some(flags) = registered_flags().filter(|_| on_main && default) else { return Ok(TerminationHandler { - signal, - previous: None, + flags: None, + stop, + watcher: None, }); - } - let handler: Bound<'_, PyCFunction> = PyCFunction::new_closure( - py, - None, - None, - |_: &Bound<'_, PyTuple>, _: Option<&Bound<'_, PyDict>>| -> PyResult<()> { - Err(CommandTerminated::new_err(TERMINATED_MESSAGE)) - }, - )?; - signal.call_method1("signal", (&number, handler))?; + }; + let main = MainThread { + ident: native_field(&main_thread, "ident")?.extract()?, + handles_interrupt: action(SIGINT_NUMBER)?.is_callable(), + }; + flags.terminated.store(false, Ordering::SeqCst); + let watched: Arc = Arc::clone(&stop); + let watcher: JoinHandle<()> = std::thread::Builder::new() + .name("sqlbuild-termination".to_owned()) + .spawn(move || watch(flags, &watched, main))?; + flags.default.store(false, Ordering::SeqCst); Ok(TerminationHandler { - signal, - previous: Some(current), + flags: Some(flags), + stop, + watcher: Some(watcher), }) } -/// The signal that interrupted the command: SIGTERM for [`CommandTerminated`], SIGINT for any other `KeyboardInterrupt`. +/// Whether SIGTERM arrived while the command runs. +pub(crate) fn terminated() -> bool { + SIGTERM_FLAGS + .get() + .and_then(Option::as_ref) + .is_some_and(|flags| flags.terminated.load(Ordering::SeqCst)) +} + +/// Whether SIGTERM arrived while the command runs, so a statement error can be reported as the termination it is. +#[pyfunction] +fn termination_requested() -> bool { + terminated() +} + +/// The signal that interrupted the command: SIGTERM once one arrived, SIGINT for any other `KeyboardInterrupt`. pub(crate) fn interrupting_signal(py: Python<'_>, error: &PyErr) -> Option { - if error.is_instance_of::(py) { - return Some(SIGTERM_NUMBER); + if !error.is_instance_of::(py) { + return None; + } + Some( + if terminated() || error.is_instance_of::(py) { + SIGTERM_NUMBER + } else { + SIGINT_NUMBER + }, + ) +} + +/// The exit code a run stopped by `signal` ends with. +pub(crate) fn signal_exit_code(signal: i64) -> i64 { + if signal == SIGTERM_NUMBER { + TERMINATED_EXIT_CODE + } else { + INTERRUPTED_EXIT_CODE } - error - .is_instance_of::(py) - .then_some(SIGINT_NUMBER) } -/// Write the termination notice to stderr; a stream that cannot be written changes nothing. -pub(crate) fn report_termination(py: Python<'_>) { - let written = imported(py, "sys", "stderr").and_then(|stderr| { - stderr.call_method1("write", (format!("error: {TERMINATED_MESSAGE}\n"),))?; - stderr.call_method0("flush") - }); - drop(written); +/// After SIGTERM: drop the interrupt it left pending for the main thread, write the termination notice to stderr and give the conventional exit code. A stream that cannot be written changes nothing. +pub(crate) fn terminated_exit(py: Python<'_>) -> i64 { + drop(py.check_signals()); + drop(write_stderr(py, &format!("error: {TERMINATED_MESSAGE}\n"))); + TERMINATED_EXIT_CODE } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( "CommandTerminated", module.py().get_type::(), - ) + )?; + module.add_function(wrap_pyfunction!(termination_requested, module)?) } diff --git a/fensu.toml b/fensu.toml index d05e32d057..def6ed272d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -797,6 +797,11 @@ modules = ["sqlbuild.runtime.execution_limits.main.execution_deadline_scope"] symbols = ["execution_deadline_scope"] reason = "Native code resolves these through the sqlbuild._native_host name table." +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main.cancel_running_statements"] +symbols = ["cancel_running_statements"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + [[targets.python.dead_code.roots]] modules = ["sqlbuild.runtime.execution_limits.models"] symbols = ["RunItemTimeouts"] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md index d020fc3125..4c7cd9a531 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/recall.md @@ -154,8 +154,9 @@ can contain sensitive SQL, output and audit evidence. | `unknown` | Recall has neither an end nor a heartbeat for it | A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it -records its real end time and its output, prints `error: terminated by SIGTERM`, and exits with -code 143. Recall keeps heartbeating while the command discovers and compiles the project, so a run +cancels the statements it is running, records its real end time, its output and exit code 143, +prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process +at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index af1025705b..eaf5e10e58 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -4271,6 +4271,7 @@ def scope_expected_model_names( def supports_color() -> bool: ... def parse_cli_invocation(argv: Sequence[str], *, use_color: bool) -> ParsedCliInvocation: ... def run_cli(argv: Sequence[str] | None = None, handlers: object | None = None) -> int: ... +def termination_requested() -> bool: ... def scope_test_ctes( texts: list[tuple[str, str]], syntax: dict[str, object] ) -> list[tuple[str | None, list[tuple[str, str]]]]: ... diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index 8becdaf2da..e1ba79de4b 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -573,6 +573,10 @@ "sqlbuild.runtime.execution_limits.main.execution_deadline_scope", "execution_deadline_scope", ), + "cancel_running_statements": ( + "sqlbuild.runtime.execution_limits.main.cancel_running_statements", + "cancel_running_statements", + ), "EventDispatcher": ( "sqlbuild.runtime.observability.classes.event_dispatcher", "EventDispatcher", diff --git a/src/sqlbuild/adapter/contract/classes/connection.py b/src/sqlbuild/adapter/contract/classes/connection.py index 56afb59516..ecd94b1f40 100644 --- a/src/sqlbuild/adapter/contract/classes/connection.py +++ b/src/sqlbuild/adapter/contract/classes/connection.py @@ -7,6 +7,7 @@ from collections.abc import Generator from typing import Any, ClassVar +from sqlbuild._native import CommandTerminated, termination_requested from sqlbuild.adapter.contract.main._attribute_failed_sql import attribute_failed_sql from sqlbuild.adapter.contract.models import QueryResult from sqlbuild.runtime.execution_limits.exceptions import ( @@ -24,6 +25,8 @@ ) from sqlbuild.runtime.observability.classes.statement_lifecycle import StatementLifecycle +_TERMINATED_MESSAGE: str = "terminated by SIGTERM" + class ConnectionMixin(ABC): """Manages adapter connection lifecycle and transaction control.""" @@ -44,6 +47,8 @@ def execute(self, *, connection: Any, sql: str) -> Any: with tracked_statement_connection(adapter=self, connection=connection): result: Any = self._execute(connection=connection, sql=sql) except Exception as error: + if termination_requested(): + raise CommandTerminated(_TERMINATED_MESSAGE) from error try: enforce_execution_deadline(after_failure=True) except (ExecutionDurationLimitError, ItemTimeoutError) as deadline_error: diff --git a/src/sqlbuild/runtime/execution_limits/_helpers/running.py b/src/sqlbuild/runtime/execution_limits/_helpers/running.py new file mode 100644 index 0000000000..40197a4247 --- /dev/null +++ b/src/sqlbuild/runtime/execution_limits/_helpers/running.py @@ -0,0 +1,5 @@ +"""The process-wide record of connections running a statement, so a termination can cancel them.""" + +from sqlbuild.runtime.execution_limits.classes.active_connections import ActiveConnections + +running_statements: ActiveConnections = ActiveConnections() diff --git a/src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py b/src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py new file mode 100644 index 0000000000..d7a605f903 --- /dev/null +++ b/src/sqlbuild/runtime/execution_limits/main/_cancel_running_statements.py @@ -0,0 +1,14 @@ +"""Cancel every statement the process is running, as a termination request does.""" + +from __future__ import annotations + +from sqlbuild.runtime.execution_limits._helpers.cancellation import cancel_connection +from sqlbuild.runtime.execution_limits._helpers.running import running_statements + + +def cancel_running_statements() -> None: + """Ask each connection with a statement in flight to cancel it.""" + + _ = running_statements.cancel_running( + cancel=lambda adapter, connection: cancel_connection(adapter=adapter, connection=connection) + ) diff --git a/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py b/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py index 4d54c446df..abeaaee289 100644 --- a/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py +++ b/src/sqlbuild/runtime/execution_limits/main/tracked_statement_connection.py @@ -10,15 +10,16 @@ current_invocation_connections_context, current_item_deadline_context, ) +from sqlbuild.runtime.execution_limits._helpers.running import running_statements from sqlbuild.runtime.execution_limits.classes.active_connections import ActiveConnections from sqlbuild.runtime.execution_limits.classes.item_deadline import ItemDeadline @contextmanager def tracked_statement_connection(*, adapter: Any, connection: Any) -> Generator[None]: - """Record that a statement runs on `connection` so a deadline can cancel it there.""" + """Record that a statement runs on `connection` so a deadline or termination can cancel it.""" - registries: list[ActiveConnections] = [] + registries: list[ActiveConnections] = [running_statements] invocation: ActiveConnections | None = current_invocation_connections_context.get() if invocation is not None: registries.append(invocation) diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 6488e0eed8..7ce0ce0d6c 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -362,6 +362,7 @@ class RecordedSelectionTestCase: @dataclass(frozen=True) class TerminatedCommandTestCase: description: str + orders_model: str environment: dict[str, str] wait_sql: str expected_output: bytes @@ -387,3 +388,10 @@ class CostAttributionTestCase: description: str collected: str expected_cost_status: str + + +@dataclass(frozen=True) +class ClosingTerminationTestCase: + description: str + shutdown_timeout: str + expected_exit_code: int diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 916af0dbc5..7b4fbf0819 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -260,17 +260,45 @@ def write_slow_import_sink(project_dir: Path) -> Path: def terminated_build( project_dir: Path, *, wait_sql: str, **env: str -) -> subprocess.CompletedProcess[bytes]: - """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row.""" +) -> tuple[subprocess.CompletedProcess[bytes], float]: + """A build sent SIGTERM, as an orchestrator stops it, once `wait_sql` returns a row, and the + seconds it took to stop.""" process: subprocess.Popen[bytes] = start(project_dir, "build", **env) try: _ = wait_for(wait_sql, project_dir, timeout=120) process.send_signal(signal.SIGTERM) + sent: float = time.monotonic() stdout, stderr = process.communicate(timeout=60) except subprocess.TimeoutExpired: raise AssertionError(f"a terminated build kept running:\n{hang_dump(process)}") from None - return subprocess.CompletedProcess(process.args, process.returncode, stdout, stderr) + completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( + process.args, process.returncode, stdout, stderr + ) + return completed, time.monotonic() - sent + + +def terminated_while_closing(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]: + """A build sent SIGTERM while it waits for a locked Recall store at exit, and the seconds it + took to stop.""" + + with held_store_lock(project_dir): + process: subprocess.Popen[bytes] = start(project_dir, "build", PYTHONUNBUFFERED="1") + assert process.stdout is not None + printed: bytes = b"" + while b"Completed" not in printed and process.poll() is None: + printed += process.stdout.readline() + time.sleep(1) + process.send_signal(signal.SIGTERM) + sent: float = time.monotonic() + try: + stdout, stderr = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + raise AssertionError(f"a closing build kept running:\n{hang_dump(process)}") from None + completed: subprocess.CompletedProcess[bytes] = subprocess.CompletedProcess( + process.args, process.returncode, printed + stdout, stderr + ) + return completed, time.monotonic() - sent def interrupted_build(project_dir: Path, **env: str) -> tuple[int | None, float]: diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index b640a329cf..ce401a8a89 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -6,6 +6,7 @@ from tests.integration.src.sqlbuild.cli.recall._test_types import ( AuditEvidenceTestCase, + ClosingTerminationTestCase, CostAttributionTestCase, FailedStatementSqlTestCase, RecordedSelectionTestCase, @@ -13,9 +14,11 @@ ) from tests.integration.src.sqlbuild.cli.recall.helpers import ( EMITTED_TAIL, + ORDERS_MODEL, recall_rows, run, terminated_build, + terminated_while_closing, write_project, write_slow_import_sink, ) @@ -85,6 +88,7 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( [ TerminatedCommandTestCase( description="during project discovery", + orders_model=ORDERS_MODEL, environment={"ORDERS_IMPORT_SECONDS": "60"}, wait_sql=( "SELECT invocation_id FROM invocations WHERE status = 'running' " @@ -94,12 +98,26 @@ def test_given_selectors_when_building_then_the_effective_selection_is_recorded( ), TerminatedCommandTestCase( description="while a task runs", + orders_model=ORDERS_MODEL, environment={"ORDERS_TASK_SLEEP_SECONDS": "60"}, wait_sql=( "SELECT invocation_id FROM output_tail WHERE text LIKE '%after the warning%'" ), expected_output=EMITTED_TAIL, ), + TerminatedCommandTestCase( + description="while a long warehouse statement runs", + orders_model=( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT sum(range % 7) AS order_id FROM range(80000000000)\n" + ), + environment={}, + wait_sql=( + "SELECT statement_id FROM statements " + "WHERE statement_kind = 'CREATE' AND ended_at IS NULL" + ), + expected_output=b"", + ), ], ids=lambda case: case.description, ) @@ -107,8 +125,9 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit test_case: TerminatedCommandTestCase, tmp_path: Path ) -> None: project_dir: Path = write_slow_import_sink(write_project(tmp_path / "orders_shop")) + _ = (project_dir / "models" / "orders.sql").write_text(test_case.orders_model, encoding="utf-8") - terminated: subprocess.CompletedProcess[bytes] = terminated_build( + terminated, stop_seconds = terminated_build( project_dir, wait_sql=test_case.wait_sql, SQLBUILD_RECALL_HEARTBEAT_SECONDS="0.5", @@ -116,18 +135,55 @@ def test_given_sigterm_when_a_build_is_stopped_then_it_is_recorded_cancelled_wit ) recorded: list[list[object]] = recall_rows( project_dir, - "SELECT i.status, i.signal_number, i.ended_at_estimated, i.ended_at > i.started_at, " - "o.flush_complete FROM invocations i JOIN output_invocations o USING (invocation_id)", + "SELECT i.status, i.signal_number, i.exit_code, i.ended_at_estimated, " + "i.ended_at > i.started_at, o.flush_complete " + "FROM invocations i JOIN output_invocations o USING (invocation_id)", + ) + starts: list[list[object]] = recall_rows( + project_dir, "SELECT COUNT(*) FROM recall_events WHERE event_type = 'invocation_started'" ) output: subprocess.CompletedProcess[bytes] = run(project_dir, "recall", "output", "last") assert terminated.returncode == 143 + assert stop_seconds < 10 assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") - assert recorded == [["cancelled", 15, 0, 1, 1]] + assert b"Traceback" not in terminated.stderr + assert recorded == [["cancelled", 15, 143, 0, 1, 1]] + assert starts == [[1]] assert test_case.expected_output in output.stdout assert b"may be incomplete" not in output.stderr +@pytest.mark.parametrize( + "test_case", + [ + ClosingTerminationTestCase( + description="a build waiting for a locked store at exit", + shutdown_timeout="60s", + expected_exit_code=143, + ), + ], + ids=lambda case: case.description, +) +def test_given_sigterm_while_recall_closes_when_a_build_has_finished_then_it_exits_143( + test_case: ClosingTerminationTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_project( + tmp_path / "orders_shop", + extra_toml=f'\n[recall]\nshutdown_timeout = "{test_case.shutdown_timeout}"\n', + ) + + terminated, stop_seconds = terminated_while_closing(project_dir) + + assert b"Completed" in terminated.stdout + assert terminated.returncode == test_case.expected_exit_code, terminated.stderr.decode( + errors="replace" + ) + assert stop_seconds < 10 + assert b"Traceback" not in terminated.stderr + assert terminated.stderr.endswith(b"error: terminated by SIGTERM\n") + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/concepts/observability/recall.mdx b/website/src/content/docs/docs/concepts/observability/recall.mdx index af623d8ade..b459907887 100644 --- a/website/src/content/docs/docs/concepts/observability/recall.mdx +++ b/website/src/content/docs/docs/concepts/observability/recall.mdx @@ -133,8 +133,9 @@ can contain sensitive SQL, output and audit evidence. | `unknown` | Recall has neither an end nor a heartbeat for it | A command stopped by SIGTERM, as an orchestrator stops a job, ends the way Ctrl-C ends it: it -records its real end time and its output, prints `error: terminated by SIGTERM`, and exits with -code 143. Recall keeps heartbeating while the command discovers and compiles the project, so a run +cancels the statements it is running, records its real end time, its output and exit code 143, +prints `error: terminated by SIGTERM`, and exits with code 143. A second SIGTERM ends the process +at once. Recall keeps heartbeating while the command discovers and compiles the project, so a run stopped in any phase is never mistaken for one that died. An `abandoned` run cannot record its end: its `ended_at` is its last heartbeat and `ended_at_estimated` is `1`. From 467905ec43dd300186f453fb5db254c7d221cf30 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:47:40 +0000 Subject: [PATCH 17/30] fix(recall): exit 143 on SIGTERM wherever the command is, without a traceback Map a termination at the top of the CLI entry after its watcher has finished, so SIGTERM during argument parsing, Recall start-up or the Recall wait at exit ends the same way as during dispatch. --- .../src/bindings/_helpers/cli/entry.rs | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index e6277324f4..a6416f5892 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -823,13 +823,7 @@ fn parsed_invocation<'py>( Ok(_) | Err(_) => None, }; let finished: PyResult<()> = finish_invocation_recall(py, &recall, exit_status); - let exit_code: Bound<'py, PyAny> = match dispatched { - Err(error) if error.is_instance_of::(py) => { - report_termination(py); - TERMINATED_EXIT_CODE.into_bound_py_any(py)? - } - dispatched => dispatched?, - }; + let exit_code: Bound<'py, PyAny> = dispatched?; finished?; Ok(exit_code) } @@ -870,7 +864,21 @@ fn run_cli<'py>( .get_item(pyo3::types::PySlice::new(py, 1, isize::MAX, 1))?, }; let argv: Bound<'py, PyAny> = offered_launch_record(&argv)?; - let _termination: TerminationHandler<'py> = handle_sigterm(py)?; + let mut termination: TerminationHandler = handle_sigterm(py)?; + let ran: PyResult> = invoke(py, &argv, handlers); + if termination.settle(py) { + drop(ran); + return terminated_exit(py).into_bound_py_any(py); + } + ran +} + +/// Run one parsed `argv` inside its invocation, external-context and diagnostics scopes. +fn invoke<'py>( + py: Python<'py>, + argv: &Bound<'py, PyAny>, + handlers: Option>, +) -> PyResult> { let external: Bound<'py, PyAny> = external_context(py)?; let invocation_scope: Bound<'py, PyAny> = host::entry(py, ExecutorHost::InvocationScope)?.call0()?; @@ -888,7 +896,7 @@ fn run_cli<'py>( )], )?; within(&diagnostics, |_| { - parsed_invocation(&argv, handlers.clone(), identity) + parsed_invocation(argv, handlers.clone(), identity) }) }) }) From c300f524d8fc5f046c2c730457fe4505695b5405 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:47:41 +0000 Subject: [PATCH 18/30] fix(recall): record a cancelled run's exit code and its lifecycle events exactly once Cancelled runs record exit code 143, or 130 for Ctrl+C. The Recall sink notes which of the invocation's own events it recorded, so an interrupt during the start event no longer records a second start. SIGTERM in a scenario run cancels its statements as Ctrl+C does. --- .../src/bindings/_helpers/cli/entry.rs | 71 +++++++++------- .../src/bindings/_helpers/recall/constants.rs | 3 + .../src/bindings/_helpers/recall/session.rs | 4 + .../src/bindings/_helpers/recall/sinks.rs | 53 +++++++++--- .../sqlbuild/cli/commands/main/_test_types.py | 9 ++ .../src/sqlbuild/cli/commands/main/helpers.py | 82 +++++++++++++++++++ .../commands/main/test_command_termination.py | 51 ++++++++++++ 7 files changed, 232 insertions(+), 41 deletions(-) create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index a6416f5892..a50a1eac2f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -483,6 +483,7 @@ fn subscribe_recall( session, replacements: Arc::clone(&recall.replacements), running_sql: Arc::default(), + invocation_events: Arc::clone(&recall.invocation_events), }, )?; dispatcher.call_method1("subscribe_history", (sink,))?; @@ -583,13 +584,13 @@ fn observed_dispatch<'py>( payload.set_item("error_message", message)?; if let Some(signal) = interrupting_signal(py, error) { payload.set_item("signal_number", signal)?; + payload.set_item("exit_code", signal_exit_code(signal))?; } if let Some(status) = take_cost_status() { payload.set_item("cost_status", status)?; } Ok(payload) }; - let announced: std::cell::Cell = std::cell::Cell::new(false); let scope: Bound<'py, PyAny> = observability.clone().into_any(); let observed: PyResult> = within(&scope, |dispatcher| { let publish = |event: Bound<'py, PyAny>| -> PyResult<()> { @@ -639,44 +640,51 @@ fn observed_dispatch<'py>( } } }); - match (&observed, &invocation.recall.session) { - (Err(error), Some(session)) - if !announced.get() && interrupting_signal(py, error).is_some() => - { - let recorded: PyResult<()> = started_payload().and_then(|started| { - record_interrupted_setup( - py, - session, - (&started_at, started), - failed_payload(error)?, - ) - }); - drop(recorded); - } - _ => {} + if let (Err(error), Some(session)) = (&observed, &invocation.recall.session) + && interrupting_signal(py, error).is_some() + && !invocation.recall.invocation_events.ended() + { + let started: Option>> = + (!invocation.recall.invocation_events.started()).then(started_payload); + let recorded: PyResult<()> = started.transpose().and_then(|started| { + record_interrupted_end( + py, + session, + started.map(|payload| (started_at, payload)), + failed_payload(error)?, + ) + }); + drop(recorded); } observed } -/// Record an invocation interrupted while its observability was still being set up, such as during project discovery, so Recall shows it cancelled instead of running. -fn record_interrupted_setup( +/// Record the end of an invocation interrupted before its own lifecycle events were all recorded, such as during project discovery, so Recall shows it cancelled instead of running. `started` holds the start event's time and payload when that event was not recorded either. +fn record_interrupted_end( py: Python<'_>, session: &sqlbuild_recall::writer::models::RecallSession, - (started_at, started): (&Bound<'_, PyAny>, Bound<'_, PyDict>), + started: Option<(SystemTime, Bound<'_, PyDict>)>, failed: Bound<'_, PyDict>, ) -> PyResult<()> { - let started: Bound<'_, PyAny> = call_with( - &host::entry(py, ExecutorHost::CreateLifecycleEvent)?, - &[ - ( - "event_type", - PyString::new(py, "invocation_started").into_any(), - ), - ("payload", started.into_any()), - ("occurred_at", started_at.clone()), - ], - )?; - publish_lifecycle_event(py, session, &started)?; + if let Some((started_at, payload)) = started { + let seconds: f64 = started_at + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0.0, |elapsed| elapsed.as_secs_f64()); + let occurred_at: Bound<'_, PyDateTime> = + PyDateTime::from_timestamp(py, seconds, Some(&PyTzInfo::utc(py)?.to_owned()))?; + let started: Bound<'_, PyAny> = call_with( + &host::entry(py, ExecutorHost::CreateLifecycleEvent)?, + &[ + ( + "event_type", + PyString::new(py, "invocation_started").into_any(), + ), + ("payload", payload.into_any()), + ("occurred_at", occurred_at.into_any()), + ], + )?; + publish_lifecycle_event(py, session, &started)?; + } publish_lifecycle_event( py, session, @@ -772,6 +780,7 @@ fn parsed_invocation<'py>( session, replacements: Arc::clone(&recall.replacements), running_sql: Arc::default(), + invocation_events: Arc::clone(&recall.invocation_events), }, )? .into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs index 1330869523..2a1af6996c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/constants.rs @@ -116,5 +116,8 @@ pub(crate) const DEBUG_COMMAND: &str = "debug"; /// The namespace attribute of `sqb debug --no-connection`. pub(crate) const NO_CONNECTION_ATTRIBUTE: &str = "no_connection"; /// The statement terminal events, at which the sink settles a statement's SQL. +pub(crate) const INVOCATION_STARTED: &str = "invocation_started"; +pub(crate) const INVOCATION_COMPLETED: &str = "invocation_completed"; +pub(crate) const INVOCATION_FAILED: &str = "invocation_failed"; pub(crate) const STATEMENT_COMPLETED: &str = "statement_completed"; pub(crate) const STATEMENT_FAILED: &str = "statement_failed"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 8555774aaf..9d53f63348 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -50,6 +50,7 @@ use crate::bindings::_helpers::recall::constants::{ }; use crate::bindings::_helpers::recall::events::escaped_text; use crate::bindings::_helpers::recall::replacements::{RecallReplacements, in_replacement_sink}; +use crate::bindings::_helpers::recall::sinks::InvocationEvents; use crate::bindings::_helpers::records::fields::field as native_field; /// What the invocation records: its session, when enabled, and the resolved target and adapter. @@ -66,6 +67,8 @@ pub(crate) struct InvocationRecall { pub(crate) replacements: Arc, /// A Recall configuration mistake the command fails with before it runs. pub(crate) configuration_error: Option, + /// The invocation's own lifecycle events the session has recorded. + pub(crate) invocation_events: Arc, } /// The text of an optional attribute value. @@ -501,6 +504,7 @@ pub(crate) fn start_invocation_recall( settings: Some((settings, project_dir)), replacements: Arc::default(), configuration_error: None, + invocation_events: Arc::default(), }) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs index 363e973c0f..29a01a6a17 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/sinks.rs @@ -1,6 +1,7 @@ //! The native subscriber that publishes every lifecycle fact and artifact into Recall. use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use pyo3::exceptions::PyException; @@ -12,7 +13,10 @@ use sqlbuild_recall::records::main::parse_event::parse_event; use sqlbuild_recall::writer::models::RecallSession; use crate::bindings::_helpers::native_host::host; -use crate::bindings::_helpers::recall::constants::{STATEMENT_COMPLETED, STATEMENT_FAILED}; +use crate::bindings::_helpers::recall::constants::{ + INVOCATION_COMPLETED, INVOCATION_FAILED, INVOCATION_STARTED, STATEMENT_COMPLETED, + STATEMENT_FAILED, +}; use crate::bindings::_helpers::recall::events::{escaped_text, event_object}; use crate::bindings::_helpers::recall::replacements::RecallReplacements; @@ -47,12 +51,40 @@ pub(crate) fn publish_lifecycle_event( /// The SQL and resource of a statement that has not reached its terminal event yet. type RunningSql = HashMap)>; +/// Which of an invocation's own lifecycle events the sink has recorded, so an interrupted run records each exactly once. +#[derive(Debug, Default)] +pub(crate) struct InvocationEvents { + started: AtomicBool, + ended: AtomicBool, +} + +impl InvocationEvents { + /// Whether `invocation_started` was recorded. + pub(crate) fn started(&self) -> bool { + self.started.load(Ordering::SeqCst) + } + + /// Whether `invocation_completed` or `invocation_failed` was recorded. + pub(crate) fn ended(&self) -> bool { + self.ended.load(Ordering::SeqCst) + } + + fn note(&self, event_type: &str) { + if event_type == INVOCATION_STARTED { + self.started.store(true, Ordering::SeqCst); + } else if event_type == INVOCATION_COMPLETED || event_type == INVOCATION_FAILED { + self.ended.store(true, Ordering::SeqCst); + } + } +} + /// Publishes lifecycle events, through the Recall replacement sink on the delivery thread when one is configured, and SQL artifacts of the active session; delivery is best effort. The SQL of a statement that fails or is cancelled is queued again with its terminal event's priority. #[pyclass(module = "sqlbuild._native", frozen)] pub(crate) struct RecallLifecycleSink { pub(crate) session: Arc, pub(crate) replacements: Arc, pub(crate) running_sql: Arc>, + pub(crate) invocation_events: Arc, } #[pymethods] @@ -99,11 +131,7 @@ impl RecallLifecycleSink { } /// Forget a statement's SQL at its terminal event; a failed one's SQL is queued again first. - fn settle_statement(&self, event: &Bound<'_, PyAny>) -> PyResult<()> { - let Some(event_type) = event.getattr_opt("event_type")? else { - return Ok(()); - }; - let event_type: String = event_type.str()?.to_string(); + fn settle_statement(&self, event: &Bound<'_, PyAny>, event_type: &str) -> PyResult<()> { if event_type != STATEMENT_COMPLETED && event_type != STATEMENT_FAILED { return Ok(()); } @@ -125,14 +153,19 @@ impl RecallLifecycleSink { } fn publish(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - self.settle_statement(event)?; - if self + let event_type: String = match event.getattr_opt("event_type")? { + Some(event_type) => event_type.str()?.to_string(), + None => String::new(), + }; + self.settle_statement(event, &event_type)?; + if !self .replacements .deliver_lifecycle(py, &self.session, event)? { - return Ok(()); + publish_lifecycle_event(py, &self.session, event)?; } - publish_lifecycle_event(py, &self.session, event) + self.invocation_events.note(&event_type); + Ok(()) } } diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index 438e4e9d1b..d5825b942e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1,5 +1,6 @@ from __future__ import annotations +import signal from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path @@ -1335,3 +1336,11 @@ class RuleFailureArtifactsTestCase: expected_exit_code: int expected_open_orders: str expected_manifest: bool + + +@dataclass(frozen=True) +class SignalledScenarioTestCase: + description: str + sent_signal: signal.Signals + expected_exit_code: int + expected_last_line: str diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index d04bf7b97c..0a3476eb66 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -4,10 +4,14 @@ import json import os import shutil +import signal +import subprocess import sys import tempfile import threading +import time from collections.abc import Callable +from itertools import takewhile from pathlib import Path from types import ModuleType from typing import Any, cast @@ -1259,3 +1263,81 @@ def open_orders_model(sql: str) -> str: """An `open_orders` table model whose query is `sql`.""" return f'MODEL (\n description "Open orders",\n materialized table,\n);\n\n{sql}\n' + + +_ENDLESS_ORDER_TOTAL: str = "(SELECT SUM(big.n) FROM range(1000000000000) AS big(n)) * 0" + + +def write_endless_scenario_project(project_dir: Path) -> Path: + """A DuckDB project whose scenario builds a model that runs until it is cancelled.""" + + files: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "scenario_signals"\n' + 'adapter = "duckdb"\n\n' + "[connection]\n" + 'database = "scenario_signals.duckdb"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n" + " description: Raw orders.\n" + " schema: main\n" + " table: raw_orders\n" + ), + "models/order_totals.sql": ( + "MODEL (description 'Order totals.', materialized table, sql_analysis false);\n\n" + f"SELECT SUM(o.amount) + {_ENDLESS_ORDER_TOTAL} AS total_amount " + 'FROM __source("raw_orders") AS o\n' + ), + "tests/scenarios/totals.sql": ( + "SCENARIO (description 'Order totals add up.');\n\n" + "WITH\n" + "__source__raw_orders AS (SELECT 1 AS id, 10 AS amount),\n" + "__expected__order_totals AS (SELECT 10 AS total_amount)\n" + "SELECT 1\n" + ), + } + for relative, contents in files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + return project_dir + + +def signalled_scenario_run( + *, project_dir: Path, sent_signal: signal.Signals +) -> tuple[int, str, float]: + """Exit code and output of `sqb scenario test` sent `sent_signal` while its model runs, and + the seconds it took to stop.""" + + process: subprocess.Popen[str] = subprocess.Popen( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + "scenario", + "test", + "--concurrency", + "1", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={**os.environ, "PYTHONUNBUFFERED": "1"}, + preexec_fn=_default_interrupt_handling, + ) + assert process.stdout is not None + started: str = "".join(takewhile(lambda line: "order_totals START" not in line, process.stdout)) + time.sleep(1.0) + process.send_signal(sent_signal) + sent: float = time.monotonic() + stdout, stderr = process.communicate(timeout=120) + return process.returncode, started + stdout + stderr, time.monotonic() - sent + + +def _default_interrupt_handling() -> None: + """Give the spawned command default Ctrl-C handling even when its parent ignores SIGINT.""" + + _ = signal.signal(signal.SIGINT, signal.SIG_DFL) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py b/tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py new file mode 100644 index 0000000000..7157e48cfc --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_command_termination.py @@ -0,0 +1,51 @@ +"""Real CLI coverage for how a scenario run stops on SIGINT and SIGTERM.""" + +from __future__ import annotations + +import signal +from pathlib import Path + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + SignalledScenarioTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + signalled_scenario_run, + write_endless_scenario_project, +) + +_STOP_NOTICE: str = "Interrupted; cleaning up running scenarios..." + + +@pytest.mark.parametrize( + "test_case", + [ + SignalledScenarioTestCase( + description="Ctrl+C cancels the running statement", + sent_signal=signal.SIGINT, + expected_exit_code=-signal.SIGINT, + expected_last_line="KeyboardInterrupt", + ), + SignalledScenarioTestCase( + description="SIGTERM cancels the running statement the same way", + sent_signal=signal.SIGTERM, + expected_exit_code=143, + expected_last_line="error: terminated by SIGTERM", + ), + ], + ids=lambda case: case.description, +) +def test_given_a_running_scenario_when_signalled_then_its_statement_is_cancelled_promptly( + test_case: SignalledScenarioTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_endless_scenario_project(tmp_path / "scenario_signals") + + exit_code, output, stop_seconds = signalled_scenario_run( + project_dir=project_dir, sent_signal=test_case.sent_signal + ) + + assert exit_code == test_case.expected_exit_code, output + assert stop_seconds < 15, output + assert output.count(_STOP_NOTICE) == 1, output + assert output.splitlines()[-1] == test_case.expected_last_line, output From db4c7def98b5136fb8001246498438b8db632509 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:20:07 +0000 Subject: [PATCH 19/30] fix(rules): judge every column of a macro call's query as it was written Replace the exemption of SQL macro output with the ticket's rule: a plain slot read counts as unqualified, an aliased read as qualified, and any other macro output as the macro writes it. A query the macro writes whole is judged once, in the macro file; a query mixing caller and macro columns is judged at the call, so an expression or column-list macro's bare output beside a hand-qualified column fails again. --- .../src/lint_text/_helpers/sql_diagnostics.rs | 30 ++++--- .../src/lint_text/_helpers/sql_requests.rs | 78 ++++++++++++++++--- .../sqlbuild-rules/src/lint_text/constants.rs | 2 + .../src/sql_lint/_helpers/additional.rs | 55 +++++++++---- .../src/sql_lint/_helpers/engine.rs | 8 +- crates/sqlbuild-rules/src/sql_lint/models.rs | 15 +++- .../sql_lint/tests/test_lint_exemptions.rs | 53 +++++++++---- .../src/sql_lint/tests/test_types.rs | 7 +- .../docs/concepts/macros/sql-macros.md | 13 +++- .../main/test_macro_slot_qualification.py | 63 ++++++++++++++- .../docs/docs/concepts/macros/sql-macros.mdx | 13 +++- 11 files changed, 268 insertions(+), 69 deletions(-) diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs index 3552e6ccf1..57ad766e7a 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_diagnostics.rs @@ -200,15 +200,24 @@ pub(crate) fn without_generated_frame( } /// Whether lint text offset `offset` lies in SQL a `@sql.` call generated in the authored file. -fn from_sql_macro(body: &LintSqlBody, (text, dialect): (&str, &str), offset: i64) -> bool { +fn from_sql_macro(body: &LintSqlBody, file: (&str, &str), offset: i64) -> bool { + sql_macro_call_at(body, file, offset).is_some() +} + +/// The `@sql.` call in the authored file that generated lint text offset `offset`, if one did. +fn sql_macro_call_at( + body: &LintSqlBody, + (text, dialect): (&str, &str), + offset: i64, +) -> Option { let (mapped, generated) = map_offset(offset, &body.passes); if !generated { - return false; + return None; } match usize::try_from(body.body_start + mapped) { Ok(start) => interpolation_text_at(text, start, dialect) - .is_some_and(|token| token.starts_with(SQL_CALL_PREFIX)), - Err(_) => false, + .filter(|token| token.starts_with(SQL_CALL_PREFIX)), + Err(_before_the_file) => None, } } @@ -231,12 +240,12 @@ fn forward_pass(offset: i64, spans: &[ExpansionSpan], end: bool) -> i64 { offset + shift } -/// Character ranges of the lint text that SQL macro calls generated, so rules about the author's style judge that SQL in its macro file instead. +/// Character ranges of the lint text that SQL macro calls generated, each with the text of the call that generated it. pub(crate) fn sql_macro_output_spans( body: &LintSqlBody, file: (&str, &str), -) -> Vec<(usize, usize)> { - let mut spans: Vec<(usize, usize)> = Vec::new(); +) -> Vec<(usize, usize, String)> { + let mut spans: Vec<(usize, usize, String)> = Vec::new(); for (index, pass) in body.passes.iter().enumerate() { for span in pass { let (mut start, mut end): (i64, i64) = (span.output_start, span.output_end); @@ -244,8 +253,11 @@ pub(crate) fn sql_macro_output_spans( start = forward_pass(start, later, false); end = forward_pass(end, later, true); } - if end > start && from_sql_macro(body, file, start) { - spans.push((start as usize, end as usize)); + if end <= start { + continue; + } + if let Some(call) = sql_macro_call_at(body, file, start) { + spans.push((start as usize, end as usize, call)); } } } diff --git a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs index c9a226f485..7835d03b1d 100644 --- a/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs +++ b/crates/sqlbuild-rules/src/lint_text/_helpers/sql_requests.rs @@ -11,7 +11,7 @@ use crate::lint_text::_helpers::sql_diagnostics::{ sql_macro_output_spans, without_generated_comments, }; use crate::lint_text::constants::{ - CROSS_KEYWORD, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, + CROSS_KEYWORD, CTE_ARGUMENT, INVALID_BATCH, LINE_COMMENT_PREFIX, LINT_API_VERSION, OMITTED_CEREMONIAL_SELECT_SQL, }; use crate::lint_text::models::{LintSqlBody, LintSqlOptions}; @@ -39,6 +39,63 @@ fn mentions_cross_join(text: &str) -> bool { .any(|window| window.eq_ignore_ascii_case(CROSS_KEYWORD)) } +/// The names a `@sql.` call passes as whole arguments, such as `orders` in `@sql.net_amount(orders)`: the relations a plain slot read in its output may qualify with. Slot mappings and the reserved `__cte` name pass none. +fn call_relations(call: &str) -> Vec { + let Some(open) = call.find('(') else { + return Vec::new(); + }; + let close: usize = call + .rfind(')') + .filter(|&close| close > open) + .unwrap_or(call.len()); + let mut names: Vec = Vec::new(); + for argument in top_level_arguments(&call[open + 1..close]) { + if let Some(name) = passed_name(argument) { + names.push(name.to_owned()); + } + } + names +} + +/// The bare name one call argument passes, if it passes one: the value of `name := value` or the whole argument. +fn passed_name(argument: &str) -> Option<&str> { + let value: &str = match argument.split_once(":=") { + Some((name, _)) if name.contains('.') || name.trim().eq_ignore_ascii_case(CTE_ARGUMENT) => { + return None; + } + Some((_, value)) => value, + None => argument, + }; + let name: &str = value.trim().trim_matches('"'); + let bare: bool = !name.is_empty() + && name + .chars() + .all(|character| character.is_alphanumeric() || character == '_'); + bare.then_some(name) +} + +/// `arguments` split at its top-level commas, outside brackets and quotes. +fn top_level_arguments(arguments: &str) -> Vec<&str> { + let mut parts: Vec<&str> = Vec::new(); + let (mut depth, mut quote, mut start): (usize, Option, usize) = (0, None, 0); + for (index, character) in arguments.char_indices() { + match (quote, character) { + (Some(open), _) if character == open => quote = None, + (Some(_), _) => {} + (None, '\'' | '"') => quote = Some(character), + (None, '(' | '[') => depth += 1, + (None, ')' | ']') => depth = depth.saturating_sub(1), + (None, ',') if depth == 0 => { + parts.push(&arguments[start..index]); + start = index + 1; + } + _ => {} + } + } + parts.push(&arguments[start..]); + parts +} + fn strings(values: &[String]) -> Value { Value::from(values.to_vec()) } @@ -51,13 +108,15 @@ pub(crate) fn lint_payload( ) -> Value { let lint_text: Cow<'_, str> = without_generated_comments(body, (file, &options.dialect)); let inserted: usize = OMITTED_CEREMONIAL_SELECT_SQL.chars().count(); - let exempt: Vec = sql_macro_output_spans(body, (file, &options.dialect)) + let outputs: Vec = sql_macro_output_spans(body, (file, &options.dialect)) .into_iter() - .map(|(start, end)| match omitted { - Some(offset) if start >= offset => (start + inserted, end + inserted), - _ => (start, end), + .map(|(start, end, call)| { + let (start, end): (usize, usize) = match omitted { + Some(offset) if start >= offset => (start + inserted, end + inserted), + _ => (start, end), + }; + serde_json::json!({"start": start, "end": end, "relations": call_relations(&call)}) }) - .map(|(start, end)| Value::from(vec![start, end])) .collect(); let sql: String = match omitted { Some(offset) => { @@ -72,11 +131,8 @@ pub(crate) fn lint_payload( let _ = payload.insert("version".to_string(), Value::from(LINT_API_VERSION)); let _ = payload.insert("sql".to_string(), Value::from(sql)); let _ = payload.insert("dialect".to_string(), Value::from(options.dialect.clone())); - if !exempt.is_empty() { - let _ = payload.insert( - "qualification_exempt_spans".to_string(), - Value::from(exempt), - ); + if !outputs.is_empty() { + let _ = payload.insert("sql_macro_outputs".to_string(), Value::from(outputs)); } if let Some(enabled) = &options.enabled_rules { let _ = payload.insert("enabled_rules".to_string(), strings(enabled)); diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 0f11c5857c..f49086dad9 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -94,6 +94,8 @@ pub(crate) const LINT_API_VERSION: i64 = 1; pub(crate) const UNUSED_CTE_CODE: &str = "SQBRSQL005"; pub(crate) const LONG_LITERAL_CODE: &str = "SQBRSQL044"; pub(crate) const MIXED_QUALIFICATION_CODE: &str = "SQBRSQL028"; +/// The reserved argument naming a CTE-list call's CTEs; it passes no relation. +pub(crate) const CTE_ARGUMENT: &str = "__cte"; /// The help for mixed qualification in a SQL macro body, where slot reads are qualified through an alias. pub(crate) const MACRO_MIXED_QUALIFICATION_REMEDIATION: &str = "Use one consistent qualification style in this query scope. A plain slot read such as `@rel.column` counts as unqualified; to qualify it, read through an alias: `FROM @rel AS s`, then `s.@rel.column`."; pub(crate) const NATIVE_LINT_ERROR_CODE: &str = "L003"; diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs index 828448edc7..67fd258715 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs @@ -9,7 +9,7 @@ use crate::sql_lint::_helpers::engine::{ significant_before, token_depths, }; use crate::sql_lint::constants::RELATION_MODIFIER_KEYWORDS; -use crate::sql_lint::models::{AdditionalFactOptions, AdditionalQueryFacts}; +use crate::sql_lint::models::{AdditionalFactOptions, AdditionalQueryFacts, SqlMacroOutput}; use rustc_hash::{FxHashMap, FxHashSet}; use sqlbuild_core::constants::{CLOSE_PAREN, OPEN_PAREN, SQL_WILDCARD}; @@ -36,7 +36,7 @@ struct ReferenceFactContext<'a> { query: &'a QuerySlice<'a>, outer_relations: &'a FxHashSet, external_identifiers: &'a HashSet, - qualification_exempt_spans: &'a [(usize, usize)], + sql_macro_outputs: &'a [SqlMacroOutput], query_start: usize, query_end: usize, } @@ -52,7 +52,7 @@ struct SelectFactContext<'a> { tokens: &'a [Token], depths: &'a [usize], external_identifiers: &'a HashSet, - qualification_exempt_spans: &'a [(usize, usize)], + sql_macro_outputs: &'a [SqlMacroOutput], dependency_identifiers: &'a HashSet, allows_ceremonial_select: bool, allows_dynamic_output_star: bool, @@ -174,7 +174,7 @@ pub(super) fn collect_additional_facts( tokens, depths: &depths, external_identifiers: options.external_identifiers, - qualification_exempt_spans: options.qualification_exempt_spans, + sql_macro_outputs: options.sql_macro_outputs, dependency_identifiers: options.dependency_identifiers, allows_ceremonial_select: options.allows_ceremonial_select, allows_dynamic_output_star: options.allows_dynamic_output_star, @@ -342,7 +342,7 @@ fn collect_select_additional_facts(context: &SelectFactContext<'_>) -> Additiona query: &reference_slice, outer_relations: &outer_relations, external_identifiers: context.external_identifiers, - qualification_exempt_spans: context.qualification_exempt_spans, + sql_macro_outputs: context.sql_macro_outputs, query_start: select_index, query_end: end, }; @@ -674,7 +674,7 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer .extend(inconsistent_qualification( tokens, (&qualified, &unqualified), - context.qualification_exempt_spans, + context.sql_macro_outputs, )); } facts.unknown_relation_qualifiers.extend( @@ -694,26 +694,41 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer facts } -/// The unqualified references of a single-relation scope that also qualifies some, judged as written: a plain macro slot read such as `@orders.amount` counts as unqualified though it reads through its relation, and SQL another file authored is judged there. +/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. A plain macro slot read such as `@orders.amount` counts as unqualified though it expands to read through its relation: in a macro file it lints as a parameter sentinel pair, and in a call's output as a qualifier naming a relation the call passes. A scope whose every reference a macro call generated is judged once, in the macro file. fn inconsistent_qualification( tokens: &[Token], (qualified, unqualified): (&[usize], &[usize]), - exempt: &[(usize, usize)], + outputs: &[SqlMacroOutput], ) -> Vec { - let authored = |index: &&usize| { - let start: usize = tokens[**index].span.start; - !exempt + let output_of = |index: usize| -> Option { + let start: usize = tokens[index].span.start; + outputs .iter() - .any(|&(exempt_start, exempt_end)| exempt_start <= start && start < exempt_end) + .position(|output| output.start <= start && start < output.end) }; - let (plain_reads, qualified): (Vec, Vec) = qualified - .iter() - .filter(authored) - .partition(|&&index| is_plain_slot_read(tokens, index)); - let mut unqualified: Vec = unqualified.iter().filter(authored).copied().collect(); + let mut references = qualified.iter().chain(unqualified).copied(); + if let Some(first) = references.next() + && let Some(output) = output_of(first) + && references.all(|index| output_of(index) == Some(output)) + { + return Vec::new(); + } + let plain_read = |index: &usize| { + is_plain_slot_read(tokens, *index) + || output_of(*index).is_some_and(|output| { + let qualifier: String = unquoted_name(&tokens[*index].text); + outputs[output] + .relations + .iter() + .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) + }) + }; + let (plain_reads, qualified): (Vec, Vec) = + qualified.iter().copied().partition(plain_read); if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { return Vec::new(); } + let mut unqualified: Vec = unqualified.to_vec(); unqualified.extend(plain_reads); unqualified.sort_unstable(); unqualified @@ -722,6 +737,12 @@ fn inconsistent_qualification( .collect() } +/// An identifier's name without its quotes. +fn unquoted_name(text: &str) -> String { + text.trim_matches(|character| matches!(character, '"' | '`' | '[' | ']')) + .to_owned() +} + /// Whether the qualified reference at `index` is a plain SQL macro slot read, both its relation and its slot a macro parameter. fn is_plain_slot_read(tokens: &[Token], index: usize) -> bool { let is_parameter = |token: &Token| { diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs index ef93075a49..18a586fc5f 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/engine.rs @@ -12,7 +12,7 @@ use polyglot_sql::{ use crate::sql_lint::constants::LINT_API_VERSION; use crate::sql_lint::models::{ AdditionalFactOptions, CrossJoinRequest, LintDiagnostic, LintEdit, LintRequest, LintResponse, - LintRuleMetadata, QueryFacts, + LintRuleMetadata, QueryFacts, SqlMacroOutput, }; use crate::sql_lint::_helpers::additional::collect_additional_facts; @@ -337,7 +337,7 @@ struct DiagnosticContext<'a> { struct FactBuildOptions<'a> { external_identifiers: &'a HashSet, - qualification_exempt_spans: &'a [(usize, usize)], + sql_macro_outputs: &'a [SqlMacroOutput], dependency_identifiers: &'a HashSet, externally_referenced_ctes: &'a HashSet, allows_ceremonial_select: bool, @@ -407,7 +407,7 @@ pub(crate) fn lint(request: LintRequest) -> Result { .collect(); let options = FactBuildOptions { external_identifiers: &external_identifiers, - qualification_exempt_spans: &request.qualification_exempt_spans, + sql_macro_outputs: &request.sql_macro_outputs, dependency_identifiers: &dependency_identifiers, externally_referenced_ctes: &externally_referenced_ctes, allows_ceremonial_select: request.allows_ceremonial_select, @@ -518,7 +518,7 @@ fn build_facts( tokens, &AdditionalFactOptions { external_identifiers: options.external_identifiers, - qualification_exempt_spans: options.qualification_exempt_spans, + sql_macro_outputs: options.sql_macro_outputs, dependency_identifiers: options.dependency_identifiers, allows_ceremonial_select: options.allows_ceremonial_select, allows_dynamic_output_star: options.allows_dynamic_output_star, diff --git a/crates/sqlbuild-rules/src/sql_lint/models.rs b/crates/sqlbuild-rules/src/sql_lint/models.rs index 57bf44bf3d..59ea49ceee 100644 --- a/crates/sqlbuild-rules/src/sql_lint/models.rs +++ b/crates/sqlbuild-rules/src/sql_lint/models.rs @@ -83,16 +83,25 @@ pub(crate) struct LintRequest { pub relation_keys: HashMap>>, #[serde(default)] pub relation_names: HashMap, - /// Character ranges of SQL another file authored, such as SQL macro output, whose column qualification is judged in that file instead. + /// The SQL each `@sql.` call in the query generated, so column qualification is judged as the macro wrote it. #[serde(default)] - pub qualification_exempt_spans: Vec<(usize, usize)>, + pub sql_macro_outputs: Vec, +} + +/// The character range one `@sql.` call's output fills, and the names its arguments pass, which a plain slot read such as `@rel.amount` expands to qualify with. +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)] +pub(crate) struct SqlMacroOutput { + pub start: usize, + pub end: usize, + #[serde(default)] + pub relations: Vec, } #[derive(Debug)] pub(crate) struct AdditionalFactOptions<'a> { pub external_identifiers: &'a HashSet, pub dependency_identifiers: &'a HashSet, - pub qualification_exempt_spans: &'a [(usize, usize)], + pub sql_macro_outputs: &'a [SqlMacroOutput], pub allows_ceremonial_select: bool, pub allows_dynamic_output_star: bool, pub allows_empty_fixture_star: bool, diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs index b058d8ee14..f4dfda04da 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs @@ -2,38 +2,65 @@ use serde_json::{Value, json}; use crate::sql_lint::main::engine::lint_json; use crate::sql_lint::tests::test_types::{ - DynamicOutputStarLintTestCase, QualificationExemptLintTestCase, + DynamicOutputStarLintTestCase, SqlMacroOutputLintTestCase, }; #[test] -fn given_exempt_macro_output_when_linting_qualification_then_only_authored_columns_are_judged() +fn given_sql_macro_output_when_linting_qualification_then_each_column_counts_as_written() -> Result<(), String> { let test_cases = [ - QualificationExemptLintTestCase { - description: "a caller's bare column beside qualified macro output passes", - sql: "SELECT order_id, orders.amount - orders.discount AS net FROM orders", - exempt: "orders.amount - orders.discount", + SqlMacroOutputLintTestCase { + description: "a bare column from an expression macro beside a qualified column fails", + sql: "SELECT o.order_id, status AS s FROM orders AS o", + output: "status", + relations: &[], + expected_count: 1, + }, + SqlMacroOutputLintTestCase { + description: "plain slot reads count as unqualified beside a bare caller column", + sql: "SELECT order_id, o.amount - o.discount AS net FROM orders AS o", + output: "o.amount - o.discount", + relations: &["o"], expected_count: 0, }, - QualificationExemptLintTestCase { - description: "the caller's own mix still fails", - sql: "SELECT order_id, orders.status, orders.amount - orders.discount AS net FROM orders", - exempt: "orders.amount - orders.discount", + SqlMacroOutputLintTestCase { + description: "plain slot reads beside a hand-qualified column fail", + sql: "SELECT o.order_id, o.amount - o.discount AS net FROM orders AS o", + output: "o.amount - o.discount", + relations: &["o"], + expected_count: 2, + }, + SqlMacroOutputLintTestCase { + description: "an aliased read counts as qualified beside a bare caller column", + sql: "SELECT order_id, s.amount AS a FROM orders AS s", + output: "s.amount", + relations: &["orders"], expected_count: 1, }, + SqlMacroOutputLintTestCase { + description: "a query scope the macro generated whole is judged in the macro file", + sql: "WITH c AS (SELECT a, w.b FROM w) SELECT * FROM c", + output: "SELECT a, w.b FROM w", + relations: &[], + expected_count: 0, + }, ]; for test_case in test_cases { let start: usize = test_case .sql - .find(test_case.exempt) - .ok_or("exempt text should be in the SQL")?; + .find(test_case.output) + .ok_or("the output should be in the SQL")?; let response = lint_json( &json!({ "version": 1, "sql": test_case.sql, "dialect": "duckdb", "enabled_rules": ["SQBRSQL028"], - "qualification_exempt_spans": [[start, start + test_case.exempt.len()]], + "sql_macro_outputs": [{ + "start": start, + "end": start + test_case.output.len(), + "relations": test_case.relations, + }], }) .to_string(), )?; diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs index 41b6cd742f..6a3348bde3 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_types.rs @@ -55,11 +55,12 @@ pub(crate) struct PlainSqlLintTestCase { pub expected_count: usize, } -pub(crate) struct QualificationExemptLintTestCase { +pub(crate) struct SqlMacroOutputLintTestCase { pub(crate) description: &'static str, pub(crate) sql: &'static str, - /// The text whose characters are another file's SQL. - pub(crate) exempt: &'static str, + /// The text a `@sql.` call generated, and the names the call passed. + pub(crate) output: &'static str, + pub(crate) relations: &'static [&'static str], pub(crate) expected_count: usize, } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index 2900c107b4..b4d36c76fd 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -171,8 +171,10 @@ MACRO ( SELECT @src.created_at, @src.updated_at, @src.updated_at > @src.created_at AS was_updated ``` -`SELECT o.order_id, @sql.audit_columns(o) FROM orders AS o` compiles to -`SELECT o.order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. +`SELECT order_id, @sql.audit_columns(o) FROM orders AS o` compiles to +`SELECT order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. +The macro's plain reads count as unqualified, so the caller writes its own columns bare too (see +[reading slots](#reading-slots)). Use a column list when the same ordered set of columns repeats, for example in every branch of a `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. @@ -317,8 +319,11 @@ fully typed. `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. - Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and - unqualified columns, counts every plain read as unqualified and every `alias.@weekly...` read as - qualified, and judges a macro's SQL in the macro file rather than at each call. A step written + unqualified columns, counts every plain read as unqualified, every `alias.@weekly...` read as + qualified, and any other column a macro outputs as the macro writes it. A query the macro writes + whole, such as a CTE of a CTE-list macro, is judged once, in the macro file. Where a call's output + joins the caller's query, as with expression and column-list macros, that query is judged at the + call, so a plain read beside a hand-qualified column such as `o.order_id` fails. A step written only in plain reads, such as `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, passes; to qualify, read every slot through an alias. diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py index b7e39b835e..1ac91849b4 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py @@ -59,7 +59,7 @@ absent_fragments=("--> models/refund_rates.sql",), ), MacroSlotQualificationTestCase( - description="an expression macro's plain reads leave the caller's style to the caller", + description="an expression macro's plain reads count as unqualified beside bare columns", files={ "models/orders.sql": ( 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' @@ -81,6 +81,67 @@ expected_fragments=("Project compiled",), absent_fragments=("SQBRSQL028",), ), + MacroSlotQualificationTestCase( + description="an expression macro's plain reads beside a hand-qualified column fail", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, CAST(10 AS DOUBLE) AS amount, " + "CAST(1 AS DOUBLE) AS discount\n" + ), + "models/_macros/sql/net_amount.sql": ( + 'MACRO (\n description "Order amount after discount",\n' + " arguments (orders RELATION (amount DOUBLE, discount DOUBLE)),\n);\n\n" + "@orders.amount - @orders.discount\n" + ), + "models/order_totals.sql": ( + 'MODEL (\n description "Net amount per order",\n materialized table,\n);\n\n' + "SELECT o.order_id, @sql.net_amount(o) AS net\n" + 'FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/order_totals.sql"), + absent_fragments=("--> models/_macros/sql/net_amount.sql",), + ), + MacroSlotQualificationTestCase( + description="an expression macro's bare column beside a hand-qualified column fails", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" + ), + "models/_macros/sql/status_value.sql": ( + 'MACRO (\n description "The order status",\n returns VARCHAR,\n);\n\nstatus\n' + ), + "models/open_orders.sql": ( + 'MODEL (\n description "Open orders",\n materialized table,\n);\n\n' + 'SELECT o.order_id, @sql.status_value() AS s FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + absent_fragments=(), + ), + MacroSlotQualificationTestCase( + description="a column-list macro's bare output beside a hand-qualified column fails", + files={ + "models/orders.sql": ( + 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' + "SELECT 1 AS order_id, 'open' AS status\n" + ), + "models/_macros/sql/order_status.sql": ( + 'MACRO (\n description "The order status column",\n);\n\nSELECT status\n' + ), + "models/open_orders.sql": ( + 'MODEL (\n description "Open orders",\n materialized table,\n);\n\n' + 'SELECT o.order_id, @sql.order_status() FROM __ref("orders") AS o\n' + ), + }, + expected_exit_code=1, + expected_fragments=("error[SQBRSQL028]", "--> models/open_orders.sql"), + absent_fragments=(), + ), MacroSlotQualificationTestCase( description="a caller that mixes styles itself still fails", files={ diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 7fd384e3c5..54e4bee69e 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -156,8 +156,10 @@ MACRO ( SELECT @src.created_at, @src.updated_at, @src.updated_at > @src.created_at AS was_updated ``` -`SELECT o.order_id, @sql.audit_columns(o) FROM orders AS o` compiles to -`SELECT o.order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. +`SELECT order_id, @sql.audit_columns(o) FROM orders AS o` compiles to +`SELECT order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. +The macro's plain reads count as unqualified, so the caller writes its own columns bare too (see +[reading slots](#reading-slots)). Use a column list when the same ordered set of columns repeats, for example in every branch of a `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. @@ -302,8 +304,11 @@ fully typed. `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. - Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and - unqualified columns, counts every plain read as unqualified and every `alias.@weekly...` read as - qualified, and judges a macro's SQL in the macro file rather than at each call. A step written + unqualified columns, counts every plain read as unqualified, every `alias.@weekly...` read as + qualified, and any other column a macro outputs as the macro writes it. A query the macro writes + whole, such as a CTE of a CTE-list macro, is judged once, in the macro file. Where a call's output + joins the caller's query, as with expression and column-list macros, that query is judged at the + call, so a plain read beside a hand-qualified column such as `o.order_id` fails. A step written only in plain reads, such as `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, passes; to qualify, read every slot through an alias. From e908c445ea8115e54fc9fe7b5662e4b435860ff0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:26:47 +0000 Subject: [PATCH 20/30] fix(recall): collect failing audits' rows only when recall.audit_evidence keeps them The invocation's audit_evidence setting now gates violation sampling itself: off collects none and error collects only for error outcomes, so those rows reach neither the warehouse audit history nor the audit_completed event. Measurement evidence is unchanged. --- .../src/entries/_helpers/executor.rs | 5 +++ crates/sqlbuild-host/src/entries/models.rs | 1 + .../src/bindings/_helpers/cli/entry.rs | 43 +++++++++++-------- .../src/bindings/_helpers/recall/session.rs | 6 ++- crates/sqlbuild-recall/src/settings/models.rs | 10 +++++ fensu.toml | 7 ++- src/sqlbuild/_native_host.py | 6 ++- src/sqlbuild/_native_host.pyi | 9 ++++ .../auditing/_helpers/evidence_setting.py | 27 ++++++++++++ .../executor/auditing/_helpers/execution.py | 3 +- src/sqlbuild/executor/auditing/constants.py | 3 ++ .../auditing/main/_audit_evidence_scope.py | 20 +++++++++ .../src/sqlbuild/cli/recall/_test_types.py | 3 ++ .../cli/recall/test_recall_run_facts.py | 39 ++++++++++++++--- 14 files changed, 155 insertions(+), 27 deletions(-) create mode 100644 src/sqlbuild/executor/auditing/_helpers/evidence_setting.py create mode 100644 src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py diff --git a/crates/sqlbuild-host/src/entries/_helpers/executor.rs b/crates/sqlbuild-host/src/entries/_helpers/executor.rs index e810c3be3a..1087c31918 100644 --- a/crates/sqlbuild-host/src/entries/_helpers/executor.rs +++ b/crates/sqlbuild-host/src/entries/_helpers/executor.rs @@ -709,6 +709,11 @@ pub(crate) const ENTRIES: &[HostEntry] = &[ kind: HostKind::Function, edge: HostEdge::Executor, }, + HostEntry { + name: "audit_evidence_scope", + kind: HostKind::Function, + edge: HostEdge::Executor, + }, HostEntry { name: "cancel_running_statements", kind: HostKind::Function, diff --git a/crates/sqlbuild-host/src/entries/models.rs b/crates/sqlbuild-host/src/entries/models.rs index 900ee396f8..86621a8546 100644 --- a/crates/sqlbuild-host/src/entries/models.rs +++ b/crates/sqlbuild-host/src/entries/models.rs @@ -342,6 +342,7 @@ pub enum ExecutorHost { RunStatus, ExecutionDurationLimitError, ExecutionDeadlineScope, + AuditEvidenceScope, CancelRunningStatements, EventDispatcher, OperationLifecycle, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs index a50a1eac2f..3962b79206 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs @@ -810,23 +810,32 @@ fn parsed_invocation<'py>( Ok(handled_dispatch(&invocation)?.unbind()) }, )?; - let dispatched: PyResult> = if policy.creates_project { - call_with(&operation.into_any(), &[]) - } else { - call_with( - &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, - &[ - ("args", args), - ("identity", identity.clone()), - ("operation", operation.into_any()), - ("recall", recall_sink), - ( - "capture", - PyBool::new(py, policy.captured).to_owned().into_any(), - ), - ], - ) - }; + let evidence_scope: Bound<'py, PyAny> = call_with( + &host::entry(py, ExecutorHost::AuditEvidenceScope)?, + &[( + "mode", + PyString::new(py, recall.audit_evidence.setting()).into_any(), + )], + )?; + let dispatched: PyResult> = within(&evidence_scope, |_| { + if policy.creates_project { + call_with(&operation.into_any(), &[]) + } else { + call_with( + &host::entry(py, FacadeHost::DispatchWithComputeLogs)?, + &[ + ("args", args), + ("identity", identity.clone()), + ("operation", operation.into_any()), + ("recall", recall_sink), + ( + "capture", + PyBool::new(py, policy.captured).to_owned().into_any(), + ), + ], + ) + } + }); let exit_status: Option = match &dispatched { Ok(code) if code.is_instance_of::() => Some(code.extract().unwrap_or(1)), Ok(_) | Err(_) => None, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs index 9d53f63348..85714984a5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs @@ -24,7 +24,7 @@ use sqlbuild_recall::schema::main::render_script::render_script; use sqlbuild_recall::schema::models::{DdlOptions, Dialect}; use sqlbuild_recall::settings::main::resolve_recall_settings::resolve_recall_settings; use sqlbuild_recall::settings::models::{ - RecallSettings, SettingsRequest, StoreLocation, StorePurpose, + AuditEvidence, RecallSettings, SettingsRequest, StoreLocation, StorePurpose, }; use sqlbuild_recall::store::main::observe_store_file::observe_store_file; use sqlbuild_recall::store::main::quarantine_store::quarantine_store; @@ -69,6 +69,8 @@ pub(crate) struct InvocationRecall { pub(crate) configuration_error: Option, /// The invocation's own lifecycle events the session has recorded. pub(crate) invocation_events: Arc, + /// Which failing audits collect evidence rows, as `recall.audit_evidence` sets it. + pub(crate) audit_evidence: AuditEvidence, } /// The text of an optional attribute value. @@ -496,6 +498,7 @@ pub(crate) fn start_invocation_recall( } else { None }; + let audit_evidence: AuditEvidence = settings.audit_evidence; Ok(InvocationRecall { session, target: settings.target.clone(), @@ -505,6 +508,7 @@ pub(crate) fn start_invocation_recall( replacements: Arc::default(), configuration_error: None, invocation_events: Arc::default(), + audit_evidence, }) } diff --git a/crates/sqlbuild-recall/src/settings/models.rs b/crates/sqlbuild-recall/src/settings/models.rs index 4e3d6a788e..376fd8dba8 100644 --- a/crates/sqlbuild-recall/src/settings/models.rs +++ b/crates/sqlbuild-recall/src/settings/models.rs @@ -83,6 +83,16 @@ impl AuditEvidence { } } + /// The `recall.audit_evidence` value naming this mode. + #[must_use] + pub fn setting(self) -> &'static str { + match self { + Self::WarnAndError => "warn-and-error", + Self::Error => "error", + Self::Off => "off", + } + } + /// Whether an audit with `outcome` keeps its evidence. #[must_use] pub fn keeps(self, outcome: &str) -> bool { diff --git a/fensu.toml b/fensu.toml index def6ed272d..084eda8612 100644 --- a/fensu.toml +++ b/fensu.toml @@ -798,7 +798,12 @@ symbols = ["execution_deadline_scope"] reason = "Native code resolves these through the sqlbuild._native_host name table." [[targets.python.dead_code.roots]] -modules = ["sqlbuild.runtime.execution_limits.main.cancel_running_statements"] +modules = ["sqlbuild.executor.auditing.main._audit_evidence_scope"] +symbols = ["audit_evidence_scope"] +reason = "Native code resolves these through the sqlbuild._native_host name table." + +[[targets.python.dead_code.roots]] +modules = ["sqlbuild.runtime.execution_limits.main._cancel_running_statements"] symbols = ["cancel_running_statements"] reason = "Native code resolves these through the sqlbuild._native_host name table." diff --git a/src/sqlbuild/_native_host.py b/src/sqlbuild/_native_host.py index e1ba79de4b..94b3745544 100644 --- a/src/sqlbuild/_native_host.py +++ b/src/sqlbuild/_native_host.py @@ -573,8 +573,12 @@ "sqlbuild.runtime.execution_limits.main.execution_deadline_scope", "execution_deadline_scope", ), + "audit_evidence_scope": ( + "sqlbuild.executor.auditing.main._audit_evidence_scope", + "audit_evidence_scope", + ), "cancel_running_statements": ( - "sqlbuild.runtime.execution_limits.main.cancel_running_statements", + "sqlbuild.runtime.execution_limits.main._cancel_running_statements", "cancel_running_statements", ), "EventDispatcher": ( diff --git a/src/sqlbuild/_native_host.pyi b/src/sqlbuild/_native_host.pyi index 7d3105f8d3..c68e33d507 100644 --- a/src/sqlbuild/_native_host.pyi +++ b/src/sqlbuild/_native_host.pyi @@ -214,6 +214,9 @@ from sqlbuild.diagnostics.main.process_resource_reporting import ( from sqlbuild.diagnostics.models import ProcessResourceUsage as ProcessResourceUsage from sqlbuild.errors.contracts.exceptions import ExecutorInputError as ExecutorInputError from sqlbuild.errors.contracts.exceptions import SharedInputError as SharedInputError +from sqlbuild.executor.auditing.main._audit_evidence_scope import ( + audit_evidence_scope as audit_evidence_scope, +) from sqlbuild.executor.auditing.main._current_audit_result_projection import ( current_audit_result_projection as current_audit_result_projection, ) @@ -550,6 +553,9 @@ from sqlbuild.runtime.execution_history.types import RunStatus as RunStatus from sqlbuild.runtime.execution_limits.exceptions import ( ExecutionDurationLimitError as ExecutionDurationLimitError, ) +from sqlbuild.runtime.execution_limits.main._cancel_running_statements import ( + cancel_running_statements as cancel_running_statements, +) from sqlbuild.runtime.execution_limits.main.execution_deadline_scope import ( execution_deadline_scope as execution_deadline_scope, ) @@ -560,6 +566,9 @@ from sqlbuild.runtime.observability.classes.event_dispatcher import ( from sqlbuild.runtime.observability.classes.operation_lifecycle import ( OperationLifecycle as OperationLifecycle, ) +from sqlbuild.runtime.observability.classes.statement_lifecycle import ( + StatementLifecycle as StatementLifecycle, +) from sqlbuild.runtime.observability.exceptions import ( ObservabilityValidationError as ObservabilityValidationError, ) diff --git a/src/sqlbuild/executor/auditing/_helpers/evidence_setting.py b/src/sqlbuild/executor/auditing/_helpers/evidence_setting.py new file mode 100644 index 0000000000..0bb7f3f1eb --- /dev/null +++ b/src/sqlbuild/executor/auditing/_helpers/evidence_setting.py @@ -0,0 +1,27 @@ +"""The `recall.audit_evidence` setting the current invocation's audits collect evidence rows by.""" + +from __future__ import annotations + +from contextvars import ContextVar + +from sqlbuild.compiler.auditing.types import AuditOutcome +from sqlbuild.executor.auditing.constants import ( + AUDIT_EVIDENCE_ERROR_ONLY, + AUDIT_EVIDENCE_OFF, + DEFAULT_AUDIT_EVIDENCE, +) + +current_audit_evidence: ContextVar[str] = ContextVar( + "sqlbuild_audit_evidence", default=DEFAULT_AUDIT_EVIDENCE +) + + +def keeps_audit_evidence(*, outcome: AuditOutcome) -> bool: + """Whether a failing audit with `outcome` collects the rows it flagged.""" + + mode: str = current_audit_evidence.get() + if mode == AUDIT_EVIDENCE_OFF: + return False + if mode == AUDIT_EVIDENCE_ERROR_ONLY: + return outcome == AuditOutcome.ERROR + return outcome in {AuditOutcome.WARN, AuditOutcome.ERROR} diff --git a/src/sqlbuild/executor/auditing/_helpers/execution.py b/src/sqlbuild/executor/auditing/_helpers/execution.py index 2d4aaf22a4..d22010c7c1 100644 --- a/src/sqlbuild/executor/auditing/_helpers/execution.py +++ b/src/sqlbuild/executor/auditing/_helpers/execution.py @@ -25,6 +25,7 @@ AuditRunScope, ) from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context +from sqlbuild.executor.auditing._helpers.evidence_setting import keeps_audit_evidence from sqlbuild.executor.auditing.constants import ( DEFAULT_EVIDENCE_ROW_LIMIT, MAX_EVIDENCE_SERIALIZED_BYTES, @@ -107,7 +108,7 @@ def execute_audit_impl( # noqa: PLR0913 executed_sql=executed_sql, run_scope_phase=run_scope_phase, ) - if outcome in {AuditOutcome.WARN, AuditOutcome.ERROR}: + if keeps_audit_evidence(outcome=outcome): evidence_rows, evidence_truncated = _bounded_evidence( cursor=cursor, raw_rows=rows, row_limit=_evidence_row_limit(audit) ) diff --git a/src/sqlbuild/executor/auditing/constants.py b/src/sqlbuild/executor/auditing/constants.py index 89edc8e7d6..6bad0ef9c8 100644 --- a/src/sqlbuild/executor/auditing/constants.py +++ b/src/sqlbuild/executor/auditing/constants.py @@ -2,3 +2,6 @@ DEFAULT_EVIDENCE_ROW_LIMIT: int = 20 MAX_EVIDENCE_SERIALIZED_BYTES: int = 65_536 +DEFAULT_AUDIT_EVIDENCE: str = "warn-and-error" +AUDIT_EVIDENCE_ERROR_ONLY: str = "error" +AUDIT_EVIDENCE_OFF: str = "off" diff --git a/src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py b/src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py new file mode 100644 index 0000000000..d2ead16424 --- /dev/null +++ b/src/sqlbuild/executor/auditing/main/_audit_evidence_scope.py @@ -0,0 +1,20 @@ +"""Audit evidence setting scope entrypoint.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import Token + +from sqlbuild.executor.auditing._helpers.evidence_setting import current_audit_evidence + + +@contextmanager +def audit_evidence_scope(*, mode: str) -> Iterator[None]: + """Collect failing audits' evidence rows as the `recall.audit_evidence` `mode` says.""" + + token: Token[str] = current_audit_evidence.set(mode) + try: + yield + finally: + current_audit_evidence.reset(token) diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 7ce0ce0d6c..62838d5545 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -379,8 +379,11 @@ class FailedStatementSqlTestCase: class AuditEvidenceTestCase: description: str recall_toml: str + severity: str + expected_exit_code: int expected_evidence: list[object] expected_artifacts: list[list[object]] + expected_warehouse_evidence: list[tuple[object, ...]] @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py index ce401a8a89..de0da599de 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_recall_run_facts.py @@ -2,6 +2,7 @@ import subprocess from pathlib import Path +import duckdb import pytest from tests.integration.src.sqlbuild.cli.recall._test_types import ( @@ -223,20 +224,38 @@ def test_given_a_failed_statement_when_recalled_then_its_sql_is_readable( AuditEvidenceTestCase( description="a warning audit's violating rows are kept by default", recall_toml="", + severity="warn", + expected_exit_code=0, expected_evidence=["warn", 2, 2, 0, 1], expected_artifacts=[["audit_evidence", "application/json"]], + expected_warehouse_evidence=[('[{"email":null},{"email":null}]',)], ), AuditEvidenceTestCase( - description="error-only evidence skips warnings", + description="error-only evidence does not collect a warning's rows", recall_toml='\n[recall]\naudit_evidence = "error"\n', - expected_evidence=["warn", 2, 2, 0, 0], + severity="warn", + expected_exit_code=0, + expected_evidence=["warn", 2, 0, 0, 0], expected_artifacts=[], + expected_warehouse_evidence=[(None,)], ), AuditEvidenceTestCase( - description="evidence can be turned off", + description="error-only evidence collects a failing audit's rows", + recall_toml='\n[recall]\naudit_evidence = "error"\n', + severity="error", + expected_exit_code=1, + expected_evidence=["error", 2, 2, 0, 1], + expected_artifacts=[["audit_evidence", "application/json"]], + expected_warehouse_evidence=[('[{"email":null},{"email":null}]',)], + ), + AuditEvidenceTestCase( + description="evidence turned off collects no rows", recall_toml='\n[recall]\naudit_evidence = "off"\n', - expected_evidence=["warn", 2, 2, 0, 0], + severity="error", + expected_exit_code=1, + expected_evidence=["error", 2, 0, 0, 0], expected_artifacts=[], + expected_warehouse_evidence=[(None,)], ), ], ids=lambda case: case.description, @@ -245,9 +264,16 @@ def test_given_a_failing_audit_when_recalled_then_its_evidence_follows_the_setti test_case: AuditEvidenceTestCase, tmp_path: Path ) -> None: project_dir: Path = write_project(tmp_path / "orders_shop", extra_toml=test_case.recall_toml) - _ = (project_dir / "models" / "customers.sql").write_text(CUSTOMERS_MODEL, encoding="utf-8") + _ = (project_dir / "models" / "customers.sql").write_text( + CUSTOMERS_MODEL.replace("severity warn", f"severity {test_case.severity}"), + encoding="utf-8", + ) built: subprocess.CompletedProcess[bytes] = run(project_dir, "build", "customers") + with duckdb.connect(str(project_dir / "warehouse.duckdb"), read_only=True) as warehouse: + warehouse_evidence: list[tuple[object, ...]] = warehouse.execute( + "SELECT evidence_json FROM dev._sqlbuild_audit_results" + ).fetchall() evidence: list[list[object]] = recall_rows( project_dir, "SELECT outcome, violation_count, evidence_count, evidence_truncated, " @@ -259,8 +285,9 @@ def test_given_a_failing_audit_when_recalled_then_its_evidence_follows_the_setti "ON a.invocation_id = r.invocation_id AND a.ref_id = r.event_id", ) - assert built.returncode == 0, built.stderr.decode(errors="replace") + assert built.returncode == test_case.expected_exit_code, built.stderr.decode(errors="replace") assert evidence == [test_case.expected_evidence] + assert warehouse_evidence == test_case.expected_warehouse_evidence assert [row[:2] for row in artifacts] == test_case.expected_artifacts assert [json.loads(str(row[2])) for row in artifacts] == [ [{"email": None}, {"email": None}] for _ in test_case.expected_artifacts From bcfd1b31057c320b9c8f1634b877079dc5b4b8fe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:30:45 +0000 Subject: [PATCH 21/30] fix(query): leave the type cell empty when an adapter reports no type --- .../bindings/_helpers/query_command/query.rs | 4 +++- .../contract/classes/duckdb_backed_adapter.py | 4 +++- .../clickhouse/classes/clickhouse_adapter.py | 4 +++- .../commands/_helpers/query/test_output.py | 20 +++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index e345a934c3..70b9f75ef7 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -276,7 +276,9 @@ fn number(value: &Bound<'_, PyAny>, decimal: &Bound<'_, PyAny>) -> PyResult, width: usize) -> PyResult>> { - let types: Vec> = native_field(result, "column_types")?.extract()?; + let types: Vec> = native_field(result, "column_types")? + .extract() + .unwrap_or_default(); Ok(if types.len() == width { types } else { diff --git a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py index 3b8c3660e8..4909d3869c 100644 --- a/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py +++ b/src/sqlbuild/adapter/contract/classes/duckdb_backed_adapter.py @@ -1127,7 +1127,9 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) - column_types: tuple[str | None, ...] = tuple(str(column[1]) for column in description) + column_types: tuple[str | None, ...] = tuple( + None if column[1] is None else str(column[1]) for column in description + ) if limit is None: return QueryResult( columns=columns, diff --git a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py index 65f5850936..0820e86f25 100644 --- a/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py +++ b/src/sqlbuild/adapters/clickhouse/classes/clickhouse_adapter.py @@ -230,7 +230,9 @@ def query(self, *, connection: Any, sql: str, limit: int | None) -> QueryResult: if description is None: return QueryResult() columns: tuple[str, ...] = tuple(str(column[0]) for column in description) - column_types: tuple[str | None, ...] = tuple(str(column[1]) for column in description) + column_types: tuple[str | None, ...] = tuple( + None if column[1] is None else str(column[1]) for column in description + ) if limit is None: return QueryResult( columns=columns, rows=tuple(cursor.fetchall()), column_types=column_types diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py index 8fbe090eb0..41cae555f3 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/query/test_output.py @@ -58,6 +58,26 @@ limit=20, expected_output='[{"id": 1, "name": "alice"}]\n', ), + QueryOutputTestCase( + description="leaves a type cell empty where the adapter reports no type", + result=QueryResult( + columns=("order_id", "status"), + rows=((7, "open"),), + column_types=("INTEGER", None), + ), + output_format=None, + limit=20, + expected_output=( + "┌──────────┬────────┐\n" + "│ order_id │ status │\n" + "│ INTEGER │ │\n" + "├──────────┼────────┤\n" + "│ 7 │ open │\n" + "└──────────┴────────┘\n" + "\n" + "1 row\n" + ), + ), QueryOutputTestCase( description="renders a box with the warehouse types when the result fits", result=QueryResult( From fc654316036ad21b2e4157eab292e0f035b0de45 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:30:47 +0000 Subject: [PATCH 22/30] fix(compile): remove a stale manifest when Rules fail A compile whose Rules fail now removes an earlier target/manifest.json, so target/ no longer pairs new compiled SQL with a manifest from a previous compile. --- .../src/bindings/_helpers/compile_command/target_writer.rs | 7 ++++++- .../.agents/skills/sqlbuild/references/docs/cli/compile.md | 3 ++- .../cli/commands/main/test_compile_rule_failures.py | 4 ++-- website/src/content/docs/docs/cli/compile.mdx | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index c05f847b39..b0f65c4705 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -806,7 +806,7 @@ fn rule_error_lines( Ok(lines) } -/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. Nothing else under `target/` changes: no manifest, tests, cache or stale-file removal. +/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest is removed, since it no longer describes `target/`; nothing else changes: no tests, cache or stale-file removal. pub(crate) fn write_rule_failed_models<'py>( target_dir: &Bound<'py, PyAny>, (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -815,6 +815,11 @@ pub(crate) fn write_rule_failed_models<'py>( let py: Python<'py> = target_dir.py(); let errors: HashMap> = rule_error_lines(diagnostics, project_dir)?; let _ = prepared_target(target_dir)?; + let manifest: PathBuf = target_dir.extract::()?.join(MANIFEST_FILE); + match std::fs::remove_file(&manifest) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => return Err(error.into()), + _ => {} + } let compiled_dir: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; let mut batch: ArtifactBatch = ArtifactBatch::new(true, None); for model in native_field(project, "models")?.try_iter()? { diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 09c41541ae..7fa5671837 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -95,7 +95,8 @@ configuration, use [`sqb rules run`](rules.md). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. Each +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a +`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros generated. A model with Rule errors starts with a header that marks it not buildable and lists its findings: diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index f7314304ce..f57bd81d90 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -64,7 +64,7 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu "sqlbuild_project.toml": _PROJECT, "models/orders.sql": _ORDERS, "models/open_orders.sql": open_orders_model( - 'SELECT o.order_id, status FROM __ref("orders") AS o' + 'SELECT o.order_id, o.status FROM __ref("orders") AS o' ), } for relative, contents in files.items(): @@ -82,7 +82,7 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu _ = (tmp_path / "models" / "open_orders.sql").write_text( open_orders_model(test_case.open_orders_sql), encoding="utf-8" ) - (tmp_path / "target" / "manifest.json").unlink(missing_ok=True) + assert (tmp_path / "target" / "manifest.json").exists() exit_code: int = main(compile_args) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 1708045b28..90b30e2038 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -79,7 +79,8 @@ configuration, use [`sqb rules run`](/docs/cli/rules). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. Each +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a +`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros generated. A model with Rule errors starts with a header that marks it not buildable and lists its findings: From 096c6b64ab5423989691dbcb63add52cb46a459b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 01:46:56 +0000 Subject: [PATCH 23/30] refactor: lower the native crossing budgets the batch raised Read the compile graph's project and write the audit warning with the summary through fewer Python calls, reuse the stdout terminal and Decimal checks the query output already has, and set the budgets to the new counts. --- .../_helpers/audit_command/command.rs | 9 +++++---- .../_helpers/compile_driver/command.rs | 2 +- .../_helpers/presentation_text/terminal.rs | 5 +++++ .../bindings/_helpers/query_command/query.rs | 19 +++++++------------ .../_helpers/query_command/query_values.rs | 3 ++- .../_helpers/query_command/text_output.rs | 6 +++--- fensu.toml | 12 +++++------- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs index ade1c89087..bfa9c04ff5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs @@ -507,10 +507,11 @@ fn complete( let style: CliStyle = CliStyle { use_color: native_field(invocation, "use_color")?.is_truthy()?, }; - if let Some(warning) = projection_warning(py, style)? { - stream.call_method1("write", (warning,))?; - } - stream.call_method1("write", (render_audit_summary(counts, style),))?; + let warning: String = projection_warning(py, style)?.unwrap_or_default(); + stream.call_method1( + "write", + (format!("{warning}{}", render_audit_summary(counts, style)),), + )?; stream.call_method0("flush")?; let payload: Bound<'_, PyAny> = audit_payload( &PyTuple::new(py, results)?.into_any(), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs index 28a7a4ea93..6b7a54e1b5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs @@ -363,7 +363,7 @@ fn compiled_with_status<'py>( write_rule_failed_models( &target_dir(&project_dir)?, ( - &native_field(&analysis, "graph")?.getattr("project")?, + &native_field(&native_field(&analysis, "graph")?, "project")?, &project_dir, ), &native_field(&analysis, "diagnostics")?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs index 369db30e07..b4f6b07b65 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/presentation_text/terminal.rs @@ -10,6 +10,11 @@ pub(crate) fn stdout_supports_color(py: Python<'_>) -> PyResult { if std::env::var_os(NO_COLOR_VARIABLE).is_some() { return Ok(false); } + stdout_is_tty(py) +} + +/// Whether `sys.stdout` is a terminal. +pub(crate) fn stdout_is_tty(py: Python<'_>) -> PyResult { let stdout = py.import("sys")?.getattr("stdout")?; if !stdout.hasattr("isatty")? { return Ok(false); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs index 70b9f75ef7..e659c3c734 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs @@ -12,7 +12,7 @@ use sqlbuild_core::query_command::models::QueryDisplay; use sqlbuild_host::entries::models::AdapterHost; use crate::bindings::_helpers::boundary::python_calls::keywords; -use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; +use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with}; use crate::bindings::_helpers::command_runtime::connection::resolve_project_connection_config; use crate::bindings::_helpers::native_host::connections::connection_failure; use crate::bindings::_helpers::native_host::host; @@ -20,7 +20,7 @@ use crate::bindings::_helpers::query_command::project::{ ProjectAdapter, ProjectInputs, project_adapter, }; use crate::bindings::_helpers::query_command::query_values::{ - display_text, json_cell, python_json_rows, strict_width, + decimal_instance, display_text, json_cell, python_json_rows, strict_width, }; use crate::bindings::_helpers::query_command::text_output::{LONG_FORMAT, render_text}; use crate::bindings::_helpers::records::construct::error_class; @@ -244,11 +244,7 @@ fn render_json( } /// Whether every value of the column at `index` is a number or NULL, and at least one is a number. -fn numeric_column( - rows: &[Vec>], - index: usize, - decimal: &Bound<'_, PyAny>, -) -> PyResult { +fn numeric_column(rows: &[Vec>], index: usize) -> PyResult { let mut seen: bool = false; for row in rows { let Some(value) = row.get(index) else { @@ -257,7 +253,7 @@ fn numeric_column( if value.is_none() { continue; } - if !number(value, decimal)? { + if !number(value)? { return Ok(false); } seen = true; @@ -265,13 +261,13 @@ fn numeric_column( Ok(seen) } -fn number(value: &Bound<'_, PyAny>, decimal: &Bound<'_, PyAny>) -> PyResult { +fn number(value: &Bound<'_, PyAny>) -> PyResult { if value.is_instance_of::() { return Ok(false); } Ok(value.is_instance_of::() || value.is_instance_of::() - || value.is_instance(decimal)?) + || decimal_instance(value)?) } /// The column types the adapter reported, where it knows them. @@ -307,11 +303,10 @@ fn render( for row in &rows { strict_width(columns.len(), row.len())?; } - let decimal: Bound<'_, PyAny> = imported(py, "decimal", "Decimal")?; let display = QueryDisplay { types: column_types(result, columns.len())?, numeric: (0..columns.len()) - .map(|index| numeric_column(&rows, index, &decimal)) + .map(|index| numeric_column(&rows, index)) .collect::>()?, columns, rows: texts(&rows, display_text)?, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs index 811f812b3e..716b4b38b0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs @@ -18,7 +18,8 @@ fn python_text(value: &Bound<'_, PyAny>) -> PyResult { Ok(value.str()?.to_string_lossy().into_owned()) } -fn decimal_instance(value: &Bound<'_, PyAny>) -> PyResult { +/// Whether `value` is a `decimal.Decimal`. +pub(crate) fn decimal_instance(value: &Bound<'_, PyAny>) -> PyResult { value.is_instance(&imported(value.py(), "decimal", "Decimal")?) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs index 68055a186a..114ac23eb3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs @@ -4,8 +4,8 @@ use pyo3::prelude::{PyResult, Python}; use sqlbuild_core::query_command::main::render_query_text::render_query_text; use sqlbuild_core::query_command::models::{QueryDisplay, QueryLayout}; -use crate::bindings::_helpers::boundary::python_calls::imported; -use crate::bindings::_helpers::build_progress::terminal::{terminal_columns, tty_flag}; +use crate::bindings::_helpers::build_progress::terminal::terminal_columns; +use crate::bindings::_helpers::presentation_text::terminal::stdout_is_tty; /// The format that always prints one record per row. pub(crate) const LONG_FORMAT: &str = "long"; @@ -14,7 +14,7 @@ const PIPED_WIDTH: usize = 120; /// The terminal's width when stdout is one, else [`PIPED_WIDTH`]. fn output_width(py: Python<'_>) -> PyResult { - if !tty_flag(&imported(py, "sys", "stdout")?).unwrap_or(false) { + if !stdout_is_tty(py).unwrap_or(false) { return Ok(PIPED_WIDTH); } terminal_columns(py) diff --git a/fensu.toml b/fensu.toml index 084eda8612..d64899dc9b 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1659,7 +1659,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/analysis_session/session.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/cursor_intrinsics.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/attachments/reference_markers.rs=3", - "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=25", + "crates/sqlbuild-python/src/bindings/_helpers/audit_command/command.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/auditing/identity.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/auditing/measurement.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/auditing/schema_audits.rs=1", @@ -1696,12 +1696,12 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/check/check_results.rs=13", "crates/sqlbuild-python/src/bindings/_helpers/cli/argv.rs=18", "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=18", - "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=30", + "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=27", "crates/sqlbuild-python/src/bindings/_helpers/cli/launch_state.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/cli/namespace.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=21", "crates/sqlbuild-python/src/bindings/_helpers/cli/surrogates.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=13", + "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_execution.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=1", @@ -1740,7 +1740,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_models.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_project.rs=6", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/assembly_sql_tests.rs=6", - "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=25", + "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/command.rs=24", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/config_validation.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/deferral.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/compile_driver/diff.rs=3", @@ -2098,9 +2098,8 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/python_values/values.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/query_command/dag.rs=3", "crates/sqlbuild-python/src/bindings/_helpers/query_command/project.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=15", + "crates/sqlbuild-python/src/bindings/_helpers/query_command/query.rs=14", "crates/sqlbuild-python/src/bindings/_helpers/query_command/query_values.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/query_command/text_output.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/recall/console.rs=10", "crates/sqlbuild-python/src/bindings/_helpers/recall/replacements.rs=8", "crates/sqlbuild-python/src/bindings/_helpers/recall/session.rs=3", @@ -2281,7 +2280,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/cli/dispatch.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/cli/entry.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs=12", - "crates/sqlbuild-python/src/bindings/_helpers/cli/termination.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_command.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_pipeline.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/clone/clone_progress.rs=3", From 24525487f7d8986e81e0e9662bdcf9ea976d1d7e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:45:49 +0000 Subject: [PATCH 24/30] test(postgres): expect the column types the query result now reports --- tests/integration/src/sqlbuild/adapters/postgres/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py index 18a0f9c776..44e11aa80f 100644 --- a/tests/integration/src/sqlbuild/adapters/postgres/test_client.py +++ b/tests/integration/src/sqlbuild/adapters/postgres/test_client.py @@ -81,6 +81,7 @@ def test_given_table_when_describing_then_postgres_returns_columns_in_order( expected_result=QueryResult( columns=("id", "name"), rows=((1, "hello"),), + column_types=("int4", "text"), ), ) ], From 7449859c91edba7095ef176ab573d033e864d479 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:07:39 +0000 Subject: [PATCH 25/30] test(compile): record compiled SQL in the Rules failure goldens --- .../compiler/failure/rule-custom-finding.json | 16 +++++++++++++++- ...ule-unused-cte-after-variable-and-macro.json | 17 ++++++++++++++++- .../compiler/failure/rule-unused-cte.json | 17 ++++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/goldens/compiler/failure/rule-custom-finding.json b/tests/goldens/compiler/failure/rule-custom-finding.json index fae7d107f4..1f2b7f5aa9 100644 --- a/tests/goldens/compiler/failure/rule-custom-finding.json +++ b/tests/goldens/compiler/failure/rule-custom-finding.json @@ -22,6 +22,20 @@ ] } ], - "compiled": {}, + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "-- sqb: compile failed for this model; not buildable", + "-- XSQBRDIFF001 at models/staging/stg_orders.sql:1:1: Models avoid the stg_ prefix", + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, "manifest": null } diff --git a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json index 38d48563d4..4c5cbf2d49 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json +++ b/tests/goldens/compiler/failure/rule-unused-cte-after-variable-and-macro.json @@ -25,6 +25,21 @@ ] } ], - "compiled": {}, + "compiled": { + "models/marts/customer_totals.sql": [ + "-- sqb: compile failed for this model; not buildable", + "-- SQBRSQL005 at models/marts/customer_totals.sql:5:117: CTE is unreachable from the final query", + "WITH totals AS (SELECT customer_id, (amount * 100) AS cents, 2 AS floor_cents FROM __ref(\"stg_orders\")), spare AS (SELECT 1 AS one)", + "SELECT customer_id, SUM(cents) AS total_amount", + "FROM totals WHERE cents > floor_cents", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, "manifest": null } diff --git a/tests/goldens/compiler/failure/rule-unused-cte.json b/tests/goldens/compiler/failure/rule-unused-cte.json index 7bf21cd0f3..6eb544614e 100644 --- a/tests/goldens/compiler/failure/rule-unused-cte.json +++ b/tests/goldens/compiler/failure/rule-unused-cte.json @@ -25,6 +25,21 @@ ] } ], - "compiled": {}, + "compiled": { + "models/marts/customer_totals.sql": [ + "-- sqb: compile failed for this model; not buildable", + "-- SQBRSQL005 at models/marts/customer_totals.sql:5:6: CTE is unreachable from the final query", + "WITH spare AS (SELECT 1 AS one)", + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, "manifest": null } From 2f666d7f1e53bf76c258581ade8c974d3c008dd2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:08:31 +0000 Subject: [PATCH 26/30] test(query): expect the boxed default and JSON for exact target output --- .../src/sqlbuild/cli/commands/main/query/test_query.py | 9 ++++++--- .../cli/commands/main/query/test_query_project_inputs.py | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py index 72b8c5ac14..9c53a27ef0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query.py @@ -75,9 +75,12 @@ def test_given_query_command_when_running_then_outputs_result( query_file_path="queries/orders.sql", query_sql="SELECT 7 AS order_id, 'gaufre' AS label\n", expected_stdout_fragment=( - "-[ RECORD 1 ]---------------------------+\n" - "order_id | 7\n" - "label | gaufre\n\n" + "┌──────────┬─────────┐\n" + "│ order_id │ label │\n" + "│ INTEGER │ VARCHAR │\n" + "├──────────┼─────────┤\n" + "│ 7 │ gaufre │\n" + "└──────────┴─────────┘\n\n" "1 row\n" ), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py index a2efbe07fc..25872dca8c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/query/test_query_project_inputs.py @@ -149,8 +149,8 @@ def test_given_project_when_running_command_then_project_discovery_runs_only_whe ( QueryTargetConnectionTestCase( description="default target connection", - command=("query", "SELECT label FROM destination", "--format", "long"), - expected_stdout_fragment="label | dev\n", + command=("query", "SELECT label FROM destination", "--format", "json"), + expected_stdout_fragment='[{"label": "dev"}]\n', ), QueryTargetConnectionTestCase( description="explicit target connection", @@ -158,11 +158,11 @@ def test_given_project_when_running_command_then_project_discovery_runs_only_whe "query", "SELECT label FROM destination", "--format", - "long", + "json", "--target", "prod", ), - expected_stdout_fragment="label | prod\n", + expected_stdout_fragment='[{"label": "prod"}]\n', ), ), ids=lambda case: case.description, From 34487c6b2b83298a894feae2b6c5165b7dab0cd5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:44:11 +0000 Subject: [PATCH 27/30] fix(compile): remove stale compiled files when Rules fail A compile whose Rules fail now leaves only this compile's model files under target/compiled/, so a model removed since the last compile no longer keeps its compiled file and a warm compile matches an uncached one. --- .../_helpers/compile_command/target_writer.rs | 14 +++++++------- .../skills/sqlbuild/references/docs/cli/compile.md | 8 ++++---- .../commands/main/test_compile_rule_failures.py | 3 +++ website/src/content/docs/docs/cli/compile.mdx | 8 ++++---- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index b0f65c4705..55db2827f3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -806,7 +806,7 @@ fn rule_error_lines( Ok(lines) } -/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest is removed, since it no longer describes `target/`; nothing else changes: no tests, cache or stale-file removal. +/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest and every other compiled file are removed, since they no longer describe `target/`; no tests or cache are written. pub(crate) fn write_rule_failed_models<'py>( target_dir: &Bound<'py, PyAny>, (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -822,6 +822,7 @@ pub(crate) fn write_rule_failed_models<'py>( } let compiled_dir: PathBuf = joined(target_dir, &[COMPILED_DIR])?.extract()?; let mut batch: ArtifactBatch = ArtifactBatch::new(true, None); + let mut written: Vec = Vec::new(); for model in native_field(project, "models")?.try_iter()? { let model: Bound<'py, PyAny> = model?; let (relative_path, query_sql): (String, String) = @@ -838,13 +839,12 @@ pub(crate) fn write_rule_failed_models<'py>( Some(lines) => format!("{RULE_FAILED_HEADER}\n{}\n{query_sql}", lines.join("\n")), None => query_sql, }; - batch.queue_sql_at( - py, - native_model_output_path(&compiled_dir, &relative_path), - &sql, - )?; + let path: PathBuf = native_model_output_path(&compiled_dir, &relative_path); + batch.queue_sql_at(py, path.clone(), &sql)?; + written.push(path); } - batch.flush(py) + batch.flush(py)?; + remove_stale(target_dir, written, None) } /// Plan uncached SQL tests and report their errors without writing artifacts or cache. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 7fa5671837..f4f2792e84 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -95,10 +95,10 @@ configuration, use [`sqb rules run`](rules.md). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a -`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each -model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros -generated. A model with Rule errors starts with a header that marks it not buildable and lists +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. +Each model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that +macros generated. Other files under `target/compiled/` and a `target/manifest.json` from an earlier +compile are removed, since they no longer match. A model with Rule errors starts with a header that marks it not buildable and lists its findings: ```sql diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py index f57bd81d90..b39902a37e 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_rule_failures.py @@ -66,6 +66,7 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu "models/open_orders.sql": open_orders_model( 'SELECT o.order_id, o.status FROM __ref("orders") AS o' ), + "models/retired_orders.sql": _ORDERS, } for relative, contents in files.items(): path: Path = tmp_path / relative @@ -83,6 +84,7 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu open_orders_model(test_case.open_orders_sql), encoding="utf-8" ) assert (tmp_path / "target" / "manifest.json").exists() + (tmp_path / "models" / "retired_orders.sql").unlink() exit_code: int = main(compile_args) @@ -96,3 +98,4 @@ def test_given_rule_findings_when_compiling_then_models_are_written_with_a_failu "SELECT 1 AS order_id, 'open' AS status\n" ) assert (tmp_path / "target" / "manifest.json").exists() is test_case.expected_manifest + assert sorted(path.name for path in compiled.iterdir()) == ["open_orders.sql", "orders.sql"] diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index 90b30e2038..bf8dad2015 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -79,10 +79,10 @@ configuration, use [`sqb rules run`](/docs/cli/rules). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a -`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each -model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros -generated. A model with Rule errors starts with a header that marks it not buildable and lists +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. +Each model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that +macros generated. Other files under `target/compiled/` and a `target/manifest.json` from an earlier +compile are removed, since they no longer match. A model with Rule errors starts with a header that marks it not buildable and lists its findings: ```sql From fe731abf61fc90fa3e5f8d7243aa36815b12614a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:44:12 +0000 Subject: [PATCH 28/30] fix(rules): let a macro's reads through a passed relation fit either qualification style In a caller's query, a column a macro call reads through the relation it was passed, such as o.created_at from @src.created_at, no longer counts as unqualified. The slot decides how that read expands, so a hand-qualified caller using such a macro passes again, while a macro's bare column beside hand-qualified columns still fails. --- .../src/sql_lint/_helpers/additional.rs | 26 ++++++++++--------- .../sql_lint/tests/test_lint_exemptions.rs | 13 +++++++--- .../docs/concepts/macros/sql-macros.md | 22 ++++++++-------- .../main/test_macro_slot_qualification.py | 10 +++---- .../docs/docs/concepts/macros/sql-macros.mdx | 22 ++++++++-------- 5 files changed, 51 insertions(+), 42 deletions(-) diff --git a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs index 67fd258715..f5649d280e 100644 --- a/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs +++ b/crates/sqlbuild-rules/src/sql_lint/_helpers/additional.rs @@ -694,7 +694,7 @@ fn collect_reference_facts(context: &ReferenceFactContext<'_>) -> AdditionalQuer facts } -/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. A plain macro slot read such as `@orders.amount` counts as unqualified though it expands to read through its relation: in a macro file it lints as a parameter sentinel pair, and in a call's output as a qualifier naming a relation the call passes. A scope whose every reference a macro call generated is judged once, in the macro file. +/// The unqualified references of a single-relation scope that also qualifies some, each judged as it was written. In a macro file a plain slot read such as `@orders.amount` lints as a parameter sentinel pair and counts as unqualified. In a call's output, a read qualified by a relation the call passes takes neither style: the slot decides how it expands, not the caller or the macro author. A scope whose every reference a macro call generated is judged once, in the macro file. fn inconsistent_qualification( tokens: &[Token], (qualified, unqualified): (&[usize], &[usize]), @@ -713,18 +713,20 @@ fn inconsistent_qualification( { return Vec::new(); } - let plain_read = |index: &usize| { - is_plain_slot_read(tokens, *index) - || output_of(*index).is_some_and(|output| { - let qualifier: String = unquoted_name(&tokens[*index].text); - outputs[output] - .relations - .iter() - .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) - }) + let passed_relation_read = |index: &usize| { + output_of(*index).is_some_and(|output| { + let qualifier: String = unquoted_name(&tokens[*index].text); + outputs[output] + .relations + .iter() + .any(|relation| relation.eq_ignore_ascii_case(&qualifier)) + }) }; - let (plain_reads, qualified): (Vec, Vec) = - qualified.iter().copied().partition(plain_read); + let (plain_reads, qualified): (Vec, Vec) = qualified + .iter() + .copied() + .filter(|index| !passed_relation_read(index)) + .partition(|&index| is_plain_slot_read(tokens, index)); if qualified.is_empty() || (unqualified.is_empty() && plain_reads.is_empty()) { return Vec::new(); } diff --git a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs index f4dfda04da..46f0ca78e0 100644 --- a/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs +++ b/crates/sqlbuild-rules/src/sql_lint/tests/test_lint_exemptions.rs @@ -17,18 +17,25 @@ fn given_sql_macro_output_when_linting_qualification_then_each_column_counts_as_ expected_count: 1, }, SqlMacroOutputLintTestCase { - description: "plain slot reads count as unqualified beside a bare caller column", + description: "reads through a passed relation pass beside a bare caller column", sql: "SELECT order_id, o.amount - o.discount AS net FROM orders AS o", output: "o.amount - o.discount", relations: &["o"], expected_count: 0, }, SqlMacroOutputLintTestCase { - description: "plain slot reads beside a hand-qualified column fail", + description: "reads through a passed relation pass beside a hand-qualified column", sql: "SELECT o.order_id, o.amount - o.discount AS net FROM orders AS o", output: "o.amount - o.discount", relations: &["o"], - expected_count: 2, + expected_count: 0, + }, + SqlMacroOutputLintTestCase { + description: "the caller's own mix beside reads through a passed relation fails", + sql: "SELECT o.order_id, status, o.amount AS a FROM orders AS o", + output: "o.amount", + relations: &["o"], + expected_count: 1, }, SqlMacroOutputLintTestCase { description: "an aliased read counts as qualified beside a bare caller column", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md index b4d36c76fd..28bf4feea9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros/sql-macros.md @@ -171,10 +171,8 @@ MACRO ( SELECT @src.created_at, @src.updated_at, @src.updated_at > @src.created_at AS was_updated ``` -`SELECT order_id, @sql.audit_columns(o) FROM orders AS o` compiles to -`SELECT order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. -The macro's plain reads count as unqualified, so the caller writes its own columns bare too (see -[reading slots](#reading-slots)). +`SELECT o.order_id, @sql.audit_columns(o) FROM orders AS o` compiles to +`SELECT o.order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. Use a column list when the same ordered set of columns repeats, for example in every branch of a `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. @@ -319,13 +317,15 @@ fully typed. `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. - Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and - unqualified columns, counts every plain read as unqualified, every `alias.@weekly...` read as - qualified, and any other column a macro outputs as the macro writes it. A query the macro writes - whole, such as a CTE of a CTE-list macro, is judged once, in the macro file. Where a call's output - joins the caller's query, as with expression and column-list macros, that query is judged at the - call, so a plain read beside a hand-qualified column such as `o.order_id` fails. A step written - only in plain reads, such as `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, - passes; to qualify, read every slot through an alias. + unqualified columns, judges a query the macro writes whole, such as a CTE of a CTE-list macro, + once, in the macro file. There every plain read counts as unqualified and every + `alias.@weekly...` read as qualified. A step written only in plain reads, such as + `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, passes; to qualify, read + every slot through an alias. +- Where a call's output joins the caller's query, as with expression and column-list macros, the + query is judged at the call. A read through the relation the call passes, such as `o.created_at` + above, fits either style; any other column counts as the macro writes it, so a bare `status` + beside a hand-qualified `o.order_id` fails. A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, could bind its bare columns to the subquery's relations instead of the one the slot reads. The diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py index 1ac91849b4..3c739adeb2 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_macro_slot_qualification.py @@ -59,7 +59,7 @@ absent_fragments=("--> models/refund_rates.sql",), ), MacroSlotQualificationTestCase( - description="an expression macro's plain reads count as unqualified beside bare columns", + description="an expression macro's plain reads pass beside bare caller columns", files={ "models/orders.sql": ( 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' @@ -82,7 +82,7 @@ absent_fragments=("SQBRSQL028",), ), MacroSlotQualificationTestCase( - description="an expression macro's plain reads beside a hand-qualified column fail", + description="an expression macro's plain reads pass beside hand-qualified columns", files={ "models/orders.sql": ( 'MODEL (\n description "One row per order",\n materialized table,\n);\n\n' @@ -100,9 +100,9 @@ 'FROM __ref("orders") AS o\n' ), }, - expected_exit_code=1, - expected_fragments=("error[SQBRSQL028]", "--> models/order_totals.sql"), - absent_fragments=("--> models/_macros/sql/net_amount.sql",), + expected_exit_code=0, + expected_fragments=("Project compiled",), + absent_fragments=("SQBRSQL028",), ), MacroSlotQualificationTestCase( description="an expression macro's bare column beside a hand-qualified column fails", diff --git a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx index 54e4bee69e..7948e1daeb 100644 --- a/website/src/content/docs/docs/concepts/macros/sql-macros.mdx +++ b/website/src/content/docs/docs/concepts/macros/sql-macros.mdx @@ -156,10 +156,8 @@ MACRO ( SELECT @src.created_at, @src.updated_at, @src.updated_at > @src.created_at AS was_updated ``` -`SELECT order_id, @sql.audit_columns(o) FROM orders AS o` compiles to -`SELECT order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. -The macro's plain reads count as unqualified, so the caller writes its own columns bare too (see -[reading slots](#reading-slots)). +`SELECT o.order_id, @sql.audit_columns(o) FROM orders AS o` compiles to +`SELECT o.order_id, o.created_at, o.updated_at, o.updated_at > o.created_at AS was_updated FROM orders AS o`. Use a column list when the same ordered set of columns repeats, for example in every branch of a `UNION ALL`. The call takes no alias (`P021`): each inserted column is named in the macro. @@ -304,13 +302,15 @@ fully typed. `USING (@weekly.week)` for a column the type does not declare. - In a column-list macro, `o.@weekly.keys` outputs the group's columns under their own names. - Rules judge a read as you wrote it. `SQBRSQL028`, which rejects a query mixing qualified and - unqualified columns, counts every plain read as unqualified, every `alias.@weekly...` read as - qualified, and any other column a macro outputs as the macro writes it. A query the macro writes - whole, such as a CTE of a CTE-list macro, is judged once, in the macro file. Where a call's output - joins the caller's query, as with expression and column-list macros, that query is judged at the - call, so a plain read beside a hand-qualified column such as `o.order_id` fails. A step written - only in plain reads, such as `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, - passes; to qualify, read every slot through an alias. + unqualified columns, judges a query the macro writes whole, such as a CTE of a CTE-list macro, + once, in the macro file. There every plain read counts as unqualified and every + `alias.@weekly...` read as qualified. A step written only in plain reads, such as + `SELECT @weekly.keys, @weekly.refund_count AS refunds FROM @weekly`, passes; to qualify, read + every slot through an alias. +- Where a call's output joins the caller's query, as with expression and column-list macros, the + query is judged at the call. A read through the relation the call passes, such as `o.created_at` + above, fits either style; any other column counts as the macro writes it, so a bare `status` + beside a hand-qualified `o.order_id` fails. A plain group read inside a subquery, such as `EXISTS (SELECT @weekly.keys FROM refunds AS r ...)`, could bind its bare columns to the subquery's relations instead of the one the slot reads. The From a3895d738f11ef45d94240288ee51e9aa48df950 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:44:14 +0000 Subject: [PATCH 29/30] perf(recall): read SQL references once in the ClickHouse statements view sql_available and sql_unavailable_reason now share one read of the per-invocation SQL references, so each source table is read once per lookup. --- .../src/schema/_helpers/clickhouse_views.rs | 19 +++++++++++-------- .../test_clickhouse_recall_views.py | 12 +++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs index 6a55d15a51..638fe61891 100644 --- a/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs +++ b/crates/sqlbuild-recall/src/schema/_helpers/clickhouse_views.rs @@ -343,7 +343,7 @@ FROM folded", ) } -/// Two folds over one read of the events: by resource, which names the resource and gathers its statement events, then by statement. Both group by `invocation_id` and `invocation_started_at`, so a filter on either reaches the read; `sql_available` needs a reference to the digest in the statement's own invocation and the stored SQL, which is shared across invocations. +/// Two folds over one read of the events: by resource, which names the resource and gathers its statement events, then by statement. Both group by `invocation_id` and `invocation_started_at`, so a filter on either reaches the read; `sql_available` needs a reference to the digest in the statement's own invocation and the stored SQL, which is shared across invocations. The references and the stored SQL are each read once. fn statements(options: &DdlOptions) -> String { let events = retained(Table::Events, options, ""); let artifacts = clickhouse_table(table_spec(Table::Artifacts).name, options); @@ -430,17 +430,20 @@ SELECT if(has_terminal, last_row_count, NULL) AS row_count, if(has_terminal, last_affected_rows, NULL) AS affected_rows, first_sql_digest AS sql_digest, - (invocation_id, concat('sha256:', first_sql_digest)) IN ( - SELECT invocation_id, digest FROM {refs} - WHERE digest IN (SELECT digest FROM {artifacts}) - ) AS sql_available, + sql_referenced + AND concat('sha256:', first_sql_digest) IN (SELECT digest FROM {artifacts}) + AS sql_available, multiIf( sql_available, NULL, - (invocation_id, concat('sha256:', first_sql_digest)) - IN (SELECT invocation_id, digest FROM {refs}), 'expired_or_not_captured', + sql_referenced, 'expired_or_not_captured', 'not_captured' ) AS sql_unavailable_reason -FROM folded" +FROM ( + SELECT *, + (invocation_id, concat('sha256:', first_sql_digest)) + IN (SELECT invocation_id, digest FROM {refs}) AS sql_referenced + FROM folded +)" ) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py index cb76fc1447..58c19acd4e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py @@ -211,14 +211,20 @@ def test_given_a_reader_when_querying_then_pinned_settings_apply_unless_it_may_n ], expected_added_columns=( ( + ("ended_at_estimated", "Nullable(UInt8)"), + ("signal_number", "Nullable(Int64)"), + ("cost_status", "Nullable(String)"), ("origin", "Nullable(String)"), ("command_mode", "Nullable(String)"), + ("selection_count", "Nullable(Int64)"), + ("selection_truncated", "Nullable(UInt8)"), + ("selector_files", "Nullable(String)"), ("integration", "Nullable(String)"), ("integration_job", "Nullable(String)"), ("integration_run_id", "Nullable(String)"), ("integration_retry", "Nullable(Int64)"), ), - (), + (("sql_unavailable_reason", "Nullable(String)"),), (), (), ), @@ -345,9 +351,9 @@ def test_given_representative_invocations_when_views_are_rewritten_then_rows_and expected_primary_key_reads=3, ), ClickHouseRecallTableReadsTestCase( - description="statements reads the events once and the stored SQL once", + description="statements reads the events, SQL references and stored SQL once each", view="statements", - expected_tables=("recall_artifacts", "recall_events"), + expected_tables=("recall_artifact_refs", "recall_artifacts", "recall_events"), expected_primary_key_reads=1, ), ClickHouseRecallTableReadsTestCase( From 528232261b8c45e4b8fe61c4bcaf6f9ff9e5a87a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:47:39 +0000 Subject: [PATCH 30/30] fix(compile): keep other artifacts when Rules fail and remove only retired models A compile whose Rules fail writes each model's SQL and removes only the compiled file of a model the project no longer has, using a record of the model files compiles wrote. Audits, tests and other artifacts stay as the last passing compile left them, so a failed compile publishes nothing else and a later delta or warm compile matches a full one. --- .../_helpers/compile_command/target_writer.rs | 120 +++++++++++++++++- .../sqlbuild/references/docs/cli/compile.md | 11 +- website/src/content/docs/docs/cli/compile.mdx | 11 +- 3 files changed, 125 insertions(+), 17 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs index 55db2827f3..11fe1c157d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compile_command/target_writer.rs @@ -1,6 +1,6 @@ //! Write compiled project output to `target/`: models, functions, audits, SQL tests, manifest. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; use pyo3::exceptions::PyValueError; @@ -58,6 +58,9 @@ const TESTS_DIR: &str = "tests"; const MANIFEST_FILE: &str = "manifest.json"; /// The compile cache file stamping the compiled tree the last stale-file walk left. const COMPILED_TREE_STAMP_FILE: &str = "compiled-tree.bin"; +/// The model files compiles wrote under `target/compiled/`, one per line, relative to it. +const COMPILED_MODELS_FILE: &str = "compiled-models.txt"; + const SQL_FILE_SUFFIX: &str = ".sql"; const SQL_TEST_CACHE_LOGGER: &str = "sqlbuild.compile.reuse"; /// The miss reason of a SQL test the cache file holds no record for. @@ -321,8 +324,15 @@ fn write_planned_target<'py>( managed.push(path); } batch.flush(py)?; + let managed: Vec = extracted(&managed)?; + record_compiled_models( + &target_dir.extract::()?, + &compiled.extract::()?, + &managed, + record_mode(remove_stale_files), + )?; if remove_stale_files { - remove_stale(target_dir, extracted(&managed)?, None)?; + remove_stale(target_dir, managed, None)?; } write_manifest(target_dir, manifest, None)?; written_target( @@ -726,9 +736,15 @@ pub(crate) fn write_static_target<'py>( })(); let (test_paths, diagnostics) = batch.settle(py, queued)?; managed.extend(test_paths); + let mut paths: Vec = managed_models; + paths.extend(extracted(&managed)?); + record_compiled_models( + &target_dir.extract::()?, + &compiled_dir, + &paths, + record_mode(remove_stale_files), + )?; if remove_stale_files { - let mut paths: Vec = managed_models; - paths.extend(extracted(&managed)?); let cache_dir: Bound<'py, PyAny> = native_field(project, "compile_cache_dir")?; let cache_dir: Option = if cache_dir.is_none() { None @@ -806,7 +822,7 @@ fn rule_error_lines( Ok(lines) } -/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest and every other compiled file are removed, since they no longer describe `target/`; no tests or cache are written. +/// After Rules failed, write every model's compiled SQL so the expansion they checked can be read. A model with Rule errors starts with a header marking it not buildable and listing them. An earlier manifest is removed, since it no longer describes `target/`, and so is a model file an earlier compile wrote for a model the project no longer has. Every other artifact stays as the last compile left it, for the next passing compile to publish or remove. pub(crate) fn write_rule_failed_models<'py>( target_dir: &Bound<'py, PyAny>, (project, project_dir): (&Bound<'py, PyAny>, &Bound<'py, PyAny>), @@ -844,7 +860,87 @@ pub(crate) fn write_rule_failed_models<'py>( written.push(path); } batch.flush(py)?; - remove_stale(target_dir, written, None) + record_compiled_models( + &target_dir.extract::()?, + &compiled_dir, + &written, + ModelRecord::RemoveRetired, + ) +} + +fn record_mode(removes_stale_files: bool) -> ModelRecord { + if removes_stale_files { + ModelRecord::Replace + } else { + ModelRecord::Extend + } +} + +/// How a compile updates the record of the model files SQLBuild wrote under `target/compiled/`. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ModelRecord { + /// A passing compile that removed stale files: the record is exactly what it wrote. + Replace, + /// A passing compile that kept the files already there: the record gains what it wrote. + Extend, + /// A compile whose Rules failed: other recorded models are retired, so their files are removed. + RemoveRetired, +} + +/// Update the record of written model files, one path per line relative to `compiled_dir`. +fn record_compiled_models( + target: &Path, + compiled_dir: &Path, + written: &[PathBuf], + mode: ModelRecord, +) -> PyResult<()> { + let record: PathBuf = target.join(COMPILED_MODELS_FILE); + let previous: String = match std::fs::read_to_string(&record) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => return Err(error.into()), + }; + let models_dir: PathBuf = compiled_dir.join(MODELS_DIR); + let mut lines: BTreeSet = BTreeSet::new(); + for path in written { + match path.strip_prefix(&models_dir) { + Ok(relative) => { + let _ = lines.insert(format!( + "{MODELS_DIR}/{}", + relative.to_string_lossy().replace('\\', "/") + )); + } + Err(_not_a_model) => continue, + } + } + for line in previous.lines().filter(|line| !line.is_empty()) { + if lines.contains(line) { + continue; + } + match mode { + ModelRecord::Replace => {} + ModelRecord::Extend => { + let _ = lines.insert(line.to_owned()); + } + ModelRecord::RemoveRetired => { + let path: PathBuf = compiled_dir.join(line); + if !path.starts_with(&models_dir) { + continue; + } + match std::fs::remove_file(&path) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(error.into()); + } + _ => {} + } + } + } + } + let contents: String = lines.iter().map(|line| format!("{line}\n")).collect(); + if contents != previous { + std::fs::write(&record, contents)?; + } + Ok(()) } /// Plan uncached SQL tests and report their errors without writing artifacts or cache. @@ -882,7 +978,7 @@ pub(crate) fn publish_staged_target<'py>( .extract()?; let (staged, compiled): (PathBuf, PathBuf) = (staged_dir.extract()?, compiled_dir.extract()?); let (kind, published) = record_compile_timing("physical_write_ms", || { - publish_staged_artifacts(py, staged, compiled, expected) + publish_staged_artifacts(py, staged, compiled.clone(), expected.clone()) })?; if kind == PUBLISHED_STAGING_CHANGED { return Err(staging_changed( @@ -906,6 +1002,16 @@ pub(crate) fn publish_staged_target<'py>( remove_stale(target_dir, extracted(&managed)?, None)?; } } + let written: Vec = expected + .iter() + .map(|relative| compiled.join(relative)) + .collect(); + record_compiled_models( + &target_dir.extract::()?, + &compiled, + &written, + ModelRecord::Replace, + )?; write_manifest(target_dir, manifest, recorder)?; written_target( target_dir, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index f4f2792e84..1a4af4ad95 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -95,11 +95,12 @@ configuration, use [`sqb rules run`](rules.md). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. -Each model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that -macros generated. Other files under `target/compiled/` and a `target/manifest.json` from an earlier -compile are removed, since they no longer match. A model with Rule errors starts with a header that marks it not buildable and lists -its findings: +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a +`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each +model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros +generated, and the compiled file of a model you removed from the project is deleted. Other +compiled files stay as the last passing compile left them. A model with Rule errors starts with a +header that marks it not buildable and lists its findings: ```sql -- sqb: compile failed for this model; not buildable diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index bf8dad2015..3fbe67637a 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -79,11 +79,12 @@ configuration, use [`sqb rules run`](/docs/cli/rules). ### When Rules fail -A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts. -Each model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that -macros generated. Other files under `target/compiled/` and a `target/manifest.json` from an earlier -compile are removed, since they no longer match. A model with Rule errors starts with a header that marks it not buildable and lists -its findings: +A Rule finding still exits `1` and blocks the manifest, the DAG file and the other artifacts; a +`target/manifest.json` from an earlier compile is removed, since it no longer matches. Each +model's expanded SQL is still written to `target/compiled/models/`, so you can read SQL that macros +generated, and the compiled file of a model you removed from the project is deleted. Other +compiled files stay as the last passing compile left them. A model with Rule errors starts with a +header that marks it not buildable and lists its findings: ```sql -- sqb: compile failed for this model; not buildable