diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 27e8cb57cc..a5a48b0136 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -561,11 +561,15 @@ jobs: strategy: fail-fast: false matrix: + # Temporary: a one-model edit recompiles the edited model uncached until native + # rendering becomes the default; tighten edit_max_ratio back to max_ratio then. include: - kind: dense max_ratio: "1.10" + edit_max_ratio: "1.25" - kind: fresh max_ratio: "1.10" + edit_max_ratio: "1.20" steps: - uses: actions/checkout@v4 - name: Check out the comparison base @@ -596,11 +600,13 @@ jobs: env: KIND: ${{ matrix.kind }} MAX_RATIO: ${{ matrix.max_ratio }} + EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }} run: | uv run python -m scripts.compare_compile_performance \ --kind "${KIND}" \ --modes cold warm edit \ --max-ratio "${MAX_RATIO}" \ + --edit-max-ratio "${EDIT_MAX_RATIO}" \ --models 3000 \ --base-root base \ --base-python base/.venv/bin/python \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs index 31c5e16613..1928fa5ace 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs @@ -1 +1,3 @@ pub(crate) mod native_store; +pub(crate) mod registration; +pub(crate) mod sql_test_scan_store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs index 0cd6e1c9a0..fd2e81a45c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs @@ -1,4 +1,4 @@ -//! Digests and project fingerprints of the shared native store, with the GIL released. +//! Digests, project fingerprints and store files of the shared native store, with the GIL released. use pyo3::exceptions::{PyOSError, PyRuntimeError}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; @@ -8,8 +8,10 @@ use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each; use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::main::open_native_store::open_native_store; +use sqlbuild_cache::store::models::NativeStore; use std::collections::HashSet; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use crate::bindings::types::CompilerDetach; @@ -50,6 +52,34 @@ fn readable_digest(digest: &std::io::Result) -> Option { } } +/// Open one cache kind's store with the GIL released; a failed read is an `OSError`. +pub(crate) fn open_store( + py: Python<'_>, + path: &Path, + kind: &str, + environment: &str, +) -> PyResult { + py.compiler_detach(|| Ok(open_native_store(path, kind, environment))) + .map_err(PyRuntimeError::new_err)? + .map_err(|error| PyOSError::new_err(error.to_string())) +} + +/// Atomically save a store holding unsaved values with the GIL released; returns entries written. +pub(crate) fn save_store( + py: Python<'_>, + store: &NativeStore, + path: &Path, + metadata: &[u8], +) -> PyResult> { + if !store.needs_save() { + return Ok(None); + } + py.compiler_detach(|| Ok(store.save(path, metadata))) + .map_err(PyRuntimeError::new_err)? + .map(Some) + .map_err(|error| PyOSError::new_err(error.to_string())) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(content_digest, module)?)?; module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs new file mode 100644 index 0000000000..c9b123dbc1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs @@ -0,0 +1,11 @@ +//! Register the shared native store functions and classes. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cache::{native_store, sql_test_scan_store}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + native_store::register(module)?; + sql_test_scan_store::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs new file mode 100644 index 0000000000..d53b762eea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs @@ -0,0 +1,67 @@ +//! Whole per-file SQL-test scan results kept as one kind of the shared native store. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_cache::digest::main::content_digest::content_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::models::NativeStore; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; + +/// The native store kind holding SQL-test scan results. +const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; + +/// One compile's view of the stored scan results, keyed by the digest of the caller's key parts. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct SqlTestScanStore { + store: Mutex, +} + +fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("SQL test scan store is poisoned") +} + +#[pymethods] +impl SqlTestScanStore { + /// Open the store at `path` for `environment`; missing, damaged or foreign files open empty. + #[new] + fn new(py: Python<'_>, path: PathBuf, environment: &str) -> PyResult { + let store: NativeStore = open_store(py, &path, SQL_TEST_SCAN_STORE_KIND, environment)?; + Ok(Self { + store: Mutex::new(store), + }) + } + + /// The value stored under the digest of `key_parts`, or `None`. + fn get(&self, key_parts: Vec) -> PyResult>> { + let key: ContentDigest = content_digest(&key_parts); + Ok(self + .store + .lock() + .map_err(poisoned)? + .get(&key) + .map(<[u8]>::to_vec)) + } + + /// Store `value` under the digest of `key_parts`. + fn put(&self, key_parts: Vec, value: Vec) -> PyResult<()> { + let key: ContentDigest = content_digest(&key_parts); + self.store.lock().map_err(poisoned)?.put(key, value); + Ok(()) + } + + /// Atomically save the store when this compile stored a value; returns entries written. + fn save(&self, py: Python<'_>, path: PathBuf) -> PyResult> { + let store = self.store.lock().map_err(poisoned)?; + save_store(py, &store, &path, &[]) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index 718dcff75f..63bec19bab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -1,15 +1,14 @@ //! The macro call memo class: one compile's recorded calls and its optional native store. -use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::Python; use pyo3::{PyResult, pyclass, pymethods}; -use sqlbuild_cache::store::main::open_native_store::open_native_store; use sqlbuild_cache::store::models::NativeStore; use sqlbuild_render::macro_calls::models::MacroCallEntry; use std::path::PathBuf; use std::sync::Mutex; -use crate::bindings::types::CompilerDetach; +use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; use crate::bindings::_helpers::render::macro_call_rows::{EntryRow, event_from_row, event_row}; @@ -80,10 +79,7 @@ impl MacroCallMemo { path: PathBuf, environment: &str, ) -> PyResult<(Vec, usize)> { - let store: NativeStore = py - .compiler_detach(|| Ok(open_native_store(&path, MACRO_CALL_STORE_KIND, environment))) - .map_err(PyRuntimeError::new_err)? - .map_err(|error| PyOSError::new_err(error.to_string()))?; + let store: NativeStore = open_store(py, &path, MACRO_CALL_STORE_KIND, environment)?; let opened = (store.metadata().to_vec(), store.loaded_entries()); self.inner.lock().map_err(poisoned)?.attach_store(store); Ok(opened) @@ -119,13 +115,9 @@ impl MacroCallMemo { metadata: &[u8], ) -> PyResult> { let mut memo = self.inner.lock().map_err(poisoned)?; - let Some(store) = memo.store_mut().filter(|store| store.needs_save()) else { + let Some(store) = memo.store_mut() else { return Ok(None); }; - let store: &NativeStore = store; - py.compiler_detach(|| Ok(store.save(&path, metadata))) - .map_err(PyRuntimeError::new_err)? - .map(Some) - .map_err(|error| PyOSError::new_err(error.to_string())) + save_store(py, store, &path, metadata) } } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 37e0e09255..9e1dce2fcb 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -32,7 +32,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { scopes::registration::register(module)?; model_config::registration::register(module)?; render::macro_calls::register(module)?; - cache::native_store::register(module)?; + cache::registration::register(module)?; attachments::registration::register(module)?; register_analysis_stages(module)?; module.add("API_VERSION", API_VERSION)?; diff --git a/fensu.toml b/fensu.toml index 8f64aec491..a10fe5c399 100644 --- a/fensu.toml +++ b/fensu.toml @@ -177,12 +177,6 @@ path = "src/sqlbuild/runtime/event_exporting/main/command_output_sink.py" symbols = ["command_output_sink"] reason = "Bare command-output sink decorators are evaluated as decorator(function), requiring positional function calling." -[[targets.python.rule_exceptions]] -rule = "FFS120" -path = "src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py" -symbols = ["_RestrictedUnpickler.find_class"] -reason = "This override preserves pickle.Unpickler's positional find_class(module, name) protocol." - [[targets.python.rule_exceptions]] rule = "FFR402" path = "src/sqlbuild/integrations/dagster/__init__.py" diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index a92535a050..84cdd7589e 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -3,6 +3,7 @@ from __future__ import annotations import platform +from collections.abc import Mapping from pathlib import Path from scripts.compile_performance_ratio.constants import MODE_TITLES, REPORTED_PHASES @@ -16,6 +17,7 @@ def comparison_markdown( max_ratio: float, per_side_projects: bool, failures: tuple[str, ...], + mode_max_ratios: Mapping[str, float] | None = None, ) -> str: """Summarize medians, ratios and per-phase timings of every mode as Markdown tables.""" @@ -31,7 +33,8 @@ def comparison_markdown( f"### {comparison.kind} {comparison.models} models, " f"{MODE_TITLES[comparison.mode]}: head vs base (same runner)", "", - f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", + f"Runner CPU: {cpu_model()}; {runs} alternating runs each; " + f"limit {(mode_max_ratios or {}).get(comparison.mode, max_ratio):.2f}x.", "", generation, "", diff --git a/scripts/compile_performance_ratio/_helpers/verdict.py b/scripts/compile_performance_ratio/_helpers/verdict.py index 6a85f2f240..cca469c30b 100644 --- a/scripts/compile_performance_ratio/_helpers/verdict.py +++ b/scripts/compile_performance_ratio/_helpers/verdict.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +from collections.abc import Mapping from scripts.compile_performance_ratio.models import CompileComparison @@ -13,6 +14,7 @@ def ratio_failures( modes: tuple[str, ...], max_ratio: float, noise_floor_seconds: float, + mode_max_ratios: Mapping[str, float] | None = None, ) -> tuple[str, ...]: """Describe every requested mode that is unmeasured or slower than ratio plus floor allow.""" @@ -26,6 +28,7 @@ def ratio_failures( failures.append(f"{mode}: no measurement") continue prefix: str = f"{comparison.kind} {comparison.models} {mode}" + limit: float = (mode_max_ratios or {}).get(mode, max_ratio) for metric, base, head, ratio in ( ( "wall", @@ -39,8 +42,8 @@ def ratio_failures( failures.append( f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)" ) - elif head > base * max_ratio + noise_floor_seconds: - failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}") + elif head > base * limit + noise_floor_seconds: + failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {limit:.2f}") return tuple(failures) diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index fec5e8d32c..330bc2704d 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -19,6 +19,7 @@ COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, + EDIT_MODE, NOISE_FLOOR_SECONDS, PROJECT_KINDS, ) @@ -59,11 +60,15 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: runs=args.runs, modes=modes, ) + mode_max_ratios: dict[str, float] = ( + {} if args.edit_max_ratio is None else {EDIT_MODE: args.edit_max_ratio} + ) failures: tuple[str, ...] = ratio_failures( comparisons=comparisons, modes=modes, max_ratio=args.max_ratio, noise_floor_seconds=args.noise_floor_seconds, + mode_max_ratios=mode_max_ratios, ) markdown: str = comparison_markdown( comparisons=comparisons, @@ -71,6 +76,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: max_ratio=args.max_ratio, per_side_projects=per_side_projects, failures=failures, + mode_max_ratios=mode_max_ratios, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) @@ -104,6 +110,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) + parser.add_argument( + "--edit-max-ratio", + type=float, + default=None, + help="Ratio limit of the one-model edit mode only; defaults to --max-ratio.", + ) parser.add_argument( "--noise-floor-seconds", type=float, diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index bfe4eee700..03da9883f1 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -432,7 +432,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "external_sql_reference_resolver", "scope_index", "declaration_scope", - "analysis_reuse", } ) RENDER_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index c2e24834a3..aac63d480d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -122,36 +122,27 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data, the project has very many files, or it records -thousands of new model renders, compile prints `Recording compile for reuse...` and a completion -line to stderr before the report. - -### Compiling after model-only edits - -When the only changes since the stored compile are edits to existing model files, compile runs in -full but reuses the stored render of every unchanged model, together with the diagnostics and -environment reads that rendering reported. Only the edited models are rendered again. Declaration -files other than models are not read again, and an unchanged model reuses its stored column -analysis only when the analyses of its upstream models in this compile match the ones it was stored -with. Contracts, semantic validation, and rules still run for the whole project, and a model whose -upstream analyses changed is analyzed again, so downstream effects of an edit, such as a changed -column type, are reported exactly as in an uncached compile, even when another command such as -`sqb build` ran in between. Compiled -files that still hold the bytes the stored compile wrote are not read or rewritten. The output is -byte-identical to `sqb compile --no-cache`. - -Every other change renders the whole project: adding, removing, or renaming a model, and editing -macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope -folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings -is always rendered again. - -Renders are stored only when a compile result for the same project and target is already stored, -so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. -The next full compile stores them, and edits after that reuse them. Stored renders are compressed -and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints -`Loading stored renders...` and a completion line to stderr before it starts. While renders are -reused, the Python modules recorded by the previous compile stay tracked even if this compile did -not import them, so editing such a module runs the next compile in full. +report is ready. When it has to hash a lot of data or the project has very many files, compile +prints `Recording compile for reuse...` and a completion line to stderr before the report. + +### Compiling after an edit + +Any change runs the whole compile again: every model, test, and audit is rendered again, and +contracts and semantic validation run for the whole project. No render is carried over from the +previous compile, so until the native compiler ships, an edited model is recompiled without any +cache. A few caches under `target/cache/` shorten the rest: + +- An unchanged model takes its column analysis from the analysis cache when its query and the + analyses of its upstream models are unchanged. +- An unchanged SQL test file reuses its stored test scan, the CTEs and expected models read from + it, from `sql-test-scans.bin`. Entries are keyed by the file's expanded text, the adapter's SQL + lexical rules, and the SQLBuild build; a damaged or unreadable entry is scanned again. +- Rule results are reused for unchanged inputs. +- Macro call results may be reused as described in + [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). + +An edit therefore costs a full render of the project plus the analysis, test scans, and rules of +what changed. The output is byte-identical to `sqb compile --no-cache`. Known limits: @@ -171,8 +162,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or -rendered again after model-only edits and `analysis_reuse_hits` for stored model analyses reused. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings and cache counters such as `sql_test_scan_cache_hits` and +`sql_test_scan_cache_misses`. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 808371a6c9..07ccca54b1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -222,7 +222,7 @@ files or environment variables that the call does not receive. SQLBuild relies on this contract to reuse macro results: -- [Compile reuse](../cli/compile.md) replays stored renders without running macros again. +- [Compile reuse](../cli/compile.md) replays an unchanged compile without running macros again. - A compile may run each distinct macro call once and reuse its result for identical calls elsewhere in the project. - Macro call results may be kept between compiles under `target/cache/`. They are recomputed when diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 251e98b73f..af327366cc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -293,6 +293,12 @@ class MacroCallMemo: def store_stats(self) -> tuple[int, int]: ... def save_store(self, path: str, metadata: bytes) -> int | None: ... +class SqlTestScanStore: + def __init__(self, path: str, environment: str) -> None: ... + def get(self, key_parts: list[str]) -> bytes | None: ... + def put(self, key_parts: list[str], value: bytes) -> None: ... + def save(self, path: str) -> int | None: ... + def content_digest(parts: list[str]) -> str: ... def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... def digest_files(paths: list[str]) -> list[str | None]: ... diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index 45cbc47e1b..1f21ac3d72 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -34,7 +34,6 @@ from sqlbuild.cli.output.models import ( WrittenTarget, ) -from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, @@ -122,7 +121,6 @@ def _analyze_compile_project( sql_analysis_enabled_override=( False if profile_flags.skip_discovery_sql_analysis else None ), - declaration_reuse=COMPILE_RENDER_REUSE.claim_discovery(), ) reject_unit_test_selectors(select=select, exclude=exclude, discovered_inputs=discovered_inputs) discover_ms: int = elapsed_ms(discover_start) @@ -328,7 +326,7 @@ def write_compile_dag_artifact( python_graph=python_graph, ) _ = resolved_dag_path.write_text(dag_json, encoding="utf-8") - _ = COMPILE_ARTIFACT_WRITES.written( + COMPILE_ARTIFACT_WRITES.written( path=resolved_dag_path, contents=dag_json.replace("\n", os.linesep).encode("utf-8") ) _ = complete_compile_phase( diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index da8b849949..8fda15011b 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -688,7 +688,7 @@ def _write_sql(*, path: Path, sql: str, check_existing: bool = True) -> None: def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): - _ = COMPILE_ARTIFACT_WRITES.written( + COMPILE_ARTIFACT_WRITES.written( path=path, contents=contents.replace(_POSIX_LINE_SEPARATOR, os.linesep).encode("utf-8"), ) @@ -703,9 +703,7 @@ def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): - unchanged: bool = COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) - if check_existing and unchanged: - return + COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) if check_existing: existing: bytes | None = _read_existing_file(path=path) if existing == contents: diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 3902e30069..7edbb6d3e1 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -28,11 +28,7 @@ CompileCommandResult, CompileWriteResult, ) -from sqlbuild.cli.compile_render_reuse.main._activated_render_reuse import activated_render_reuse -from sqlbuild.cli.compile_render_reuse.main._render_reuse_session import render_reuse_session -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile @@ -68,7 +64,6 @@ def run_compile( else reuse_attempt ) output: RecordedCompileOutput = RecordedCompileOutput() - render_reuse: CompileRenderReuse | None = render_reuse_session(attempt=attempt) status: TransientStatusReporter | None = start_compile_status( json_output=request.json_output, no_color=request.no_color, @@ -78,12 +73,7 @@ def run_compile( with ( collect_compile_timings() as detailed_timings, COMPILE_INPUT_READS.recording() as input_reads, - COMPILE_ARTIFACT_WRITES.recording( - stored=StoredArtifacts( - stamps=attempt.prior_target_files, digests=attempt.prior_target_digests - ) - ) as artifact_writes, - activated_render_reuse(render_reuse=render_reuse), + COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, PreparedCompileArtifacts( enabled=not request.profile_flags.skip_write ) as prepared_artifacts, @@ -111,7 +101,6 @@ def run_compile( artifacts_written=result.artifacts_written, dag_artifact_path=result.dag_artifact_path, json_output=request.json_output, - render_reuse=render_reuse, ) finally: output.print_stdout() diff --git a/src/sqlbuild/cli/compile_render_reuse/__init__.py b/src/sqlbuild/cli/compile_render_reuse/__init__.py deleted file mode 100644 index 1b6139a171..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Reuse unaffected renders of the stored compile in the next full compile.""" diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py deleted file mode 100644 index b89b7a1c87..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Lifecycle lines for loading a large render store before the compile status starts.""" - -from __future__ import annotations - -import sys -import time -from pathlib import Path - -from sqlbuild.cli.compile_render_reuse.constants import ( - RENDER_LOAD_DONE_MESSAGE, - RENDER_LOAD_FAILED_MESSAGE, - RENDER_LOAD_NOTICE_BYTES, - RENDER_LOAD_START_MESSAGE, -) -from sqlbuild.cli.compile_reuse.constants import BYTES_PER_MEBIBYTE, REUSE_RENDER_STATE_SUFFIX - - -def start_load_notice(*, entry_path: Path) -> float | None: - """Announce loading on stderr when the slot's stored render files are large.""" - - stored_bytes: int = sum( - _file_size(path=path) - for path in entry_path.parent.glob(f"{entry_path.stem}-*{REUSE_RENDER_STATE_SUFFIX}") - ) - if stored_bytes < RENDER_LOAD_NOTICE_BYTES: - return None - print( - RENDER_LOAD_START_MESSAGE.format(mebibytes=stored_bytes / BYTES_PER_MEBIBYTE), - file=sys.stderr, - ) - return time.monotonic() - - -def _file_size(*, path: Path) -> int: - """Size a stored render file, treating one removed by a concurrent compile as empty.""" - - try: - return path.stat().st_size - except OSError: - return 0 - - -def finish_load_notice(*, started: float | None, loaded: bool) -> None: - """Report whether an announced load produced usable renders.""" - - if started is None: - return - message: str = RENDER_LOAD_DONE_MESSAGE if loaded else RENDER_LOAD_FAILED_MESSAGE - print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py deleted file mode 100644 index f68ef0b514..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Checksummed, layered storage for the renders of the compile one stored entry describes.""" - -from __future__ import annotations - -import json -import zlib -from collections.abc import Iterable -from pathlib import Path -from typing import cast - -from sqlbuild.cli.compile_render_reuse.constants import ( - RENDER_COMPRESSION_LEVEL, - RENDER_INDEX_FIELD_COUNT, - RENDER_OVERLAY_MAX_SHARE, - RENDER_STATE_MAGIC, -) -from sqlbuild.cli.compile_render_reuse.models import ( - StoredRenderLayer, - StoredRenderLayers, - StoredRenderState, -) -from sqlbuild.cli.compile_reuse.constants import ( - REUSE_ENTRY_BYTE_ORDER, - REUSE_ENTRY_CHECKSUM_BYTES, - REUSE_ENTRY_LENGTH_BYTES, - REUSE_MAX_ENTRY_BYTES, - REUSE_RENDER_STATE_SUFFIX, -) -from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError -from sqlbuild.cli.compile_reuse.main._framed_section import framed_stored_section -from sqlbuild.cli.compile_reuse.main._read_framed_section import read_stored_section -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState - - -def read_render_state(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderState | None: - """Read verified renders when every changed path is a stored model; any fault means none.""" - - try: - newest: StoredRenderLayer | None = _read_layer(path=path, changed_paths=changed_paths) - if newest is None: - return None - if newest.base_file is None: - return _stored_state(base_file=path.name, base=newest, overlay=None) - base: StoredRenderLayer | None = _read_layer( - path=path.parent / _layer_name(newest.base_file), changed_paths=frozenset() - ) - if base is None or base.base_file is not None: - return None - return _stored_state(base_file=newest.base_file, base=base, overlay=newest) - except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError, zlib.error): - return None - - -def render_state_layer( - *, state: RenderReuseState, stored: StoredRenderLayers | None -) -> RenderStateLayer | None: - """Return an overlay on the stored base, a new base, or None when released bytes are needed.""" - - if ( - stored is not None - and state.model_paths == stored.model_paths - and stored.model_keys <= state.model_payloads.keys() - and stored.group_keys <= state.group_payloads.keys() - ): - models: list[tuple[str, memoryview]] = _held( - (path, state.model_payloads.get(path)) for path in state.model_paths - ) - groups: list[tuple[str, memoryview]] = _held(sorted(state.group_payloads.items())) - if ( - sum(len(item) for _, item in (*models, *groups)) - <= stored.base_bytes * RENDER_OVERLAY_MAX_SHARE - ): - return _layer( - model_paths=state.model_paths, - base_file=stored.base_file, - models=models, - groups=groups, - ) - if None in state.model_payloads.values() or None in state.group_payloads.values(): - return None - return _layer( - model_paths=state.model_paths, - base_file=None, - models=_held((path, state.model_payloads.get(path)) for path in state.model_paths), - groups=_held(sorted(state.group_payloads.items())), - ) - - -def _held(items: Iterable[tuple[str, memoryview | None]]) -> list[tuple[str, memoryview]]: - return [(key, payload) for key, payload in items if payload is not None] - - -def _layer( - *, - model_paths: tuple[str, ...], - base_file: str | None, - models: list[tuple[str, memoryview]], - groups: list[tuple[str, memoryview]], -) -> RenderStateLayer: - index: bytes = json.dumps( - { - "model_paths": list(model_paths), - "base": base_file, - "models": [[path, len(payload)] for path, payload in models], - "groups": [[name, len(payload)] for name, payload in groups], - }, - separators=(",", ":"), - ).encode("utf-8", "surrogateescape") - compressor: zlib._Compress = zlib.compressobj(RENDER_COMPRESSION_LEVEL) - compressed: list[bytes] = [compressor.compress(item) for _, item in (*models, *groups)] - compressed.append(compressor.flush()) - return RenderStateLayer( - chunks=( - RENDER_STATE_MAGIC, - framed_stored_section(data=index), - _section_header(payloads=compressed), - *compressed, - ), - base_file=base_file, - ) - - -def _section_header(*, payloads: Iterable[bytes]) -> bytes: - length: int = 0 - checksum: int = 0 - for payload in payloads: - length += len(payload) - checksum = zlib.crc32(payload, checksum) - return length.to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER) + checksum.to_bytes( - REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER - ) - - -def _read_layer(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderLayer | None: - with open(path, "rb") as handle: - if handle.read(len(RENDER_STATE_MAGIC)) != RENDER_STATE_MAGIC: - return None - index: object = json.loads(read_stored_section(handle=handle)) - if not isinstance(index, dict): - return None - fields: dict[str, object] = cast(dict[str, object], index) - model_paths: tuple[str, ...] = tuple(_string(item) for item in _list(fields["model_paths"])) - if not changed_paths.issubset(model_paths): - return None - compressed: bytes = read_stored_section(handle=handle) - if handle.read(1) or len(compressed) > REUSE_MAX_ENTRY_BYTES: - return None - base: object = fields["base"] - model_rows: list[tuple[str, int]] = _rows(fields["models"]) - group_rows: list[tuple[str, int]] = _rows(fields["groups"]) - size: int = sum(length for _, length in (*model_rows, *group_rows)) - if size > REUSE_MAX_ENTRY_BYTES: - return None - decompressor: zlib._Decompress = zlib.decompressobj() - payload: bytes = decompressor.decompress(compressed, size + 1) - del compressed - if len(payload) != size or not decompressor.eof or decompressor.unconsumed_tail: - return None - view: memoryview = memoryview(payload) - offset: int = 0 - models: dict[str, memoryview] = {} - for name, length in model_rows: - models[name] = view[offset : offset + length] - offset += length - groups: dict[str, memoryview] = {} - for name, length in group_rows: - groups[name] = view[offset : offset + length] - offset += length - return StoredRenderLayer( - model_paths=model_paths, - base_file=None if base is None else _string(base), - models=models, - groups=groups, - size=len(payload), - ) - - -def _stored_state( - *, base_file: str, base: StoredRenderLayer, overlay: StoredRenderLayer | None -) -> StoredRenderState: - newest: StoredRenderLayer = base if overlay is None else overlay - state: RenderReuseState = RenderReuseState( - model_paths=newest.model_paths, - model_payloads=dict(base.models) if overlay is None else {**base.models, **overlay.models}, - group_payloads=dict(base.groups) if overlay is None else {**base.groups, **overlay.groups}, - ) - return StoredRenderState( - state=state, - layers=StoredRenderLayers( - model_paths=newest.model_paths, - model_keys=frozenset(state.model_payloads), - group_keys=frozenset(state.group_payloads), - base_file=base_file, - base_bytes=base.size, - overlay_models=frozenset() if overlay is None else frozenset(overlay.models), - overlay_groups=frozenset() if overlay is None else frozenset(overlay.groups), - ), - ) - - -def _layer_name(name: str) -> str: - if Path(name).name != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): - raise CompileReuseEntryError("invalid stored render base name") - return name - - -def _rows(value: object) -> list[tuple[str, int]]: - rows: list[tuple[str, int]] = [] - for row in _list(value): - items: list[object] = _list(row) - if len(items) != RENDER_INDEX_FIELD_COUNT: - raise CompileReuseEntryError("invalid stored render index") - length: object = items[1] - if not isinstance(length, int) or isinstance(length, bool) or length < 0: - raise CompileReuseEntryError("invalid stored render length") - rows.append((_string(items[0]), length)) - return rows - - -def _list(value: object) -> list[object]: - if not isinstance(value, list): - raise CompileReuseEntryError("invalid stored render list") - return cast(list[object], value) - - -def _string(value: object) -> str: - if not isinstance(value, str): - raise CompileReuseEntryError("invalid stored render string") - return value diff --git a/src/sqlbuild/cli/compile_render_reuse/constants.py b/src/sqlbuild/cli/compile_render_reuse/constants.py deleted file mode 100644 index be5e190f0a..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/constants.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Render storage format constants.""" - -from __future__ import annotations - -RENDER_STATE_MAGIC: bytes = b"SQBRENDER2\n" -RENDER_COMPRESSION_LEVEL: int = 1 -RENDER_OVERLAY_MAX_SHARE: float = 0.25 -RENDER_INDEX_FIELD_COUNT: int = 2 -RENDER_LOAD_NOTICE_BYTES: int = 8 * 1024 * 1024 -RENDER_LOAD_START_MESSAGE: str = "Loading stored renders ({mebibytes:.0f} MiB)..." -RENDER_LOAD_DONE_MESSAGE: str = "Loaded stored renders ({seconds:.1f} s)" -RENDER_LOAD_FAILED_MESSAGE: str = ( - "Stored renders not reused; rendering every model again ({seconds:.1f} s)" -) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/__init__.py b/src/sqlbuild/cli/compile_render_reuse/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py b/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py deleted file mode 100644 index a40d737c78..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public entry for exposing one compile's render reuse to the rendering it runs.""" - -from __future__ import annotations - -from contextlib import AbstractContextManager, nullcontext - -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse -from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE - - -def activated_render_reuse( - *, render_reuse: CompileRenderReuse | None -) -> AbstractContextManager[object]: - """Activate the render reuse session for the compile block, or do nothing without one.""" - - return ( - nullcontext() - if render_reuse is None - else COMPILE_RENDER_REUSE.activated(render_reuse.session) - ) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py deleted file mode 100644 index 399a33b9e5..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Public entry for the render reuse one full compile runs with.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.compile_render_reuse._helpers.load_notice import ( - finish_load_notice, - start_load_notice, -) -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import read_render_state -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse, StoredRenderState -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession - - -def render_reuse_session(*, attempt: CompileReuseAttempt) -> CompileRenderReuse | None: - """Record renders once a compile is stored, reusing those the changed paths cannot affect.""" - - if ( - attempt.outcome is not CompileReuseOutcome.MISS - or attempt.entry_path is None - or not attempt.prior_entry - ): - return None - stored: StoredRenderState | None = ( - None - if attempt.changed_paths is None or attempt.render_state_path is None - else _loaded_render_state( - entry_path=attempt.entry_path, - path=attempt.render_state_path, - changed_paths=attempt.changed_paths, - ) - ) - return CompileRenderReuse( - session=CompileRenderReuseSession( - prior=None if stored is None else stored.state, - changed_paths=attempt.changed_paths, - retained_models=frozenset() if stored is None else stored.layers.overlay_models, - retained_groups=frozenset() if stored is None else stored.layers.overlay_groups, - ), - stored=None if stored is None else stored.layers, - ) - - -def _loaded_render_state( - *, entry_path: Path, path: Path, changed_paths: frozenset[str] -) -> StoredRenderState | None: - started: float | None = start_load_notice(entry_path=entry_path) - stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) - _ = finish_load_notice(started=started, loaded=stored is not None) - return stored diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py deleted file mode 100644 index 83467f8b70..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public entry for the render layer a finished compile stores with its entry.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import render_state_layer -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState - - -def stored_render_layer(*, render_reuse: CompileRenderReuse | None) -> RenderStateLayer | None: - """Return this compile's renders as an overlay on the stored ones, or as a new base.""" - - state: RenderReuseState | None = ( - None if render_reuse is None else render_reuse.session.stored_state() - ) - if render_reuse is None or state is None: - return None - layer: RenderStateLayer | None = render_state_layer(state=state, stored=render_reuse.stored) - if layer is not None: - return layer - complete: RenderReuseState | None = render_reuse.session.stored_state(complete=True) - return None if complete is None else render_state_layer(state=complete, stored=None) diff --git a/src/sqlbuild/cli/compile_render_reuse/models.py b/src/sqlbuild/cli/compile_render_reuse/models.py deleted file mode 100644 index a7595e63c1..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/models.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Stored renders of a previous compile and the render reuse of one full compile.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.models import RenderReuseState - - -@dataclass(frozen=True) -class StoredRenderLayer: - """One decoded render file: its index and its payloads by key.""" - - model_paths: tuple[str, ...] - base_file: str | None - models: dict[str, memoryview] - groups: dict[str, memoryview] - size: int - - -@dataclass(frozen=True) -class StoredRenderLayers: - """How a stored compile's renders are split between a base file and one overlay.""" - - model_paths: tuple[str, ...] - model_keys: frozenset[str] - group_keys: frozenset[str] - base_file: str - base_bytes: int - overlay_models: frozenset[str] - overlay_groups: frozenset[str] - - -@dataclass(frozen=True) -class StoredRenderState: - """Renders a stored compile recorded, merged across layers, with their layout.""" - - state: RenderReuseState - layers: StoredRenderLayers - - -@dataclass(frozen=True) -class CompileRenderReuse: - """One full compile's render reuse session with the stored renders it started from.""" - - session: CompileRenderReuseSession - stored: StoredRenderLayers | None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index dac7203b29..5b9872a59f 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -9,14 +9,12 @@ from pathlib import Path from sqlbuild.cli.compile_reuse._helpers.entry_file import ( - entry_render_state_path, read_entry_header, read_entry_stdout, rewrite_entry_inputs, ) from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, - changed_project_paths, compare_project_files, needs_refresh, restamped_paths, @@ -106,16 +104,7 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ) attempt = replace( attempt, - prior_entry=True, - prior_modules=header.inputs.modules, - prior_target_files=header.inputs.target_files, - prior_target_digests=header.inputs.target_digests, restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), - render_state_path=( - None - if attempt.changed_paths is None or attempt.entry_path is None - else entry_render_state_path(path=attempt.entry_path, header=header) - ), ) return replace( attempt, @@ -168,22 +157,12 @@ def _checked_attempt( unchanged: bool = comparison.unchanged and target_files_unchanged( stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree ) - changed_paths: frozenset[str] | None = None - verified: dict[str, str] = comparison.verified - if identity_unchanged and not unchanged: - changed_paths, verified = changed_project_paths( - project_dir=project_dir, - stored=inputs.project_files, - current=attempt.snapshot, - verified=comparison.verified, - ) return replace( attempt, outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, digests=carried_forward_digests( - stored=inputs.project_files, current=attempt.snapshot, verified=verified + stored=inputs.project_files, current=attempt.snapshot, verified=comparison.verified ), - changed_paths=changed_paths, ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py index aceecdc384..9ffe463834 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py @@ -20,15 +20,13 @@ REUSE_ENTRY_SUFFIX, REUSE_FORMAT_VERSION, REUSE_MAX_ENTRY_BYTES, - REUSE_MAX_RENDER_STATES, REUSE_MAX_STORED_ENTRIES, - REUSE_RENDER_STATE_SUFFIX, + REUSE_RETIRED_RENDER_SUFFIX, REUSE_STDOUT_SEPARATOR, REUSE_STDOUT_SUFFIX, ) from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError from sqlbuild.cli.compile_reuse.models import ( - RenderStateLayer, SettingsEnvironmentInputs, StoredCompileHeader, StoredCompileInputs, @@ -51,7 +49,7 @@ def read_entry_header(*, path: Path) -> StoredCompileHeader | None: with open(path, "rb") as handle: if handle.read(len(REUSE_ENTRY_MAGIC)) != REUSE_ENTRY_MAGIC: return None - metadata: bytes = read_framed_section(handle=handle) + metadata: bytes = _read_section(handle=handle) if handle.read(1): return None payload: object = json.loads(metadata) @@ -83,24 +81,13 @@ def write_entry( inputs: StoredCompileInputs, output: StoredCompileOutput, stdout: str, - render_state: RenderStateLayer | None = None, ) -> None: - """Publish one entry, its stdout, and its renders atomically, naming them in the entry.""" + """Publish one entry and its stdout atomically, filling in the stdout file and checksum.""" encoded_stdout: bytes = stdout.encode("utf-8", "surrogateescape") - stamp: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}" - stdout_file: str = f"{stamp}{REUSE_STDOUT_SUFFIX}" - render_state_file: str | None = ( - None - if render_state is None - or sum(len(chunk) for chunk in render_state.chunks) > REUSE_MAX_ENTRY_BYTES - else f"{stamp}{REUSE_RENDER_STATE_SUFFIX}" - ) + stdout_file: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}{REUSE_STDOUT_SUFFIX}" stored_output: StoredCompileOutput = replace( - output, - stdout_file=stdout_file, - stdout_checksum=zlib.crc32(encoded_stdout), - render_state_file=render_state_file, + output, stdout_file=stdout_file, stdout_checksum=zlib.crc32(encoded_stdout) ) metadata: bytes = _metadata(inputs=inputs, output=stored_output) if len(metadata) + len(encoded_stdout) > REUSE_MAX_ENTRY_BYTES: @@ -108,31 +95,14 @@ def write_entry( return try: path.parent.mkdir(parents=True, exist_ok=True) - _publish_file(path=path.parent / stdout_file, contents=(encoded_stdout,)) - if render_state is not None and render_state_file is not None: - _publish_file(path=path.parent / render_state_file, contents=render_state.chunks) - _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) - _remove_stdout_files(entry_path=path, keep=(stdout_file,)) - _remove_render_files( - entry_path=path, - keep=( - () - if render_state is None or render_state_file is None - else (render_state_file, render_state.base_file) - ), - ) + _publish_file(path=path.parent / stdout_file, contents=encoded_stdout) + _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) + _remove_side_files(entry_path=path, keep=stdout_file) _prune_entries(directory=path.parent, keep=path) except OSError: remove_entry(path=path) -def entry_render_state_path(*, path: Path, header: StoredCompileHeader) -> Path | None: - """Return where the entry's renders are stored, when it stored any.""" - - name: str | None = header.output.render_state_file - return None if name is None else path.parent / name - - def rewrite_entry_inputs( *, path: Path, header: StoredCompileHeader, inputs: StoredCompileInputs ) -> None: @@ -140,7 +110,7 @@ def rewrite_entry_inputs( metadata: bytes = _metadata(inputs=inputs, output=header.output) with contextlib.suppress(OSError): - _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) + _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) def remove_entry(*, path: Path) -> None: @@ -148,18 +118,16 @@ def remove_entry(*, path: Path) -> None: with contextlib.suppress(OSError): path.unlink(missing_ok=True) - _remove_stdout_files(entry_path=path, keep=()) - _remove_render_files(entry_path=path, keep=()) + _remove_side_files(entry_path=path, keep=None) -def _publish_file(*, path: Path, contents: tuple[bytes | memoryview, ...]) -> None: +def _publish_file(*, path: Path, contents: bytes) -> None: descriptor, temporary_path = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent ) try: with os.fdopen(descriptor, "wb") as handle: - for chunk in contents: - _ = handle.write(chunk) + _ = handle.write(contents) os.replace(temporary_path, path) except BaseException: with contextlib.suppress(OSError): @@ -174,48 +142,31 @@ def _metadata(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> by def _framed_metadata(*, metadata: bytes) -> bytes: - return REUSE_ENTRY_MAGIC + framed_section(data=metadata) - - -def framed_section(*, data: bytes) -> bytes: - """Prefix one section with its length and checksum so a reader can verify it.""" - return b"".join( ( - len(data).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), - zlib.crc32(data).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), - data, + REUSE_ENTRY_MAGIC, + len(metadata).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), + zlib.crc32(metadata).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), + metadata, ) ) -def _remove_stdout_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: - _remove_entry_files(entry_path=entry_path, suffix=REUSE_STDOUT_SUFFIX, keep=keep) - +def _remove_side_files(*, entry_path: Path, keep: str | None) -> None: + """Remove every file stored beside an entry except keep, including older releases' files.""" -def _remove_render_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: - _remove_entry_files(entry_path=entry_path, suffix=REUSE_RENDER_STATE_SUFFIX, keep=keep) - - -def _remove_entry_files(*, entry_path: Path, suffix: str, keep: tuple[str | None, ...]) -> None: prefix: str = f"{entry_path.stem}{REUSE_STDOUT_SEPARATOR}" try: candidates: list[Path] = list(entry_path.parent.iterdir()) except OSError: return for candidate in candidates: - if ( - candidate.name.startswith(prefix) - and candidate.suffix == suffix - and candidate.name not in keep - ): + if candidate.name.startswith(prefix) and candidate.name != keep: with contextlib.suppress(OSError): candidate.unlink() -def read_framed_section(*, handle: BinaryIO) -> bytes: - """Read one length- and checksum-framed section, raising when it is damaged.""" - +def _read_section(*, handle: BinaryIO) -> bytes: length: int = int.from_bytes(handle.read(REUSE_ENTRY_LENGTH_BYTES), REUSE_ENTRY_BYTE_ORDER) checksum: int = int.from_bytes(handle.read(REUSE_ENTRY_CHECKSUM_BYTES), REUSE_ENTRY_BYTE_ORDER) if length > REUSE_MAX_ENTRY_BYTES: @@ -229,6 +180,10 @@ def read_framed_section(*, handle: BinaryIO) -> bytes: def _prune_entries(*, directory: Path, keep: Path) -> None: entries: list[tuple[int, Path]] = [] for candidate in directory.iterdir(): + if candidate.suffix == REUSE_RETIRED_RENDER_SUFFIX: + with contextlib.suppress(OSError): + candidate.unlink() + continue if candidate.suffix != REUSE_ENTRY_SUFFIX or candidate == keep: continue with contextlib.suppress(OSError): @@ -236,8 +191,6 @@ def _prune_entries(*, directory: Path, keep: Path) -> None: entries.sort(reverse=True) for _, stale in entries[REUSE_MAX_STORED_ENTRIES - 1 :]: remove_entry(path=stale) - for _, older in entries[REUSE_MAX_RENDER_STATES - 1 : REUSE_MAX_STORED_ENTRIES - 1]: - _remove_render_files(entry_path=older, keep=()) def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dict[str, object]: @@ -254,14 +207,12 @@ def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dic for path, item in inputs.project_files.items() }, "target_files": {path: list(stamp) for path, stamp in inputs.target_files.items()}, - "target_digests": inputs.target_digests, "stderr_lines": list(output.stderr_lines), "exit_code": output.exit_code, "timings_span": None if output.timings_span is None else list(output.timings_span), "stdout_length": output.stdout_length, "stdout_checksum": output.stdout_checksum, "stdout_file": output.stdout_file, - "render_state_file": output.render_state_file, "target_tree": inputs.target_tree, "settings_inputs": [ { @@ -310,10 +261,6 @@ def _header(*, payload: object) -> StoredCompileHeader: _settings_inputs(item) for item in _list(fields["settings_inputs"]) ), settings_digest=_string(fields["settings_digest"]), - target_digests={ - _string(path): _string(digest) - for path, digest in _mapping(fields.get("target_digests", {})).items() - }, ), output=StoredCompileOutput( stderr_lines=tuple(_string(line) for line in _list(fields["stderr_lines"])), @@ -322,7 +269,6 @@ def _header(*, payload: object) -> StoredCompileHeader: stdout_length=_integer(fields["stdout_length"]), stdout_checksum=_integer(fields["stdout_checksum"]), stdout_file=_stdout_file(fields["stdout_file"]), - render_state_file=_render_state_file(fields.get("render_state_file")), ), ) @@ -345,15 +291,6 @@ def _stdout_file(value: object) -> str: return name -def _render_state_file(value: object) -> str | None: - if value is None: - return None - name: str = _string(value) - if os.path.basename(name) != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): - raise CompileReuseEntryError("invalid stored render file name") - return name - - def _timings_span(value: object) -> tuple[int, int] | None: if value is None: return None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py index e68d78330c..5c0472b33c 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py @@ -101,37 +101,6 @@ def compare_project_files( return ProjectFilesComparison(unchanged=True, verified=verified) -def changed_project_paths( - *, - project_dir: str, - stored: dict[str, StoredProjectFile], - current: dict[str, FileStamp], - verified: dict[str, str], -) -> tuple[frozenset[str], dict[str, str]]: - """Return every path added, removed, or changed since the stored compile, and new digests.""" - - digests: dict[str, str] = dict(verified) - changed: set[str] = {path for path in stored if path not in current} - for relative_path, stamp in current.items(): - previous: StoredProjectFile | None = stored.get(relative_path) - if previous is None or not _same_identity(stamp=stamp, recorded=previous.stamp): - changed.add(relative_path) - continue - if stamp.kind in _PRESENCE_ONLY_KINDS or (stamp == previous.stamp and not previous.racy): - continue - if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: - changed.add(relative_path) - continue - digest: str | None = digests.get(relative_path) or file_digest( - path=os.path.join(project_dir, relative_path) - ) - if digest is None or digest != previous.digest: - changed.add(relative_path) - else: - digests[relative_path] = digest - return frozenset(changed), digests - - def carried_forward_digests( *, stored: dict[str, StoredProjectFile], diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index 3c792b9ba5..a676b33f60 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -71,19 +71,6 @@ def loaded_module_stamps(*, covered_paths: frozenset[str]) -> tuple[tuple[str, i return tuple(sorted(stamps.values())) -def with_carried_module_stamps( - *, - current: tuple[tuple[str, int, int], ...], - carried: tuple[tuple[str, int, int], ...], - covered_paths: frozenset[str], -) -> tuple[tuple[str, int, int], ...]: - """Add stored stamps of modules a reused render imported but this process did not load.""" - - stamps: dict[str, tuple[str, int, int]] = {stamp[0]: stamp for stamp in carried} - stamps.update((stamp[0], stamp) for stamp in current) - return tuple(sorted(stamp for path, stamp in stamps.items() if path not in covered_paths)) - - def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool: """Return whether every recorded module file still has the same stat identity.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index 853013c52f..7bed444a1d 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -8,8 +8,6 @@ import time from pathlib import Path -from sqlbuild.cli.compile_render_reuse.main._render_state_layer import stored_render_layer -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.entry_file import remove_entry, write_entry from sqlbuild.cli.compile_reuse._helpers.project_files import ( pending_digest_bytes, @@ -23,7 +21,6 @@ loaded_module_stamps, settings_inputs_digest, tracked_environment_names, - with_carried_module_stamps, ) from sqlbuild.cli.compile_reuse._helpers.target_files import verified_target_files from sqlbuild.cli.compile_reuse._helpers.timings import compile_timings_span @@ -35,7 +32,6 @@ REUSE_STORE_DONE_MESSAGE, REUSE_STORE_NOTICE_BYTES, REUSE_STORE_NOTICE_PATHS, - REUSE_STORE_NOTICE_RENDERS, REUSE_STORE_SKIPPED_MESSAGE, REUSE_STORE_START_MESSAGE, ) @@ -62,7 +58,6 @@ def write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, - render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" @@ -71,6 +66,7 @@ def write_compile_entry( try: _write_compile_entry( attempt=attempt, + entry_path=attempt.entry_path, output=output, exit_code=exit_code, input_reads=input_reads, @@ -79,7 +75,6 @@ def write_compile_entry( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, - render_reuse=render_reuse, ) except Exception: _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) @@ -89,6 +84,7 @@ def write_compile_entry( def _write_compile_entry( *, attempt: CompileReuseAttempt, + entry_path: Path, output: RecordedCompileOutput, exit_code: int, input_reads: CompileInputReads, @@ -97,11 +93,7 @@ def _write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, - render_reuse: CompileRenderReuse | None, ) -> None: - if attempt.entry_path is None: - return - entry_path: Path = attempt.entry_path stdout: str | None = output.stdout settings: SettingsInputsResult = provider_settings_inputs( settings_classes=input_reads.settings_classes @@ -117,10 +109,7 @@ def _write_compile_entry( remove_entry(path=entry_path) return project_dir: str = str(attempt.project_dir) - notice_started: float | None = _start_notice( - attempt=attempt, - pending_renders=0 if render_reuse is None else render_reuse.session.pending_renders(), - ) + notice_started: float | None = _start_notice(attempt=attempt) digests: dict[str, str] = with_missing_digests( project_dir=project_dir, snapshot=attempt.snapshot, @@ -154,8 +143,10 @@ def _write_compile_entry( environment_names=environment_names, environment_digest=environment_digest(names=environment_names), search_path=attempt.search_path, - modules=_module_stamps( - attempt=attempt, project_dir=project_dir, render_reuse=render_reuse + modules=loaded_module_stamps( + covered_paths=frozenset( + os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot + ) ), project_files=stored_project_files( snapshot=attempt.snapshot, digests=digests, snapshot_ns=attempt.snapshot_ns @@ -164,11 +155,6 @@ def _write_compile_entry( target_tree=artifacts_written, settings_inputs=settings.inputs, settings_digest=settings_inputs_digest(inputs=settings.inputs), - target_digests={ - path: artifact.digest - for path, artifact in artifact_writes.artifacts.items() - if artifact.digest is not None and path in target_files - }, ), output=StoredCompileOutput( stderr_lines=output.stderr_lines, @@ -179,40 +165,21 @@ def _write_compile_entry( stdout_file="", ), stdout=stdout, - render_state=stored_render_layer(render_reuse=render_reuse), ) _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) -def _module_stamps( - *, attempt: CompileReuseAttempt, project_dir: str, render_reuse: CompileRenderReuse | None -) -> tuple[tuple[str, int, int], ...]: - covered_paths: frozenset[str] = frozenset( - os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot - ) - current: tuple[tuple[str, int, int], ...] = loaded_module_stamps(covered_paths=covered_paths) - if render_reuse is None or not render_reuse.session.reused_any(): - return current - return with_carried_module_stamps( - current=current, carried=attempt.prior_modules, covered_paths=covered_paths - ) - - -def _start_notice(*, attempt: CompileReuseAttempt, pending_renders: int) -> float | None: +def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: pending_bytes: int = pending_digest_bytes( snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped ) - if ( - pending_bytes < REUSE_STORE_NOTICE_BYTES - and len(attempt.snapshot) < REUSE_STORE_NOTICE_PATHS - and pending_renders < REUSE_STORE_NOTICE_RENDERS + if pending_bytes < REUSE_STORE_NOTICE_BYTES and len(attempt.snapshot) < ( + REUSE_STORE_NOTICE_PATHS ): return None print( REUSE_STORE_START_MESSAGE.format( - paths=len(attempt.snapshot), - mebibytes=pending_bytes / BYTES_PER_MEBIBYTE, - renders=pending_renders, + paths=len(attempt.snapshot), mebibytes=pending_bytes / BYTES_PER_MEBIBYTE ), file=sys.stderr, ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py index 4cbddde38a..72d1fd4ecf 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py @@ -32,11 +32,11 @@ def snapshot_target_files( if entry.is_dir(follow_symlinks=False): pending.append(entry.path) continue - stamp: FileStamp | None = path_stamp(path=entry.path) + stamp: FileStamp | None = _path_stamp(path=entry.path) if stamp is not None: stamps[entry.path] = stamp for path in extra_paths: - extra: FileStamp | None = path_stamp(path=path) + extra: FileStamp | None = _path_stamp(path=path) if extra is not None: stamps[path] = extra return stamps @@ -81,7 +81,7 @@ def _target_stamps(*, project_dir: str, paths: tuple[str, ...], tree: bool) -> d """Stamp the whole compiled tree plus extra paths, or only the given paths.""" if not tree: - return {path: stamp for path in paths if (stamp := path_stamp(path=path)) is not None} + return {path: stamp for path in paths if (stamp := _path_stamp(path=path)) is not None} compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep extra_paths: tuple[str, ...] = tuple( path for path in paths if not path.startswith(compiled_prefix) @@ -103,9 +103,7 @@ def _compiled_root(*, project_dir: str) -> str: return os.path.join(project_dir, TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME) -def path_stamp(*, path: str) -> FileStamp | None: - """Return the stat identity of one path, or None when it cannot be stated.""" - +def _path_stamp(*, path: str) -> FileStamp | None: try: status: os.stat_result = os.lstat(path) except OSError: diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py index cba8640bd6..069c0f24a9 100644 --- a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py +++ b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py @@ -7,14 +7,10 @@ import threading from collections.abc import Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.models import RecordedArtifact -if TYPE_CHECKING: - from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts - class CompileArtifactWriteRegistry: """Deliver artifact writes from every thread, including background staging threads.""" @@ -23,41 +19,31 @@ def __init__(self, *, digest_size: int) -> None: self._digest_size: int = digest_size self._lock: threading.Lock = threading.Lock() self._recorders: tuple[CompileArtifactWrites, ...] = () - self._stored: StoredArtifacts | None = None @contextmanager - def recording( - self, *, stored: StoredArtifacts | None = None - ) -> Iterator[CompileArtifactWrites]: - """Record artifact writes until the block exits, trusting unchanged stored artifacts.""" + def recording(self) -> Iterator[CompileArtifactWrites]: + """Record artifact writes until the block exits.""" recorder: CompileArtifactWrites = CompileArtifactWrites() with self._lock: self._recorders = (*self._recorders, recorder) - if stored is not None: - self._stored = stored try: yield recorder finally: with self._lock: self._recorders = tuple(item for item in self._recorders if item is not recorder) - if stored is not None: - self._stored = None - def written(self, *, path: str | os.PathLike[str], contents: bytes) -> bool: - """Note the exact bytes an artifact holds; return whether it already holds them.""" + def written(self, *, path: str | os.PathLike[str], contents: bytes) -> None: + """Note the exact bytes an artifact holds after a write or an unchanged-file check.""" recorders: tuple[CompileArtifactWrites, ...] = self._recorders if not recorders: - return False - absolute_path: str = os.path.abspath(path) + return artifact: RecordedArtifact = RecordedArtifact( digest=hashlib.blake2b(contents, digest_size=self._digest_size).hexdigest() ) for recorder in recorders: - recorder.record(path=absolute_path, artifact=artifact) - stored: StoredArtifacts | None = self._stored - return stored is not None and stored.holds(path=absolute_path, digest=artifact.digest) + recorder.record(path=os.path.abspath(path), artifact=artifact) def kept(self, *, path: str | os.PathLike[str], size: int, mtime_ns: int) -> None: """Note an earlier artifact kept as-is after its size and mtime were validated.""" diff --git a/src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py b/src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py deleted file mode 100644 index 0fc105cfbb..0000000000 --- a/src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Artifacts a stored compile left in target/, trusted while their stat identity is unchanged.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.target_files import path_stamp -from sqlbuild.cli.compile_reuse.types import FileStamp - - -class StoredArtifacts: - """Answer whether an artifact still holds the bytes a stored compile recorded for it.""" - - def __init__(self, *, stamps: dict[str, FileStamp], digests: dict[str, str]) -> None: - self._stamps: dict[str, FileStamp] = stamps - self._digests: dict[str, str] = digests - - def holds(self, *, path: str, digest: str | None) -> bool: - """Return whether the file at path is unchanged since it was stored with this digest.""" - - return ( - digest is not None - and self._digests.get(path) == digest - and self.unchanged_since_stored(path=path) - ) - - def unchanged_since_stored(self, *, path: str) -> bool: - """Return whether the file at path still has the stat identity it was stored with.""" - - stamp: FileStamp | None = self._stamps.get(path) - return stamp is not None and path_stamp(path=path) == stamp diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 2bd4a76bac..d4da1c210d 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -19,9 +19,8 @@ REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" +REUSE_RETIRED_RENDER_SUFFIX: str = ".render" REUSE_STDOUT_SEPARATOR: str = "-" -REUSE_RENDER_STATE_SUFFIX: str = ".render" -REUSE_MAX_RENDER_STATES: int = 2 REUSE_ENTRY_LENGTH_BYTES: int = 8 REUSE_ENTRY_CHECKSUM_BYTES: int = 4 REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" @@ -95,10 +94,9 @@ MILLISECONDS_PER_SECOND: int = 1000 REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 REUSE_STORE_NOTICE_PATHS: int = 100_000 -REUSE_STORE_NOTICE_RENDERS: int = 4_000 BYTES_PER_MEBIBYTE: int = 1024 * 1024 REUSE_STORE_START_MESSAGE: str = ( - "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash, {renders} renders)..." + "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." ) REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" REUSE_STORE_SKIPPED_MESSAGE: str = ( diff --git a/src/sqlbuild/cli/compile_reuse/main/_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_framed_section.py deleted file mode 100644 index 9259128048..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_framed_section.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for framing one stored section with its length and checksum.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.entry_file import framed_section - - -def framed_stored_section(*, data: bytes) -> bytes: - """Prefix one section with its length and checksum so a reader can verify it.""" - - return framed_section(data=data) diff --git a/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py deleted file mode 100644 index 08583cd0f7..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public entry for reading one framed stored section.""" - -from __future__ import annotations - -from typing import BinaryIO - -from sqlbuild.cli.compile_reuse._helpers.entry_file import read_framed_section - - -def read_stored_section(*, handle: BinaryIO) -> bytes: - """Read one length- and checksum-framed section, raising when it is damaged.""" - - return read_framed_section(handle=handle) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py index 0fbf395025..0736318972 100644 --- a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py +++ b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py @@ -4,7 +4,6 @@ from pathlib import Path -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput @@ -23,7 +22,6 @@ def write_reusable_compile( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, - render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable full compile, or drop the stored one when it cannot be reused.""" @@ -37,5 +35,4 @@ def write_reusable_compile( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, - render_reuse=render_reuse, ) diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index 16f143dfb5..cc2bd55c4b 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -81,7 +81,6 @@ class StoredCompileInputs: target_tree: bool settings_inputs: tuple[SettingsEnvironmentInputs, ...] settings_digest: str - target_digests: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) @@ -94,7 +93,6 @@ class StoredCompileOutput: stdout_length: int stdout_checksum: int stdout_file: str - render_state_file: str | None = None @dataclass(frozen=True) @@ -130,17 +128,3 @@ class CompileReuseAttempt: snapshot_ns: int = 0 digests: dict[str, str] = field(default_factory=dict) restamped: frozenset[str] = frozenset() - changed_paths: frozenset[str] | None = None - render_state_path: Path | None = None - prior_entry: bool = False - prior_modules: tuple[tuple[str, int, int], ...] = () - prior_target_files: dict[str, FileStamp] = field(default_factory=dict) - prior_target_digests: dict[str, str] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RenderStateLayer: - """One render layer to publish: its framed chunks, and the base layer it extends.""" - - chunks: tuple[bytes | memoryview, ...] - base_file: str | None diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py index fedb337ce2..5323f6d344 100644 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompileModelInput, ModelAnalysisCaching +from sqlbuild.compiler.compile.models import AnalysisCacheContext, CompileModelInput from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -22,6 +22,6 @@ class NativeModelAnalysisRequest: inference_profile: ExpressionInferenceProfile allow_compact_analysis: bool rich_type_inference: bool - analysis_caching: ModelAnalysisCaching | None + analysis_cache: AnalysisCacheContext | None complete_binding_schemas: dict[str, dict[str, str]] dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py index 337da09d48..b084a8d591 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py @@ -7,11 +7,7 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.classes.binding_dataflow import BindingDataflow -from sqlbuild.compiler.compile.models import ( - CompileModelInput, - DataflowReuse, - PolyglotAnalysisResult, -) +from sqlbuild.compiler.compile.models import CompileModelInput, PolyglotAnalysisResult from sqlbuild.compiler.compile.models import ( ModelSqlAnalysis as _ModelSqlAnalysis, ) @@ -30,7 +26,8 @@ def analyze_binding_waves( *, requests: tuple[_ModelSqlAnalysisRequest, ...], names: tuple[str, ...], - reuse: DataflowReuse, + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], shapes: dict[str, dict[str, str]], types: dict[str, dict[str, str]], nullability: dict[str, dict[str, InferredNullability]], @@ -43,7 +40,8 @@ def analyze_binding_waves( return BindingDataflow( requests=requests, names=names, - reuse=reuse, + cached=cached, + previous_signatures=previous_signatures, shapes=shapes, types=types, nullability=nullability, @@ -57,7 +55,8 @@ def analyze_binding_dataflow( *, requests: tuple[_ModelSqlAnalysisRequest, ...], names: tuple[str, ...], - reuse: DataflowReuse, + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], shapes: dict[str, dict[str, str]], types: dict[str, dict[str, str]], nullability: dict[str, dict[str, InferredNullability]], @@ -71,7 +70,8 @@ def analyze_binding_dataflow( BindingDataflow, requests=requests, names=names, - reuse=reuse, + cached=cached, + previous_signatures=previous_signatures, shapes=shapes, types=types, nullability=nullability, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index fabfe7f530..b4360ccdd3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -19,7 +19,9 @@ model_analysis_cache_key, model_analysis_output_signature, read_compact_analysis_cache_candidate, + read_model_analyses, record_analysis_cache_metrics, + write_model_analyses, ) from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults from sqlbuild.compiler.compile._helpers.analysis.compact import ( @@ -54,6 +56,7 @@ build_column_nullability_by_table, build_complete_binding_schemas, published_model_shape, + upstream_signatures, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( build_declared_column_types as _build_column_types_by_table, @@ -103,7 +106,6 @@ report_mocks_reading_referencing_helpers, ) from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint -from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) @@ -139,11 +141,9 @@ CompileSqlFunctionInput, CompileSqlScenarioInput, CompileSqlTestInput, - DataflowReuse, DynamicColumnContractProof, InferredColumn, MacroContext, - ModelAnalysisCaching, PolyglotAnalysisResult, ) from sqlbuild.compiler.compile.models import ( @@ -353,14 +353,7 @@ def assemble_compiled_project( inference_profile=profile, allow_compact_analysis=allow_compact_analysis, rich_type_inference=rich_type_inference, - analysis_caching=( - None - if analysis_cache is None - else ModelAnalysisCaching( - cache=analysis_cache, - reuse=inputs.analysis_reuse if analysis_model_names is None else None, - ) - ), + analysis_cache=analysis_cache, complete_binding_schemas=complete_binding_schemas, ) model_sql_analysis_by_name = analyze_model_sql_by_engine( @@ -662,14 +655,11 @@ def _analyze_model_sql_in_parallel( inference_profile: ExpressionInferenceProfile, allow_compact_analysis: bool, rich_type_inference: bool, - analysis_caching: ModelAnalysisCaching | None, + analysis_cache: AnalysisCacheContext | None, complete_binding_schemas: dict[str, dict[str, str]], ) -> dict[str, _ModelSqlAnalysis]: if not model_inputs: return {} - analysis_cache: AnalysisCacheContext | None = ( - None if analysis_caching is None else analysis_caching.cache - ) analyzed_model_names: frozenset[str] = frozenset( _model_name(model_input) for model_input in model_inputs ) @@ -683,13 +673,6 @@ def _analyze_model_sql_in_parallel( ) for model_input in model_inputs ) - stored: StoredModelAnalyses = StoredModelAnalyses( - caching=analysis_caching, - requests=requests, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) request_cache_keys: tuple[str, ...] = tuple( request.cache_key for request in requests if request.cache_key is not None ) @@ -698,7 +681,7 @@ def _analyze_model_sql_in_parallel( for request in requests ) compact_batch_plan: CompactAnalysisCachePlan | None = None - if analysis_cache is not None and not stored.served: + if analysis_cache is not None: compact_batch_plan = build_compact_analysis_cache_plan( context=analysis_cache, models=tuple( @@ -726,24 +709,32 @@ def _analyze_model_sql_in_parallel( else None ) if compact_candidate is not None: - completed_analyses: dict[str, PolyglotAnalysisResult] = ( - {} - if analysis_cache is None - else stored.read_every_model(context=analysis_cache, requests=requests) - ) - if analysis_cache is not None and all( - key in completed_analyses for key in request_cache_keys - ): - record_analysis_cache_metrics( - batch_hits=len(requests), entry_hits=0, misses=0, bypasses=0 + completed_analyses: dict[str, PolyglotAnalysisResult] = {} + if analysis_cache is not None: + completed_analyses, _, _ = read_model_analyses( + context=analysis_cache, + cache_keys=request_cache_keys, + model_names=tuple(_model_name(request.model_input) for request in requests), + upstream_model_names_by_key={ + request.cache_key: _referenced_model_names( + model_input=request.model_input, + available_names=analyzed_model_names, + ) + for request in requests + if request.cache_key is not None + }, ) - return { - _model_name(request.model_input): _ModelSqlAnalysis( - polyglot_analysis=completed_analyses[cache_key], - placeholders=request.placeholders, + if all(key in completed_analyses for key in request_cache_keys): + record_analysis_cache_metrics( + batch_hits=len(requests), entry_hits=0, misses=0, bypasses=0 ) - for request, cache_key in zip(requests, request_cache_keys, strict=True) - } + return { + _model_name(request.model_input): _ModelSqlAnalysis( + polyglot_analysis=completed_analyses[cache_key], + placeholders=request.placeholders, + ) + for request, cache_key in zip(requests, request_cache_keys, strict=True) + } try: compact_analyses: tuple[_ModelSqlAnalysis, ...] = _analyze_model_sql_requests( requests=requests, @@ -781,7 +772,6 @@ def _analyze_model_sql_in_parallel( complete_binding_schemas=complete_binding_schemas, inference_profile=inference_profile, ) - stored.record_none() return { _model_name(model_input): analysis for model_input, analysis in zip(model_inputs, compact_analyses, strict=True) @@ -793,7 +783,7 @@ def _analyze_model_sql_in_parallel( previous_signatures: dict[str, str] cached_output_signatures_by_key: dict[str, str] cached_analyses, previous_signatures, cached_output_signatures_by_key = ( - stored.read( + read_model_analyses( context=analysis_cache, cache_keys=entry_cache_keys, model_names=tuple(_model_name(request.model_input) for request in requests), @@ -835,11 +825,8 @@ def _analyze_model_sql_in_parallel( analyses, cached_analyses = analyze_binding_dataflow( requests=requests, names=tuple(_model_name(request.model_input) for request in requests), - reuse=DataflowReuse( - cached=cached_analyses, - previous_signatures=previous_signatures, - served={name: analysis.dependencies for name, analysis in stored.served.items()}, - ), + cached=cached_analyses, + previous_signatures=previous_signatures, shapes=complete_binding_schemas, types=column_types_by_table, nullability=column_nullability_by_table, @@ -938,11 +925,6 @@ def _analyze_model_sql_in_parallel( invalidated_names: set[str] = _downstream_model_names( model_inputs=model_inputs, changed_names=changed_signature_names, - ) | _with_downstream( - model_inputs=model_inputs, - names=stored.stale_names( - requests=requests, cached=cached_analyses, current_signatures=current_signatures_by_name - ), ) if invalidated_names and not dependency_ordered: invalidated_requests: tuple[_ModelSqlAnalysisRequest, ...] = tuple( @@ -1004,17 +986,33 @@ def _analyze_model_sql_in_parallel( for model_name, invalidated_analysis in invalidated_analyses_by_name.items() } ) - stored.publish( - context=analysis_cache, - requests=requests, - results=tuple(analysis.polyglot_analysis for analysis in analyses), - cached=cached_analyses, - cached_output_signatures=cached_output_signatures_by_key, - invalidated_names=invalidated_names, - latest_by_model=analyses_to_record_by_name, - previous_signatures=previous_signatures, - current_signatures=current_signatures_by_name, - ) + with record_compile_timing("cache_publication_ms"): + write_model_analyses( + context=analysis_cache, + analyses_by_key={ + request.cache_key: analysis.polyglot_analysis + for request, analysis in zip(requests, analyses, strict=True) + if request.cache_key is not None + and ( + request.cache_key not in cached_analyses + or _model_name(request.model_input) in invalidated_names + ) + }, + latest_analyses_by_model=analyses_to_record_by_name, + dependency_signatures_by_key={ + request.cache_key: upstream_signatures( + model_input=request.model_input, + signatures=current_signatures_by_name, + available_names=analyzed_model_names, + ) + for request in requests + if request.cache_key is not None + and ( + request.cache_key not in cached_analyses + or _model_name(request.model_input) in invalidated_names + ) + }, + ) return { _model_name(model_input): analysis for model_input, analysis in zip(model_inputs, analyses, strict=True) @@ -1384,10 +1382,6 @@ def _should_recover_cte_facts(model_input: CompileModelInput) -> bool: ) -def _with_downstream(*, model_inputs: tuple[CompileModelInput, ...], names: set[str]) -> set[str]: - return names | _downstream_model_names(model_inputs=model_inputs, changed_names=names) - - def _model_name(model_input: CompileModelInput) -> str: return model_input.model_file.file_path.stem diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 85f3713d60..0513354e15 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -390,3 +390,15 @@ def referenced_model_names( } ) ) + + +def upstream_signatures( + *, model_input: CompileModelInput, signatures: dict[str, str], available_names: frozenset[str] +) -> dict[str, str]: + """Return the output signatures of the analyzed models this model references.""" + + return { + name: signatures[name] + for name in referenced_model_names(model_input=model_input, available_names=available_names) + if name in signatures + } diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 0ace8f95ed..17e155d9d1 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -2,9 +2,7 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import dataclass, field -from functools import partial from pathlib import Path from sqlbuild.compiler.attachments.constants import DEFAULT_POLICY, INSTANCE_POLICY @@ -42,14 +40,10 @@ render_parameterized_sql, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics -from sqlbuild.compiler.compile._helpers.render.reuse import reused_or_rendered from sqlbuild.compiler.compile._helpers.render.sql_vars import ( expand_authored_sql_result, ) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.constants import ( - RENDER_REUSE_ATTACHED_AUDITS_GROUP, - RENDER_REUSE_SINGULAR_AUDITS_GROUP, SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, ) from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -143,7 +137,6 @@ def build_project_audit_inputs( model_inputs: tuple[CompileModelInput, ...], source_inputs: tuple[CompileSourceInput, ...], seed_inputs: tuple[CompileSeedInput, ...], - render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: """Resolve built-in-aware generic audits and build every project audit input.""" @@ -159,7 +152,6 @@ def build_project_audit_inputs( source_inputs=source_inputs, generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, - render_reuse=render_reuse, ) return audit_inputs, diagnostics @@ -173,7 +165,6 @@ def build_audit_inputs( generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] | None = None, seed_inputs: tuple[CompileSeedInput, ...] = (), - render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileAuditInput, ...]: """Build compile-time audit inputs from discovered SQL audit blocks.""" @@ -204,38 +195,20 @@ def build_audit_inputs( sql_lexical_syntax=sql_lexical_syntax, ) audit_inputs: list[CompileAuditInput] = list( - reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SINGULAR_AUDITS_GROUP, - render=partial( - _build_singular_audit_inputs, - discovered_inputs=discovered_inputs, - context=attachment_context, - ), + _build_singular_audit_inputs( + discovered_inputs=discovered_inputs, context=attachment_context ) ) model_input: CompileModelInput for model_input in model_inputs: if model_input.schema_entry is None: continue - render: Callable[[], tuple[CompileAuditInput, ...]] = partial( - build_model_attached_audit_inputs, model_input=model_input, context=attachment_context - ) audit_inputs.extend( - render() - if render_reuse is None - else render_reuse.model_audits(model_input=model_input, render=render) + build_model_attached_audit_inputs(model_input=model_input, context=attachment_context) ) audit_inputs.extend( - reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_ATTACHED_AUDITS_GROUP, - render=partial( - _build_source_and_seed_audit_inputs, - source_inputs=source_inputs, - seed_inputs=seed_inputs, - context=attachment_context, - ), + _build_source_and_seed_audit_inputs( + source_inputs=source_inputs, seed_inputs=seed_inputs, context=attachment_context ) ) return tuple(audit_inputs) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 78564c72cb..0e900cedab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -4,9 +4,7 @@ import inspect import re -from collections.abc import Callable from dataclasses import dataclass, field, fields, replace -from functools import partial from inspect import Parameter, Signature from pathlib import Path from typing import Any, cast @@ -91,7 +89,6 @@ expand_effective_vars, expand_template_data, ) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_FULL_REFRESH_CONFIG_KEY, @@ -446,7 +443,6 @@ def build_model_inputs( defer_model_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, reference_cache_dir: Path | None = None, - render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileModelInput, ...]: """Attach schema metadata to discovered model files.""" @@ -464,7 +460,6 @@ def build_model_inputs( external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, legacy_schema_files=legacy_schema_files, - render_reuse=render_reuse, ) @@ -477,7 +472,6 @@ def _build_model_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, extract_references: SqlReferenceExtractor, legacy_schema_files: tuple[DiscoveredSchemaFile, ...], - render_reuse: CompileRenderReuseSession | None, ) -> tuple[CompileModelInput, ...]: effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings @@ -537,11 +531,7 @@ def _build_model_inputs( sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( discovered_inputs.sql_hook_files ) - render_files: tuple[DiscoveredSqlModelFile, ...] = tuple( - model_file - for model_file in discovered_inputs.model_files - if render_reuse is None or not render_reuse.has_reusable_model(model_file=model_file) - ) + render_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files prepared_var_substituted_sqls: dict[Path, str | None] = dict( zip( (model_file.file_path for model_file in render_files), @@ -594,22 +584,14 @@ def _build_model_inputs( model_inputs: list[CompileModelInput] = [] model_file: DiscoveredSqlModelFile for model_file in discovered_inputs.model_files: - reused: CompileModelInput | None = ( - None if render_reuse is None else render_reuse.reused_model(model_file=model_file) - ) - if reused is not None: - model_inputs.append(reused) - continue - render: Callable[[], CompileModelInput] = partial( - _build_model_input, - loop=loop, - model_file=model_file, - prepared_var_substituted_sql=prepared_var_substituted_sqls.get(model_file.file_path), - ) model_inputs.append( - render() - if render_reuse is None - else render_reuse.rendered_model(model_file=model_file, render=render) + _build_model_input( + loop=loop, + model_file=model_file, + prepared_var_substituted_sql=prepared_var_substituted_sqls.get( + model_file.file_path + ), + ) ) validate_declared_schema_models_are_attached( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 84b8dbc1cc..12a8478da9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -18,6 +18,7 @@ from sqlbuild.compiler.compile._helpers.sql_tests.extraction_errors import ( validate_authored_test_cte_names, ) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( DeclarationScopeBuild, @@ -46,6 +47,8 @@ def build_declaration_scope( ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" + scan_cache: SqlTestScanCache = SqlTestScanCache(cache_dir=compile_cache_dir) + validate_authored_test_cte_names( test_files=discovered_inputs.test_files, syntax=sql_lexical_syntax ) @@ -54,7 +57,7 @@ def build_declaration_scope( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, + scan_cache=scan_cache, ) if native_scope is not None: return native_scope @@ -76,7 +79,7 @@ def build_declaration_scope( discovered_inputs=discovered_inputs, index=index, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, + scan_cache=scan_cache, ) if has_scoped_relationship_declarations and (discovered_inputs.test_files or discovered_inputs.scenario_files) @@ -97,6 +100,7 @@ def build_declaration_scope( scope_index=index, loaded_macros=loaded_macros, ), + sql_test_scans=scan_cache, ) @@ -105,7 +109,7 @@ def _build_native_declaration_scope( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None, + scan_cache: SqlTestScanCache, ) -> DeclarationScopeBuild | None: """Build the scope natively with Python's errors and order, or None for the Python stage.""" @@ -126,7 +130,7 @@ def _build_native_declaration_scope( facts, fault = extract_scope_relationship_facts( discovered_inputs=discovered_inputs, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, + scan_cache=scan_cache, ) if fault is not None: raise CompileInputError(fault) @@ -145,6 +149,7 @@ def _build_native_declaration_scope( loaded_macros=loaded_macros, lookup=lookup, ), + sql_test_scans=scan_cache, ) @@ -166,6 +171,7 @@ def rebind_declaration_scope( scope_index=scope.index, loaded_macros=loaded_macros, ), + sql_test_scans=scope.sql_test_scans, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 46758adddc..f722b7a28c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -2,10 +2,12 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path from typing import cast +import orjson + from sqlbuild.compiler.compile._helpers.render.macros import ( find_macro_call_names, find_nested_macro_call_names, @@ -17,10 +19,10 @@ extract_sql_test_expected_model_names, extract_unclassified_sql_test_ctes, ) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import ( MACRO_ACTUAL_TEST_CTE_NAME, - SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, - SQL_TEST_FACT_CACHE_NAMESPACE, + SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, ) from sqlbuild.compiler.compile.models import ( CompileSqlTestCte, @@ -29,7 +31,6 @@ ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.scopes.main._native_expected_model_names import ( native_expected_model_names, ) @@ -55,6 +56,7 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] +type _BlockNames = tuple[str, ...] | Exception def build_scope_relationship_grants( @@ -62,24 +64,19 @@ def build_scope_relationship_grants( discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex, sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, + scan_cache: SqlTestScanCache | None = None, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" lookup: ScopeLookup = build_scope_lookup(index=index) shared_declarations: _SharedDeclarations = _shared_declarations_by_directory(lookup=lookup) - with FactCacheStore( - root=compile_cache_dir, - namespace=SQL_TEST_FACT_CACHE_NAMESPACE, - algorithm=SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, - ) as fact_cache: - test_grants, test_faults = _test_relationship_grants( - discovered_inputs=discovered_inputs, - lookup=lookup, - shared_declarations=shared_declarations, - fact_cache=fact_cache, - syntax=sql_lexical_syntax, - ) + test_grants, test_faults = _test_relationship_grants( + discovered_inputs=discovered_inputs, + lookup=lookup, + shared_declarations=shared_declarations, + scan_cache=scan_cache or SqlTestScanCache(cache_dir=None), + syntax=sql_lexical_syntax, + ) scenario_grants, scenario_faults = _scenario_relationship_grants( discovered_inputs=discovered_inputs, lookup=lookup, @@ -96,98 +93,43 @@ def extract_scope_relationship_facts( *, discovered_inputs: DiscoveredProjectInputs, sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, + scan_cache: SqlTestScanCache | None = None, ) -> tuple[tuple[RelationshipFact, ...], str | None]: """Return the relationship names grants are resolved from, and the first extraction fault.""" facts: list[RelationshipFact] = [] faults: list[ScopeRelationshipFault] = [] - with FactCacheStore( - root=compile_cache_dir, - namespace=SQL_TEST_FACT_CACHE_NAMESPACE, - algorithm=SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, - ) as fact_cache: - cache_keys: dict[int, str] = ( - { - file_index: _expected_models_fact_key( - test_file=test_file, fact_cache=fact_cache, syntax=sql_lexical_syntax - ) - for file_index, test_file in enumerate(discovered_inputs.test_files) - } - if fact_cache.enabled - else {} - ) - cached_names: dict[str, object] = fact_cache.read_many( - tuple( - (_expected_models_fact_slot(discovered_inputs.test_files[file_index]), cache_key) - for file_index, cache_key in cache_keys.items() - ) - ) - cached_by_file: list[tuple[tuple[str, ...], ...] | None] = [ - _cached_expected_names( - value=cached_names.get(cache_keys[file_index]) - if file_index in cache_keys - else None, - block_count=len(test_file.blocks), - ) - for file_index, test_file in enumerate(discovered_inputs.test_files) - ] - scanned_names: dict[tuple[int, int], tuple[str, ...]] = _scanned_expected_names( - test_files=discovered_inputs.test_files, - cached_by_file=cached_by_file, - syntax=sql_lexical_syntax, - ) - for file_index, test_file in enumerate(discovered_inputs.test_files): - cache_key: str | None = cache_keys.get(file_index) - cached_file_names: tuple[tuple[str, ...], ...] | None = cached_by_file[file_index] - extracted_names: list[tuple[str, ...]] = [] - for block_index, block in enumerate(test_file.blocks): - scanned: tuple[str, ...] | None = scanned_names.get((file_index, block_index)) - try: - expected_names: tuple[str, ...] = ( - cached_file_names[block_index] - if cached_file_names is not None - else scanned - if scanned is not None - else extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=sql_lexical_syntax, - mode=block.mode, - ) - ) - extracted_names.append(expected_names) - called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) - facts.append( - RelationshipFact( - resource=ResourceIdentity( - ResourceKind.TEST, block.name or test_file.relative_path.stem - ), - expected_models=expected_names, - called_macros=called_macros, - tested_macros=( - _tested_macro_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=sql_lexical_syntax, - ) - if block.mode is SqlTestMode.MACRO - else () - ), - ) + names_by_file: list[list[_BlockNames]] = _expected_names_by_file( + test_files=discovered_inputs.test_files, + scan_cache=scan_cache or SqlTestScanCache(cache_dir=None), + syntax=sql_lexical_syntax, + scan_natively=True, + ) + for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): + for block, block_names in zip(test_file.blocks, file_names, strict=True): + try: + expected_names: tuple[str, ...] = _names_or_raise(block_names) + called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) + facts.append( + RelationshipFact( + resource=ResourceIdentity( + ResourceKind.TEST, block.name or test_file.relative_path.stem + ), + expected_models=expected_names, + called_macros=called_macros, + tested_macros=( + _tested_macro_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=sql_lexical_syntax, + ) + if block.mode is SqlTestMode.MACRO + else () + ), ) - except Exception as error: - faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - if ( - cache_key is not None - and cached_file_names is None - and len(extracted_names) == len(test_file.blocks) - ): - fact_cache.stage( - key=cache_key, - slot=_expected_models_fact_slot(test_file), - value=tuple(extracted_names), ) + except Exception as error: + faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) scenario_names: list[tuple[str, ...] | None] = native_expected_model_names( sqls=[scenario.sql_body for scenario in discovered_inputs.scenario_files], syntax=sql_lexical_syntax, @@ -211,17 +153,111 @@ def extract_scope_relationship_facts( return tuple(facts), faults[0].message if faults else None +def _expected_names_by_file( + *, + test_files: Sequence[DiscoveredSqlTestFile], + scan_cache: SqlTestScanCache, + syntax: SqlLexicalSyntax, + scan_natively: bool, +) -> list[list[_BlockNames]]: + """Return each block's expected-model names or extraction error, reusing whole stored files.""" + + stored: list[tuple[tuple[str, ...], ...] | None] = [ + scan_cache.read( + algorithm=SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, + syntax=syntax, + parts=_expected_names_parts(test_file), + decode=_stored_expected_names(block_count=len(test_file.blocks)), + ) + for test_file in test_files + ] + scanned: dict[tuple[int, int], tuple[str, ...]] = ( + _scanned_expected_names(test_files=test_files, stored=stored, syntax=syntax) + if scan_natively + else {} + ) + names_by_file: list[list[_BlockNames]] = [] + for file_index, test_file in enumerate(test_files): + stored_names: tuple[tuple[str, ...], ...] | None = stored[file_index] + if stored_names is not None: + names_by_file.append(list(stored_names)) + continue + file_names: list[_BlockNames] = [] + for block_index, block in enumerate(test_file.blocks): + native_names: tuple[str, ...] | None = scanned.get((file_index, block_index)) + if native_names is not None: + file_names.append(native_names) + continue + try: + file_names.append( + extract_sql_test_expected_model_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=syntax, + mode=block.mode, + ) + ) + except Exception as error: + file_names.append(error) + if not any(isinstance(names, Exception) for names in file_names): + scan_cache.write( + algorithm=SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, + syntax=syntax, + parts=_expected_names_parts(test_file), + value=orjson.dumps(file_names), + ) + names_by_file.append(file_names) + return names_by_file + + +def _expected_names_parts(test_file: DiscoveredSqlTestFile) -> list[str]: + parts: list[str] = [str(test_file.relative_path)] + for block in test_file.blocks: + parts.extend((block.sql_body, block.mode.value)) + return parts + + +def _stored_expected_names( + *, block_count: int +) -> Callable[[bytes], tuple[tuple[str, ...], ...] | None]: + """Decode one file's stored names, rejecting anything that is not their exact shape.""" + + def decode(value: bytes) -> tuple[tuple[str, ...], ...] | None: + try: + decoded: object = orjson.loads(value) + except orjson.JSONDecodeError: + return None + if not isinstance(decoded, list) or len(cast(list[object], decoded)) != block_count: + return None + blocks: list[tuple[str, ...]] = [] + for names in cast(list[object], decoded): + if not isinstance(names, list) or not all( + isinstance(name, str) for name in cast(list[object], names) + ): + return None + blocks.append(tuple(cast(list[str], names))) + return tuple(blocks) + + return decode + + +def _names_or_raise(names: _BlockNames) -> tuple[str, ...]: + if isinstance(names, Exception): + raise names + return names + + def _scanned_expected_names( *, test_files: Sequence[DiscoveredSqlTestFile], - cached_by_file: Sequence[tuple[tuple[str, ...], ...] | None], + stored: Sequence[tuple[tuple[str, ...], ...] | None], syntax: SqlLexicalSyntax, ) -> dict[tuple[int, int], tuple[str, ...]]: - """Scan uncached model-mode blocks natively, keeping only the names the scan reproduces.""" + """Scan model-mode blocks of unstored files natively, keeping only names the scan reproduces.""" pending: list[tuple[int, int]] = [] for file_index, test_file in enumerate(test_files): - if cached_by_file[file_index] is not None: + if stored[file_index] is not None: continue pending.extend( (file_index, block_index) @@ -257,47 +293,21 @@ def _test_relationship_grants( discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup, shared_declarations: _SharedDeclarations, - fact_cache: FactCacheStore, + scan_cache: SqlTestScanCache, syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] - cache_keys: dict[int, str] = ( - { - file_index: _expected_models_fact_key( - test_file=test_file, fact_cache=fact_cache, syntax=syntax - ) - for file_index, test_file in enumerate(discovered_inputs.test_files) - } - if fact_cache.enabled - else {} - ) - cached_names: dict[str, object] = fact_cache.read_many( - tuple( - (_expected_models_fact_slot(discovered_inputs.test_files[file_index]), cache_key) - for file_index, cache_key in cache_keys.items() - ) + names_by_file: list[list[_BlockNames]] = _expected_names_by_file( + test_files=discovered_inputs.test_files, + scan_cache=scan_cache, + syntax=syntax, + scan_natively=False, ) - for file_index, test_file in enumerate(discovered_inputs.test_files): - cache_key: str | None = cache_keys.get(file_index) - cached_file_names: tuple[tuple[str, ...], ...] | None = _cached_expected_names( - value=None if cache_key is None else cached_names.get(cache_key), - block_count=len(test_file.blocks), - ) - extracted_names: list[tuple[str, ...]] = [] - for block_index, block in enumerate(test_file.blocks): + for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): + for block, block_names in zip(test_file.blocks, file_names, strict=True): try: - expected_names: tuple[str, ...] = ( - cached_file_names[block_index] - if cached_file_names is not None - else extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=syntax, - mode=block.mode, - ) - ) - extracted_names.append(expected_names) + expected_names: tuple[str, ...] = _names_or_raise(block_names) grants.extend( _expected_model_grants( lookup=lookup, @@ -323,43 +333,9 @@ def _test_relationship_grants( ) except Exception as error: faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - if ( - cache_key is not None - and cached_file_names is None - and len(extracted_names) == len(test_file.blocks) - ): - fact_cache.stage( - key=cache_key, - slot=_expected_models_fact_slot(test_file), - value=tuple(extracted_names), - ) return tuple(grants), tuple(faults) -def _expected_models_fact_key( - *, test_file: DiscoveredSqlTestFile, fact_cache: FactCacheStore, syntax: SqlLexicalSyntax -) -> str: - parts: list[str] = [str(test_file.relative_path), syntax.cache_key] - for block in test_file.blocks: - parts.extend((block.sql_body, block.mode.value)) - return fact_cache.key(*parts) - - -def _cached_expected_names( - *, value: object, block_count: int -) -> tuple[tuple[str, ...], ...] | None: - if not isinstance(value, tuple) or len(value) != block_count: - return None - for names in value: - if not isinstance(names, tuple) or not all(isinstance(name, str) for name in names): - return None - return cast(tuple[tuple[str, ...], ...], value) - - -def _expected_models_fact_slot(test_file: DiscoveredSqlTestFile) -> str: - return f"expected:{test_file.relative_path}" - - def _scenario_relationship_grants( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index b143019c3b..17b278dce8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -46,9 +46,10 @@ sql_test_reads, ) from sqlbuild.compiler.compile._helpers.sql_tests.native import ( - extract_expanded_sql_tests_cached, + extract_expanded_sql_tests, extract_unexpanded_sql_test, ) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestExtractionError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -118,13 +119,13 @@ def build_test_inputs_with_cache( declaration_expansion: DeclarationExpansionContext, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...], - compile_cache_dir: Path | None, sql_lexical_syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache, ) -> tuple[CompileSqlTestInput, ...]: - """Build SQL test inputs inside the compile timing boundary.""" + """Build SQL test inputs inside the compile timing boundary, then save the scan store.""" with record_compile_timing("test_input_compile_ms"): - return build_test_inputs( + test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs( discovered_inputs=discovered_inputs, effective_vars=effective_vars, macro_context=macro_context, @@ -132,9 +133,11 @@ def build_test_inputs_with_cache( declaration_expansion=declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, - compile_cache_dir=compile_cache_dir, sql_lexical_syntax=sql_lexical_syntax, + scan_cache=scan_cache, ) + scan_cache.save() + return test_inputs def build_test_inputs( @@ -147,7 +150,7 @@ def build_test_inputs( sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = (), - compile_cache_dir: Path | None = None, + scan_cache: SqlTestScanCache | None = None, ) -> tuple[CompileSqlTestInput, ...]: """Build compile-time test inputs from discovered SQL-native test blocks.""" @@ -308,12 +311,12 @@ def build_test_inputs( ) ) try: - test_ctes_batch: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests_cached( + test_ctes_batch: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( tests=tuple( (test.sql_body, str(test.test_file.relative_path), test.mode) for test in expanded_tests ), - cache_root=compile_cache_dir, + scan_cache=scan_cache, syntax=sql_lexical_syntax, ) except SqlTestExtractionError as error: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py index 3d7b954235..384a535141 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py @@ -2,6 +2,8 @@ from __future__ import annotations +import os +import shutil from pathlib import Path from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars @@ -9,8 +11,17 @@ resolve_early_model_templates, resolve_run_id, ) +from sqlbuild.compiler.compile.constants import ( + COMPILE_CACHE_DISABLE_ENV_VAR, + COMPILE_CACHE_DISABLE_VALUE, + RETIRED_FACT_CACHE_DIRECTORY_NAME, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root +from sqlbuild.compiler.frontier.constants import ( + COMPILER_CACHE_DIRECTORY_NAME, + ENGINE_CACHE_NAMESPACE_SUFFIXES, +) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import TargetConfig @@ -46,6 +57,36 @@ def build_compile_target_context( return target_name, target_config, cache_dir +def compile_cache_root( + *, project_dir: Path | None, target_config: TargetConfig | None, no_cache: bool +) -> Path | None: + """Return the shared compile-cache directory, or None when caching is disabled.""" + + if ( + no_cache + or project_dir is None + or (target_config is not None and target_config.compile_cache is False) + or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE + ): + return None + root: Path = compiler_cache_directory(project_dir) + _remove_retired_fact_caches(root=root) + return root + + +def _remove_retired_fact_caches(*, root: Path) -> None: + """Delete the per-file fact caches older releases kept under every engine's compiler root.""" + + for suffix in ENGINE_CACHE_NAMESPACE_SUFFIXES.values(): + retired: Path = ( + root.parent + / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" + / (RETIRED_FACT_CACHE_DIRECTORY_NAME) + ) + if retired.is_dir(): + shutil.rmtree(retired, ignore_errors=True) + + def build_effective_target_namespace( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index 9a7ee0f036..ae6080acb7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -21,23 +21,6 @@ _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None ) -_ACTIVE_TAPS: ContextVar[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = ContextVar( - "sqlbuild_tapped_compile_diagnostics", default=() -) - - -@contextmanager -def tapped_compile_diagnostics() -> Iterator[list[tuple[tuple[str, ...], CompilerDiagnostic]]]: - """Also note every diagnostic reported inside the block, in report order, for replay.""" - - tap: list[tuple[tuple[str, ...], CompilerDiagnostic]] = [] - token: Token[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = _ACTIVE_TAPS.set( - (*_ACTIVE_TAPS.get(), tap) - ) - try: - yield tap - finally: - _ACTIVE_TAPS.reset(token) @contextmanager @@ -71,8 +54,6 @@ def collecting(*args: P.args, **kwargs: P.kwargs) -> R: def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: """Record a diagnostic in the active collector, or raise it when none is active.""" - for tap in _ACTIVE_TAPS.get(): - tap.append((key, diagnostic)) collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() if collector is not None: collector.add(key=key, diagnostic=diagnostic) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/reuse.py b/src/sqlbuild/compiler/compile/_helpers/render/reuse.py deleted file mode 100644 index 70132c44da..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/reuse.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Route project-wide renders through the active render reuse session, if any.""" - -from __future__ import annotations - -from collections.abc import Callable -from functools import partial -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs -from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( - build_scenario_inputs, - build_test_inputs_with_cache, -) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import ( - COMPILE_RENDER_REUSE, - RENDER_REUSE_SCENARIOS_GROUP, - RENDER_REUSE_SOURCES_GROUP, - RENDER_REUSE_TESTS_GROUP, -) -from sqlbuild.compiler.compile.models import ( - CompileSourceInput, - CompileSqlFunctionInput, - CompileSqlScenarioInput, - CompileSqlTestInput, - ModelInputBuildContext, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def reused_or_rendered[T]( - *, render_reuse: CompileRenderReuseSession | None, name: str, render: Callable[[], T] -) -> T: - """Return the stored render of one project-wide group, or render and record it.""" - - return render() if render_reuse is None else render_reuse.group(name=name, render=render) - - -def claimed_render_reuse( - *, discovered_inputs: DiscoveredProjectInputs -) -> CompileRenderReuseSession | None: - """Claim the active render reuse session, if any, and plan it for the discovered models.""" - - render_reuse: CompileRenderReuseSession | None = COMPILE_RENDER_REUSE.claim() - if render_reuse is not None: - render_reuse.plan_models(model_files=discovered_inputs.model_files) - return render_reuse - - -def build_sql_resource_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - sql_function_inputs: tuple[CompileSqlFunctionInput, ...], - no_sql_validation: bool, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - compile_cache_dir: Path | None, - render_reuse: CompileRenderReuseSession | None, -) -> tuple[ - tuple[CompileSourceInput, ...], - tuple[CompileSqlTestInput, ...], - tuple[CompileSqlScenarioInput, ...], -]: - """Render sources, tests, and scenarios, each reused whole after model-only edits.""" - - source_inputs: tuple[CompileSourceInput, ...] = reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SOURCES_GROUP, - render=partial( - build_source_inputs, - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - effective_settings=context.effective_settings, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=context.declaration_expansion, - no_sql_validation=no_sql_validation, - ), - ) - test_inputs: tuple[CompileSqlTestInput, ...] = reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_TESTS_GROUP, - render=partial( - build_test_inputs_with_cache, - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, - compile_cache_dir=compile_cache_dir, - sql_lexical_syntax=context.sql_lexical_syntax, - ), - ) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SCENARIOS_GROUP, - render=partial( - build_scenario_inputs, - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=context.sql_lexical_syntax, - ), - ) - return source_inputs, test_inputs, scenario_inputs diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 8c2a4aa13f..1577857686 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,16 +2,14 @@ from __future__ import annotations -from pathlib import Path +from collections.abc import Callable from typing import Protocol, cast import orjson import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import ( - SQL_TEST_FACT_CACHE_ALGORITHM, - SQL_TEST_FACT_CACHE_NAMESPACE, -) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import SQL_TEST_CTE_SCAN_ALGORITHM from sqlbuild.compiler.compile.exceptions import ( CompileInputError, NativeSqlTestResponseError, @@ -24,7 +22,6 @@ CompileSqlTestCtes, ) from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -46,104 +43,93 @@ def extract_sql_tests_json(self, request_json: str) -> str: ... _AUTHORED_CTE_LENGTH: int = 3 -def extract_expanded_sql_tests_cached( +def extract_expanded_sql_tests( *, tests: tuple[tuple[str, str, SqlTestMode], ...], - cache_root: Path | None, syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache | None = None, ) -> tuple[CompileSqlTestCtes, ...]: - """Reuse exact per-file extraction facts and extract only tests of changed files natively.""" - - with FactCacheStore( - root=cache_root, - namespace=SQL_TEST_FACT_CACHE_NAMESPACE, - algorithm=SQL_TEST_FACT_CACHE_ALGORITHM, - ) as fact_cache: - if not fact_cache.enabled: - return extract_expanded_sql_tests(tests=tests, syntax=syntax) - indexes_by_file: dict[str, list[int]] = {} - for index, (_sql, file_label, _mode) in enumerate(tests): - indexes_by_file.setdefault(file_label, []).append(index) - keys_by_file: dict[str, str] = { - file_label: fact_cache.key( - file_label, - syntax.cache_key, - *_file_test_key_parts(tests=tests, indexes=indexes), + """Extract expanded tests natively, reusing whole stored results of unchanged files.""" + + store: SqlTestScanCache = scan_cache or SqlTestScanCache(cache_dir=None) + indexes_by_file: dict[str, list[int]] = {} + for index, (_sql, file_label, _mode) in enumerate(tests): + indexes_by_file.setdefault(file_label, []).append(index) + results: list[CompileSqlTestCtes | None] = [None] * len(tests) + parts_by_file: dict[str, list[str]] = {} + for file_label, indexes in indexes_by_file.items(): + parts: list[str] = [file_label] + for index in indexes: + parts.extend((tests[index][0], tests[index][2].value)) + modes: tuple[SqlTestMode, ...] = tuple(tests[index][2] for index in indexes) + stored: tuple[CompileSqlTestCtes, ...] | None = store.read( + algorithm=SQL_TEST_CTE_SCAN_ALGORITHM, + syntax=syntax, + parts=parts, + decode=_stored_file_tests(modes=modes), + ) + if stored is None: + parts_by_file[file_label] = parts + continue + for index, test_ctes in zip(indexes, stored, strict=True): + results[index] = test_ctes + missing: list[int] = [] + for file_label in parts_by_file: + missing.extend(indexes_by_file[file_label]) + missing.sort() + if missing: + try: + items: list[object] = _native_batch( + tests=[ + { + "sql": tests[index][0], + "fileLabel": tests[index][1], + "mode": tests[index][2].value, + "raw": False, + } + for index in missing + ], + syntax=syntax, ) - for file_label, indexes in indexes_by_file.items() - } - cached: dict[str, object] = fact_cache.read_many(tuple(keys_by_file.items())) - results: list[CompileSqlTestCtes | None] = [None] * len(tests) - missing_files: list[str] = [] - for file_label, indexes in indexes_by_file.items(): - file_results: tuple[CompileSqlTestCtes, ...] | None = _cached_file_tests( - value=cached.get(keys_by_file[file_label]), - modes=tuple(tests[index][2] for index in indexes), + except SqlTestExtractionError as error: + error.test_index = missing[error.test_index] + raise + items_by_index: dict[int, object] = dict(zip(missing, items, strict=True)) + for file_label, parts in parts_by_file.items(): + file_items: list[object] = [ + items_by_index[index] for index in indexes_by_file[file_label] + ] + for index, item in zip(indexes_by_file[file_label], file_items, strict=True): + results[index] = _test_ctes_from_native(item) + store.write( + algorithm=SQL_TEST_CTE_SCAN_ALGORITHM, + syntax=syntax, + parts=parts, + value=orjson.dumps(file_items), ) - if file_results is None: - missing_files.append(file_label) - continue - for index, test_ctes in zip(indexes, file_results, strict=True): - results[index] = test_ctes - missing_indexes: list[int] = [] - for file_label in missing_files: - missing_indexes.extend(indexes_by_file[file_label]) - if missing_indexes: - try: - extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tests=tuple(tests[index] for index in missing_indexes), syntax=syntax - ) - except SqlTestExtractionError as error: - error.test_index = missing_indexes[error.test_index] - raise - for index, test_ctes in zip(missing_indexes, extracted, strict=True): - results[index] = test_ctes - for file_label in missing_files: - fact_cache.stage( - key=keys_by_file[file_label], - slot=file_label, - value=tuple(results[index] for index in indexes_by_file[file_label]), - ) - return tuple(result for result in results if result is not None) - - -def _file_test_key_parts( - *, tests: tuple[tuple[str, str, SqlTestMode], ...], indexes: list[int] -) -> list[str]: - parts: list[str] = [] - for index in indexes: - sql, _file_label, mode = tests[index] - parts.extend((sql, mode.value)) - return parts - - -def _cached_file_tests( - *, value: object, modes: tuple[SqlTestMode, ...] -) -> tuple[CompileSqlTestCtes, ...] | None: - if not isinstance(value, tuple) or len(value) != len(modes): - return None - for test_ctes, mode in zip(value, modes, strict=True): - if not isinstance(test_ctes, CompileSqlTestCtes) or test_ctes.mode is not mode: - return None - return cast(tuple[CompileSqlTestCtes, ...], value) + return tuple(result for result in results if result is not None) -def extract_expanded_sql_tests( - *, - tests: tuple[tuple[str, str, SqlTestMode], ...], - syntax: SqlLexicalSyntax, -) -> tuple[CompileSqlTestCtes, ...]: - """Extract and classify expanded tests natively under the adapter's lexical rules.""" +def _stored_file_tests( + *, modes: tuple[SqlTestMode, ...] +) -> Callable[[bytes], tuple[CompileSqlTestCtes, ...] | None]: + """Decode one file's stored tests, rejecting anything that is not their exact shape.""" - if not tests: - return () - expanded: tuple[tuple[str, str, SqlTestMode, bool], ...] = tuple( - (sql, file_label, mode, False) for sql, file_label, mode in tests - ) - extracted: tuple[tuple[CompileSqlTestCtes, bool], ...] = _extract_natively( - tests=expanded, syntax=syntax - ) - return tuple(test_ctes for test_ctes, _invalid_calls in extracted) + def decode(value: bytes) -> tuple[CompileSqlTestCtes, ...] | None: + try: + items: object = orjson.loads(value) + if not isinstance(items, list) or len(cast(list[object], items)) != len(modes): + return None + decoded: tuple[CompileSqlTestCtes, ...] = tuple( + _test_ctes_from_native(item) for item in cast(list[object], items) + ) + except (orjson.JSONDecodeError, CompileInputError): + return None + if any(test_ctes.mode is not mode for test_ctes, mode in zip(decoded, modes, strict=True)): + return None + return decoded + + return decode def extract_unexpanded_sql_test( diff --git a/src/sqlbuild/compiler/compile/classes/binding_dataflow.py b/src/sqlbuild/compiler/compile/classes/binding_dataflow.py index 638f39f370..0909ba58c7 100644 --- a/src/sqlbuild/compiler/compile/classes/binding_dataflow.py +++ b/src/sqlbuild/compiler/compile/classes/binding_dataflow.py @@ -19,9 +19,7 @@ published_model_shape, referenced_model_names, ) -from sqlbuild.compiler.compile.classes.stored_model_analyses import dependencies_current from sqlbuild.compiler.compile.models import ( - DataflowReuse, ModelSqlAnalysis, ModelSqlAnalysisRequest, PolyglotAnalysisResult, @@ -44,7 +42,8 @@ def __init__( *, requests: tuple[ModelSqlAnalysisRequest, ...], names: tuple[str, ...], - reuse: DataflowReuse, + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], shapes: dict[str, dict[str, str]], types: dict[str, dict[str, str]], nullability: dict[str, dict[str, InferredNullability]], @@ -54,15 +53,8 @@ def __init__( ) -> None: self._requests: tuple[ModelSqlAnalysisRequest, ...] = requests self._names: tuple[str, ...] = names + self._cached: dict[str, PolyglotAnalysisResult] = cached self._by_name: dict[str, ModelSqlAnalysisRequest] = dict(zip(names, requests, strict=True)) - served_keys: frozenset[str | None] = frozenset( - self._by_name[name].cache_key for name in reuse.served if name in self._by_name - ) - self._cached: dict[str, PolyglotAnalysisResult] = { - key: analysis for key, analysis in reuse.cached.items() if key not in served_keys - } - self._served: dict[str, dict[str, str]] = reuse.served - self._signatures: dict[str, str] = {} available_names: frozenset[str] = frozenset(names) self._dependencies: dict[str, tuple[str, ...]] = { name: referenced_model_names( @@ -70,11 +62,11 @@ def __init__( ) for name, request in self._by_name.items() } - self._previous_signatures: dict[str, str] = reuse.previous_signatures + self._previous_signatures: dict[str, str] = previous_signatures self._complete_shapes: dict[str, dict[str, str]] = dict(shapes) self._available_types: dict[str, dict[str, str]] = dict(types) self._available_nullability: dict[str, dict[str, InferredNullability]] = dict(nullability) - self._reusable: dict[str, PolyglotAnalysisResult] = dict(reuse.cached) + self._reusable: dict[str, PolyglotAnalysisResult] = dict(cached) self._profile: ExpressionInferenceProfile = profile self._analyze: Callable[..., tuple[ModelSqlAnalysis, ...]] = analyze self._complete: Callable[..., tuple[ModelSqlAnalysis, ...]] = complete @@ -259,8 +251,6 @@ def _analyze_ready(self, ready: tuple[str, ...]) -> None: self._changed.add(name) if request.cache_key is not None: self._reusable.pop(request.cache_key, None) - if not self._served_parents_final(name) and request.cache_key is not None: - self._reusable.pop(request.cache_key, None) bindings: dict[str, dict[str, str]] | None = ( None if request.binding_schema is None @@ -298,24 +288,11 @@ def _analyze_ready(self, ready: tuple[str, ...]) -> None: self._results[name] = result self._publish(name=name, request=request, result=result) - def _served_parents_final(self, name: str) -> bool: - """Return whether a stored analysis, if served, was made from the parents' final state.""" - - dependencies: dict[str, str] | None = self._served.get(name) - return dependencies is None or dependencies_current( - dependencies=dependencies, - parent_signatures={ - parent: self._signatures.get(parent) for parent in self._dependencies[name] - }, - ) - def _publish( self, *, name: str, request: ModelSqlAnalysisRequest, result: ModelSqlAnalysis ) -> None: analysis: PolyglotAnalysisResult = result.polyglot_analysis - signature: str = model_analysis_output_signature(analysis) - self._signatures[name] = signature - if signature != self._previous_signatures.get(name): + if model_analysis_output_signature(analysis) != self._previous_signatures.get(name): self._changed.add(name) required: frozenset[str] = binding_relation_names(request.model_input.references) star_known: bool = not analysis.has_star or ( diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py b/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py deleted file mode 100644 index ad40b52907..0000000000 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Process-wide access to the render reuse session of the running compile command.""" - -from __future__ import annotations - -import threading -from collections.abc import Iterator -from contextlib import contextmanager -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession - - -class CompileRenderReuseRegistry: - """Expose one compile command's render reuse session to the rendering code it runs.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._session: CompileRenderReuseSession | None = None - - @contextmanager - def activated(self, session: CompileRenderReuseSession) -> Iterator[CompileRenderReuseSession]: - """Make the session available to rendering until the block exits.""" - - with self._lock: - previous: CompileRenderReuseSession | None = self._session - self._session = session - try: - yield session - finally: - with self._lock: - self._session = previous - - def claim(self) -> CompileRenderReuseSession | None: - """Hand the active session to the first project render only, so it is used once.""" - - with self._lock: - session: CompileRenderReuseSession | None = self._session - if session is None or not session.claim(): - return None - return session - - def claim_discovery(self) -> CompileRenderReuseSession | None: - """Hand the active session to the first project discovery only, so it is used once.""" - - with self._lock: - session: CompileRenderReuseSession | None = self._session - if session is None or not session.claim_discovery(): - return None - return session diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py deleted file mode 100644 index f0aca97da8..0000000000 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Reuse unaffected rendered inputs from the previous compile and record this compile's.""" - -from __future__ import annotations - -import pickle -import threading -from collections.abc import Callable -from dataclasses import replace -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - report_compile_diagnostic, - tapped_compile_diagnostics, -) -from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads -from sqlbuild.compiler.compile.constants import ( - COMPILE_INPUT_READS, - RENDER_REUSE_ANALYSIS_PREFIX, - RENDER_REUSE_DECLARATIONS_PREFIX, - RENDER_REUSE_MODEL_AUDITS_PREFIX, -) -from sqlbuild.compiler.compile.models import ( - CompileAuditInput, - CompileModelInput, - CompilerDiagnostic, - RenderReuseState, - StoredModelAnalysis, - StoredRender, -) -from sqlbuild.compiler.discovery.models import DiscoveredDeclarationFiles, DiscoveredSqlModelFile -from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError -from sqlbuild.compiler.fact_cache.main._dump_fact_payload import dumped_fact_payload -from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload -from sqlbuild.compiler.profiling.main._metric import record_compile_metric - -_LOAD_ERRORS: tuple[type[Exception], ...] = ( - FactCachePayloadError, - pickle.UnpicklingError, - AttributeError, - EOFError, - ImportError, - IndexError, - KeyError, - TypeError, - ValueError, - RecursionError, - MemoryError, - OverflowError, -) - - -class CompileRenderReuseSession: - """Serve stored renders of inputs a change set cannot affect, and record every render.""" - - def __init__( - self, - *, - prior: RenderReuseState | None, - changed_paths: frozenset[str] | None, - retained_models: frozenset[str] = frozenset(), - retained_groups: frozenset[str] = frozenset(), - ) -> None: - self._lock: threading.Lock = threading.Lock() - self._claimed: bool = False - self._discovery_claimed: bool = False - self._prior: RenderReuseState | None = prior if changed_paths is not None else None - self._changed_paths: frozenset[str] = changed_paths or frozenset() - self._reusable: bool = False - self._model_paths: tuple[str, ...] | None = None - self._retained_models: frozenset[str] = retained_models - self._retained_groups: frozenset[str] = retained_groups - self._model_records: dict[str, memoryview | None] = {} - self._group_records: dict[str, memoryview | None] = {} - self._decoded_models: dict[str, StoredRender] = {} - self._decoded_groups: dict[str, StoredRender] = {} - self._pending_models: list[tuple[str, StoredRender]] = [] - self._pending_groups: list[tuple[str, StoredRender]] = [] - self._serialized_models: dict[str, memoryview] = {} - self._serialized_groups: dict[str, memoryview] = {} - self._reused_paths: set[str] = set() - self._stored_analyses: dict[str, StoredModelAnalysis] = {} - self._stored_analysis_payloads: dict[str, memoryview] = {} - - def claim(self) -> bool: - """Return True for the first claim only; later project renders neither reuse nor record.""" - - with self._lock: - claimed: bool = self._claimed - self._claimed = True - return not claimed - - def claim_discovery(self) -> bool: - """Return True for the first discovery claim only; later discoveries run in full.""" - - with self._lock: - claimed: bool = self._discovery_claimed - self._discovery_claimed = True - return not claimed - - def declaration_files( - self, - *, - variant: str, - discover: Callable[[], DiscoveredDeclarationFiles], - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], - ) -> DiscoveredDeclarationFiles: - """Reuse stored declaration files after model-only edits, or discover and record them.""" - - name: str = f"{RENDER_REUSE_DECLARATIONS_PREFIX}{variant}" - prior: RenderReuseState | None = self._prior - payload: memoryview | None = ( - prior.group_payloads.get(name) - if prior is not None - and edits_only_models(changed_paths=self._changed_paths, model_paths=prior.model_paths) - else None - ) - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if ( - stored is not None - and payload is not None - and isinstance(stored.value, DiscoveredDeclarationFiles) - ): - model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() - _replay(stored=stored) - self._group_records[name] = payload - self._decoded_groups[name] = stored - return replace(stored.value, model_files=model_files) - with ( - COMPILE_INPUT_READS.recording() as reads, - tapped_compile_diagnostics() as reported, - ): - declarations: DiscoveredDeclarationFiles = discover() - recorded: StoredRender | None = _stored_render( - reads=reads, - value=replace(declarations, model_files=(), native_session=None), - reported=reported, - ) - if recorded is not None: - self._pending_groups.append((name, recorded)) - return declarations - - def plan_models(self, *, model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: - """Reuse stored renders only when the change set touches nothing but existing models.""" - - model_paths: tuple[str, ...] = tuple(str(item.relative_path) for item in model_files) - self._model_paths = model_paths - prior: RenderReuseState | None = self._prior - self._reusable = ( - prior is not None - and prior.model_paths == model_paths - and self._changed_paths.issubset(model_paths) - ) - - def has_reusable_model(self, *, model_file: DiscoveredSqlModelFile) -> bool: - """Return whether a stored render of this model may stand in for rendering it.""" - - path: str = str(model_file.relative_path) - return ( - self._reusable - and self._prior is not None - and path not in self._changed_paths - and path in self._prior.model_payloads - ) - - def reused_model(self, *, model_file: DiscoveredSqlModelFile) -> CompileModelInput | None: - """Return the stored render of an unchanged model, replaying what rendering reported.""" - - if not self.has_reusable_model(model_file=model_file) or self._prior is None: - return None - path: str = str(model_file.relative_path) - payload: memoryview | None = self._prior.model_payloads[path] - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if stored is None or not isinstance(stored.value, CompileModelInput): - return None - _replay(stored=stored) - self._model_records[path] = payload - self._decoded_models[path] = stored - self._reused_paths.add(path) - record_compile_metric(metric="render_reuse_hits", value=1) - return replace(stored.value, model_file=model_file) - - def rendered_model( - self, - *, - model_file: DiscoveredSqlModelFile, - render: Callable[[], CompileModelInput], - ) -> CompileModelInput: - """Render one model and record its render with what rendering reported and read.""" - - with ( - COMPILE_INPUT_READS.recording() as reads, - tapped_compile_diagnostics() as reported, - ): - model_input: CompileModelInput = render() - record_compile_metric(metric="render_reuse_misses", value=1) - stored: StoredRender | None = _stored_render( - reads=reads, value=replace(model_input, model_file=None), reported=reported - ) - if stored is not None: - self._pending_models.append((str(model_file.relative_path), stored)) - return model_input - - def model_audits( - self, - *, - model_input: CompileModelInput, - render: Callable[[], tuple[CompileAuditInput, ...]], - ) -> tuple[CompileAuditInput, ...]: - """Return a reused model's stored attached audits, or render and record them.""" - - path: str = str(model_input.model_file.relative_path) - return self.group( - name=f"{RENDER_REUSE_MODEL_AUDITS_PREFIX}{path}", - render=render, - reusable=path in self._reused_paths, - ) - - def group[T](self, *, name: str, render: Callable[[], T], reusable: bool = True) -> T: - """Return a stored project-wide render no model edit can affect, or render and record it.""" - - payload: memoryview | None = ( - self._prior.group_payloads.get(name) - if reusable and self._reusable and self._prior is not None - else None - ) - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if stored is not None and payload is not None: - _replay(stored=stored) - self._group_records[name] = payload - self._decoded_groups[name] = stored - return cast(T, stored.value) - with ( - COMPILE_INPUT_READS.recording() as reads, - tapped_compile_diagnostics() as reported, - ): - value: T = render() - rendered: StoredRender | None = _stored_render(reads=reads, value=value, reported=reported) - if rendered is not None: - self._pending_groups.append((name, rendered)) - return value - - def pending_renders(self) -> int: - """Return how many new renders storing this compile still has to serialize.""" - - return len(self._pending_models) + len(self._pending_groups) - - def reused_any(self) -> bool: - """Return whether any render was reused, so imports it made were skipped this run.""" - - return bool(self._decoded_models or self._decoded_groups) - - def reused_model_names(self) -> frozenset[str]: - """Return the names of models whose stored renders this compile reused.""" - - return frozenset(Path(path).stem for path in self._reused_paths) - - def stored_analyses(self) -> dict[str, StoredModelAnalysis]: - """Return stored analyses of reused models by model name.""" - - return self._stored_analyses - - def record_analyses(self, *, analyses: dict[str, StoredModelAnalysis | None]) -> None: - """Record every model's analysis for the next compile; absent or None drops it.""" - - for path in self._model_paths or (): - name: str = f"{RENDER_REUSE_ANALYSIS_PREFIX}{path}" - model_name: str = Path(path).stem - analysis: StoredModelAnalysis | None = analyses.get(model_name) - stored: StoredRender = StoredRender( - value=analysis, diagnostics=(), environment_names=() - ) - if analysis is not None and analysis is self._stored_analyses.get(model_name): - self._group_records[name] = self._stored_analysis_payloads.get(name) - self._decoded_groups[name] = stored - else: - self._pending_groups.append((name, stored)) - - def release_stored(self) -> None: - """Drop stored base bytes once every render is decoded, recording reused renders as None.""" - - self._keep_stored_analyses() - self._prior = None - self._reusable = False - for records, retained in ( - (self._model_records, self._retained_models), - (self._group_records, self._retained_groups), - ): - for key in records.keys() - retained: - records[key] = None - - def _keep_stored_analyses(self) -> None: - prior: RenderReuseState | None = self._prior - if prior is None or not self._reusable: - return - for path in self._reused_paths: - name: str = f"{RENDER_REUSE_ANALYSIS_PREFIX}{path}" - payload: memoryview | None = prior.group_payloads.get(name) - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if payload is None or stored is None: - continue - if not isinstance(stored.value, StoredModelAnalysis): - continue - self._stored_analyses[Path(path).stem] = stored.value - if name in self._retained_groups: - self._stored_analysis_payloads[name] = memoryview(bytes(payload)) - - def stored_state(self, *, complete: bool = False) -> RenderReuseState | None: - """Serialize this compile's renders; with complete, re-serialize released reused ones.""" - - if self._model_paths is None: - return None - self.release_stored() - self._serialize_pending() - if complete: - self._model_records = _reserialized( - records=self._model_records, decoded=self._decoded_models - ) - self._group_records = _reserialized( - records=self._group_records, decoded=self._decoded_groups - ) - return RenderReuseState( - model_paths=self._model_paths, - model_payloads={**self._model_records, **self._serialized_models}, - group_payloads={**self._group_records, **self._serialized_groups}, - ) - - def _serialize_pending(self) -> None: - for pending, serialized in ( - (self._pending_models, self._serialized_models), - (self._pending_groups, self._serialized_groups), - ): - for key, stored in pending: - payload: bytes | None = _dumped(stored) - if payload is not None: - serialized[key] = memoryview(payload) - pending.clear() - - -def edits_only_models(*, changed_paths: frozenset[str], model_paths: tuple[str, ...]) -> bool: - """Return whether every changed path is a model file of the stored compile.""" - - return changed_paths.issubset(model_paths) - - -def _reserialized( - *, records: dict[str, memoryview | None], decoded: dict[str, StoredRender] -) -> dict[str, memoryview | None]: - complete: dict[str, memoryview | None] = {} - key: str - payload: memoryview | None - for key, payload in records.items(): - if payload is None: - dumped: bytes | None = _dumped(decoded[key]) - if dumped is None: - continue - payload = memoryview(dumped) - complete[key] = payload - return complete - - -def _stored_render( - *, - reads: CompileInputReads, - value: object, - reported: list[tuple[tuple[str, ...], CompilerDiagnostic]], -) -> StoredRender | None: - """Keep one render for reuse unless it read the run identity or provider settings.""" - - if reads.read_run_id or reads.settings_classes: - return None - return StoredRender( - value=value, diagnostics=tuple(reported), environment_names=reads.environment_names - ) - - -def _dumped(stored: StoredRender) -> bytes | None: - try: - return dumped_fact_payload(stored) - except (pickle.PicklingError, TypeError, AttributeError, RecursionError): - return None - - -def _replay(*, stored: StoredRender) -> None: - for name in stored.environment_names: - COMPILE_INPUT_READS.environment_read(name) - for key, diagnostic in stored.diagnostics: - report_compile_diagnostic(key=key, diagnostic=diagnostic) - - -def _loaded_render(payload: memoryview) -> StoredRender | None: - try: - loaded: object = loaded_fact_payload(payload) - except _LOAD_ERRORS: - return None - return loaded if isinstance(loaded, StoredRender) else None diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py new file mode 100644 index 0000000000..2e0cd9bd95 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py @@ -0,0 +1,85 @@ +"""Per-file SQL-test scan results reused across compiles from the shared native store.""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Sequence +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.constants import ( + SQL_TEST_SCAN_STORE_FILE_NAME, + SQL_TEST_SCAN_STORE_VERSION, +) +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity +from sqlbuild.compiler.profiling.main._metric import record_compile_metric +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + + +class SqlTestScanCache: + """Whole per-file scan results keyed by installed code, scan algorithm, rules and text.""" + + def __init__(self, *, cache_dir: Path | None) -> None: + self._path: Path | None = ( + None if cache_dir is None else cache_dir / SQL_TEST_SCAN_STORE_FILE_NAME + ) + self._store: _native.SqlTestScanStore | None = None + self._hits: int = 0 + self._misses: int = 0 + + def read[T]( + self, + *, + algorithm: str, + syntax: SqlLexicalSyntax, + parts: Sequence[str], + decode: Callable[[bytes], T | None], + ) -> T | None: + """Return one file's decoded stored result, or None when it must be scanned again.""" + + store: _native.SqlTestScanStore | None = self._opened() + if store is None: + return None + found: bytes | None = store.get([algorithm, syntax.cache_key, *parts]) + decoded: T | None = None if found is None else decode(found) + if decoded is None: + self._misses += 1 + else: + self._hits += 1 + return decoded + + def write( + self, *, algorithm: str, syntax: SqlLexicalSyntax, parts: Sequence[str], value: bytes + ) -> None: + """Store one file's complete scan result; callers never store failed scans.""" + + store: _native.SqlTestScanStore | None = self._opened() + if store is not None: + store.put([algorithm, syntax.cache_key, *parts], value) + + def save(self) -> None: + """Persist results stored by this compile and report hit and miss counts.""" + + record_compile_metric(metric="sql_test_scan_cache_hits", value=self._hits) + record_compile_metric(metric="sql_test_scan_cache_misses", value=self._misses) + if self._store is None or self._path is None: + return + try: + _ = self._store.save(str(self._path)) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("SQL test scan store not saved: %s", error) + + def _opened(self) -> _native.SqlTestScanStore | None: + if self._path is None: + return None + if self._store is None: + environment: str = _native.content_digest( + [SQL_TEST_SCAN_STORE_VERSION, compiled_code_identity()] + ) + try: + self._store = _native.SqlTestScanStore(str(self._path), environment) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("SQL test scan store not loaded: %s", error) + self._path = None + return None + return self._store diff --git a/src/sqlbuild/compiler/compile/classes/stored_model_analyses.py b/src/sqlbuild/compiler/compile/classes/stored_model_analyses.py deleted file mode 100644 index 1204f1e697..0000000000 --- a/src/sqlbuild/compiler/compile/classes/stored_model_analyses.py +++ /dev/null @@ -1,452 +0,0 @@ -"""Serve unchanged models' analyses from the previous compile instead of the analysis cache.""" - -from __future__ import annotations - -import hashlib -from collections import Counter - -import orjson - -from sqlbuild.compiler.compile._helpers.analysis.cache import ( - model_analysis_output_signature, - read_model_analyses, - write_model_analyses, -) -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import referenced_model_names -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompileModelInput, - ModelAnalysisCaching, - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, - StoredAnalysisScope, - StoredModelAnalysis, -) -from sqlbuild.compiler.compile.types import ModelAnalysisReuse -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.profiling.main._metric import record_compile_metric -from sqlbuild.compiler.profiling.main.record import record_compile_timing - - -class StoredModelAnalyses: - """Analyses a compile may serve from the previous compile, and what it records for the next.""" - - def __init__( - self, - *, - caching: ModelAnalysisCaching | None, - requests: tuple[ModelSqlAnalysisRequest, ...], - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], - ) -> None: - self._scope: StoredAnalysisScope | None = _stored_analysis_scope( - caching=caching, - requests=requests, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) - - @property - def served(self) -> dict[str, StoredModelAnalysis]: - """Return stored analyses that may serve their own model, whose cache key no other uses.""" - - return {} if self._scope is None else self._scope.stored - - def stale_names( - self, - *, - requests: tuple[ModelSqlAnalysisRequest, ...], - cached: dict[str, PolyglotAnalysisResult], - current_signatures: dict[str, str], - ) -> set[str]: - """Return served models whose parents' final signatures differ from their stored ones.""" - - analyzed_names: frozenset[str] = frozenset( - _name(request.model_input) for request in requests - ) - served: dict[str, StoredModelAnalysis] = self.served - return { - name - for request in requests - if (name := _name(request.model_input)) in served - and request.cache_key is not None - and cached.get(request.cache_key) is served[name].analysis - and not dependencies_current( - dependencies=served[name].dependencies, - parent_signatures=_parent_signatures( - request=request, - signatures=current_signatures, - analyzed_names=analyzed_names, - ), - ) - } - - def read( - self, - *, - context: AnalysisCacheContext, - cache_keys: tuple[str, ...], - model_names: tuple[str, ...], - upstream_model_names_by_key: dict[str, tuple[str, ...]], - ) -> tuple[dict[str, PolyglotAnalysisResult], dict[str, str], dict[str, str]]: - """Read the analysis cache for keys without a stored analysis; serve the rest.""" - - stored: dict[str, StoredModelAnalysis] = self.served - stored_by_key: dict[str, StoredModelAnalysis] = { - analysis.cache_key: analysis for analysis in stored.values() - } - analyses, signatures, output_signatures = read_model_analyses( - context=context, - cache_keys=tuple(key for key in cache_keys if key not in stored_by_key), - model_names=model_names, - upstream_model_names_by_key={ - key: names - for key, names in upstream_model_names_by_key.items() - if key not in stored_by_key - }, - ) - for key in cache_keys: - served: StoredModelAnalysis | None = stored_by_key.get(key) - if served is not None: - analyses[key] = served.analysis - output_signatures[key] = served.output_signature - return analyses, signatures, output_signatures - - def read_every_model( - self, *, context: AnalysisCacheContext, requests: tuple[ModelSqlAnalysisRequest, ...] - ) -> dict[str, PolyglotAnalysisResult]: - """Read every model's analysis from the cache; record them when the cache held them all.""" - - analyzed_names: frozenset[str] = frozenset( - _name(request.model_input) for request in requests - ) - cache_keys: tuple[str, ...] = tuple( - request.cache_key for request in requests if request.cache_key is not None - ) - analyses, signatures, output_signatures = read_model_analyses( - context=context, - cache_keys=cache_keys, - model_names=tuple(_name(request.model_input) for request in requests), - upstream_model_names_by_key={ - request.cache_key: referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - for request in requests - if request.cache_key is not None - }, - ) - if all(key in analyses for key in cache_keys): - self._record( - requests=requests, - cached=analyses, - cached_output_signatures=output_signatures, - written={}, - written_dependencies={}, - previous_signatures=signatures, - current_signatures={ - _name(request.model_input): output_signatures[request.cache_key] - for request in requests - if request.cache_key is not None - }, - ) - return analyses - - def record_none(self) -> None: - """Record no analyses, so the next compile analyzes or reads every model from the cache.""" - - if self._scope is not None: - self._scope.reuse.record_analyses(analyses={}) - - def publish( - self, - *, - context: AnalysisCacheContext, - requests: tuple[ModelSqlAnalysisRequest, ...], - results: tuple[PolyglotAnalysisResult, ...], - cached: dict[str, PolyglotAnalysisResult], - cached_output_signatures: dict[str, str], - invalidated_names: set[str], - latest_by_model: dict[str, PolyglotAnalysisResult], - previous_signatures: dict[str, str], - current_signatures: dict[str, str], - ) -> None: - """Write new and invalidated analyses to the cache, then record them for the next run.""" - - analyzed_names: frozenset[str] = frozenset( - _name(request.model_input) for request in requests - ) - written: dict[str, PolyglotAnalysisResult] = { - request.cache_key: result - for request, result in zip(requests, results, strict=True) - if request.cache_key is not None - and (request.cache_key not in cached or _name(request.model_input) in invalidated_names) - } - written_dependencies: dict[str, dict[str, str]] = { - key: _dependency_signatures( - request=request, - current_signatures=current_signatures, - analyzed_names=analyzed_names, - ) - for request in requests - if (key := request.cache_key) is not None and key in written - } - record_compile_metric( - metric="analysis_reuse_hits", - value=sum( - request.cache_key is not None - and (served := self.served.get(_name(request.model_input))) is not None - and cached.get(request.cache_key) is served.analysis - and _name(request.model_input) not in invalidated_names - for request in requests - ), - ) - with record_compile_timing("cache_publication_ms"): - write_model_analyses( - context=context, - analyses_by_key=written, - latest_analyses_by_model=latest_by_model, - dependency_signatures_by_key=written_dependencies, - ) - self._record( - requests=requests, - cached=cached, - cached_output_signatures=cached_output_signatures, - written=written, - written_dependencies=written_dependencies, - previous_signatures=previous_signatures, - current_signatures=current_signatures, - ) - - def _record( - self, - *, - requests: tuple[ModelSqlAnalysisRequest, ...], - cached: dict[str, PolyglotAnalysisResult], - cached_output_signatures: dict[str, str], - written: dict[str, PolyglotAnalysisResult], - written_dependencies: dict[str, dict[str, str]], - previous_signatures: dict[str, str], - current_signatures: dict[str, str], - ) -> None: - if self._scope is None: - return - self._scope.reuse.record_analyses( - analyses=_recorded_analyses( - scope=self._scope, - requests=requests, - cached=cached, - cached_output_signatures=cached_output_signatures, - written=written, - written_dependencies=written_dependencies, - previous_signatures=previous_signatures, - current_signatures=current_signatures, - ) - ) - - -def dependencies_current( - *, dependencies: dict[str, str], parent_signatures: dict[str, str | None] -) -> bool: - """Return whether a stored analysis was made from parents with these final signatures.""" - - return dependencies == parent_signatures - - -def _stored_analysis_scope( - *, - caching: ModelAnalysisCaching | None, - requests: tuple[ModelSqlAnalysisRequest, ...], - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], -) -> StoredAnalysisScope | None: - if caching is None or caching.reuse is None: - return None - context: str = analysis_reuse_context( - cache=caching.cache, - model_names=frozenset(_name(request.model_input) for request in requests), - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) - stored: dict[str, StoredModelAnalysis] = _reusable_stored_analyses( - reuse=caching.reuse, context=context, requests=requests - ) - record_compile_metric(metric="analysis_reuse_hits", value=0) - return StoredAnalysisScope(reuse=caching.reuse, context=context, stored=stored) - - -def analysis_reuse_context( - *, - cache: AnalysisCacheContext, - model_names: frozenset[str], - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], -) -> str: - """Digest every analysis input shared by all models, including non-model relations.""" - - relations: dict[str, list[object]] = { - name: _relation_inputs( - name=name, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) - for name in ( - column_types_by_table.keys() - | column_nullability_by_table.keys() - | complete_binding_schemas.keys() - ) - - model_names - } - return hashlib.sha256( - orjson.dumps( - { - "shared_fingerprint": cache.shared_fingerprint, - "signature_namespace": cache.signature_namespace, - "relations": relations, - }, - option=orjson.OPT_SORT_KEYS, - ) - ).hexdigest() - - -def _relation_inputs( - *, - name: str, - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], -) -> list[object]: - nullability: dict[str, InferredNullability] = column_nullability_by_table.get(name, {}) - return [ - column_types_by_table.get(name), - {column: value.value for column, value in nullability.items()}, - complete_binding_schemas.get(name), - ] - - -def _reusable_stored_analyses( - *, - reuse: ModelAnalysisReuse, - context: str, - requests: tuple[ModelSqlAnalysisRequest, ...], -) -> dict[str, StoredModelAnalysis]: - reused_names: frozenset[str] = reuse.reused_model_names() - stored: dict[str, StoredModelAnalysis] = reuse.stored_analyses() - key_counts: Counter[str | None] = Counter(request.cache_key for request in requests) - reusable: dict[str, StoredModelAnalysis] = {} - for request in requests: - name: str = _name(request.model_input) - analysis: StoredModelAnalysis | None = stored.get(name) - if ( - analysis is not None - and analysis.context == context - and name in reused_names - and request.cache_key is not None - and analysis.cache_key == request.cache_key - and key_counts[request.cache_key] == 1 - ): - reusable[name] = analysis - return reusable - - -def _parent_signatures( - *, - request: ModelSqlAnalysisRequest, - signatures: dict[str, str], - analyzed_names: frozenset[str], -) -> dict[str, str | None]: - return { - upstream_name: signatures.get(upstream_name) - for upstream_name in referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - } - - -def _dependency_signatures( - *, - request: ModelSqlAnalysisRequest, - current_signatures: dict[str, str], - analyzed_names: frozenset[str], -) -> dict[str, str]: - return { - upstream_name: current_signatures[upstream_name] - for upstream_name in referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - if upstream_name in current_signatures - } - - -def _recorded_analyses( - *, - scope: StoredAnalysisScope, - requests: tuple[ModelSqlAnalysisRequest, ...], - cached: dict[str, PolyglotAnalysisResult], - cached_output_signatures: dict[str, str], - written: dict[str, PolyglotAnalysisResult], - written_dependencies: dict[str, dict[str, str]], - previous_signatures: dict[str, str], - current_signatures: dict[str, str], -) -> dict[str, StoredModelAnalysis | None]: - analyzed_names: frozenset[str] = frozenset(_name(request.model_input) for request in requests) - recorded: dict[str, StoredModelAnalysis | None] = {} - for request in requests: - name: str = _name(request.model_input) - key: str | None = request.cache_key - written_analysis: PolyglotAnalysisResult | None = None if key is None else written.get(key) - cached_analysis: PolyglotAnalysisResult | None = None if key is None else cached.get(key) - previous: StoredModelAnalysis | None = scope.stored.get(name) - upstream: tuple[str, ...] = referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - parents: dict[str, str | None] = _parent_signatures( - request=request, signatures=current_signatures, analyzed_names=analyzed_names - ) - if key is None or (written_analysis is None and cached_analysis is None): - recorded[name] = None - elif written_analysis is not None: - recorded[name] = StoredModelAnalysis( - context=scope.context, - cache_key=key, - analysis=written_analysis, - output_signature=model_analysis_output_signature(written_analysis), - dependencies=written_dependencies.get(key, {}), - signature=current_signatures.get(name), - ) - elif ( - previous is not None - and previous.cache_key == key - and previous.analysis is cached_analysis - and previous.signature == current_signatures.get(name) - and dependencies_current(dependencies=previous.dependencies, parent_signatures=parents) - ): - recorded[name] = previous - elif cached_analysis is not None and all( - upstream_name in previous_signatures - and previous_signatures[upstream_name] == parents[upstream_name] - for upstream_name in upstream - ): - recorded[name] = StoredModelAnalysis( - context=scope.context, - cache_key=key, - analysis=cached_analysis, - output_signature=cached_output_signatures.get(key) - or model_analysis_output_signature(cached_analysis), - dependencies={ - upstream_name: previous_signatures[upstream_name] for upstream_name in upstream - }, - signature=current_signatures.get(name), - ) - else: - recorded[name] = None - return recorded - - -def _name(model_input: CompileModelInput) -> str: - return model_input.model_file.file_path.stem diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 75ac536a3f..d4777ac0ee 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -7,7 +7,6 @@ from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( CompileInputReadRegistry, ) -from sqlbuild.compiler.compile.classes.render_reuse_registry import CompileRenderReuseRegistry from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind @@ -177,18 +176,6 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() -COMPILE_RENDER_REUSE: CompileRenderReuseRegistry = CompileRenderReuseRegistry() -RENDER_REUSE_SOURCES_GROUP: str = "sources" -RENDER_REUSE_TESTS_GROUP: str = "tests" -RENDER_REUSE_SCENARIOS_GROUP: str = "scenarios" -RENDER_REUSE_SINGULAR_AUDITS_GROUP: str = "singular_audits" -RENDER_REUSE_ATTACHED_AUDITS_GROUP: str = "source_and_seed_audits" -RENDER_REUSE_MODEL_AUDITS_PREFIX: str = "model_audits:" -RENDER_REUSE_DECLARATIONS_PREFIX: str = "declarations:" -RENDER_REUSE_ANALYSIS_PREFIX: str = "analysis:" -SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" -SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v3" -SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" COMPACT_ANALYSIS_RESPONSE_LENGTH: int = 2 COMPACT_RELATION_STUB_PREFIX: str = "__sqlbuild_project_input_" COMPACT_REFERENCE_MARKER_PATTERN: re.Pattern[str] = re.compile( @@ -200,3 +187,8 @@ COMPACT_ANALYSIS_FACT_LENGTH: int = 6 COMPACT_ANALYSIS_SOURCE_LENGTH: int = 3 MODEL_SCHEMA_CONFIG_KEY: str = "schema" +SQL_TEST_SCAN_STORE_FILE_NAME: str = "sql-test-scans.bin" +RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" +SQL_TEST_SCAN_STORE_VERSION: str = "sql-test-scan-store-v1" +SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" +SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: str = "sql-test-expected-models-v1" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 2b857fdbdb..819da6a2e5 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -18,6 +18,11 @@ from sqlbuild.compiler.compile._helpers.attachment.references import ( validate_table_function_call_arities, ) +from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs +from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( + build_scenario_inputs, + build_test_inputs_with_cache, +) from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) @@ -39,11 +44,6 @@ build_public_model_schema_index, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile._helpers.render.reuse import ( - build_sql_resource_inputs, - claimed_render_reuse, -) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAuditInput, @@ -126,15 +126,12 @@ def build_compile_inputs( _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), ) resolved_run_id: str = resolve_run_id(selected_run_id=run_id) - render_reuse: CompileRenderReuseSession | None = claimed_render_reuse( - discovered_inputs=discovered_inputs - ) loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) declaration_scope: DeclarationScopeBuild = build_declaration_scope( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, - compile_cache_dir=compile_cache_dir, sql_lexical_syntax=adapter_context.sql_lexical_syntax, + compile_cache_dir=compile_cache_dir, ) model_context: ModelInputBuildContext = ModelInputBuildContext( effective_vars=effective_vars, @@ -156,7 +153,6 @@ def build_compile_inputs( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=compile_cache_dir, - render_reuse=render_reuse, ) model_context = model_build.context seed_inputs: tuple[CompileSeedInput, ...] = build_seed_inputs(discovered_inputs) @@ -170,17 +166,34 @@ def build_compile_inputs( adapter_context.python_functions_inherit_default_namespace ), ) - source_inputs: tuple[CompileSourceInput, ...] - test_inputs: tuple[CompileSqlTestInput, ...] - scenario_inputs: tuple[CompileSqlScenarioInput, ...] - source_inputs, test_inputs, scenario_inputs = build_sql_resource_inputs( + source_inputs: tuple[CompileSourceInput, ...] = build_source_inputs( discovered_inputs=discovered_inputs, - context=model_context, - sql_function_inputs=sql_function_inputs, + effective_vars=model_context.effective_vars, + effective_settings=model_context.effective_settings, + macro_context=model_context.macro_context, + loaded_macros=model_context.loaded_macros, + declaration_expansion=model_context.declaration_expansion, no_sql_validation=no_sql_validation, + ) + test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs_with_cache( + discovered_inputs=discovered_inputs, + effective_vars=model_context.effective_vars, + macro_context=model_context.macro_context, + loaded_macros=model_context.loaded_macros, + declaration_expansion=model_context.declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, - compile_cache_dir=compile_cache_dir, - render_reuse=render_reuse, + sql_function_inputs=sql_function_inputs, + sql_lexical_syntax=model_context.sql_lexical_syntax, + scan_cache=declaration_scope.sql_test_scans, + ) + scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( + discovered_inputs=discovered_inputs, + effective_vars=model_context.effective_vars, + macro_context=model_context.macro_context, + loaded_macros=model_context.loaded_macros, + declaration_expansion=model_context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=model_context.sql_lexical_syntax, ) audit_inputs: tuple[CompileAuditInput, ...] audit_diagnostics: tuple[CompilerDiagnostic, ...] @@ -190,10 +203,7 @@ def build_compile_inputs( model_inputs=model_build.inputs, source_inputs=source_inputs, seed_inputs=seed_inputs, - render_reuse=render_reuse, ) - if render_reuse is not None: - render_reuse.release_stored() diagnostics: tuple[CompilerDiagnostic, ...] = ( *audit_diagnostics, *model_build.diagnostics, @@ -243,7 +253,6 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, scope_index=declaration_scope.index, declaration_scope=declaration_scope, - analysis_reuse=render_reuse, ) @@ -283,7 +292,6 @@ def _build_models_with_declarations( defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, reference_cache_dir: Path | None, - render_reuse: CompileRenderReuseSession | None, ) -> ModelInputScopeBuild: public_enums: dict[str, EnumDeclaration] public_constants: dict[str, ConstantDeclaration] @@ -309,7 +317,6 @@ def _build_models_with_declarations( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=reference_cache_dir, - render_reuse=render_reuse, ) return ModelInputScopeBuild( inputs=model_inputs, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 62a07a1fcc..e7f4fd938c 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -13,6 +13,7 @@ from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( @@ -23,7 +24,6 @@ DiagnosticPhase, DiagnosticSeverity, FunctionLanguage, - ModelAnalysisReuse, SqlTestMode, TypedSqlValueRenderer, ) @@ -334,6 +334,7 @@ class DeclarationScopeBuild: loaded_macros: dict[str, LoadedMacro] index: ScopeIndex resolver: DeclarationScopeResolver + sql_test_scans: SqlTestScanCache @dataclass(frozen=True) @@ -826,44 +827,6 @@ class ModelSqlAnalysis: dynamic_column_contract: DynamicColumnContractProof | None = None -@dataclass(frozen=True) -class DataflowReuse: - """Analyses a binding dataflow may reuse, and the signatures that decide whether it may.""" - - cached: dict[str, PolyglotAnalysisResult] - previous_signatures: dict[str, str] - served: dict[str, dict[str, str]] - - -@dataclass(frozen=True) -class StoredModelAnalysis: - """One model's analysis as the analysis cache held it after a compile, kept for reuse.""" - - context: str - cache_key: str - analysis: PolyglotAnalysisResult - output_signature: str - dependencies: dict[str, str] - signature: str | None - - -@dataclass(frozen=True) -class StoredAnalysisScope: - """Stored analyses one compile may serve, and the session recording its own analyses.""" - - reuse: ModelAnalysisReuse - context: str - stored: dict[str, StoredModelAnalysis] - - -@dataclass(frozen=True) -class ModelAnalysisCaching: - """The analysis cache one compile reads and writes, and its stored analysis reuse.""" - - cache: AnalysisCacheContext - reuse: ModelAnalysisReuse | None = None - - @dataclass(frozen=True) class ModelSqlAnalysisRequest: """Inputs to one model's SQL analysis.""" @@ -978,7 +941,6 @@ class CompileProjectInputs: external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None scope_index: ScopeIndex = field(default_factory=ScopeIndex) declaration_scope: DeclarationScopeBuild | None = field(default=None, repr=False, compare=False) - analysis_reuse: ModelAnalysisReuse | None = field(default=None, repr=False, compare=False) @dataclass(frozen=True) @@ -1748,21 +1710,3 @@ class SqlAnalysisOptOutRequest: query_sql: str placeholders: dict[str, str] | None project_config_path: Path - - -@dataclass(frozen=True) -class StoredRender: - """One stored render with the diagnostics it reported and environment variables it read.""" - - value: object - diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] - environment_names: tuple[str, ...] - - -@dataclass(frozen=True) -class RenderReuseState: - """Serialized renders of one compile by model path and group; None marks released bytes.""" - - model_paths: tuple[str, ...] - model_payloads: dict[str, memoryview | None] - group_payloads: dict[str, memoryview | None] diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index d3faf5c129..b8e69b9a86 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -13,7 +13,6 @@ CompileProjectInputs, SqlReferenceOrigin, SqlReferenceScan, - StoredModelAnalysis, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -51,22 +50,6 @@ class CompileInputsReadyCallback(Protocol): def __call__(self, *, inputs: CompileProjectInputs) -> None: ... -class ModelAnalysisReuse(Protocol): - """Analyses a previous compile stored for models whose renders this compile reused.""" - - def reused_model_names(self) -> frozenset[str]: - """Return the names of models whose stored renders this compile reused.""" - ... - - def stored_analyses(self) -> dict[str, StoredModelAnalysis]: - """Return stored analyses of reused models by model name.""" - ... - - def record_analyses(self, *, analyses: dict[str, StoredModelAnalysis | None]) -> None: - """Record every model's analysis for the next compile; absent or None drops it.""" - ... - - class AttachedAuditTargetKind(StrEnum): MODEL = "model" SOURCE = "source" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index c8cff23b66..0a91af1c9b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Callable -from functools import partial from pathlib import Path from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( @@ -70,7 +69,6 @@ DiscoveryFileFault, TolerantScopeDiscovery, ) -from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -82,33 +80,17 @@ def build_discovered_project_inputs( local_config: LocalConfig, sql_analysis_enabled: bool, extract_output_column_locations: bool = True, - declaration_reuse: DeclarationFilesReuse | None = None, ) -> DiscoveredProjectInputs: """Discover all project files and functions into one inputs bundle.""" with OperationLifecycle( operation_kind="project", operation_name="discovery_declaration_parse" ) as declaration_lifecycle: - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]] = partial( - _discover_model_files, + declarations: DiscoveredDeclarationFiles = _discover_declaration_files( project_dir=project_dir, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, ) - discover: Callable[[], DiscoveredDeclarationFiles] = partial( - _discover_declaration_files, - project_dir=project_dir, - discover_models=discover_models, - ) - declarations: DiscoveredDeclarationFiles = ( - discover() - if declaration_reuse is None - else declaration_reuse.declaration_files( - variant=f"{int(sql_analysis_enabled)}{int(extract_output_column_locations)}", - discover=discover, - discover_models=discover_models, - ) - ) declaration_lifecycle.completed( metadata={ "item_count": sum( @@ -216,13 +198,18 @@ def build_discovered_project_inputs( def _discover_declaration_files( *, project_dir: Path, - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], + sql_analysis_enabled: bool, + extract_output_column_locations: bool, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): source_files: tuple[DiscoveredSourceFile, ...] = discover_native_source_files( project_dir=project_dir ) - model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() + model_files: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=sql_analysis_enabled, + extract_output_column_locations=extract_output_column_locations, + ) return DiscoveredDeclarationFiles( source_files=source_files, model_files=model_files, @@ -243,19 +230,6 @@ def _discover_declaration_files( ) -def _discover_model_files( - *, - project_dir: Path, - sql_analysis_enabled: bool, - extract_output_column_locations: bool, -) -> tuple[DiscoveredSqlModelFile, ...]: - return discover_native_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - - def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: """Aggregate bounded scope inputs while retaining independent authored faults.""" diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index 560e3ed11b..5b387ef28a 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -23,7 +23,6 @@ ) from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -35,7 +34,6 @@ def discover_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None = None, extract_output_column_locations: bool = True, - declaration_reuse: DeclarationFilesReuse | None = None, ) -> DiscoveredProjectInputs: """Load all raw project inputs from disk before semantic resolution.""" @@ -47,7 +45,6 @@ def discover_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=sql_analysis_enabled_override, extract_output_column_locations=extract_output_column_locations, - declaration_reuse=declaration_reuse, ), ) @@ -57,14 +54,12 @@ def _discover_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None, extract_output_column_locations: bool, - declaration_reuse: DeclarationFilesReuse | None, ) -> DiscoveredProjectInputs: with OperationLifecycle(operation_kind="project", operation_name="discovery_project_assembly"): return _assemble_discovered_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=sql_analysis_enabled_override, extract_output_column_locations=extract_output_column_locations, - declaration_reuse=declaration_reuse, ) @@ -73,7 +68,6 @@ def _assemble_discovered_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None, extract_output_column_locations: bool, - declaration_reuse: DeclarationFilesReuse | None, ) -> DiscoveredProjectInputs: project_config: ProjectConfig = load_project_config(project_dir=project_dir) local_config: LocalConfig = load_local_config(project_dir=project_dir) @@ -102,7 +96,6 @@ def _assemble_discovered_project_inputs( local_config=local_config, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, - declaration_reuse=declaration_reuse, ) validate_discovered_inputs(discovered_inputs) from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index 1dcdded57a..daff13b0a5 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import Field from enum import StrEnum from pathlib import Path @@ -11,10 +10,6 @@ from sqlbuild.compiler.scopes.types import ScopeKind if TYPE_CHECKING: - from sqlbuild.compiler.discovery.models import ( - DiscoveredDeclarationFiles, - DiscoveredSqlModelFile, - ) from sqlbuild.provider.classes.provider import Provider type ProjectProvider = Provider @@ -57,17 +52,3 @@ class DirectorySnapshotEntry(NamedTuple): name: str is_dir: bool is_walkable_dir: bool - - -class DeclarationFilesReuse(Protocol): - """Reuse the non-model declaration files a previous compile parsed when only models changed.""" - - def declaration_files( - self, - *, - variant: str, - discover: Callable[[], DiscoveredDeclarationFiles], - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], - ) -> DiscoveredDeclarationFiles: - """Return stored declaration files with freshly discovered models, or discover all.""" - ... diff --git a/src/sqlbuild/compiler/fact_cache/__init__.py b/src/sqlbuild/compiler/fact_cache/__init__.py deleted file mode 100644 index 996c10d790..0000000000 --- a/src/sqlbuild/compiler/fact_cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Content-addressed local cache for deterministic per-file compile facts.""" diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/__init__.py b/src/sqlbuild/compiler/fact_cache/_helpers/__init__.py deleted file mode 100644 index e3ff32475e..0000000000 --- a/src/sqlbuild/compiler/fact_cache/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache helpers.""" diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/publication.py b/src/sqlbuild/compiler/fact_cache/_helpers/publication.py deleted file mode 100644 index 53cabf2787..0000000000 --- a/src/sqlbuild/compiler/fact_cache/_helpers/publication.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Background publication of fact rows in multi-row statements so writes rarely take the GIL.""" - -from __future__ import annotations - -import sqlite3 -import threading -import zlib -from contextlib import closing -from pathlib import Path - -from sqlbuild.compiler.fact_cache.classes.fact_publication_registry import ( - FactPublicationRegistry, -) -from sqlbuild.compiler.fact_cache.constants import ( - FACT_CACHE_CREATE_TABLE_SQL, - FACT_CACHE_INSERT_CHUNK_ROWS, - FACT_CACHE_INSERT_ROW_SQL, - FACT_CACHE_INSERT_SQL, - FACT_CACHE_SQLITE_TIMEOUT_SECONDS, - FACT_CACHE_WRITE_PRAGMAS, - FACT_CACHE_WRITER_THREAD_NAME, -) - -_PUBLICATIONS: FactPublicationRegistry = FactPublicationRegistry() - - -def publish_fact_rows( - *, database_path: Path, rows: tuple[tuple[str, str, str, bytes], ...] -) -> None: - """Publish (slot, key, digest, payload) rows after earlier publications to the same file.""" - - def write(previous: threading.Thread | None) -> None: - _write_rows(database_path=database_path, rows=rows, previous=previous) - - _PUBLICATIONS.start( - database_path=database_path, write=write, thread_name=FACT_CACHE_WRITER_THREAD_NAME - ) - - -def await_fact_publication(*, database_path: Path | None = None) -> None: - """Wait until pending publications to one database, or to every database, are durable.""" - - _PUBLICATIONS.wait(database_path=database_path) - - -def entry_digest(*, cache_key: str, payload: bytes) -> str: - """Return the CRC-32 checksum binding one payload to its exact cache key.""" - - return f"{zlib.crc32(payload, zlib.crc32(cache_key.encode() + b'\0')):08x}" - - -def _write_rows( - *, - database_path: Path, - rows: tuple[tuple[str, str, str, bytes], ...], - previous: threading.Thread | None, -) -> None: - if previous is not None: - previous.join() - try: - _insert_records(database_path=database_path, records=rows) - except sqlite3.DatabaseError as error: - if type(error) is sqlite3.DatabaseError: - _replace_unreadable_database(database_path=database_path, records=rows) - except (sqlite3.Error, OSError): - return - - -def _replace_unreadable_database( - *, database_path: Path, records: tuple[tuple[str, str, str, bytes], ...] -) -> None: - try: - database_path.unlink(missing_ok=True) - _insert_records(database_path=database_path, records=records) - except (OSError, sqlite3.Error): - return - - -def _insert_records( - *, database_path: Path, records: tuple[tuple[str, str, str, bytes], ...] -) -> None: - database_path.parent.mkdir(parents=True, exist_ok=True) - with ( - closing( - sqlite3.connect(database_path, timeout=FACT_CACHE_SQLITE_TIMEOUT_SECONDS) - ) as connection, - connection, - ): - for pragma in FACT_CACHE_WRITE_PRAGMAS: - _ = connection.execute(pragma) - _ = connection.execute(FACT_CACHE_CREATE_TABLE_SQL) - for start in range(0, len(records), FACT_CACHE_INSERT_CHUNK_ROWS): - chunk: tuple[tuple[str, str, str, bytes], ...] = records[ - start : start + FACT_CACHE_INSERT_CHUNK_ROWS - ] - parameters: list[str | bytes] = [] - for record in chunk: - parameters.extend(record) - _ = connection.execute( - FACT_CACHE_INSERT_SQL + ",".join([FACT_CACHE_INSERT_ROW_SQL] * len(chunk)), - parameters, - ) diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py b/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py deleted file mode 100644 index 757e20e635..0000000000 --- a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Pickle codec restricted to SQLBuild value types for local fact cache payloads.""" - -from __future__ import annotations - -import dataclasses -import enum -import io -import pickle -from typing import override - -from sqlbuild.compiler.fact_cache.constants import ( - FACT_CACHE_ALLOWED_GLOBALS, - FACT_CACHE_ALLOWED_PACKAGE, - FACT_CACHE_PICKLE_PROTOCOL, -) -from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError - - -class _RestrictedUnpickler(pickle.Unpickler): - """Resolve only immutable-value classes, never functions or arbitrary callables.""" - - @override - def find_class(self, module: str, name: str) -> type: - if (module, name) in FACT_CACHE_ALLOWED_GLOBALS: - resolved: object = super().find_class(module, name) - if isinstance(resolved, type): - return resolved - if module == FACT_CACHE_ALLOWED_PACKAGE or module.startswith( - f"{FACT_CACHE_ALLOWED_PACKAGE}." - ): - resolved = super().find_class(module, name) - if isinstance(resolved, type) and ( - dataclasses.is_dataclass(resolved) - or issubclass(resolved, enum.Enum) - or (issubclass(resolved, tuple) and hasattr(resolved, "_fields")) - ): - return resolved - raise FactCachePayloadError(f"fact cache payload references disallowed {module}.{name}") - - -def dump_fact_payload(value: object) -> bytes: - """Serialize one cache value with the fixed fact-cache protocol.""" - - return pickle.dumps(value, protocol=FACT_CACHE_PICKLE_PROTOCOL) - - -def load_fact_payload(payload: bytes | memoryview) -> object: - """Deserialize one verified cache payload through the restricted class allowlist.""" - - return _RestrictedUnpickler(io.BytesIO(payload)).load() diff --git a/src/sqlbuild/compiler/fact_cache/classes/__init__.py b/src/sqlbuild/compiler/fact_cache/classes/__init__.py deleted file mode 100644 index 595a1c6848..0000000000 --- a/src/sqlbuild/compiler/fact_cache/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache classes.""" diff --git a/src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py b/src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py deleted file mode 100644 index cd0809f5ca..0000000000 --- a/src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py +++ /dev/null @@ -1,189 +0,0 @@ -"""SQLite-backed content-addressed store for deterministic per-file compile facts.""" - -from __future__ import annotations - -import hashlib -import hmac -import pickle -import sqlite3 -from collections.abc import Sequence -from contextlib import closing -from pathlib import Path -from types import TracebackType -from typing import Any - -from sqlbuild.compiler.fact_cache._helpers.code_identity import installed_code_identity -from sqlbuild.compiler.fact_cache._helpers.publication import ( - await_fact_publication, - entry_digest, - publish_fact_rows, -) -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import ( - dump_fact_payload, - load_fact_payload, -) -from sqlbuild.compiler.fact_cache.constants import ( - FACT_CACHE_DATABASE_SUFFIX, - FACT_CACHE_DIGEST_BYTES, - FACT_CACHE_DIRECTORY_PREFIX, - FACT_CACHE_MAX_ENTRY_BYTES, - FACT_CACHE_QUERY_CHUNK_SIZE, - FACT_CACHE_SQLITE_TIMEOUT_SECONDS, - FACT_CACHE_VERSION, -) -from sqlbuild.compiler.profiling.main._metric import record_compile_metric - -_READ_ERRORS: tuple[type[BaseException], ...] = ( - OSError, - sqlite3.DatabaseError, - pickle.UnpicklingError, - AttributeError, - EOFError, - ImportError, - IndexError, - KeyError, - TypeError, - ValueError, - RecursionError, - MemoryError, -) - - -class FactCacheStore: - """Reuse exact facts keyed by every input and the producing code; faults are misses.""" - - def __init__(self, *, root: Path | None, namespace: str, algorithm: str) -> None: - self._database_path: Path | None = ( - None - if root is None - else root - / f"{FACT_CACHE_DIRECTORY_PREFIX}{FACT_CACHE_VERSION}" - / f"{namespace}{FACT_CACHE_DATABASE_SUFFIX}" - ) - self._key_prefix: bytes = ( - f"{FACT_CACHE_VERSION}\0{namespace}\0{algorithm}\0".encode() - + (installed_code_identity().encode() if root is not None else b"") - + b"\0" - ) - self._key_hasher: Any = hashlib.blake2b( - self._key_prefix, digest_size=FACT_CACHE_DIGEST_BYTES - ) - self._pending: dict[str, tuple[str, str, bytes]] = {} - self._hits: int = 0 - self._misses: int = 0 - - @property - def enabled(self) -> bool: - """Return whether this invocation reads and publishes cached facts.""" - - return self._database_path is not None - - def __enter__(self) -> FactCacheStore: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exc_value, traceback - try: - if exc_type is None: - self._write_pending() - finally: - self._pending.clear() - if self._database_path is not None: - record_compile_metric(metric="fact_cache_hits", value=self._hits) - record_compile_metric(metric="fact_cache_misses", value=self._misses) - - def key(self, *parts: str | bytes) -> str: - """Return the exact identity of one fact computed from the given ordered inputs.""" - - digest: Any = self._key_hasher.copy() - for part in parts: - encoded: bytes = ( - part if isinstance(part, bytes) else part.encode("utf-8", "surrogatepass") - ) - digest.update(len(encoded).to_bytes(8, "little")) - digest.update(encoded) - return str(digest.hexdigest()) - - def read_many(self, entries: Sequence[tuple[str, str]]) -> dict[str, object]: - """Return verified cached values for requested (slot, key) pairs; faults are misses.""" - - found: dict[str, object] = {} - if self._database_path is None or not entries: - return found - expected_keys: dict[str, str] = dict(entries) - await_fact_publication(database_path=self._database_path) - if self._database_path.is_file(): - try: - with closing( - sqlite3.connect( - f"file:{self._database_path}?mode=ro", - uri=True, - timeout=FACT_CACHE_SQLITE_TIMEOUT_SECONDS, - ) - ) as connection: - slots: tuple[str, ...] = tuple(expected_keys) - for start in range(0, len(slots), FACT_CACHE_QUERY_CHUNK_SIZE): - chunk: tuple[str, ...] = slots[start : start + FACT_CACHE_QUERY_CHUNK_SIZE] - placeholders: str = ",".join("?" for _ in chunk) - rows: list[tuple[object, object, object, object]] = connection.execute( - f"SELECT slot, cache_key, digest, payload FROM fact " - f"WHERE slot IN ({placeholders})", - chunk, - ).fetchall() - for slot, cache_key, digest, payload in rows: - if not isinstance(slot, str) or expected_keys.get(slot) != cache_key: - continue - value: object | None = _verified_value( - cache_key=cache_key, digest=digest, payload=payload - ) - if value is not None: - found[expected_keys[slot]] = value - except _READ_ERRORS: - found = {} - self._hits += sum(1 for key in expected_keys.values() if key in found) - self._misses += sum(1 for key in expected_keys.values() if key not in found) - return found - - def stage(self, *, key: str, slot: str, value: object) -> None: - """Queue one freshly computed fact for publication after a successful invocation.""" - - if self._database_path is None: - return - try: - payload: bytes = dump_fact_payload(value) - except (pickle.PicklingError, TypeError, AttributeError, RecursionError, ValueError): - return - if len(payload) > FACT_CACHE_MAX_ENTRY_BYTES: - return - self._pending[slot] = (key, entry_digest(cache_key=key, payload=payload), payload) - - def _write_pending(self) -> None: - if self._database_path is None or not self._pending: - return - publish_fact_rows( - database_path=self._database_path, - rows=tuple( - (slot, key, digest, payload) - for slot, (key, digest, payload) in self._pending.items() - ), - ) - - -def _verified_value(*, cache_key: object, digest: object, payload: object) -> object | None: - if ( - not isinstance(cache_key, str) - or not isinstance(digest, str) - or not isinstance(payload, bytes) - or len(payload) > FACT_CACHE_MAX_ENTRY_BYTES - or not hmac.compare_digest(digest, entry_digest(cache_key=cache_key, payload=payload)) - ): - return None - try: - return load_fact_payload(payload) - except _READ_ERRORS: - return None diff --git a/src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py b/src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py deleted file mode 100644 index 13824c2e4a..0000000000 --- a/src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Process-wide ordering of background fact publications per cache database.""" - -from __future__ import annotations - -import threading -from collections.abc import Callable -from pathlib import Path - - -class FactPublicationRegistry: - """Serialize publications per database and let readers wait for pending writes.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._writers: dict[Path, threading.Thread] = {} - - def start( - self, - *, - database_path: Path, - write: Callable[[threading.Thread | None], None], - thread_name: str, - ) -> None: - """Start one writer that first waits for the previous writer of the same database.""" - - with self._lock: - previous: threading.Thread | None = self._writers.get(database_path) - writer: threading.Thread = threading.Thread( - target=write, args=(previous,), name=thread_name - ) - self._writers[database_path] = writer - writer.start() - - def wait(self, *, database_path: Path | None) -> None: - """Wait for the latest writer of one database, or of every database when None.""" - - with self._lock: - writers: tuple[threading.Thread, ...] = tuple( - writer - for path, writer in self._writers.items() - if database_path is None or path == database_path - ) - for writer in writers: - writer.join() diff --git a/src/sqlbuild/compiler/fact_cache/constants.py b/src/sqlbuild/compiler/fact_cache/constants.py deleted file mode 100644 index 4bcf1840f9..0000000000 --- a/src/sqlbuild/compiler/fact_cache/constants.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Stable constants for the local compile fact cache.""" - -from __future__ import annotations - -FACT_CACHE_VERSION: int = 1 -FACT_CACHE_DIRECTORY_PREFIX: str = "facts-v" -FACT_CACHE_DATABASE_SUFFIX: str = ".sqlite3" -FACT_CACHE_DIGEST_BYTES: int = 32 -FACT_CACHE_MAX_ENTRY_BYTES: int = 16 * 1024 * 1024 -FACT_CACHE_SQLITE_TIMEOUT_SECONDS: float = 0.1 -FACT_CACHE_QUERY_CHUNK_SIZE: int = 500 -FACT_CACHE_PICKLE_PROTOCOL: int = 5 -FACT_CACHE_CREATE_TABLE_SQL: str = """ -CREATE TABLE IF NOT EXISTS fact ( - slot TEXT PRIMARY KEY, - cache_key TEXT NOT NULL, - digest TEXT NOT NULL, - payload BLOB NOT NULL -) -""" -FACT_CACHE_INSERT_SQL: str = ( - "INSERT OR REPLACE INTO fact (slot, cache_key, digest, payload) VALUES " -) -FACT_CACHE_INSERT_ROW_SQL: str = "(?, ?, ?, ?)" -FACT_CACHE_PARAMETERS_PER_ROW: int = 4 -SQLITE_PRE_3_32_MAX_BOUND_PARAMETERS: int = 999 -FACT_CACHE_INSERT_CHUNK_ROWS: int = ( - SQLITE_PRE_3_32_MAX_BOUND_PARAMETERS // FACT_CACHE_PARAMETERS_PER_ROW -) -FACT_CACHE_WRITE_PRAGMAS: tuple[str, ...] = ( - "PRAGMA synchronous = OFF", - "PRAGMA journal_mode = MEMORY", -) -FACT_CACHE_WRITER_THREAD_NAME: str = "sqlbuild-fact-cache-writer" -FACT_CACHE_ALLOWED_PACKAGE: str = "sqlbuild" -FACT_CACHE_ALLOWED_GLOBALS: frozenset[tuple[str, str]] = frozenset( - { - ("pathlib", "Path"), - ("pathlib", "PosixPath"), - ("pathlib", "PurePath"), - ("pathlib", "PurePosixPath"), - ("pathlib", "PureWindowsPath"), - ("pathlib", "WindowsPath"), - ("pathlib._local", "Path"), - ("pathlib._local", "PosixPath"), - ("pathlib._local", "PurePath"), - ("pathlib._local", "PurePosixPath"), - ("pathlib._local", "PureWindowsPath"), - ("pathlib._local", "WindowsPath"), - ("decimal", "Decimal"), - ("datetime", "date"), - ("datetime", "datetime"), - ("datetime", "time"), - ("datetime", "timedelta"), - ("datetime", "timezone"), - ("builtins", "frozenset"), - ("builtins", "set"), - ("builtins", "complex"), - ("collections", "OrderedDict"), - } -) diff --git a/src/sqlbuild/compiler/fact_cache/exceptions.py b/src/sqlbuild/compiler/fact_cache/exceptions.py deleted file mode 100644 index 37bf4d202f..0000000000 --- a/src/sqlbuild/compiler/fact_cache/exceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Fact cache exceptions.""" - -from __future__ import annotations - - -class FactCachePayloadError(ValueError): - """Raised when a stored fact payload is not a trusted value graph.""" diff --git a/src/sqlbuild/compiler/fact_cache/main/__init__.py b/src/sqlbuild/compiler/fact_cache/main/__init__.py deleted file mode 100644 index 56c61b674b..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache public entry modules.""" diff --git a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py b/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py deleted file mode 100644 index 27ffbcebd1..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Project-local compile-cache root resolution.""" - -from __future__ import annotations - -import os -from pathlib import Path - -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.spec.contracts.models import TargetConfig - - -def compile_cache_root( - *, project_dir: Path | None, target_config: TargetConfig | None, no_cache: bool -) -> Path | None: - """Return the shared compile-cache directory, or None when caching is disabled.""" - - if ( - no_cache - or project_dir is None - or (target_config is not None and target_config.compile_cache is False) - or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE - ): - return None - return compiler_cache_directory(project_dir) diff --git a/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py deleted file mode 100644 index 35f3cdd661..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for serializing one local cache value with the fact-cache codec.""" - -from __future__ import annotations - -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import dump_fact_payload - - -def dumped_fact_payload(value: object) -> bytes: - """Serialize one cache value with the fixed fact-cache protocol.""" - - return dump_fact_payload(value) diff --git a/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py deleted file mode 100644 index 1687bed1ed..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for reading one local cache value through the restricted fact-cache codec.""" - -from __future__ import annotations - -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import load_fact_payload - - -def loaded_fact_payload(payload: bytes | memoryview) -> object: - """Deserialize one verified cache payload through the restricted class allowlist.""" - - return load_fact_payload(payload) diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/code_identity.py b/src/sqlbuild/compiler/frontier/_helpers/code_identity.py similarity index 96% rename from src/sqlbuild/compiler/fact_cache/_helpers/code_identity.py rename to src/sqlbuild/compiler/frontier/_helpers/code_identity.py index 9d5616bf4b..b173a9509d 100644 --- a/src/sqlbuild/compiler/fact_cache/_helpers/code_identity.py +++ b/src/sqlbuild/compiler/frontier/_helpers/code_identity.py @@ -1,4 +1,4 @@ -"""Identity of the installed code that produces cached compile facts.""" +"""Identity of the installed SQLBuild code that produces cached compiler and rule results.""" from __future__ import annotations diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 6da274d310..6594b32e77 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -66,6 +66,7 @@ ), "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), + "sqlbuild.compiler.compile.models:DeclarationScopeBuild": frozenset({"sql_test_scans"}), "sqlbuild.compiler.compile.models:CompiledProject": frozenset({"native_session"}), } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( diff --git a/src/sqlbuild/compiler/fact_cache/main/code_identity.py b/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py similarity index 63% rename from src/sqlbuild/compiler/fact_cache/main/code_identity.py rename to src/sqlbuild/compiler/frontier/main/compiled_code_identity.py index afdc7f4ea1..103cd40e4f 100644 --- a/src/sqlbuild/compiler/fact_cache/main/code_identity.py +++ b/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py @@ -1,9 +1,9 @@ -"""Public identity of the installed compiler code for derived caches.""" +"""Identity of the installed and native code behind cached compiler and rule results.""" import hashlib import sqlbuild._native as _native -from sqlbuild.compiler.fact_cache._helpers.code_identity import installed_code_identity +from sqlbuild.compiler.frontier._helpers.code_identity import installed_code_identity def compiled_code_identity() -> str: diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 8d8a0a4009..1ff96dd869 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -22,9 +22,6 @@ "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", - "analysis_reuse_hits", - "fact_cache_hits", - "fact_cache_misses", - "render_reuse_hits", - "render_reuse_misses", + "sql_test_scan_cache_hits", + "sql_test_scan_cache_misses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index a1bddfd128..22f51e54cc 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -22,9 +22,6 @@ "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", - "analysis_reuse_hits", - "fact_cache_hits", - "fact_cache_misses", - "render_reuse_hits", - "render_reuse_misses", + "sql_test_scan_cache_hits", + "sql_test_scan_cache_misses", ] diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 60699505be..b806fb398c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -14,7 +14,7 @@ import orjson from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index a2a4ff7470..d9cd1a569b 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -27,6 +27,7 @@ ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, @@ -244,13 +245,15 @@ def start_custom_rules( dialect: str, verify_determinism: bool = False, ) -> Future[CustomRulesOutcome] | None: - """Submit selected custom rules, reusing the implementation fingerprints of their payloads.""" + """Submit selected custom rules after digesting installed code here, clear of GIL contention.""" custom_rules: tuple[Rule, ...] = _selected_custom_rules( config=config, catalogue=catalogue, custom_payloads=custom_payloads ) if not custom_rules: return None + if config.cache.enabled: + _ = compiled_code_identity() return executor.submit( evaluate_custom_rules_cached, project=project, diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py index af631e553e..d1ceddcb9f 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -10,7 +10,7 @@ import orjson -from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 9e8f5cfd02..1bf00f83ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -88,11 +88,8 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", - "analysis_reuse_hits", - "fact_cache_hits", - "fact_cache_misses", - "render_reuse_hits", - "render_reuse_misses", + "sql_test_scan_cache_hits", + "sql_test_scan_cache_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 789e841de6..5c800ff775 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -33,7 +33,7 @@ "target/run/functions/python/is_completed_order_py.sql", ), expected_exit_code=0, - expected_compiler_cache_count=4, + expected_compiler_cache_count=3, expected_local_history=False, expected_compiled_paths=( "target/compiled/models/marts/fact_orders.sql", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index b0402ffab7..bc3989c839 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -389,7 +389,7 @@ class CompileCacheInvalidationTestCase: edited_env: dict[str, str] = field(default_factory=dict) expected_failure: bool = False expected_output_change: bool = True - expected_rewarmed_fact_cache_misses: int = 0 + expected_rewarmed_analysis_cache_misses: int = 0 @dataclass(frozen=True) @@ -400,8 +400,8 @@ class CompileCacheDisabledTestCase: compile_args: tuple[str, ...] = () env: dict[str, str] = field(default_factory=dict) edit: Callable[[Path], None] = lambda _root: None - expected_fact_cache_counts: tuple[int, int] = (0, 0) - expected_fact_databases: tuple[Path, ...] = () + expected_analysis_cache_counts: tuple[int, int] = (0, 0) + expected_analysis_databases: tuple[Path, ...] = () @dataclass(frozen=True) @@ -693,12 +693,12 @@ class CompileReuseStoreFailureTestCase: @dataclass(frozen=True) class IncrementalEditStep: - """One authored edit, and whether its compile may reuse unaffected renders.""" + """One authored edit, the command run over it before its compile, and whether it replays.""" description: str edit: Callable[[Path], None] - expected_render_reuse: bool between: Callable[[Path], None] + expected_replayed: bool = False @dataclass(frozen=True) @@ -723,63 +723,43 @@ class RandomEditChainTestCase: @dataclass(frozen=True) class BrokenEditInvalidationTestCase: - """An incremental invalidation deliberately broken, and edits whose output it must change.""" + """A cache invalidation deliberately broken, and an edit whose output it must change.""" description: str - compiled_edits: tuple[Callable[[Path], None], ...] edit: Callable[[Path], None] sabotage: Callable[[pytest.MonkeyPatch], None] expected_matches_uncached: bool - after_edit: Callable[[Path, pytest.MonkeyPatch], None] @dataclass(frozen=True) -class BrokenChangeDetectionTestCase: - """A model edit compiled while change detection reports nothing changed.""" +class ExternalModuleEditTestCase: + """An outside module a macro imports while rendering, rewritten in place after a compile.""" description: str - edit: Callable[[Path], None] + initial_value: str + edited_value: str expected_matches_uncached: bool + expected_compiled_value: str @dataclass(frozen=True) -class RenderStoreNoticeTestCase: - """A render-count notice threshold and whether storing renders announces itself.""" - - description: str - notice_renders: int - expected_notice: bool - - -@dataclass(frozen=True) -class RenderSavePolicyTestCase: - """Compiles after a cold compile, and the render files and reuse each one must leave.""" - - description: str - edits: tuple[Callable[[Path], None], ...] - expected_cold_render_files: int - expected_render_files: tuple[int, ...] - expected_reused: tuple[bool, ...] - - -@dataclass(frozen=True) -class ExternalModuleEditTestCase: - """An outside module imported only while rendering, rewritten after its render is reused.""" +class SqlTestScanStoreTestCase: + """A change between two compiles, and how many stored SQL-test scans it must invalidate.""" description: str - initial_value: str - edited_value: str - expected_matches_uncached: bool - expected_compiled_value: str + change: Callable[[Path, pytest.MonkeyPatch], None] + expected_rescans: int + arrange: Callable[[pytest.MonkeyPatch], None] = lambda _monkeypatch: None + expected_matches_uncached: bool = True @dataclass(frozen=True) -class RenderLoadNoticeTestCase: - """A stored-render size threshold and whether loading the store announces itself.""" +class RetiredCompilerCacheTestCase: + """A compile over a project holding cache files older releases wrote.""" description: str - notice_bytes: int - expected_notice: bool + compile_args: tuple[str, ...] + expected_removed: bool @dataclass(frozen=True) @@ -794,7 +774,7 @@ class DenseBatchedPreparationTestCase: @dataclass(frozen=True) class SharedCacheKeyTestCase: - """A model analysis schedule under which a model shares a served model's cache key.""" + """A model analysis schedule under which two models share one analysis cache key.""" description: str schedule: Callable[[pytest.MonkeyPatch], None] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 8a1c29d7b4..0acd2a0030 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -16,7 +16,7 @@ import time import zipfile from bisect import bisect_left -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager, redirect_stdout from dataclasses import replace from io import StringIO @@ -31,8 +31,6 @@ import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.commands.main.project._compile as compile_command_module -import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice -import sqlbuild.cli.compile_render_reuse.main._render_reuse_session as render_reuse_main import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files import sqlbuild.cli.compile_reuse._helpers.store as reuse_store @@ -42,32 +40,43 @@ import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage -import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow -import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session -import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses import sqlbuild.compiler.contracts.main.validate as contract_validation +import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, ) +from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.cli.compile_reuse._helpers.entry_file import ( read_entry_header, read_entry_stdout, write_entry, ) -from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts from sqlbuild.cli.compile_reuse.constants import ( REUSE_DISABLE_ENV_VAR, REUSE_ENTRY_DIRECTORY_NAME, - REUSE_RENDER_STATE_SUFFIX, ) -from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.cli.compile_reuse.models import ( + ProjectFilesComparison, + StoredCompileHeader, + StoredCompileInputs, +) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import ( + RETIRED_FACT_CACHE_DIRECTORY_NAME, + SQL_TEST_SCAN_STORE_FILE_NAME, +) +from sqlbuild.compiler.frontier.constants import ( + COMPILER_CACHE_DIRECTORY_NAME, + COMPILER_ENGINE_ENV_VAR, + ENGINE_CACHE_NAMESPACE_SUFFIXES, +) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, @@ -2540,8 +2549,8 @@ class CompileCacheOutcome(NamedTuple): returncode: int diagnostics: tuple[str, ...] fingerprint: str - fact_cache_hits: int - fact_cache_misses: int + analysis_cache_hits: int + analysis_cache_misses: int def run_installed_sqb( @@ -2567,7 +2576,7 @@ def run_installed_sqb( def compile_cache_outcome( *, project_dir: Path, env: dict[str, str], compile_args: tuple[str, ...] = () ) -> CompileCacheOutcome: - """Compile in a fresh process and return its semantic fingerprint and fact-cache counts.""" + """Compile in a fresh process and return its semantic fingerprint and analysis-cache counts.""" result: subprocess.CompletedProcess[str] = run_installed_sqb( project_dir=project_dir, args=("compile", "--json", *compile_args), env=env @@ -2580,8 +2589,9 @@ def compile_cache_outcome( fingerprint=semantic_compile_fingerprint( payload=values, compiled_dir=project_dir / "target" / "compiled" ), - fact_cache_hits=timings.get("fact_cache_hits", 0), - fact_cache_misses=timings.get("fact_cache_misses", 0), + analysis_cache_hits=timings.get("analysis_batch_cache_hits", 0) + + timings.get("analysis_entry_cache_hits", 0), + analysis_cache_misses=timings.get("analysis_cache_misses", 0), ) @@ -3378,10 +3388,6 @@ def compile_edit_without_reuse(project_dir: Path, monkeypatch: pytest.MonkeyPatc monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, "0") -def leave_edit_uncompiled(_project_dir: Path, _monkeypatch: pytest.MonkeyPatch) -> None: - """Leave the edit for the compared compile to see first.""" - - def write_before_reuse_store( *, monkeypatch: pytest.MonkeyPatch, project_dir: Path, write: Callable[[Path], None] ) -> None: @@ -3500,12 +3506,6 @@ class IncrementalEditComparison(NamedTuple): incremental: CompileReuseRun reference: CompileReuseRun - @property - def reused_renders(self) -> int: - """Return how many model renders the incremental compile reused.""" - - return self.incremental.timings.get("render_reuse_hits", 0) - @property def matches(self) -> bool: """Return whether exit code, report, and every compiled artifact are byte-identical.""" @@ -3577,18 +3577,6 @@ def in_process_reuse_run( "introduce_error": ("introduce_error", "fix_error"), "drop_column": ("drop_column", "comment", "restore_column"), } -_MODEL_ONLY_EDIT_KINDS: frozenset[str] = frozenset( - { - "comment", - "add_column", - "type_change", - "header_change", - "introduce_error", - "fix_error", - "drop_column", - "restore_column", - } -) _DROPPED_COLUMN: str = " customer_id,\n" _KEPT_COLUMN: str = " order_id,\n" _EDIT_ANCHORS: dict[str, str] = {"type_change": " quantity", "drop_column": _DROPPED_COLUMN} @@ -3651,12 +3639,6 @@ def random_edit_plan(*, seed: int, step_count: int) -> tuple[str, ...]: ) -def is_model_only_edit(kind: str) -> bool: - """Return whether an edit kind touches model files only, so renders may be reused.""" - - return kind in _MODEL_ONLY_EDIT_KINDS - - def _swapped(contents: str, first: str, second: str) -> str: return ( contents.replace(first, _SWAP_PLACEHOLDER) @@ -3772,26 +3754,25 @@ def no_intervening_command(_root: Path) -> None: """Compile the edit directly after making it.""" -def model_edit_step( +def edit_step( description: str, edit: Callable[[Path], None], between: Callable[[Path], None] = no_intervening_command, ) -> IncrementalEditStep: - """Return an edit to model files only, whose compile must reuse unaffected renders.""" + """Return one edit, optionally followed by another command before the compared compile.""" - return IncrementalEditStep( - description=description, edit=edit, expected_render_reuse=True, between=between - ) + return IncrementalEditStep(description=description, edit=edit, between=between) -def full_edit_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: - """Return an edit beyond model contents, whose compile must render everything again.""" +def star_chain_steps(*, between: Callable[[Path], None]) -> tuple[IncrementalEditStep, ...]: + """Add a star chain, then add and remove an upstream column with a command before each.""" - return IncrementalEditStep( - description=description, - edit=edit, - expected_render_reuse=False, - between=no_intervening_command, + return ( + edit_step("star_chain_added", star_chain_added), + edit_step("upstream_column_added", staging_extra_flag, between), + edit_step("edit_after_added_column", fact_comment), + edit_step("upstream_column_removed", staging_extra_flag_removed, between), + edit_step("edit_after_removed_column", payments_comment), ) @@ -3855,7 +3836,7 @@ def star_chain_with_twin_added(root: Path) -> None: def twin_header_changed(root: Path) -> None: - """Change the twin's header only, so its render is not reused while its SQL is unchanged.""" + """Change the twin's header only, leaving its SQL and so its analysis cache key unchanged.""" replace_project_text( root, @@ -4086,86 +4067,191 @@ def stg_orders_test_edit(root: Path) -> None: ) -def keep_invalidation(_monkeypatch: pytest.MonkeyPatch) -> None: - """Leave every incremental invalidation intact.""" +def stg_orders_test_edited_again(root: Path) -> None: + """Change the staging orders SQL test's edited expected value once more.""" + + replace_project_text( + root, "tests/unit/test_stg_orders.sql", "101 AS customer_id", "102 AS customer_id" + ) + + +def adapter_switched(*, before: str, after: str) -> Callable[[Path], None]: + """Return an edit that switches the project adapter.""" + + def switch(root: Path) -> None: + replace_project_text( + root, "sqlbuild_project.toml", f'adapter = "{before}"', f'adapter = "{after}"' + ) + + return switch + + +def disable_project_reuse(monkeypatch: pytest.MonkeyPatch) -> None: + """Set the fixture environment but compile every time, so the finer caches are exercised.""" + + enable_compile_reuse(monkeypatch) + monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, "1") + + +def sql_test_scan_counts(run: CompileReuseRun) -> tuple[int, int]: + """Return the SQL-test scan store hits and misses of one compile.""" + + return run.timings["sql_test_scan_cache_hits"], run.timings["sql_test_scan_cache_misses"] + +def edit_sql_test_scan_input(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Edit one SQL test file between compiles.""" -def skip_upstream_analysis_checks(monkeypatch: pytest.MonkeyPatch) -> None: - """Serve stored analyses even when an upstream model's final signature changed.""" + stg_orders_test_edit(root) - for module in (stored_model_analyses, binding_dataflow): - monkeypatch.setattr(module, "dependencies_current", lambda **_kwargs: True) +def edit_unrelated_model(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Edit a model no SQL test text contains.""" -def reuse_declarations_after_any_edit(monkeypatch: pytest.MonkeyPatch) -> None: - """Load stored renders and reuse stored declaration files even after a non-model edit.""" + fact_comment(root) + + +def change_adapter_lexical_rules(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Give the project adapter different lexical rules that leave this project's SQL unchanged.""" - read_render_state: Callable[..., object] = render_reuse_main.read_render_state monkeypatch.setattr( - render_reuse_main, - "read_render_state", - lambda *, path, changed_paths: read_render_state(path=path, changed_paths=frozenset()), + DuckDbBackedAdapter, + "sql_lexical_syntax", + replace(DuckDbBackedAdapter.sql_lexical_syntax, triple_quoted_strings=True), ) - monkeypatch.setattr(render_reuse_session, "edits_only_models", lambda **_kwargs: True) -def trust_changed_artifacts(monkeypatch: pytest.MonkeyPatch) -> None: - """Trust stored artifact digests even when the artifact changed since it was stored.""" +def upgrade_native_build(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Pretend the native extension was rebuilt from different source.""" - monkeypatch.setattr(StoredArtifacts, "unchanged_since_stored", lambda _self, **_kwargs: True) + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") -def disable_change_detection(monkeypatch: pytest.MonkeyPatch) -> None: - """Make compile reuse believe no project file changed since the stored compile.""" +def corrupt_sql_test_scan_store(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Overwrite the stored SQL-test scans with bytes that are not a store file.""" - monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: (frozenset(), {})) + path: Path = compiler_cache_directory(root) / SQL_TEST_SCAN_STORE_FILE_NAME + assert path.is_file() + _ = path.write_bytes(b"not a native store") -def set_store_notice_renders(monkeypatch: pytest.MonkeyPatch, renders: int) -> None: - """Announce storing a compile once it has at least this many new renders to record.""" +def edit_installed_code(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Pretend the installed Python code changed, as a local edit of an editable install does.""" - monkeypatch.setattr(reuse_store, "REUSE_STORE_NOTICE_RENDERS", renders) + monkeypatch.setattr( + compiled_code_identity_module, "installed_code_identity", lambda: "edited-python-code" + ) -def prime_render_store(project_dir: Path) -> CompileReuseRun: - """After a cold compile, compile one leaf edit so the stored compile also holds renders.""" +RETIRED_RENDER_FILES: tuple[str, ...] = ( + "0123456789abcdef-0000.render", + "fedcba9876543210-1111.render", +) - fact_comment(project_dir) - return run_reuse_compile(project_dir=project_dir) +def write_retired_compiler_cache_files(project_dir: Path) -> tuple[Path, ...]: + """Leave every engine's fact cache and render files of other target slots from old releases.""" -def prime_render_store_in_process(project_dir: Path) -> int: - """In this process: compile cold, then compile one leaf edit so renders are stored.""" + written: list[Path] = [ + compiler_cache_directory(project_dir).parent + / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" + / RETIRED_FACT_CACHE_DIRECTORY_NAME + / "sql-tests.sqlite3" + for suffix in ENGINE_CACHE_NAMESPACE_SUFFIXES.values() + ] + written.extend( + compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME / name + for name in RETIRED_RENDER_FILES + ) + for path in written: + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(b"retired") + return tuple(written) + + +_ORIGINAL_SCAN_WRITE: Callable[..., None] = SqlTestScanCache.write + + +def _stored_scans_replaced(replacement: bytes) -> Callable[[pytest.MonkeyPatch], None]: + def arrange(monkeypatch: pytest.MonkeyPatch) -> None: + def write( + self: SqlTestScanCache, + *, + algorithm: str, + syntax: SqlLexicalSyntax, + parts: Sequence[str], + value: bytes, + ) -> None: + del value + _ORIGINAL_SCAN_WRITE( + self, algorithm=algorithm, syntax=syntax, parts=parts, value=replacement + ) - cold: int = compile_in_process(project_dir=project_dir) - fact_comment(project_dir) - return max(cold, compile_in_process(project_dir=project_dir)) + monkeypatch.setattr(SqlTestScanCache, "write", write) + return arrange -def render_store_files(project_dir: Path) -> int: - """Return how many stored render files the project's compile reuse folder holds.""" - return len( - list( - (compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME).glob( - f"*{REUSE_RENDER_STATE_SUFFIX}" - ) - ) +store_undecodable_scans: Callable[[pytest.MonkeyPatch], None] = _stored_scans_replaced(b"{]") +store_misshapen_scans: Callable[[pytest.MonkeyPatch], None] = _stored_scans_replaced( + b'[{"mode": "model"}]' +) + + +def ignore_test_text_in_scan_key(monkeypatch: pytest.MonkeyPatch) -> None: + """Key stored SQL-test scans by file path only, so an edited test reads a stale scan.""" + + read: Callable[..., object] = SqlTestScanCache.read + write: Callable[..., None] = SqlTestScanCache.write + + def path_only(kwargs: dict[str, Any]) -> dict[str, Any]: + return {**kwargs, "parts": kwargs["parts"][:1]} + + monkeypatch.setattr( + SqlTestScanCache, "read", lambda self, **kwargs: read(self, **path_only(kwargs)) ) + monkeypatch.setattr( + SqlTestScanCache, "write", lambda self, **kwargs: write(self, **path_only(kwargs)) + ) + +def restore_sql_test_scan_writes(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Store real scan results again after an arrangement replaced them.""" -def edit_and_compile(*, project_dir: Path, edit: Callable[[Path], None]) -> CompileReuseRun: - """Apply one edit, compile with reuse enabled, and return the run.""" + monkeypatch.setattr(SqlTestScanCache, "write", _ORIGINAL_SCAN_WRITE) - edit(project_dir) - return run_reuse_compile(project_dir=project_dir) + +def keep_invalidation(_monkeypatch: pytest.MonkeyPatch) -> None: + """Leave every cache invalidation intact.""" + + +def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: + """Make compile reuse believe no project file changed since the stored compile.""" + + monkeypatch.setattr( + reuse_attempt, + "compare_project_files", + lambda **_kwargs: ProjectFilesComparison(unchanged=True, verified={}), + ) + + +def ignore_query_in_analysis_key(monkeypatch: pytest.MonkeyPatch) -> None: + """Key cached model analyses without their query, so an edited query reads a stale analysis.""" + + analysis_key: Callable[..., str] = project_assembly.model_analysis_cache_key + + def query_blind_key(**kwargs: Any) -> str: + kwargs["query_sql"] = "" + return analysis_key(**kwargs) + + monkeypatch.setattr(project_assembly, "model_analysis_cache_key", query_blind_key) EXTERNAL_FLAVOR_MODULE: str = "extflavor" def write_external_flavor(extlib: Path, value: str) -> None: - """Write, or rewrite in place, an outside module that a macro imports only while rendering.""" + """Write, or rewrite in place, an outside module that a macro imports while rendering.""" extlib.mkdir(parents=True, exist_ok=True) (extlib / f"{EXTERNAL_FLAVOR_MODULE}.py").write_text(f"VALUE = {value!r}\n", encoding="utf-8") @@ -4197,18 +4283,6 @@ def compiled_text(*, run: CompileReuseRun, suffix: str) -> str: return run.compiled[path].decode("utf-8") -def set_render_load_notice_bytes(monkeypatch: pytest.MonkeyPatch, stored_bytes: int) -> None: - """Announce loading stored renders once their files hold at least this many bytes.""" - - monkeypatch.setattr(render_load_notice, "RENDER_LOAD_NOTICE_BYTES", stored_bytes) - - -def json_report_keys(stdout: str) -> tuple[str, ...]: - """Parse a JSON compile report and return its top-level keys.""" - - return tuple(cast(dict[str, object], json.loads(stdout))) - - _COMPILER_ENGINE_LINE: re.Pattern[str] = re.compile(r'\n "compiler_engine": "([a-z-]+)",') diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py index 38d5db9526..7d27d5d81a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py @@ -194,13 +194,13 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d cold: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) warm: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) assert cold.returncode == 0 - assert cold.fact_cache_misses > 0 + assert cold.analysis_cache_misses > 0 assert warm == CompileCacheOutcome( returncode=0, diagnostics=cold.diagnostics, fingerprint=cold.fingerprint, - fact_cache_hits=cold.fact_cache_misses, - fact_cache_misses=0, + analysis_cache_hits=cold.analysis_cache_misses, + analysis_cache_misses=0, ) test_case.edit(project_dir) @@ -217,7 +217,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d ) is test_case.expected_output_change assert edited[:3] == reference[:3] assert rewarmed[:3] == reference[:3] - assert rewarmed.fact_cache_misses == test_case.expected_rewarmed_fact_cache_misses + assert rewarmed.analysis_cache_misses == test_case.expected_rewarmed_analysis_cache_misses @pytest.mark.parametrize( @@ -240,7 +240,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d ], ids=lambda case: case.description, ) -def test_given_disabled_compile_cache_when_compiling_then_no_facts_are_read_or_written( +def test_given_disabled_compile_cache_when_compiling_then_no_analyses_are_read_or_written( cache_invalidation_project: Path, test_case: CompileCacheDisabledTestCase ) -> None: project_dir: Path = cache_invalidation_project @@ -258,11 +258,14 @@ def test_given_disabled_compile_cache_when_compiling_then_no_facts_are_read_or_w first_timings: dict[str, int] = json.loads(first.stdout)["compile_timings"] second_timings: dict[str, int] = json.loads(second.stdout)["compile_timings"] assert ( - first_timings["fact_cache_hits"], - first_timings["fact_cache_misses"], - ) == test_case.expected_fact_cache_counts + first_timings["analysis_batch_cache_hits"] + first_timings["analysis_entry_cache_hits"], + first_timings["analysis_cache_misses"], + ) == test_case.expected_analysis_cache_counts assert ( - second_timings["fact_cache_hits"], - second_timings["fact_cache_misses"], - ) == test_case.expected_fact_cache_counts - assert tuple((project_dir / "target").rglob("facts-v*")) == test_case.expected_fact_databases + second_timings["analysis_batch_cache_hits"] + second_timings["analysis_entry_cache_hits"], + second_timings["analysis_cache_misses"], + ) == test_case.expected_analysis_cache_counts + assert ( + tuple((project_dir / "target").rglob("model-analysis.sqlite3")) + == test_case.expected_analysis_databases + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py deleted file mode 100644 index f2520c73c2..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py +++ /dev/null @@ -1,305 +0,0 @@ -"""Edits that reuse stored declarations, analyses, and artifacts must match an uncached compile.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - BrokenEditInvalidationTestCase, - IncrementalEditSequenceTestCase, - SharedCacheKeyTestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( - CompileReuseRun, - IncrementalEditComparison, - analyze_in_one_batch, - analyze_one_model_at_a_time, - build_between, - compare_incremental_compile, - compile_edit_without_reuse, - compile_in_process, - compile_without_reuse_between, - compiled_artifact_tampered, - enable_compile_reuse, - fact_comment, - fact_quantity_type_declared, - fact_quantity_type_removed, - full_edit_step, - in_process_reuse_run, - keep_invalidation, - leave_edit_uncompiled, - model_edit_step, - other_target_build_between, - payments_comment, - plan_between, - prime_render_store, - prime_render_store_in_process, - replace_project_text, - reuse_declarations_after_any_edit, - run_reuse_compile, - skip_upstream_analysis_checks, - staging_column_removed, - staging_column_renamed, - staging_column_restored, - staging_comment, - staging_extra_flag, - staging_extra_flag_removed, - staging_quantity_restored, - staging_quantity_text, - staging_rename_reverted, - star_chain_added, - star_chain_with_twin_added, - stg_orders_test_edit, - trust_changed_artifacts, - twin_header_changed, -) - - -@pytest.mark.parametrize( - "test_case", - [ - IncrementalEditSequenceTestCase( - description="upstream_shape_changes", - steps=( - model_edit_step("upstream_column_removed", staging_column_removed), - model_edit_step("edit_while_downstream_broken", payments_comment), - model_edit_step("upstream_column_restored", staging_column_restored), - model_edit_step("upstream_column_renamed", staging_column_renamed), - model_edit_step("upstream_rename_reverted", staging_rename_reverted), - ), - ), - IncrementalEditSequenceTestCase( - description="contract_changes", - steps=( - model_edit_step("downstream_type_declared", fact_quantity_type_declared), - model_edit_step("upstream_type_breaks_contract", staging_quantity_text), - model_edit_step("edit_after_contract_error", fact_comment), - model_edit_step("upstream_type_restored", staging_quantity_restored), - model_edit_step("downstream_type_removed", fact_quantity_type_removed), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_plan", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step("upstream_column_added", staging_extra_flag, plan_between), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", staging_extra_flag_removed, plan_between - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_build", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step("upstream_column_added", staging_extra_flag, build_between), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", staging_extra_flag_removed, build_between - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_uncached_compile", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step( - "upstream_column_added", staging_extra_flag, compile_without_reuse_between - ), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", - staging_extra_flag_removed, - compile_without_reuse_between, - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_other_target_build", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step( - "upstream_column_added", staging_extra_flag, other_target_build_between - ), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", - staging_extra_flag_removed, - other_target_build_between, - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="declarations_and_artifacts", - steps=( - full_edit_step( - "source_type_edit", - lambda root: replace_project_text( - root, - "sources/raw.yml", - " - name: quantity\n type: INTEGER", - " - name: quantity\n type: BIGINT", - ), - ), - model_edit_step("edit_after_source", staging_comment), - model_edit_step("compiled_artifact_tampered", compiled_artifact_tampered), - full_edit_step("test_edit", stg_orders_test_edit), - model_edit_step("edit_after_test", fact_comment), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_edit_sequence_when_reusing_stored_phases_then_each_step_matches_uncached_compile( - compile_reuse_project: Path, test_case: IncrementalEditSequenceTestCase -) -> None: - project_dir: Path = compile_reuse_project - cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - primed: CompileReuseRun = prime_render_store(project_dir) - assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr - - for step in test_case.steps: - step.edit(project_dir) - step.between(project_dir) - comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) - - assert comparison.matches is test_case.expected_matches_uncached, ( - step.description, - comparison.mismatched_artifacts, - comparison.incremental.stderr, - ) - assert (comparison.reused_renders > 0) is step.expected_render_reuse, step.description - assert ( - comparison.incremental.timings.get("analysis_reuse_hits", 0) > 0 - ) is step.expected_render_reuse, step.description - - -@pytest.mark.parametrize( - "test_case", - [ - BrokenEditInvalidationTestCase( - description="intact_invalidation_after_upstream_shape_change", - compiled_edits=(), - edit=staging_column_removed, - sabotage=keep_invalidation, - expected_matches_uncached=True, - after_edit=compile_edit_without_reuse, - ), - BrokenEditInvalidationTestCase( - description="analysis_served_despite_upstream_shape_change", - compiled_edits=(), - edit=staging_column_removed, - sabotage=skip_upstream_analysis_checks, - expected_matches_uncached=False, - after_edit=compile_edit_without_reuse, - ), - BrokenEditInvalidationTestCase( - description="intact_invalidation_after_test_edit", - compiled_edits=(), - edit=stg_orders_test_edit, - sabotage=keep_invalidation, - expected_matches_uncached=True, - after_edit=leave_edit_uncompiled, - ), - BrokenEditInvalidationTestCase( - description="declarations_reused_despite_test_edit", - compiled_edits=(), - edit=stg_orders_test_edit, - sabotage=reuse_declarations_after_any_edit, - expected_matches_uncached=False, - after_edit=leave_edit_uncompiled, - ), - BrokenEditInvalidationTestCase( - description="intact_invalidation_after_tampering", - compiled_edits=(), - edit=compiled_artifact_tampered, - sabotage=keep_invalidation, - expected_matches_uncached=True, - after_edit=leave_edit_uncompiled, - ), - BrokenEditInvalidationTestCase( - description="artifact_trusted_despite_tampering", - compiled_edits=(), - edit=compiled_artifact_tampered, - sabotage=trust_changed_artifacts, - expected_matches_uncached=False, - after_edit=leave_edit_uncompiled, - ), - ], - ids=lambda case: case.description, -) -def test_given_broken_edit_invalidation_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( - compile_reuse_project: Path, - test_case: BrokenEditInvalidationTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - project_dir: Path = compile_reuse_project - enable_compile_reuse(monkeypatch) - assert prime_render_store_in_process(project_dir) == 0 - for edit in test_case.compiled_edits: - edit(project_dir) - assert compile_in_process(project_dir=project_dir) == 0 - test_case.edit(project_dir) - test_case.after_edit(project_dir, monkeypatch) - test_case.sabotage(monkeypatch) - - broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) - reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) - comparison: IncrementalEditComparison = IncrementalEditComparison( - incremental=broken, reference=reference - ) - - assert comparison.matches is test_case.expected_matches_uncached, ( - comparison.mismatched_artifacts - ) - - -@pytest.mark.parametrize( - "test_case", - [ - SharedCacheKeyTestCase( - description="dataflow_one_model_at_a_time", schedule=analyze_one_model_at_a_time - ), - SharedCacheKeyTestCase(description="one_batch", schedule=analyze_in_one_batch), - ], - ids=lambda case: case.description, -) -def test_given_model_sharing_a_served_cache_key_when_upstream_changes_then_it_matches_uncached( - compile_reuse_project: Path, - test_case: SharedCacheKeyTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - project_dir: Path = compile_reuse_project - enable_compile_reuse(monkeypatch) - test_case.schedule(monkeypatch) - star_chain_with_twin_added(project_dir) - assert prime_render_store_in_process(project_dir) == 0 - staging_extra_flag(project_dir) - twin_header_changed(project_dir) - compile_edit_without_reuse(project_dir, monkeypatch) - - incremental: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) - reference: CompileReuseRun = in_process_reuse_run( - project_dir=project_dir, capsys=capsys, args=("--no-cache",) - ) - comparison: IncrementalEditComparison = IncrementalEditComparison( - incremental=incremental, reference=reference - ) - - assert comparison.matches is test_case.expected_matches_uncached, ( - comparison.mismatched_artifacts - ) - assert incremental.timings.get("analysis_reuse_hits", 0) > 0 - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index 90bd684ca7..cea28c600b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -1,4 +1,4 @@ -"""Incremental edit compiles must match an uncached compile of the same edited project.""" +"""Compiles after an edit must match an uncached compile of the same edited project.""" from __future__ import annotations @@ -7,53 +7,80 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - BrokenChangeDetectionTestCase, + BrokenEditInvalidationTestCase, ExternalModuleEditTestCase, IncrementalEditSequenceTestCase, RandomEditChainTestCase, - RenderLoadNoticeTestCase, - RenderSavePolicyTestCase, - RenderStoreNoticeTestCase, + SharedCacheKeyTestCase, + SqlTestScanStoreTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( CompileReuseRun, IncrementalEditComparison, RandomEditChain, + adapter_switched, add_external_flavor_macro, + analyze_in_one_batch, + analyze_one_model_at_a_time, + build_between, + change_adapter_lexical_rules, compare_incremental_compile, + compile_edit_without_reuse, compile_in_process, - compile_in_process_output, + compile_without_reuse_between, + compiled_artifact_tampered, compiled_text, completed_order_udf_signature_change, - disable_change_detection, - edit_and_compile, + corrupt_sql_test_scan_store, + disable_project_reuse, + edit_installed_code, + edit_sql_test_scan_input, + edit_step, + edit_unrelated_model, enable_compile_reuse, fact_audit_added, fact_comment, fact_error_fixed, fact_error_introduced, - full_edit_step, + fact_quantity_type_declared, + fact_quantity_type_removed, + ignore_project_changes, + ignore_query_in_analysis_key, + ignore_test_text_in_scan_key, in_process_reuse_run, - is_model_only_edit, - json_report_keys, - model_edit_step, + keep_invalidation, move_project_file, order_status_nullability_change, + other_target_build_between, payment_expression_type_change, - prime_render_store, - prime_render_store_in_process, + payments_comment, + plan_between, random_edit_plan, - render_store_files, replace_project_text, + restore_sql_test_scan_writes, run_reuse_compile, - set_render_load_notice_bytes, - set_store_notice_renders, + sql_test_scan_counts, + staging_column_removed, + staging_column_renamed, + staging_column_restored, staging_comment, staging_contract_change, + staging_extra_flag, staging_header_change, staging_new_column, + staging_quantity_restored, + staging_quantity_text, + staging_rename_reverted, staging_type_change, star_chain_added, + star_chain_steps, + star_chain_with_twin_added, + stg_orders_test_edit, + stg_orders_test_edited_again, + store_misshapen_scans, + store_undecodable_scans, + twin_header_changed, + upgrade_native_build, write_external_flavor, write_generated_edit_models, write_project_file, @@ -66,20 +93,20 @@ IncrementalEditSequenceTestCase( description="model_edits", steps=( - model_edit_step("comment_without_shape_change", staging_comment), - model_edit_step("type_change_propagates_downstream", staging_type_change), - model_edit_step("new_output_column", staging_new_column), - model_edit_step("header_config_change", staging_header_change), - model_edit_step("column_contract_change", staging_contract_change), - model_edit_step("attached_audit_added", fact_audit_added), - model_edit_step("error_introduced", fact_error_introduced), - model_edit_step("error_fixed", fact_error_fixed), + edit_step("comment_without_shape_change", staging_comment), + edit_step("type_change_propagates_downstream", staging_type_change), + edit_step("new_output_column", staging_new_column), + edit_step("header_config_change", staging_header_change), + edit_step("column_contract_change", staging_contract_change), + edit_step("attached_audit_added", fact_audit_added), + edit_step("error_introduced", fact_error_introduced), + edit_step("error_fixed", fact_error_fixed), ), ), IncrementalEditSequenceTestCase( description="model_set_changes", steps=( - full_edit_step( + edit_step( "model_added", lambda root: write_project_file( root, @@ -88,8 +115,8 @@ 'SELECT order_id, quantity FROM __ref("stg_orders")\n', ), ), - model_edit_step("edit_after_add", staging_comment), - full_edit_step( + edit_step("edit_after_add", staging_comment), + edit_step( "model_renamed", lambda root: move_project_file( root, @@ -97,17 +124,17 @@ "models/marts/order_quantity_totals.sql", ), ), - full_edit_step( + edit_step( "model_removed", lambda root: (root / "models/marts/order_quantity_totals.sql").unlink(), ), - model_edit_step("edit_after_remove", fact_comment), + edit_step("edit_after_remove", fact_comment), ), ), IncrementalEditSequenceTestCase( description="declaration_edits", steps=( - full_edit_step( + edit_step( "macro_edit", lambda root: replace_project_text( root, @@ -116,14 +143,14 @@ "{quantity} * {price_cents}", ), ), - model_edit_step("edit_after_macro", fact_comment), - full_edit_step( + edit_step("edit_after_macro", fact_comment), + edit_step( "module_imported_by_macro_edit", lambda root: replace_project_text( root, "macros/_rounding.py", "_SCALE: int = 2", "_SCALE: int = 3" ), ), - full_edit_step( + edit_step( "scoped_macro_edit", lambda root: replace_project_text( root, @@ -132,7 +159,7 @@ "/ 100.0, 3)", ), ), - full_edit_step( + edit_step( "path_defaults_edit", lambda root: replace_project_text( root, @@ -141,7 +168,7 @@ '[path_defaults.staging]\nmaterialized = "table"', ), ), - full_edit_step( + edit_step( "enum_edit", lambda root: replace_project_text( root, @@ -150,7 +177,7 @@ 'WEB "online"', ), ), - full_edit_step( + edit_step( "constant_edit", lambda root: replace_project_text( root, @@ -159,7 +186,7 @@ "value 2)", ), ), - full_edit_step( + edit_step( "new_scope_folder", lambda root: write_project_file( root, @@ -169,19 +196,19 @@ ' return f"({price_cents}) * ({quantity})"\n', ), ), - model_edit_step("edit_after_declarations", staging_comment), + edit_step("edit_after_declarations", staging_comment), ), ), IncrementalEditSequenceTestCase( description="resource_edits", steps=( - full_edit_step( + edit_step( "seed_edit", lambda root: replace_project_text( root, "seeds/waffle_types.csv", "Classic Belgian", "Classic Brussels" ), ), - full_edit_step( + edit_step( "source_edit", lambda root: replace_project_text( root, @@ -190,17 +217,9 @@ " - name: quantity\n type: BIGINT", ), ), - model_edit_step("edit_after_source", staging_type_change), - full_edit_step( - "test_edit", - lambda root: replace_project_text( - root, - "tests/unit/test_stg_orders.sql", - "100 AS customer_id", - "101 AS customer_id", - ), - ), - full_edit_step( + edit_step("edit_after_source", staging_type_change), + edit_step("test_edit", stg_orders_test_edit), + edit_step( "audit_edit", lambda root: replace_project_text( root, @@ -209,44 +228,90 @@ "WHERE NOT (@expression) AND 1 = 1", ), ), - model_edit_step("edit_after_resources", fact_comment), + edit_step("edit_after_resources", fact_comment), + edit_step("compiled_artifact_tampered", compiled_artifact_tampered), ), ), IncrementalEditSequenceTestCase( description="analysis_input_edits", steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step("upstream_type_flips_downstream_shapes", staging_type_change), - model_edit_step("contract_type_edit", staging_contract_change), - full_edit_step("source_expression_edit", payment_expression_type_change), - full_edit_step("udf_signature_edit", completed_order_udf_signature_change), - full_edit_step("schema_entry_nullability_edit", order_status_nullability_change), - model_edit_step("edit_after_analysis_inputs", fact_comment), + edit_step("star_chain_added", star_chain_added), + edit_step("upstream_type_flips_downstream_shapes", staging_type_change), + edit_step("contract_type_edit", staging_contract_change), + edit_step("source_expression_edit", payment_expression_type_change), + edit_step("udf_signature_edit", completed_order_udf_signature_change), + edit_step("schema_entry_nullability_edit", order_status_nullability_change), + edit_step("edit_after_analysis_inputs", fact_comment), ), ), + IncrementalEditSequenceTestCase( + description="upstream_shape_changes", + steps=( + edit_step("upstream_column_removed", staging_column_removed), + edit_step("edit_while_downstream_broken", payments_comment), + edit_step("upstream_column_restored", staging_column_restored), + edit_step("upstream_column_renamed", staging_column_renamed), + edit_step("upstream_rename_reverted", staging_rename_reverted), + ), + ), + IncrementalEditSequenceTestCase( + description="contract_changes", + steps=( + edit_step("downstream_type_declared", fact_quantity_type_declared), + edit_step("upstream_type_breaks_contract", staging_quantity_text), + edit_step("edit_after_contract_error", fact_comment), + edit_step("upstream_type_restored", staging_quantity_restored), + edit_step("downstream_type_removed", fact_quantity_type_removed), + ), + ), + IncrementalEditSequenceTestCase( + description="sql_test_scan_inputs", + steps=( + edit_step("test_edited", stg_orders_test_edit), + edit_step("unrelated_model_edited", fact_comment), + edit_step("adapter_switched", adapter_switched(before="duckdb", after="postgres")), + edit_step("test_edited_on_other_adapter", stg_orders_test_edited_again), + edit_step("adapter_restored", adapter_switched(before="postgres", after="duckdb")), + ), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_plan", + steps=star_chain_steps(between=plan_between), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_build", + steps=star_chain_steps(between=build_between), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_uncached_compile", + steps=star_chain_steps(between=compile_without_reuse_between), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_other_target_build", + steps=star_chain_steps(between=other_target_build_between), + ), ], ids=lambda case: case.description, ) -def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches_uncached_compile( +def test_given_edit_sequence_when_compiling_with_caches_then_each_step_matches_uncached_compile( compile_reuse_project: Path, test_case: IncrementalEditSequenceTestCase ) -> None: project_dir: Path = compile_reuse_project cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - primed: CompileReuseRun = prime_render_store(project_dir) - assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr + warm: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + assert (cold.returncode, warm.returncode, warm.reused) == (0, 0, True), warm.stderr for step in test_case.steps: step.edit(project_dir) step.between(project_dir) comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) - assert comparison.incremental.reused is False, step.description + assert comparison.incremental.reused is step.expected_replayed, step.description assert comparison.matches is test_case.expected_matches_uncached, ( step.description, comparison.mismatched_artifacts, comparison.incremental.stderr, ) - assert (comparison.reused_renders > 0) is step.expected_render_reuse, step.description @pytest.mark.parametrize( @@ -257,7 +322,7 @@ def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches ], ids=lambda case: case.description, ) -def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_matches_uncached_compile( +def test_given_random_edit_chain_when_compiling_with_caches_then_each_step_matches_uncached_compile( compile_reuse_project: Path, test_case: RandomEditChainTestCase ) -> None: project_dir: Path = compile_reuse_project @@ -265,8 +330,7 @@ def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_mat project_dir=project_dir, model_count=test_case.model_count, seed=test_case.seed ) cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - primed: CompileReuseRun = prime_render_store(project_dir) - assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr + assert cold.returncode == 0, cold.stderr chain: RandomEditChain = RandomEditChain(project_dir=project_dir, seed=test_case.seed) for step, kind in enumerate( @@ -282,34 +346,61 @@ def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_mat comparison.mismatched_artifacts, comparison.incremental.stderr, ) - assert (comparison.reused_renders > 0) is is_model_only_edit(kind), (step, kind) - assert (comparison.incremental.timings.get("analysis_reuse_hits", 0) > 0) is ( - is_model_only_edit(kind) - ), (step, kind) @pytest.mark.parametrize( "test_case", [ - BrokenChangeDetectionTestCase( - description="new_output_column", + BrokenEditInvalidationTestCase( + description="intact_invalidation_after_new_output_column", edit=staging_new_column, + sabotage=keep_invalidation, + expected_matches_uncached=True, + ), + BrokenEditInvalidationTestCase( + description="compile_replayed_despite_model_edit", + edit=staging_new_column, + sabotage=ignore_project_changes, + expected_matches_uncached=False, + ), + BrokenEditInvalidationTestCase( + description="intact_invalidation_after_error_introduced", + edit=fact_error_introduced, + sabotage=keep_invalidation, + expected_matches_uncached=True, + ), + BrokenEditInvalidationTestCase( + description="stale_analysis_despite_query_edit", + edit=fact_error_introduced, + sabotage=ignore_query_in_analysis_key, + expected_matches_uncached=False, + ), + BrokenEditInvalidationTestCase( + description="intact_invalidation_after_test_edit", + edit=stg_orders_test_edit, + sabotage=keep_invalidation, + expected_matches_uncached=True, + ), + BrokenEditInvalidationTestCase( + description="stale_sql_test_scan_despite_test_edit", + edit=stg_orders_test_edit, + sabotage=ignore_test_text_in_scan_key, expected_matches_uncached=False, ), ], ids=lambda case: case.description, ) -def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( +def test_given_broken_cache_invalidation_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( compile_reuse_project: Path, - test_case: BrokenChangeDetectionTestCase, + test_case: BrokenEditInvalidationTestCase, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: project_dir: Path = compile_reuse_project enable_compile_reuse(monkeypatch) - assert prime_render_store_in_process(project_dir) == 0 + test_case.sabotage(monkeypatch) + assert compile_in_process(project_dir=project_dir) == 0 test_case.edit(project_dir) - disable_change_detection(monkeypatch) broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) @@ -317,78 +408,135 @@ def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_re incremental=broken, reference=reference ) - assert comparison.reused_renders > 0 - assert comparison.matches is test_case.expected_matches_uncached + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) @pytest.mark.parametrize( "test_case", [ - RenderStoreNoticeTestCase( - description="few_renders_store_quietly", notice_renders=100_000, expected_notice=False - ), - RenderStoreNoticeTestCase( - description="many_renders_announce_recording", notice_renders=1, expected_notice=True + SharedCacheKeyTestCase( + description="dataflow_one_model_at_a_time", schedule=analyze_one_model_at_a_time ), + SharedCacheKeyTestCase(description="one_batch", schedule=analyze_in_one_batch), ], ids=lambda case: case.description, ) -def test_given_new_renders_when_storing_after_an_edit_then_slow_recording_is_announced( +def test_given_models_sharing_an_analysis_cache_key_when_upstream_changes_then_it_matches_uncached( compile_reuse_project: Path, - test_case: RenderStoreNoticeTestCase, + test_case: SharedCacheKeyTestCase, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: + project_dir: Path = compile_reuse_project enable_compile_reuse(monkeypatch) - set_store_notice_renders(monkeypatch, test_case.notice_renders) - assert compile_in_process(project_dir=compile_reuse_project) == 0 - fact_comment(compile_reuse_project) - - code, _, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) + test_case.schedule(monkeypatch) + star_chain_with_twin_added(project_dir) + assert compile_in_process(project_dir=project_dir) == 0 + staging_extra_flag(project_dir) + twin_header_changed(project_dir) + compile_edit_without_reuse(project_dir, monkeypatch) + + incremental: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = in_process_reuse_run( + project_dir=project_dir, capsys=capsys, args=("--no-cache",) + ) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=incremental, reference=reference + ) - assert code == 0 - assert ("renders)..." in err) is test_case.expected_notice - assert ("Recorded compile for reuse" in err) is test_case.expected_notice + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) @pytest.mark.parametrize( "test_case", [ - RenderSavePolicyTestCase( - description="cold_then_two_model_edits", - edits=(fact_comment, staging_comment), - expected_cold_render_files=0, - expected_render_files=(1, 2), - expected_reused=(False, True), + SqlTestScanStoreTestCase( + description="test_file_edited", + change=edit_sql_test_scan_input, + expected_rescans=2, + ), + SqlTestScanStoreTestCase( + description="unrelated_model_edited", + change=edit_unrelated_model, + expected_rescans=0, + ), + SqlTestScanStoreTestCase( + description="adapter_lexical_rules_changed", + change=change_adapter_lexical_rules, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="native_build_upgraded", + change=upgrade_native_build, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="installed_python_code_changed", + change=edit_installed_code, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="store_file_corrupted", + change=corrupt_sql_test_scan_store, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="stored_entries_undecodable", + arrange=store_undecodable_scans, + change=restore_sql_test_scan_writes, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="stored_entries_misshapen", + arrange=store_misshapen_scans, + change=restore_sql_test_scan_writes, + expected_rescans=12, ), ], ids=lambda case: case.description, ) -def test_given_cold_compile_when_editing_then_renders_are_stored_only_once_a_compile_exists( - compile_reuse_project: Path, test_case: RenderSavePolicyTestCase +def test_given_stored_sql_test_scans_when_inputs_change_then_only_changed_files_rescan( + compile_reuse_project: Path, + test_case: SqlTestScanStoreTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], ) -> None: - cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project) - cold_render_files: int = render_store_files(compile_reuse_project) - render_files: list[int] = [] - reused: list[bool] = [] - - for edit in test_case.edits: - run: CompileReuseRun = edit_and_compile(project_dir=compile_reuse_project, edit=edit) - assert run.returncode == 0, run.stderr - render_files.append(render_store_files(compile_reuse_project)) - reused.append(run.timings.get("render_reuse_hits", 0) > 0) + project_dir: Path = compile_reuse_project + disable_project_reuse(monkeypatch) + test_case.arrange(monkeypatch) + cold: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + test_case.change(project_dir, monkeypatch) + + incremental: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = in_process_reuse_run( + project_dir=project_dir, capsys=capsys, args=("--no-cache",) + ) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=incremental, reference=reference + ) + stored: int = sum(sql_test_scan_counts(cold)) assert cold.returncode == 0, cold.stderr - assert cold_render_files == test_case.expected_cold_render_files - assert tuple(render_files) == test_case.expected_render_files - assert tuple(reused) == test_case.expected_reused + assert sql_test_scan_counts(cold) == (0, stored) + assert sql_test_scan_counts(incremental) == ( + stored - test_case.expected_rescans, + test_case.expected_rescans, + ) + assert sql_test_scan_counts(reference) == (0, 0) + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) @pytest.mark.parametrize( "test_case", [ ExternalModuleEditTestCase( - description="module_rewritten_in_place_after_reuse", + description="module_rewritten_in_place_after_a_compile", initial_value="'vanilla'", edited_value="'choco'", expected_matches_uncached=True, @@ -397,7 +545,7 @@ def test_given_cold_compile_when_editing_then_renders_are_stored_only_once_a_com ], ids=lambda case: case.description, ) -def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_then_output_is_fresh( +def test_given_module_imported_while_rendering_when_it_changes_after_a_compile_then_output_is_fresh( compile_reuse_project: Path, tmp_path: Path, test_case: ExternalModuleEditTestCase ) -> None: extlib: Path = tmp_path / "extlib" @@ -406,9 +554,7 @@ def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_ ) cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) staging_comment(compile_reuse_project) - recorded: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) - staging_comment(compile_reuse_project) - reused: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + edited: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) write_external_flavor(extlib, test_case.edited_value) comparison: IncrementalEditComparison = IncrementalEditComparison( @@ -418,8 +564,8 @@ def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_ ), ) - assert (cold.returncode, recorded.returncode, reused.returncode) == (0, 0, 0), reused.stderr - assert reused.timings.get("render_reuse_hits", 0) > 0 + assert (cold.returncode, edited.returncode) == (0, 0), edited.stderr + assert comparison.incremental.reused is False assert comparison.matches is test_case.expected_matches_uncached, ( comparison.mismatched_artifacts ) @@ -428,32 +574,5 @@ def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_ ) -@pytest.mark.parametrize( - "test_case", - [ - RenderLoadNoticeTestCase( - description="small_store_loads_quietly", notice_bytes=1 << 40, expected_notice=False - ), - RenderLoadNoticeTestCase( - description="large_store_announces_loading", notice_bytes=1, expected_notice=True - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_renders_when_loading_them_then_large_loads_are_announced_on_stderr( - compile_reuse_project: Path, - test_case: RenderLoadNoticeTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - enable_compile_reuse(monkeypatch) - set_render_load_notice_bytes(monkeypatch, test_case.notice_bytes) - assert prime_render_store_in_process(compile_reuse_project) == 0 - staging_comment(compile_reuse_project) - - code, out, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) - - assert code == 0 - assert "compile_timings" in json_report_keys(out) - assert ("Loading stored renders" in err) is test_case.expected_notice - assert ("Loaded stored renders" in err) is test_case.expected_notice +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index 5e1f98e710..11565105d4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -188,7 +188,7 @@ description="full_render_each_step", project_reuse=False, steps=_EDIT_STEPS ), MacroCallStoreEditSequenceTestCase( - description="with_project_and_render_reuse", project_reuse=True, steps=_EDIT_STEPS + description="with_project_reuse", project_reuse=True, steps=_EDIT_STEPS ), ], ids=lambda case: case.description, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py index daf0d72598..f107cbb22b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py @@ -23,6 +23,7 @@ CompileReuseReplayTestCase, CompileReuseStoreFailureTestCase, CompileReuseTimingsTestCase, + RetiredCompilerCacheTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( COMPILE_REUSE_HIT_LINE, @@ -72,6 +73,7 @@ write_orders_api_timeout_file, write_project_file, write_recording_sink, + write_retired_compiler_cache_files, ) _CONTRACT_ERROR_MODEL: str = ( @@ -807,5 +809,32 @@ def test_given_store_is_interrupted_when_compiling_then_the_interrupt_propagates assert raised.type is test_case.expected_raised +@pytest.mark.parametrize( + "test_case", + [ + RetiredCompilerCacheTestCase( + description="cached_compile_removes_them", compile_args=(), expected_removed=True + ), + RetiredCompilerCacheTestCase( + description="uncached_compile_leaves_the_cache_alone", + compile_args=("--no-cache",), + expected_removed=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_cache_files_of_older_releases_when_compiling_then_cached_compile_deletes_them( + compile_reuse_project: Path, test_case: RetiredCompilerCacheTestCase +) -> None: + retired: tuple[Path, ...] = write_retired_compiler_cache_files(compile_reuse_project) + + run: CompileReuseRun = run_reuse_compile( + project_dir=compile_reuse_project, args=test_case.compile_args + ) + + assert run.returncode == 0, run.stderr + assert [path.exists() for path in retired] == [not test_case.expected_removed] * len(retired) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 4ca693ff57..cd14c2f000 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -224,6 +224,19 @@ class RulesEditChainCase: expected_exit_code: int = 1 +@dataclass(frozen=True) +class OneModelCustomRuleEditCase: + """One model edited after a warm compile of a model-subject custom rule.""" + + description: str + edited_model: str + before: str + after: str + expected_warm_counts: tuple[int, int] + expected_edit_counts: tuple[int, int] + expected_edit_exit_code: int + + @dataclass(frozen=True) class BrokenInvalidationChainCase: """An edit chain compiled with deliberately stale fact digests against a cache-free oracle.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index c4013f5e32..94aa44f9a3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -507,6 +507,32 @@ def _stale_sources(self, key): """ +ONE_MODEL_EDIT_MODEL_COUNT: int = 6 +# One cached result per model for the custom rule, plus the cached built-in rules request. +ONE_MODEL_EDIT_CACHED_RESULTS: int = ONE_MODEL_EDIT_MODEL_COUNT + 1 +ONE_MODEL_EDIT_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n' + '[rules]\nselect = ["XSQBRONE001"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" + ), + "rules/orders.py": ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n\n" + '@rule(code="XSQBRONE001", slug="literal-order", message="Order 3 is a literal", ' + 'remediation="Select order 3 from a source.")\n' + "def literal_order(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + " source: str = ctx.sql.for_model(model).expanded.source\n" + ' return [ctx.finding(subject=model)] if "SELECT 3 " in source else []\n' + ), + **{ + f"models/orders_{index:03d}.sql": ( + f'MODEL (description "Orders {index}");\n\nSELECT {index} AS order_id\n' + ) + for index in range(ONE_MODEL_EDIT_MODEL_COUNT) + }, +} + + def custom_rules_edit_chain( *, root: Path, edits: tuple[Any, ...], environment: tuple[tuple[str, str], ...] = () ) -> tuple[ @@ -542,6 +568,14 @@ def rules_compile_outcome( ) -> tuple[int, str, str]: """Compile in a fresh process and return its exit code, diagnostics, and artifacts digest.""" + return rules_compile_run(project_dir, *arguments, environment=environment)[0] + + +def rules_compile_run( + project_dir: Path, *arguments: str, environment: tuple[tuple[str, str], ...] = () +) -> tuple[tuple[int, str, str], tuple[int, int]]: + """Compile in a fresh process; return its comparable outcome and rule cache hits and misses.""" + result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -563,10 +597,14 @@ def rules_compile_outcome( for path in sorted(filter(Path.is_file, compiled.rglob("*"))): artifacts.update(path.relative_to(compiled).as_posix().encode()) artifacts.update(path.read_bytes()) + timings: dict[str, int] = payload.get("compile_timings", {}) return ( - result.returncode, - json.dumps(payload.get("diagnostics"), sort_keys=True), - artifacts.hexdigest(), + ( + result.returncode, + json.dumps(payload.get("diagnostics"), sort_keys=True), + artifacts.hexdigest(), + ), + (timings.get("rule_cache_hits", 0), timings.get("rule_cache_misses", 0)), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py index d0c9caa44d..72661eccf3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py @@ -1,5 +1,6 @@ """Incremental custom Rules across split hosts must equal a cache-free compile after every edit.""" +import shutil from pathlib import Path import pytest @@ -10,14 +11,19 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( BrokenInvalidationChainCase, + OneModelCustomRuleEditCase, RulesCacheEditCase, RulesEditChainCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + ONE_MODEL_EDIT_CACHED_RESULTS, + ONE_MODEL_EDIT_PROJECT, STALE_SOURCE_FACTS_SITECUSTOMIZE, custom_rules_edit_chain, rule_cache_counts, rules_compile_outcome, + rules_compile_run, + write_project_files, ) CONFIG: str = "sqlbuild_project.toml" @@ -132,5 +138,51 @@ def test_given_broken_invalidation_when_applying_edit_chain_then_oracle_detects_ ) +@pytest.mark.parametrize( + "test_case", + ( + OneModelCustomRuleEditCase( + description="edit_adds_a_finding", + edited_model="models/orders_002.sql", + before="SELECT 2 AS", + after="SELECT 3 AS", + expected_warm_counts=(ONE_MODEL_EDIT_CACHED_RESULTS, 0), + expected_edit_counts=(ONE_MODEL_EDIT_CACHED_RESULTS - 1, 1), + expected_edit_exit_code=1, + ), + OneModelCustomRuleEditCase( + description="edit_removes_the_finding", + edited_model="models/orders_003.sql", + before="SELECT 3 AS", + after="SELECT 30 AS", + expected_warm_counts=(ONE_MODEL_EDIT_CACHED_RESULTS, 0), + expected_edit_counts=(ONE_MODEL_EDIT_CACHED_RESULTS - 1, 1), + expected_edit_exit_code=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_model_custom_rule_when_one_model_is_edited_then_only_that_model_reruns( + tmp_path: Path, test_case: OneModelCustomRuleEditCase +) -> None: + project_dir: Path = tmp_path / "orders" + write_project_files(project_dir=project_dir, files=ONE_MODEL_EDIT_PROJECT) + cold, cold_counts = rules_compile_run(project_dir) + _, warm_counts = rules_compile_run(project_dir) + replace_project_text(project_dir, test_case.edited_model, test_case.before, test_case.after) + + edited, edit_counts = rules_compile_run(project_dir) + oracle_dir: Path = tmp_path / "oracle" + shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) + oracle: tuple[int, str, str] = rules_compile_outcome(oracle_dir, "--no-cache") + + assert cold[0] == 1, cold[1] + assert cold_counts == (0, ONE_MODEL_EDIT_CACHED_RESULTS) + assert warm_counts == test_case.expected_warm_counts + assert edit_counts == test_case.expected_edit_counts + assert edited[0] == test_case.expected_edit_exit_code, edited[1] + assert edited == oracle + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 272473d7e4..0898ca5204 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -221,17 +221,6 @@ class TolerantDeclarationFilesTestCase: expected_declaration_faults: int -@dataclass(frozen=True) -class DeclarationReuseTestCase: - """A preview compile followed by an edit that only touches one model file.""" - - description: str - files: tuple[tuple[str, bytes], ...] - edited_path: str - edited_contents: bytes - expected_reused_session: object = None - - @dataclass(frozen=True) class FailureTextTestCase: """A project whose discovery failure message must keep its authored text unchanged.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 2c997d3792..72eb243627 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -9,17 +9,12 @@ import pytest from sqlbuild import _native -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import RENDER_REUSE_DECLARATIONS_PREFIX -from sqlbuild.compiler.compile.models import RenderReuseState from sqlbuild.compiler.discovery._helpers.filesystem import core from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, - DeclarationReuseTestCase, DeepDeclarationHeaderTestCase, GeneratedDeclarationFileTestCase, NativeDeclarationFailureTestCase, @@ -504,47 +499,5 @@ def test_given_engine_when_discovering_then_native_session_is_kept_only_for_prev ) -@pytest.mark.parametrize( - "test_case", - [ - DeclarationReuseTestCase( - description="a model edit reuses every stored declaration file", - files=_EVERY_KIND, - edited_path="models/marts/orders.sql", - edited_contents=_MODEL + b" -- edited\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_preview_compile_when_only_a_model_changes_then_declaration_files_are_reused( - test_case: DeclarationReuseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_project(project_dir=tmp_path, files=test_case.files) - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, _PREVIEW) - first_session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=None, changed_paths=None - ) - first: DiscoveredProjectInputs = discover_project_inputs( - project_dir=tmp_path, declaration_reuse=first_session - ) - first_session.plan_models(model_files=first.model_files) - state: RenderReuseState | None = first_session.stored_state() - assert state is not None - _ = (tmp_path / test_case.edited_path).write_bytes(test_case.edited_contents) - second_session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=state, changed_paths=frozenset({test_case.edited_path}) - ) - - second: DiscoveredProjectInputs = discover_project_inputs( - project_dir=tmp_path, declaration_reuse=second_session - ) - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "python") - fresh: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - - assert any(name.startswith(RENDER_REUSE_DECLARATIONS_PREFIX) for name in state.group_payloads) - assert second.native_session is test_case.expected_reused_session - assert render_stage_capture(second) == render_stage_capture(fresh) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py index 1927e81a71..459d0a388b 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py @@ -1,6 +1,6 @@ """Test-case models for the same-runner compile performance ratio guard.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from scripts.compile_performance_ratio.models import CompileComparison @@ -12,6 +12,7 @@ class RatioFailuresTestCase: modes: tuple[str, ...] max_ratio: float expected_failures: tuple[str, ...] + mode_max_ratios: dict[str, float] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py index 6da3ed0a93..74ebe3f291 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py @@ -66,6 +66,25 @@ max_ratio=1.10, expected_failures=("dense 3000 warm: wall ratio 1.518 exceeds 1.10",), ), + RatioFailuresTestCase( + description="an edit within its own looser limit passes while warm keeps the default", + comparisons=( + comparison(mode="warm", wall=(7.6, 7.6), cpu=(9.0, 9.0)), + comparison(mode="edit", wall=(8.0, 10.0), cpu=(10.0, 12.0)), + ), + modes=("warm", "edit"), + max_ratio=1.10, + mode_max_ratios={"edit": 1.30}, + expected_failures=(), + ), + RatioFailuresTestCase( + description="an edit beyond its own limit fails against that limit", + comparisons=(comparison(mode="edit", wall=(8.0, 11.0), cpu=(10.0, 10.0)),), + modes=("edit",), + max_ratio=1.10, + mode_max_ratios={"edit": 1.30}, + expected_failures=("dense 3000 edit: wall ratio 1.375 exceeds 1.30",), + ), RatioFailuresTestCase( description="a requested mode without a measurement fails", comparisons=(comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)),), @@ -102,6 +121,7 @@ def test_given_mode_comparisons_when_judging_then_reports_each_failed_limit( modes=test_case.modes, max_ratio=test_case.max_ratio, noise_floor_seconds=NOISE_FLOOR_SECONDS, + mode_max_ratios=test_case.mode_max_ratios, ) assert failures == test_case.expected_failures diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py deleted file mode 100644 index 913e647485..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Test case types for layered render storage.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class RenderLayerTestCase: - """Renders changed after a stored compile, and whether an overlay can hold them.""" - - description: str - changed_models: tuple[str, ...] - removed_models: tuple[str, ...] - expected_overlay: bool - - -@dataclass(frozen=True) -class RenderEditChainTestCase: - """Successive one-model edits, each stored on top of the previous compile.""" - - description: str - edits: tuple[str, ...] - expected_overlay_models: frozenset[str] - - -@dataclass(frozen=True) -class StoredRenderReadTestCase: - """A stored render file, possibly damaged, and whether reading it yields renders.""" - - description: str - prepare: Callable[[Path], tuple[Path, frozenset[str]]] - expected_readable: bool - - -@dataclass(frozen=True) -class RenderLoadNoticeTestCase: - """Stored render files, some removed while being sized, and the expected notice.""" - - description: str - file_sizes: tuple[int, ...] - removed_while_sizing: int - notice_bytes: int - expected_notice: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py deleted file mode 100644 index c215b98d84..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Shared builders for layered render storage unit tests.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( - read_render_state, - render_state_layer, -) -from sqlbuild.cli.compile_render_reuse.models import StoredRenderState -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState - -MODEL_PATHS: tuple[str, ...] = tuple(f"models/orders_{index:02d}.sql" for index in range(20)) - - -def publish_render_state(*, directory: Path, name: str, layer: RenderStateLayer | None) -> Path: - """Write one render layer as the store would publish it.""" - - assert layer is not None - path: Path = directory / name - path.write_bytes(b"".join(bytes(chunk) for chunk in layer.chunks)) - return path - - -def read_stored(path: Path) -> StoredRenderState: - """Read a render file that must decode.""" - - stored: StoredRenderState | None = read_render_state(path=path, changed_paths=frozenset()) - assert stored is not None - return stored - - -def stored_base(directory: Path) -> tuple[Path, StoredRenderState]: - """Publish a base render file holding every model and one group, and read it back.""" - - state: RenderReuseState = RenderReuseState( - model_paths=MODEL_PATHS, - model_payloads={path: memoryview(f"render {path}".encode() * 20) for path in MODEL_PATHS}, - group_payloads={"sources": memoryview(b"source renders" * 20)}, - ) - path: Path = publish_render_state( - directory=directory, - name="orders-base.render", - layer=render_state_layer(state=state, stored=None), - ) - return path, read_stored(path) - - -def next_render_state( - *, stored: StoredRenderState, edited: tuple[str, ...], removed: tuple[str, ...] = () -) -> RenderReuseState: - """Return the next compile's renders as a session stores them after releasing reused bytes. - - Renders reused from the stored base are None; reused overlay renders keep their bytes. - """ - - kept: dict[str, memoryview | None] = { - path: retained_payload(stored=stored, path=path, payload=payload) - for path, payload in filter( - lambda item: item[0] not in removed, stored.state.model_payloads.items() - ) - } - return RenderReuseState( - model_paths=MODEL_PATHS, - model_payloads={ - **kept, - **{path: memoryview(f"edited {path}".encode() * 20) for path in edited}, - }, - group_payloads=dict.fromkeys(stored.state.group_payloads), - ) - - -def retained_payload( - *, stored: StoredRenderState, path: str, payload: memoryview | None -) -> memoryview | None: - """Keep a reused render's bytes only when it lives in the stored overlay.""" - - return {True: payload, False: None}[path in stored.layers.overlay_models] - - -def payload_bytes(state: RenderReuseState) -> dict[str, bytes]: - """Return every model payload as bytes for comparison.""" - - return {path: bytes(payload or b"") for path, payload in state.model_payloads.items()} - - -def store_edit_chain(*, directory: Path, edits: tuple[str, ...]) -> StoredRenderState: - """Store one compile per edited model, each layered on the previous one, and read the last.""" - - stored: StoredRenderState - _, stored = stored_base(directory) - for index, edited in enumerate(edits): - path: Path = publish_render_state( - directory=directory, - name=f"orders-edit-{index}.render", - layer=render_state_layer( - state=next_render_state(stored=stored, edited=(edited,)), stored=stored.layers - ), - ) - stored = read_stored(path) - return stored - - -def chain_payload_bytes(edits: tuple[str, ...]) -> dict[str, bytes]: - """Return the model payloads an edit chain must leave readable.""" - - return { - **{path: f"render {path}".encode() * 20 for path in MODEL_PATHS}, - **{path: f"edited {path}".encode() * 20 for path in edits}, - } - - -def flipped_last_byte(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and corrupt its last payload byte.""" - - path, _ = stored_base(directory) - data: bytearray = bytearray(path.read_bytes()) - data[-1] ^= 0xFF - path.write_bytes(bytes(data)) - return path, frozenset() - - -def truncated(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and cut off its tail.""" - - path, _ = stored_base(directory) - path.write_bytes(path.read_bytes()[:-7]) - return path, frozenset() - - -def emptied(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and empty it.""" - - path, _ = stored_base(directory) - path.write_bytes(b"") - return path, frozenset() - - -def overlay_without_base(directory: Path) -> tuple[Path, frozenset[str]]: - """Store an overlay and delete the base file it layers on.""" - - base_path, stored = stored_base(directory) - overlay_path: Path = publish_render_state( - directory=directory, - name="orders-overlay.render", - layer=render_state_layer( - state=next_render_state(stored=stored, edited=(MODEL_PATHS[0],)), stored=stored.layers - ), - ) - base_path.unlink() - return overlay_path, frozenset() - - -def changed_macro(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and report a changed path that is not a stored model.""" - - path, _ = stored_base(directory) - return path, frozenset({"macros/currency.py"}) - - -def intact(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and report one stored model as changed.""" - - path, _ = stored_base(directory) - return path, frozenset({MODEL_PATHS[4]}) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py deleted file mode 100644 index 3feb8dc648..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Sizing the render store for the load notice must tolerate files removed by a concurrent compile.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_render_reuse._helpers import load_notice -from sqlbuild.cli.compile_reuse.constants import REUSE_RENDER_STATE_SUFFIX -from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( - RenderLoadNoticeTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderLoadNoticeTestCase( - description="large_store_announced", - file_sizes=(64, 64), - removed_while_sizing=0, - notice_bytes=100, - expected_notice=True, - ), - RenderLoadNoticeTestCase( - description="small_store_silent", - file_sizes=(16, 16), - removed_while_sizing=0, - notice_bytes=100, - expected_notice=False, - ), - RenderLoadNoticeTestCase( - description="file_removed_while_sizing_counts_as_empty", - file_sizes=(64, 64), - removed_while_sizing=1, - notice_bytes=100, - expected_notice=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_render_files_when_starting_load_notice_then_sizes_without_failing( - test_case: RenderLoadNoticeTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - # Arrange - entry_path: Path = tmp_path / "dev.entry" - stored_paths: list[Path] = [ - tmp_path / f"dev-{index}{REUSE_RENDER_STATE_SUFFIX}" - for index in range(len(test_case.file_sizes)) - ] - for path, size in zip(stored_paths, test_case.file_sizes, strict=True): - path.write_bytes(b"x" * size) - listed: list[Path] = sorted(stored_paths) - monkeypatch.setattr(load_notice, "RENDER_LOAD_NOTICE_BYTES", test_case.notice_bytes) - monkeypatch.setattr(Path, "glob", lambda self, pattern: iter(listed)) - for path in listed[: test_case.removed_while_sizing]: - path.unlink() - - # Act - started: float | None = load_notice.start_load_notice(entry_path=entry_path) - - # Assert - assert (started is not None) is test_case.expected_notice - assert capsys.readouterr().out == "" diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py deleted file mode 100644 index bca44fbe3e..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Layered render storage must round-trip exactly and treat every fault as no stored renders.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( - read_render_state, - render_state_layer, -) -from sqlbuild.cli.compile_render_reuse.models import StoredRenderState -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState -from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( - RenderEditChainTestCase, - RenderLayerTestCase, - StoredRenderReadTestCase, -) -from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers.helpers import ( - MODEL_PATHS, - chain_payload_bytes, - changed_macro, - emptied, - flipped_last_byte, - intact, - next_render_state, - overlay_without_base, - payload_bytes, - store_edit_chain, - stored_base, - truncated, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderLayerTestCase( - description="one_model_changed", - changed_models=(MODEL_PATHS[3],), - removed_models=(), - expected_overlay=True, - ), - RenderLayerTestCase( - description="most_models_changed", - changed_models=MODEL_PATHS[:15], - removed_models=(), - expected_overlay=False, - ), - RenderLayerTestCase( - description="stored_render_dropped", - changed_models=(MODEL_PATHS[3],), - removed_models=(MODEL_PATHS[5],), - expected_overlay=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_released_reused_renders_when_storing_then_only_a_small_overlay_needs_no_rewrite( - test_case: RenderLayerTestCase, tmp_path: Path -) -> None: - _, stored = stored_base(tmp_path) - current: RenderReuseState = next_render_state( - stored=stored, edited=test_case.changed_models, removed=test_case.removed_models - ) - - layer: RenderStateLayer | None = render_state_layer(state=current, stored=stored.layers) - - assert (layer is not None) is test_case.expected_overlay - - -@pytest.mark.parametrize( - "test_case", - [ - RenderEditChainTestCase( - description="different_models_edited", - edits=(MODEL_PATHS[1], MODEL_PATHS[2]), - expected_overlay_models=frozenset({MODEL_PATHS[1], MODEL_PATHS[2]}), - ), - RenderEditChainTestCase( - description="same_model_edited_twice", - edits=(MODEL_PATHS[1], MODEL_PATHS[1]), - expected_overlay_models=frozenset({MODEL_PATHS[1]}), - ), - ], - ids=lambda case: case.description, -) -def test_given_overlay_when_storing_again_then_earlier_overlay_renders_are_kept( - test_case: RenderEditChainTestCase, tmp_path: Path -) -> None: - stored: StoredRenderState = store_edit_chain(directory=tmp_path, edits=test_case.edits) - - assert stored.layers.overlay_models == test_case.expected_overlay_models - assert payload_bytes(stored.state) == chain_payload_bytes(test_case.edits) - - -@pytest.mark.parametrize( - "test_case", - [ - StoredRenderReadTestCase( - description="intact_with_changed_model", prepare=intact, expected_readable=True - ), - StoredRenderReadTestCase( - description="flipped_payload_byte", prepare=flipped_last_byte, expected_readable=False - ), - StoredRenderReadTestCase( - description="truncated", prepare=truncated, expected_readable=False - ), - StoredRenderReadTestCase(description="empty", prepare=emptied, expected_readable=False), - StoredRenderReadTestCase( - description="overlay_whose_base_is_missing", - prepare=overlay_without_base, - expected_readable=False, - ), - StoredRenderReadTestCase( - description="changed_path_is_not_a_stored_model", - prepare=changed_macro, - expected_readable=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_render_file_when_reading_then_only_intact_relevant_renders_are_returned( - test_case: StoredRenderReadTestCase, tmp_path: Path -) -> None: - path, changed_paths = test_case.prepare(tmp_path) - - stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) - - assert (stored is not None) is test_case.expected_readable - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index 90ea98eb84..fd9b5d9c9c 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -8,9 +8,6 @@ import pytest -from sqlbuild.cli.compile_reuse.models import StoredProjectFile -from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import digested_project_files - @dataclass(frozen=True) class ProjectFingerprintTestCase: @@ -83,16 +80,6 @@ class SettingsDigestTestCase: expected_changed: bool -@dataclass(frozen=True) -class ChangedPathsTestCase: - """One project change and every project path a change set must name for it.""" - - description: str - change: Callable[[Path], object] - expected_paths: tuple[str, ...] - store: Callable[[Path], dict[str, StoredProjectFile]] = digested_project_files - - @dataclass(frozen=True) class EngineReuseDigestTestCase: """Two engine environments for one invocation and whether their reuse keys may match.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index dd03ca35e2..766bd626ac 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -131,12 +131,6 @@ def break_link(project_dir: Path) -> None: (project_dir.parent / "shared/customers.sql").unlink() -def digested_project_files(project_dir: Path) -> dict[str, StoredProjectFile]: - """Record the project with digests and a snapshot time that leaves no file racy.""" - - return stored_project_files(project_dir=project_dir, snapshot_ns=later_snapshot_ns()) - - def stat_only_project_files(*, project_dir: Path) -> dict[str, StoredProjectFile]: """Record the project as a stored compile does by default: stat stamps without digests.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py index cb38c6869c..1de9e2e1a5 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py @@ -10,7 +10,6 @@ from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, - changed_project_paths, compare_project_files, restamped_paths, snapshot_project_files, @@ -18,7 +17,6 @@ from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile from sqlbuild.cli.compile_reuse.types import FileStamp from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( - ChangedPathsTestCase, DigestCarryForwardTestCase, LinkCycleTestCase, ProjectFingerprintTestCase, @@ -149,68 +147,13 @@ def test_given_stored_fingerprint_when_project_changes_then_change_is_detected_e ) test_case.change(project_dir) - current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) comparison: ProjectFilesComparison = compare_project_files( - project_dir=str(project_dir), stored=stored, current=current - ) - changed, _ = changed_project_paths( - project_dir=str(project_dir), stored=stored, current=current, verified={} - ) - - assert comparison.unchanged is test_case.expected_unchanged - assert (not changed) is test_case.expected_unchanged - - -@pytest.mark.parametrize( - "test_case", - [ - ChangedPathsTestCase( - description="touched_without_change", - change=touch_without_change, - expected_paths=(), - ), - ChangedPathsTestCase( - description="model_edited", - change=lambda root: write_file(root / "models/orders.sql", "SELECT 10 AS order_id\n"), - expected_paths=("models/orders.sql",), - ), - ChangedPathsTestCase( - description="same_size_and_mtime_new_content", - change=rewrite_same_size_keeping_mtime, - expected_paths=("models/orders.sql",), - ), - ChangedPathsTestCase( - description="file_added_and_removed", - change=lambda root: ( - write_file(root / "models/returns.sql", "SELECT 1\n"), - (root / "macros/currency.py").unlink(), - ), - expected_paths=("macros/currency.py", "models/returns.sql"), - ), - ChangedPathsTestCase( - description="touched_without_stored_digest", - change=touch_without_change, - expected_paths=("models/orders.sql",), - store=lambda root: stat_only_project_files(project_dir=root), - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_fingerprint_when_listing_changes_then_returns_every_changed_path( - test_case: ChangedPathsTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - stored: dict[str, StoredProjectFile] = test_case.store(project_dir) - - test_case.change(project_dir) - changed, _ = changed_project_paths( project_dir=str(project_dir), stored=stored, current=snapshot_project_files(project_dir=str(project_dir)), - verified={}, ) - assert changed == frozenset(test_case.expected_paths) + assert comparison.unchanged is test_case.expected_unchanged @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/__init__.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py deleted file mode 100644 index 7ed01cf136..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Test case types for trusting artifacts a stored compile left.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class StoredArtifactTestCase: - """A change to a stored artifact and whether its stored digest may still be trusted.""" - - description: str - change: Callable[[Path], None] - written_contents: bytes - expected_holds: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py deleted file mode 100644 index d7998b411c..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Builders for stored compile artifact unit tests.""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -from sqlbuild.cli.compile_reuse.constants import DIGEST_SIZE_BYTES - -STORED_SQL: bytes = b"SELECT order_id FROM orders\n" - - -def artifact_unchanged(_path: Path) -> None: - """Leave the stored artifact as it is.""" - - -def artifact_rewritten(path: Path) -> None: - """Rewrite the stored artifact with other contents.""" - - path.write_bytes(b"SELECT order_id FROM returns\n") - - -def artifact_removed(path: Path) -> None: - """Remove the stored artifact.""" - - path.unlink() - - -def artifact_digest(contents: bytes) -> str: - """Return the digest compile reuse records for artifact contents.""" - - return hashlib.blake2b(contents, digest_size=DIGEST_SIZE_BYTES).hexdigest() diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py deleted file mode 100644 index cac7162a49..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py +++ /dev/null @@ -1,73 +0,0 @@ -"""A stored artifact is trusted only while its stat identity and digest both still match.""" - -from __future__ import annotations - -import os -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_reuse._helpers.target_files import path_stamp -from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts -from sqlbuild.cli.compile_reuse.types import FileStamp -from tests.unit.src.sqlbuild.cli.compile_reuse.classes._test_types import StoredArtifactTestCase -from tests.unit.src.sqlbuild.cli.compile_reuse.classes.helpers import ( - STORED_SQL, - artifact_digest, - artifact_removed, - artifact_rewritten, - artifact_unchanged, -) - - -@pytest.mark.parametrize( - "test_case", - [ - StoredArtifactTestCase( - description="unchanged_same_contents", - change=artifact_unchanged, - written_contents=STORED_SQL, - expected_holds=True, - ), - StoredArtifactTestCase( - description="unchanged_new_contents", - change=artifact_unchanged, - written_contents=b"SELECT 1 AS order_id\n", - expected_holds=False, - ), - StoredArtifactTestCase( - description="rewritten_since_stored", - change=artifact_rewritten, - written_contents=STORED_SQL, - expected_holds=False, - ), - StoredArtifactTestCase( - description="removed_since_stored", - change=artifact_removed, - written_contents=STORED_SQL, - expected_holds=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_artifact_when_writing_then_trusted_only_while_stamp_and_digest_match( - tmp_path: Path, test_case: StoredArtifactTestCase -) -> None: - artifact: Path = tmp_path / "compiled" / "orders.sql" - artifact.parent.mkdir() - artifact.write_bytes(STORED_SQL) - path: str = os.fspath(artifact) - stamp: FileStamp | None = path_stamp(path=path) - assert stamp is not None - stored: StoredArtifacts = StoredArtifacts( - stamps={path: stamp}, digests={path: artifact_digest(STORED_SQL)} - ) - test_case.change(artifact) - - holds: bool = stored.holds(path=path, digest=artifact_digest(test_case.written_contents)) - - assert holds is test_case.expected_holds - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py deleted file mode 100644 index 82cd55965b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Test case types for the compile render reuse session and stored model analyses.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.models import CompilerDiagnostic - - -@dataclass(frozen=True) -class RenderReplayTestCase: - """A reused render and what reusing it must report again.""" - - description: str - model_name: str - expected_diagnostics: tuple[CompilerDiagnostic, ...] - expected_environment_names: tuple[str, ...] - - -@dataclass(frozen=True) -class ReusableModelsTestCase: - """Stored and current models, and which current models may reuse their stored render.""" - - description: str - stored_models: tuple[str, ...] - current_models: tuple[str, ...] - changed_paths: frozenset[str] - run_id_readers: frozenset[str] - expected_reusable: dict[str, bool] - - -@dataclass(frozen=True) -class CorruptRenderTestCase: - """A stored render payload that cannot be decoded.""" - - description: str - model_name: str - expected_rendered_again: bool - - -@dataclass(frozen=True) -class ReleasedRenderTestCase: - """A reused and an edited render, and which stored bytes storing the compile releases.""" - - description: str - retained_models: frozenset[str] - expected_released: dict[str, bool] - expected_query_sqls: dict[str, str] - - -@dataclass(frozen=True) -class DeclarationReuseTestCase: - """Paths changed since a stored compile, and whether its declaration files are reused.""" - - description: str - changed_paths: frozenset[str] | None - recorded_variant: str - expected_full_discoveries: int - - -@dataclass(frozen=True) -class StoredAnalysisScopeTestCase: - """Which renders a compile reused, how analyses were stored, and which may be served.""" - - description: str - reused_models: frozenset[str] - rekeyed_models: frozenset[str] - twinned_models: frozenset[str] - stored_column_types: dict[str, dict[str, str]] - expected_served: frozenset[str] - - -@dataclass(frozen=True) -class StaleServedAnalysisTestCase: - """Final parent signatures in this compile, and which served analyses they make stale.""" - - description: str - current_signatures: dict[str, str] - expected_stale: set[str] diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py deleted file mode 100644 index 9482548c3a..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py +++ /dev/null @@ -1,342 +0,0 @@ -"""Shared builders for render reuse session and stored model analysis unit tests.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import replace -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - collect_compile_diagnostics, - report_compile_diagnostic, -) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.classes.stored_model_analyses import ( - StoredModelAnalyses, - analysis_reuse_context, -) -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompileModelInput, - CompilerDiagnostic, - CompileSqlReference, - ModelAnalysisCaching, - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, - RenderReuseState, - StoredModelAnalysis, - StoredRender, -) -from sqlbuild.compiler.compile.types import CompileContextKey, DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.discovery.models import ( - DiscoveredDeclarationFiles, - DiscoveredMacroFile, - DiscoveredSqlModelFile, -) -from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload -from sqlbuild.compiler.references.types import SqlReferenceKind - -REGION_ENV_VAR: str = "ORDERS_REGION" -ORDERS_CACHE: AnalysisCacheContext = AnalysisCacheContext( - root=Path("/orders/target/cache"), shared_fingerprint="orders-fingerprint" -) -RAW_ORDERS_TYPES: dict[str, dict[str, str]] = {"raw_orders": {"order_id": "INTEGER"}} - - -def model_file(name: str, sql: str = "SELECT 1 AS order_id") -> DiscoveredSqlModelFile: - """Return one discovered SQL model.""" - - return DiscoveredSqlModelFile( - file_path=Path("/orders") / "models" / f"{name}.sql", - relative_path=Path("models") / f"{name}.sql", - contents=sql, - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql=sql, - ) - - -def missing_description(name: str) -> CompilerDiagnostic: - """Return the warning a render of the named model reports.""" - - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.WARNING, - code="P010", - message=f"Model '{name}' has no description", - ) - - -def _render(item: DiscoveredSqlModelFile) -> CompileModelInput: - report_compile_diagnostic( - key=(item.file_path.stem,), diagnostic=missing_description(item.file_path.stem) - ) - COMPILE_INPUT_READS.environment_read(REGION_ENV_VAR) - return CompileModelInput(model_file=item, query_sql=item.query_sql) - - -def _render_reading_run_id(item: DiscoveredSqlModelFile) -> CompileModelInput: - COMPILE_INPUT_READS.context_read(CompileContextKey.RUN_ID) - return _render(item) - - -def recorded_state( - model_files: tuple[DiscoveredSqlModelFile, ...], - *, - run_id_readers: frozenset[str] = frozenset(), -) -> RenderReuseState: - """Render every model through a fresh session and return what it stores.""" - - session: CompileRenderReuseSession = CompileRenderReuseSession(prior=None, changed_paths=None) - session.plan_models(model_files=model_files) - renderers: dict[bool, Callable[[DiscoveredSqlModelFile], CompileModelInput]] = { - False: _render, - True: _render_reading_run_id, - } - with collect_compile_diagnostics(): - for item in model_files: - render: Callable[[DiscoveredSqlModelFile], CompileModelInput] = renderers[ - item.file_path.stem in run_id_readers - ] - _ = session.rendered_model(model_file=item, render=lambda item=item, r=render: r(item)) - state: RenderReuseState | None = session.stored_state() - assert state is not None - return state - - -def planned_session( - *, - stored: tuple[DiscoveredSqlModelFile, ...], - current: tuple[DiscoveredSqlModelFile, ...], - changed_paths: frozenset[str] = frozenset(), - run_id_readers: frozenset[str] = frozenset(), -) -> CompileRenderReuseSession: - """Record renders for the stored models, then plan a session for the current models.""" - - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=recorded_state(stored, run_id_readers=run_id_readers), changed_paths=changed_paths - ) - session.plan_models(model_files=current) - return session - - -def corrupted_session(item: DiscoveredSqlModelFile) -> CompileRenderReuseSession: - """Plan a session whose only stored render payload cannot be decoded.""" - - state: RenderReuseState = recorded_state((item,)) - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=replace( - state, - model_payloads={str(item.relative_path): memoryview(b"not a render")}, - ), - changed_paths=frozenset(), - ) - session.plan_models(model_files=(item,)) - return session - - -def edited_session(*, retained_models: frozenset[str]) -> CompileRenderReuseSession: - """Reuse the stored orders render and render an edited customers model.""" - - orders: DiscoveredSqlModelFile = model_file("orders") - customers: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=recorded_state((orders, model_file("customers"))), - changed_paths=frozenset({str(customers.relative_path)}), - retained_models=retained_models, - ) - session.plan_models(model_files=(orders, customers)) - with collect_compile_diagnostics(): - _ = session.reused_model(model_file=orders) - _ = session.rendered_model(model_file=customers, render=lambda: _render(customers)) - return session - - -def released_paths(state: RenderReuseState | None) -> dict[str, bool]: - """Return, per stored model, whether its bytes were released.""" - - assert state is not None - return {path: payload is None for path, payload in state.model_payloads.items()} - - -def stored_query_sqls(state: RenderReuseState | None) -> dict[str, str]: - """Decode every stored model render and return its query SQL.""" - - assert state is not None - return { - path: _stored_query_sql(cast(memoryview, payload)) - for path, payload in filter(lambda item: item[1] is not None, state.model_payloads.items()) - } - - -def _stored_query_sql(payload: memoryview) -> str: - stored: StoredRender = cast(StoredRender, loaded_fact_payload(payload)) - return cast(CompileModelInput, stored.value).query_sql - - -DECLARATIONS_VARIANT: str = "10" -OTHER_DECLARATIONS_VARIANT: str = "01" - - -def declaration_files( - model_files: tuple[DiscoveredSqlModelFile, ...], -) -> DiscoveredDeclarationFiles: - """Return discovered declaration files holding one macro file and the given models.""" - - return DiscoveredDeclarationFiles( - source_files=(), - model_files=model_files, - enum_files=(), - constant_files=(), - model_schema_files=(), - sql_function_files=(), - sql_hook_files=(), - python_function_files=(), - schema_files=(), - seed_files=(), - test_files=(), - scenario_files=(), - audit_files=(), - macro_files=( - DiscoveredMacroFile( - file_path=Path("/orders/macros/currency.py"), - relative_path=Path("macros/currency.py"), - contents="def total(price: str) -> str:\n return price\n", - ), - ), - adapter_file=None, - ) - - -class CountingDiscovery: - """Discover fixed declaration files and count full and model-only discoveries.""" - - def __init__(self, model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: - self.model_files: tuple[DiscoveredSqlModelFile, ...] = model_files - self.full: int = 0 - - def discover(self) -> DiscoveredDeclarationFiles: - """Discover every declaration file.""" - - self.full += 1 - return declaration_files(self.model_files) - - def discover_models(self) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover model files only.""" - - return self.model_files - - -def declarations_state(*, recorded_variant: str) -> RenderReuseState: - """Return stored renders of two models, with declaration files discovered for one variant.""" - - stored: tuple[DiscoveredSqlModelFile, ...] = (model_file("orders"), model_file("customers")) - session: CompileRenderReuseSession = CompileRenderReuseSession(prior=None, changed_paths=None) - _ = session.declaration_files( - variant=recorded_variant, - discover=CountingDiscovery(stored).discover, - discover_models=CountingDiscovery(stored).discover_models, - ) - session.plan_models(model_files=stored) - with collect_compile_diagnostics(): - for item in stored: - _ = session.rendered_model(model_file=item, render=lambda item=item: _render(item)) - state: RenderReuseState | None = session.stored_state() - assert state is not None - return state - - -def analyzed_model(name: str, *upstream: str) -> CompileModelInput: - """Return one model input reading the given upstream models.""" - - model: DiscoveredSqlModelFile = model_file(name) - return CompileModelInput( - model_file=model, - query_sql=model.contents, - references=tuple( - CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name=upstream_name) - for upstream_name in upstream - ), - ) - - -def stored_analysis( - *, name: str, context: str, dependencies: dict[str, str] -) -> StoredModelAnalysis: - """Return the analysis a previous compile stored for one model.""" - - return StoredModelAnalysis( - context=context, - cache_key=f"{name}-key", - analysis=PolyglotAnalysisResult(analysis_succeeded=True), - output_signature=f"{name}-signature", - dependencies=dependencies, - signature=f"{name}-signature", - ) - - -def analysis_request(model_input: CompileModelInput, cache_key: str) -> ModelSqlAnalysisRequest: - """Return the analysis request of one model with the given cache key.""" - - return ModelSqlAnalysisRequest( - model_input=model_input, - query_sql=model_input.query_sql, - placeholders=None, - cache_key=cache_key, - binding_schema=None, - ) - - -def orders_analysis_context( - *, requests: tuple[ModelSqlAnalysisRequest, ...], column_types: dict[str, dict[str, str]] -) -> str: - """Return the shared analysis inputs digest of the orders project with these source types.""" - - return analysis_reuse_context( - cache=ORDERS_CACHE, - model_names=frozenset(item.model_input.model_file.file_path.stem for item in requests), - column_types_by_table=column_types, - column_nullability_by_table={}, - complete_binding_schemas={}, - ) - - -def orders_stored_analyses( - *, requests: tuple[ModelSqlAnalysisRequest, ...], reuse: RecordingAnalysisReuse -) -> StoredModelAnalyses: - """Return the stored analyses a compile of the orders project may serve.""" - - return StoredModelAnalyses( - caching=ModelAnalysisCaching(cache=ORDERS_CACHE, reuse=reuse), - requests=requests, - column_types_by_table=RAW_ORDERS_TYPES, - column_nullability_by_table={}, - complete_binding_schemas={}, - ) - - -class RecordingAnalysisReuse: - """Stored analyses and reused renders of a previous compile, recording new analyses.""" - - def __init__(self, *, reused: frozenset[str], stored: dict[str, StoredModelAnalysis]) -> None: - self._reused: frozenset[str] = reused - self._stored: dict[str, StoredModelAnalysis] = stored - self.recorded: dict[str, StoredModelAnalysis | None] | None = None - - def reused_model_names(self) -> frozenset[str]: - """Return the names of models whose renders were reused.""" - - return self._reused - - def stored_analyses(self) -> dict[str, StoredModelAnalysis]: - """Return the stored analyses by model name.""" - - return self._stored - - def record_analyses(self, *, analyses: dict[str, StoredModelAnalysis | None]) -> None: - """Keep what the compile records.""" - - self.recorded = analyses diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py deleted file mode 100644 index 1fd8865668..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Render reuse must replay what an unchanged render reported and never serve a changed one.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.models import CompileModelInput -from sqlbuild.compiler.discovery.models import DiscoveredDeclarationFiles, DiscoveredSqlModelFile -from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( - CorruptRenderTestCase, - DeclarationReuseTestCase, - ReleasedRenderTestCase, - RenderReplayTestCase, - ReusableModelsTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( - DECLARATIONS_VARIANT, - OTHER_DECLARATIONS_VARIANT, - REGION_ENV_VAR, - CountingDiscovery, - corrupted_session, - declaration_files, - declarations_state, - edited_session, - missing_description, - model_file, - planned_session, - released_paths, - stored_query_sqls, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderReplayTestCase( - description="unchanged_model", - model_name="orders", - expected_diagnostics=(missing_description("orders"),), - expected_environment_names=(REGION_ENV_VAR,), - ), - ], - ids=lambda case: case.description, -) -def test_given_unchanged_model_when_reusing_then_render_and_reports_are_replayed( - test_case: RenderReplayTestCase, -) -> None: - unchanged: DiscoveredSqlModelFile = model_file(test_case.model_name) - edited: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") - session: CompileRenderReuseSession = planned_session( - stored=(unchanged, model_file("customers")), - current=(unchanged, edited), - changed_paths=frozenset({str(edited.relative_path)}), - ) - - with ( - COMPILE_INPUT_READS.recording() as reads, - collect_compile_diagnostics() as collected, - ): - reused: CompileModelInput | None = session.reused_model(model_file=unchanged) - - assert reused == CompileModelInput(model_file=unchanged, query_sql=unchanged.query_sql) - assert reused is not None - assert reused.model_file is unchanged - assert collected.diagnostics == test_case.expected_diagnostics - assert reads.environment_names == test_case.expected_environment_names - - -@pytest.mark.parametrize( - "test_case", - [ - ReusableModelsTestCase( - description="changed_model", - stored_models=("orders", "customers"), - current_models=("orders", "customers"), - changed_paths=frozenset({"models/customers.sql"}), - run_id_readers=frozenset(), - expected_reusable={"orders": True, "customers": False}, - ), - ReusableModelsTestCase( - description="render_read_run_identity", - stored_models=("orders", "customers"), - current_models=("orders", "customers"), - changed_paths=frozenset(), - run_id_readers=frozenset({"orders"}), - expected_reusable={"orders": False, "customers": True}, - ), - ReusableModelsTestCase( - description="model_added", - stored_models=("orders",), - current_models=("orders", "customers"), - changed_paths=frozenset({"models/customers.sql"}), - run_id_readers=frozenset(), - expected_reusable={"orders": False, "customers": False}, - ), - ReusableModelsTestCase( - description="model_removed", - stored_models=("orders", "customers", "returns"), - current_models=("orders", "customers"), - changed_paths=frozenset(), - run_id_readers=frozenset(), - expected_reusable={"orders": False, "customers": False}, - ), - ReusableModelsTestCase( - description="non_model_input_changed", - stored_models=("orders", "customers"), - current_models=("orders", "customers"), - changed_paths=frozenset({"macros/currency.py"}), - run_id_readers=frozenset(), - expected_reusable={"orders": False, "customers": False}, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_renders_when_planning_then_only_unchanged_identical_model_sets_reuse( - test_case: ReusableModelsTestCase, -) -> None: - current: tuple[DiscoveredSqlModelFile, ...] = tuple(map(model_file, test_case.current_models)) - session: CompileRenderReuseSession = planned_session( - stored=tuple(map(model_file, test_case.stored_models)), - current=current, - changed_paths=test_case.changed_paths, - run_id_readers=test_case.run_id_readers, - ) - - reusable: dict[str, bool] = { - item.file_path.stem: session.has_reusable_model(model_file=item) for item in current - } - - assert reusable == test_case.expected_reusable - - -@pytest.mark.parametrize( - "test_case", - [ - CorruptRenderTestCase( - description="undecodable_payload", model_name="orders", expected_rendered_again=True - ), - ], - ids=lambda case: case.description, -) -def test_given_corrupt_stored_render_when_reusing_then_the_model_is_rendered_again( - test_case: CorruptRenderTestCase, -) -> None: - item: DiscoveredSqlModelFile = model_file(test_case.model_name) - session: CompileRenderReuseSession = corrupted_session(item) - - with collect_compile_diagnostics(): - reused: CompileModelInput | None = session.reused_model(model_file=item) - - assert (reused is None) is test_case.expected_rendered_again - - -@pytest.mark.parametrize( - "test_case", - [ - ReleasedRenderTestCase( - description="reused_from_base", - retained_models=frozenset(), - expected_released={"models/orders.sql": True, "models/customers.sql": False}, - expected_query_sqls={ - "models/orders.sql": "SELECT 1 AS order_id", - "models/customers.sql": "SELECT 2 AS customer_id", - }, - ), - ReleasedRenderTestCase( - description="reused_from_overlay", - retained_models=frozenset({"models/orders.sql"}), - expected_released={"models/orders.sql": False, "models/customers.sql": False}, - expected_query_sqls={ - "models/orders.sql": "SELECT 1 AS order_id", - "models/customers.sql": "SELECT 2 AS customer_id", - }, - ), - ], - ids=lambda case: case.description, -) -def test_given_reused_render_when_storing_then_base_bytes_are_released_and_recoverable( - test_case: ReleasedRenderTestCase, -) -> None: - session: CompileRenderReuseSession = edited_session(retained_models=test_case.retained_models) - - released: dict[str, bool] = released_paths(session.stored_state()) - complete: dict[str, str] = stored_query_sqls(session.stored_state(complete=True)) - - assert released == test_case.expected_released - assert complete == test_case.expected_query_sqls - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationReuseTestCase( - description="model_edited", - changed_paths=frozenset({"models/customers.sql"}), - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=0, - ), - DeclarationReuseTestCase( - description="model_added", - changed_paths=frozenset({"models/returns.sql"}), - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - DeclarationReuseTestCase( - description="macro_edited", - changed_paths=frozenset({"macros/currency.py"}), - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - DeclarationReuseTestCase( - description="changes_unknown", - changed_paths=None, - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - DeclarationReuseTestCase( - description="declarations_stored_for_other_discovery", - changed_paths=frozenset({"models/customers.sql"}), - recorded_variant=OTHER_DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_declarations_when_discovering_then_only_model_only_edits_reuse_them( - test_case: DeclarationReuseTestCase, -) -> None: - current: tuple[DiscoveredSqlModelFile, ...] = ( - model_file("orders"), - model_file("customers", "SELECT 2 AS customer_id"), - ) - discovery: CountingDiscovery = CountingDiscovery(current) - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=declarations_state(recorded_variant=test_case.recorded_variant), - changed_paths=test_case.changed_paths, - ) - - with collect_compile_diagnostics(): - discovered: DiscoveredDeclarationFiles = session.declaration_files( - variant=DECLARATIONS_VARIANT, - discover=discovery.discover, - discover_models=discovery.discover_models, - ) - - assert discovered == declaration_files(current) - assert discovery.full == test_case.expected_full_discoveries - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py deleted file mode 100644 index 79d647a281..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py +++ /dev/null @@ -1,166 +0,0 @@ -"""A stored analysis is served only while its inputs and its parents' final signatures match.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses -from sqlbuild.compiler.compile.models import ( - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, - StoredModelAnalysis, -) -from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( - StaleServedAnalysisTestCase, - StoredAnalysisScopeTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( - RAW_ORDERS_TYPES, - RecordingAnalysisReuse, - analysis_request, - analyzed_model, - orders_analysis_context, - orders_stored_analyses, - stored_analysis, -) - -_CHAIN: tuple[ModelSqlAnalysisRequest, ...] = ( - analysis_request(analyzed_model("stg_orders"), "stg_orders-key"), - analysis_request(analyzed_model("int_orders", "stg_orders"), "int_orders-key"), - analysis_request(analyzed_model("fct_orders", "int_orders"), "fct_orders-key"), -) -_CHAIN_DEPENDENCIES: dict[str, dict[str, str]] = { - "stg_orders": {}, - "int_orders": {"stg_orders": "stg_orders-signature"}, - "fct_orders": {"int_orders": "int_orders-signature"}, -} -_STORED_SIGNATURES: dict[str, str] = { - "stg_orders": "stg_orders-signature", - "int_orders": "int_orders-signature", - "fct_orders": "fct_orders-signature", -} -_ALL_MODELS: frozenset[str] = frozenset(_CHAIN_DEPENDENCIES) - - -@pytest.mark.parametrize( - "test_case", - [ - StoredAnalysisScopeTestCase( - description="nothing_changed", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset(), - twinned_models=frozenset(), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=_ALL_MODELS, - ), - StoredAnalysisScopeTestCase( - description="root_edited", - reused_models=frozenset({"int_orders", "fct_orders"}), - rekeyed_models=frozenset(), - twinned_models=frozenset(), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=frozenset({"int_orders", "fct_orders"}), - ), - StoredAnalysisScopeTestCase( - description="cache_key_changed", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset({"int_orders"}), - twinned_models=frozenset(), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=frozenset({"stg_orders", "fct_orders"}), - ), - StoredAnalysisScopeTestCase( - description="cache_key_shared_with_another_model", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset(), - twinned_models=frozenset({"fct_orders"}), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=frozenset({"stg_orders", "int_orders"}), - ), - StoredAnalysisScopeTestCase( - description="source_types_changed", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset(), - twinned_models=frozenset(), - stored_column_types={"raw_orders": {"order_id": "BIGINT"}}, - expected_served=frozenset(), - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_analyses_when_scoping_reuse_then_only_identical_inputs_are_served( - test_case: StoredAnalysisScopeTestCase, -) -> None: - context: str = orders_analysis_context( - requests=_CHAIN, column_types=test_case.stored_column_types - ) - stored: dict[str, StoredModelAnalysis] = { - name: stored_analysis(name=name, context=context, dependencies=dependencies) - for name, dependencies in _CHAIN_DEPENDENCIES.items() - } - requests: tuple[ModelSqlAnalysisRequest, ...] = tuple( - analysis_request( - request.model_input, - f"{request.model_input.model_file.file_path.stem}-key" - + "-edited" - * (request.model_input.model_file.file_path.stem in test_case.rekeyed_models), - ) - for request in _CHAIN - ) + tuple( - analysis_request(analyzed_model(f"{name}_twin", "int_orders"), f"{name}-key") - for name in sorted(test_case.twinned_models) - ) - - served: StoredModelAnalyses = orders_stored_analyses( - requests=requests, - reuse=RecordingAnalysisReuse(reused=test_case.reused_models, stored=stored), - ) - - assert frozenset(served.served) == test_case.expected_served - - -@pytest.mark.parametrize( - "test_case", - [ - StaleServedAnalysisTestCase( - description="parents_unchanged", - current_signatures=_STORED_SIGNATURES, - expected_stale=set(), - ), - StaleServedAnalysisTestCase( - description="root_signature_changed", - current_signatures={**_STORED_SIGNATURES, "stg_orders": "stg_orders-widened"}, - expected_stale={"int_orders"}, - ), - StaleServedAnalysisTestCase( - description="middle_signature_changed", - current_signatures={**_STORED_SIGNATURES, "int_orders": "int_orders-widened"}, - expected_stale={"fct_orders"}, - ), - ], - ids=lambda case: case.description, -) -def test_given_served_analyses_when_parents_are_final_then_changed_parents_make_children_stale( - test_case: StaleServedAnalysisTestCase, -) -> None: - context: str = orders_analysis_context(requests=_CHAIN, column_types=RAW_ORDERS_TYPES) - stored: dict[str, StoredModelAnalysis] = { - name: stored_analysis(name=name, context=context, dependencies=dependencies) - for name, dependencies in _CHAIN_DEPENDENCIES.items() - } - served: StoredModelAnalyses = orders_stored_analyses( - requests=_CHAIN, reuse=RecordingAnalysisReuse(reused=_ALL_MODELS, stored=stored) - ) - cached: dict[str, PolyglotAnalysisResult] = { - analysis.cache_key: analysis.analysis for analysis in stored.values() - } - - stale: set[str] = served.stale_names( - requests=_CHAIN, cached=cached, current_signatures=test_case.current_signatures - ) - - assert stale == test_case.expected_stale - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py b/tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py deleted file mode 100644 index 89bbeb8c34..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache tests.""" diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py deleted file mode 100644 index 611c1f64d4..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache class tests.""" diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py deleted file mode 100644 index 78a760c91b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class FactCacheReadTestCase: - """One read of a fact published for the ("orders", "SELECT 1") inputs.""" - - description: str - read_key_parts: tuple[str, ...] - read_algorithm: str - expected_hit: bool - expected_metrics: tuple[int, int] - - -@dataclass(frozen=True) -class FactCacheCodeIdentityTestCase: - """One change to the installed code that produced a published fact.""" - - description: str - next_code_identity: str - expected_found_count: int - - -@dataclass(frozen=True) -class FactCacheCorruptionTestCase: - """One way a persisted fact row can become untrustworthy.""" - - description: str - corrupt: Callable[[Path], None] - expected_found_count: int - expected_republished_hit: bool - - -@dataclass(frozen=True) -class FactCacheRetentionTestCase: - """A sequence of (key parts, slot) publications and the rows that must remain.""" - - description: str - publications: tuple[tuple[tuple[str, ...], str], ...] - stale_key_parts: tuple[str, ...] - expected_slots: list[str] - expected_stale_found_count: int - - -@dataclass(frozen=True) -class FactCacheNoPersistenceTestCase: - """One invocation shape that must leave no fact database behind.""" - - description: str - expected_enabled: bool - expected_database_files: list[Path] - - -@dataclass(frozen=True) -class FactCacheWriterErrorTestCase: - """One SQLite error raised while publishing after an earlier fact was stored.""" - - description: str - error: Exception - expected_earlier_found_count: int - - -@dataclass(frozen=True) -class FactCacheParameterLimitTestCase: - """One SQLite bound-parameter limit and a publication larger than one statement allows.""" - - description: str - max_bound_parameters: int - fact_count: int - expected_stored_count: int diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py deleted file mode 100644 index 853b5dc9b3..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Helpers for fact cache store unit tests.""" - -from __future__ import annotations - -import pickle -import sqlite3 -from contextlib import closing -from pathlib import Path - -from sqlbuild.compiler.fact_cache._helpers.publication import ( - await_fact_publication, - entry_digest, -) -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore -from sqlbuild.spec.contracts.models import SourceLocation - -FACT_NAMESPACE: str = "unit" -FACT_ALGORITHM: str = "unit-facts-v1" -FACT_VALUE: tuple[SourceLocation, ...] = ( - SourceLocation(path=Path("models/orders.sql"), line=3, column=5, end_line=3, end_column=12), -) - - -def fact_database(root: Path) -> Path: - return next(root.rglob(f"{FACT_NAMESPACE}.sqlite3")) - - -def publish_fact( - root: Path, *, key_parts: tuple[str, ...], slot: str, value: object = FACT_VALUE -) -> str: - with FactCacheStore(root=root, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store: - key: str = store.key(*key_parts) - store.stage(key=key, slot=slot, value=value) - await_fact_publication() - return key - - -def read_fact( - root: Path | None, - *, - key_parts: tuple[str, ...], - slot: str = "orders", - algorithm: str = FACT_ALGORITHM, -) -> dict[str, object]: - with FactCacheStore(root=root, namespace=FACT_NAMESPACE, algorithm=algorithm) as store: - return store.read_many(((slot, store.key(*key_parts)),)) - - -def stored_fact_slots(root: Path) -> list[str]: - with closing(sqlite3.connect(fact_database(root))) as connection: - rows: list[tuple[str]] = connection.execute( - "SELECT slot FROM fact ORDER BY slot" - ).fetchall() - return [slot for (slot,) in rows] - - -def _overwrite_verified_payload(database: Path, payload: bytes) -> None: - with closing(sqlite3.connect(database)) as connection, connection: - cache_key: str = connection.execute("SELECT cache_key FROM fact").fetchone()[0] - digest: str = entry_digest(cache_key=cache_key, payload=payload) - _ = connection.execute("UPDATE fact SET payload = ?, digest = ?", (payload, digest)) - - -def flip_payload_byte(root: Path) -> None: - database: Path = fact_database(root) - with closing(sqlite3.connect(database)) as connection, connection: - payload: bytes = connection.execute("SELECT payload FROM fact").fetchone()[0] - _ = connection.execute("UPDATE fact SET payload = ?", (payload[:-1] + b"\x00",)) - - -def write_untrusted_global(root: Path) -> None: - _overwrite_verified_payload(fact_database(root), pickle.dumps(Path.home, protocol=5)) - - -def truncate_verified_payload(root: Path) -> None: - _overwrite_verified_payload(fact_database(root), b"\x80\x05") - - -def replace_database_with_garbage(root: Path) -> None: - fact_database(root).write_bytes(b"not a sqlite database") - - -def publish_facts(root: Path, *, slots: tuple[str, ...]) -> None: - with FactCacheStore(root=root, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store: - for slot in slots: - store.stage(key=store.key(slot), slot=slot, value=FACT_VALUE) - await_fact_publication() diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py deleted file mode 100644 index 69d738f151..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py +++ /dev/null @@ -1,314 +0,0 @@ -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable -from pathlib import Path - -import pytest - -from sqlbuild.compiler.fact_cache._helpers import publication -from sqlbuild.compiler.fact_cache.classes import fact_cache_store -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore -from sqlbuild.compiler.profiling.main.collect import collect_compile_timings -from tests.unit.src.sqlbuild.compiler.fact_cache.classes._test_types import ( - FactCacheCodeIdentityTestCase, - FactCacheCorruptionTestCase, - FactCacheNoPersistenceTestCase, - FactCacheParameterLimitTestCase, - FactCacheReadTestCase, - FactCacheRetentionTestCase, - FactCacheWriterErrorTestCase, -) -from tests.unit.src.sqlbuild.compiler.fact_cache.classes.helpers import ( - FACT_ALGORITHM, - FACT_NAMESPACE, - FACT_VALUE, - flip_payload_byte, - publish_fact, - publish_facts, - read_fact, - replace_database_with_garbage, - stored_fact_slots, - truncate_verified_payload, - write_untrusted_global, -) - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheReadTestCase( - description="same_inputs_hit", - read_key_parts=("orders", "SELECT 1"), - read_algorithm=FACT_ALGORITHM, - expected_hit=True, - expected_metrics=(1, 0), - ), - FactCacheReadTestCase( - description="changed_input_misses", - read_key_parts=("orders", "SELECT 2"), - read_algorithm=FACT_ALGORITHM, - expected_hit=False, - expected_metrics=(0, 1), - ), - FactCacheReadTestCase( - description="extra_empty_input_misses", - read_key_parts=("orders", "SELECT 1", ""), - read_algorithm=FACT_ALGORITHM, - expected_hit=False, - expected_metrics=(0, 1), - ), - FactCacheReadTestCase( - description="concatenated_inputs_miss", - read_key_parts=("ordersSELECT 1",), - read_algorithm=FACT_ALGORITHM, - expected_hit=False, - expected_metrics=(0, 1), - ), - FactCacheReadTestCase( - description="changed_algorithm_misses", - read_key_parts=("orders", "SELECT 1"), - read_algorithm="unit-facts-v2", - expected_hit=False, - expected_metrics=(0, 1), - ), - ), - ids=lambda case: case.description, -) -def test_given_published_fact_when_reading_then_hits_only_for_identical_inputs( - tmp_path: Path, test_case: FactCacheReadTestCase -) -> None: - key: str = publish_fact(tmp_path, key_parts=("orders", "SELECT 1"), slot="orders") - - with collect_compile_timings() as timings: - found: dict[str, object] = read_fact( - tmp_path, key_parts=test_case.read_key_parts, algorithm=test_case.read_algorithm - ) - - assert (found == {key: FACT_VALUE}) is test_case.expected_hit - assert ( - timings.metrics["fact_cache_hits"], - timings.metrics["fact_cache_misses"], - ) == test_case.expected_metrics - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheCodeIdentityTestCase( - description="next_release", next_code_identity="next-release", expected_found_count=0 - ), - ), - ids=lambda case: case.description, -) -def test_given_published_fact_when_producing_code_changes_then_misses( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: FactCacheCodeIdentityTestCase -) -> None: - _ = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - monkeypatch.setattr( - fact_cache_store, "installed_code_identity", lambda: test_case.next_code_identity - ) - - assert len(read_fact(tmp_path, key_parts=("orders",))) == test_case.expected_found_count - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheCorruptionTestCase( - description="payload_bytes_changed", - corrupt=flip_payload_byte, - expected_found_count=0, - expected_republished_hit=True, - ), - FactCacheCorruptionTestCase( - description="verified_payload_references_function", - corrupt=write_untrusted_global, - expected_found_count=0, - expected_republished_hit=True, - ), - FactCacheCorruptionTestCase( - description="verified_payload_truncated", - corrupt=truncate_verified_payload, - expected_found_count=0, - expected_republished_hit=True, - ), - FactCacheCorruptionTestCase( - description="database_file_replaced", - corrupt=replace_database_with_garbage, - expected_found_count=0, - expected_republished_hit=True, - ), - ), - ids=lambda case: case.description, -) -def test_given_corrupted_fact_when_reading_then_misses_and_next_publication_repairs_it( - tmp_path: Path, test_case: FactCacheCorruptionTestCase -) -> None: - _ = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - test_case.corrupt(tmp_path) - - assert len(read_fact(tmp_path, key_parts=("orders",))) == test_case.expected_found_count - - key: str = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - assert ( - read_fact(tmp_path, key_parts=("orders",)) == {key: FACT_VALUE} - ) is test_case.expected_republished_hit - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheRetentionTestCase( - description="same_slot_replaced", - publications=( - (("orders", "v1"), "orders"), - (("orders", "v2"), "orders"), - (("customers", "v1"), "customers"), - ), - stale_key_parts=("orders", "v1"), - expected_slots=["customers", "orders"], - expected_stale_found_count=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_new_fact_for_same_slot_when_publishing_then_previous_fact_is_pruned( - tmp_path: Path, test_case: FactCacheRetentionTestCase -) -> None: - for key_parts, slot in test_case.publications: - _ = publish_fact(tmp_path, key_parts=key_parts, slot=slot) - - assert stored_fact_slots(tmp_path) == test_case.expected_slots - assert ( - len(read_fact(tmp_path, key_parts=test_case.stale_key_parts)) - == test_case.expected_stale_found_count - ) - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheNoPersistenceTestCase( - description="disabled_root", expected_enabled=False, expected_database_files=[] - ), - ), - ids=lambda case: case.description, -) -def test_given_disabled_root_when_staging_then_nothing_is_persisted( - tmp_path: Path, test_case: FactCacheNoPersistenceTestCase -) -> None: - with FactCacheStore(root=None, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store: - store.stage(key=store.key("orders"), slot="orders", value=FACT_VALUE) - assert store.enabled is test_case.expected_enabled - assert store.read_many((("orders", store.key("orders")),)) == {} - - assert list(tmp_path.iterdir()) == test_case.expected_database_files - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheNoPersistenceTestCase( - description="failed_invocation", expected_enabled=True, expected_database_files=[] - ), - ), - ids=lambda case: case.description, -) -def test_given_failed_invocation_when_exiting_then_staged_facts_are_discarded( - tmp_path: Path, test_case: FactCacheNoPersistenceTestCase -) -> None: - with ( - pytest.raises(RuntimeError), - FactCacheStore(root=tmp_path, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store, - ): - assert store.enabled is test_case.expected_enabled - store.stage(key=store.key("orders"), slot="orders", value=FACT_VALUE) - raise RuntimeError("compile failed") - - assert list(tmp_path.rglob("*.sqlite3")) == test_case.expected_database_files - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheWriterErrorTestCase( - description="busy_database_keeps_rows", - error=sqlite3.OperationalError("database is locked"), - expected_earlier_found_count=1, - ), - FactCacheWriterErrorTestCase( - description="integrity_error_keeps_rows", - error=sqlite3.IntegrityError("constraint failed"), - expected_earlier_found_count=1, - ), - FactCacheWriterErrorTestCase( - description="interface_error_keeps_rows", - error=sqlite3.InterfaceError("bad parameter"), - expected_earlier_found_count=1, - ), - FactCacheWriterErrorTestCase( - description="unreadable_database_is_replaced", - error=sqlite3.DatabaseError("file is not a database"), - expected_earlier_found_count=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_writer_error_when_publishing_then_only_corruption_discards_rows_and_nothing_escapes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: FactCacheWriterErrorTestCase -) -> None: - _ = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - escaped: list[threading.ExceptHookArgs] = [] - monkeypatch.setattr(threading, "excepthook", escaped.append) - - def failing_insert(**_: object) -> None: - raise test_case.error - - monkeypatch.setattr(publication, "_insert_records", failing_insert) - _ = publish_fact(tmp_path, key_parts=("customers",), slot="customers") - monkeypatch.undo() - - assert escaped == [] - assert len(read_fact(tmp_path, key_parts=("orders",))) == ( - test_case.expected_earlier_found_count - ) - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheParameterLimitTestCase( - description="sqlite_before_3_32", - max_bound_parameters=999, - fact_count=600, - expected_stored_count=600, - ), - ), - ids=lambda case: case.description, -) -def test_given_sqlite_parameter_limit_when_publishing_many_facts_then_all_are_stored( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: FactCacheParameterLimitTestCase -) -> None: - connect: Callable[..., sqlite3.Connection] = sqlite3.connect - - def limited_connect(database: Path, timeout: float = 5.0) -> sqlite3.Connection: - connection: sqlite3.Connection = connect(database, timeout=timeout) - _ = connection.setlimit( - sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, test_case.max_bound_parameters - ) - return connection - - monkeypatch.setattr(sqlite3, "connect", limited_connect) - slots: tuple[str, ...] = tuple(f"order_{index:04d}" for index in range(test_case.fact_count)) - - publish_facts(tmp_path, slots=slots) - - stored: list[str] = stored_fact_slots(tmp_path) - assert len(stored) == test_case.expected_stored_count - assert stored == list(slots) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py index 0507124821..4c0da94975 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -6,8 +6,8 @@ from enum import StrEnum from pathlib import Path, PurePosixPath +from sqlbuild.compiler.compile._helpers.attachment.target import compile_cache_root from sqlbuild.compiler.compile.models import CompactLineageFacts -from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index b07bbd8c1c..f1be1b4fac 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -113,36 +113,27 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data, the project has very many files, or it records -thousands of new model renders, compile prints `Recording compile for reuse...` and a completion -line to stderr before the report. - -### Compiling after model-only edits - -When the only changes since the stored compile are edits to existing model files, compile runs in -full but reuses the stored render of every unchanged model, together with the diagnostics and -environment reads that rendering reported. Only the edited models are rendered again. Declaration -files other than models are not read again, and an unchanged model reuses its stored column -analysis only when the analyses of its upstream models in this compile match the ones it was stored -with. Contracts, semantic validation, and rules still run for the whole project, and a model whose -upstream analyses changed is analyzed again, so downstream effects of an edit, such as a changed -column type, are reported exactly as in an uncached compile, even when another command such as -`sqb build` ran in between. Compiled -files that still hold the bytes the stored compile wrote are not read or rewritten. The output is -byte-identical to `sqb compile --no-cache`. - -Every other change renders the whole project: adding, removing, or renaming a model, and editing -macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope -folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings -is always rendered again. - -Renders are stored only when a compile result for the same project and target is already stored, -so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. -The next full compile stores them, and edits after that reuse them. Stored renders are compressed -and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints -`Loading stored renders...` and a completion line to stderr before it starts. While renders are -reused, the Python modules recorded by the previous compile stay tracked even if this compile did -not import them, so editing such a module runs the next compile in full. +report is ready. When it has to hash a lot of data or the project has very many files, compile +prints `Recording compile for reuse...` and a completion line to stderr before the report. + +### Compiling after an edit + +Any change runs the whole compile again: every model, test, and audit is rendered again, and +contracts and semantic validation run for the whole project. No render is carried over from the +previous compile, so until the native compiler ships, an edited model is recompiled without any +cache. A few caches under `target/cache/` shorten the rest: + +- An unchanged model takes its column analysis from the analysis cache when its query and the + analyses of its upstream models are unchanged. +- An unchanged SQL test file reuses its stored test scan, the CTEs and expected models read from + it, from `sql-test-scans.bin`. Entries are keyed by the file's expanded text, the adapter's SQL + lexical rules, and the SQLBuild build; a damaged or unreadable entry is scanned again. +- Rule results are reused for unchanged inputs. +- Macro call results may be reused as described in + [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). + +An edit therefore costs a full render of the project plus the analysis, test scans, and rules of +what changed. The output is byte-identical to `sqb compile --no-cache`. Known limits: @@ -163,8 +154,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or -rendered again after model-only edits and `analysis_reuse_hits` for stored model analyses reused. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings and cache counters such as `sql_test_scan_cache_hits` and +`sql_test_scan_cache_misses`. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 8e3fda3c32..a79d1fcfff 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -212,7 +212,7 @@ files or environment variables that the call does not receive. SQLBuild relies on this contract to reuse macro results: -- [Compile reuse](/docs/cli/compile) replays stored renders without running macros again. +- [Compile reuse](/docs/cli/compile) replays an unchanged compile without running macros again. - A compile may run each distinct macro call once and reuse its result for identical calls elsewhere in the project. - Macro call results may be kept between compiles under `target/cache/`. They are recomputed when