From 12a8e6658e2c72019e233724eb0aa3e24ffacd8a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:41:26 +0100 Subject: [PATCH 1/5] refactor(compile): drop fine-grained edit reuse in favour of coarse caches Remove render reuse after model-only edits, the discovery, analysis and artifact-write reuse built on it, and the per-file compile fact cache. Every edit now compiles the whole project; whole-project no-change reuse, the analysis cache, the macro-call store and the rules caches remain. The expanded SQL-test extraction and expected-model scans run uncached; both are lexical native scans. Rule caches take the code identity from the rule engine. The edit-sequence oracle covers the remaining caches, including sequences that advance the analysis cache between compiles and sabotaged invalidation that the oracle must detect. --- fensu.toml | 6 - scripts/compiler_differential/constants.py | 1 - .../sqlbuild/references/docs/cli/compile.md | 46 +- .../references/docs/concepts/macros.md | 2 +- .../cli/commands/_helpers/compile/pipeline.py | 4 +- .../_helpers/compile/target_writer.py | 6 +- .../cli/commands/main/project/_compile.py | 13 +- .../cli/compile_render_reuse/__init__.py | 1 - .../compile_render_reuse/_helpers/__init__.py | 0 .../_helpers/load_notice.py | 49 -- .../_helpers/render_state_file.py | 230 --------- .../cli/compile_render_reuse/constants.py | 14 - .../cli/compile_render_reuse/main/__init__.py | 0 .../main/_activated_render_reuse.py | 20 - .../main/_render_reuse_session.py | 53 -- .../main/_render_state_layer.py | 23 - .../cli/compile_render_reuse/models.py | 48 -- .../cli/compile_reuse/_helpers/attempt.py | 23 +- .../cli/compile_reuse/_helpers/entry_file.py | 106 +--- .../compile_reuse/_helpers/project_files.py | 31 -- .../_helpers/runtime_identity.py | 13 - .../cli/compile_reuse/_helpers/store.py | 55 +-- .../compile_reuse/_helpers/target_files.py | 10 +- .../compile_artifact_write_registry.py | 26 +- .../compile_reuse/classes/stored_artifacts.py | 29 -- src/sqlbuild/cli/compile_reuse/constants.py | 5 +- .../cli/compile_reuse/main/_framed_section.py | 11 - .../main/_read_framed_section.py | 13 - .../main/_write_reusable_compile.py | 3 - src/sqlbuild/cli/compile_reuse/models.py | 16 - .../compiler/analysis_session/models.py | 4 +- .../_helpers/assembly/binding_waves.py | 18 +- .../compile/_helpers/assembly/project.py | 126 +++-- .../_helpers/assembly/semantic_shapes.py | 12 + .../compile/_helpers/attachment/audits.py | 37 +- .../compile/_helpers/attachment/core.py | 34 +- .../_helpers/attachment/declaration_scope.py | 6 - .../attachment/scope_relationships.py | 227 ++------- .../compile/_helpers/attachment/sql_tests.py | 8 +- .../compile/_helpers/attachment/target.py | 22 +- .../compile/_helpers/diagnostics/collector.py | 19 - .../compiler/compile/_helpers/render/reuse.py | 111 ----- .../compile/_helpers/sql_tests/native.py | 88 ---- .../compile/classes/binding_dataflow.py | 35 +- .../compile/classes/render_reuse_registry.py | 50 -- .../compile/classes/render_reuse_session.py | 396 --------------- .../compile/classes/stored_model_analyses.py | 452 ------------------ src/sqlbuild/compiler/compile/constants.py | 13 - .../compile/main/_build_compile_inputs.py | 53 +- src/sqlbuild/compiler/compile/models.py | 58 --- src/sqlbuild/compiler/compile/types.py | 17 - .../_helpers/filesystem/aggregation.py | 42 +- .../compiler/discovery/main/discover.py | 7 - src/sqlbuild/compiler/discovery/types.py | 19 - src/sqlbuild/compiler/fact_cache/__init__.py | 1 - .../compiler/fact_cache/_helpers/__init__.py | 1 - .../fact_cache/_helpers/publication.py | 102 ---- .../fact_cache/_helpers/restricted_pickle.py | 50 -- .../compiler/fact_cache/classes/__init__.py | 1 - .../fact_cache/classes/fact_cache_store.py | 189 -------- .../classes/fact_publication_registry.py | 44 -- src/sqlbuild/compiler/fact_cache/constants.py | 61 --- .../compiler/fact_cache/exceptions.py | 7 - .../compiler/fact_cache/main/__init__.py | 1 - .../fact_cache/main/_compile_cache_root.py | 28 -- .../fact_cache/main/_dump_fact_payload.py | 11 - .../fact_cache/main/_load_fact_payload.py | 11 - .../compiler/fact_cache/main/code_identity.py | 14 - src/sqlbuild/compiler/profiling/constants.py | 5 - src/sqlbuild/compiler/profiling/types.py | 5 - .../_helpers/engine/custom_rules.py | 2 +- .../_helpers/run}/code_identity.py | 14 +- .../rule_engine/_helpers/run/native_memo.py | 2 +- .../main/build/test_compile_json_behavior.py | 5 - .../cli/commands/main/compile/_test_types.py | 56 +-- .../cli/commands/main/compile/helpers.py | 166 ++----- .../test_compile_cache_invalidation.py | 27 +- .../test_compile_incremental_edit_phases.py | 305 ------------ .../compile/test_compile_incremental_edits.py | 329 ++++++------- .../compile/test_compile_macro_call_store.py | 2 +- .../sqlbuild/cli/commands/main/test_rules.py | 4 - .../compiler/discovery/_test_types.py | 11 - .../test_native_declaration_files.py | 47 -- .../cli/compile_render_reuse/__init__.py | 0 .../compile_render_reuse/_helpers/__init__.py | 0 .../_helpers/_test_types.py | 46 -- .../compile_render_reuse/_helpers/helpers.py | 167 ------- .../_helpers/test_load_notice.py | 68 --- .../_helpers/test_render_state_file.py | 136 ------ .../cli/compile_reuse/_helpers/_test_types.py | 13 - .../cli/compile_reuse/_helpers/helpers.py | 6 - .../_helpers/test_project_files.py | 59 +-- .../cli/compile_reuse/classes/__init__.py | 0 .../cli/compile_reuse/classes/_test_types.py | 17 - .../cli/compile_reuse/classes/helpers.py | 32 -- .../classes/test_stored_artifacts.py | 73 --- .../compiler/compile/classes/__init__.py | 0 .../compiler/compile/classes/_test_types.py | 79 --- .../compiler/compile/classes/helpers.py | 342 ------------- .../classes/test_render_reuse_session.py | 254 ---------- .../classes/test_stored_model_analyses.py | 166 ------- .../sqlbuild/compiler/fact_cache/__init__.py | 1 - .../compiler/fact_cache/classes/__init__.py | 1 - .../fact_cache/classes/_test_types.py | 74 --- .../compiler/fact_cache/classes/helpers.py | 87 ---- .../classes/test_fact_cache_store.py | 314 ------------ .../src/sqlbuild/compiler/frontier/helpers.py | 2 +- website/src/content/docs/docs/cli/compile.mdx | 46 +- .../src/content/docs/docs/concepts/macros.mdx | 2 +- 109 files changed, 530 insertions(+), 5667 deletions(-) delete mode 100644 src/sqlbuild/cli/compile_render_reuse/__init__.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/constants.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/main/__init__.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py delete mode 100644 src/sqlbuild/cli/compile_render_reuse/models.py delete mode 100644 src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py delete mode 100644 src/sqlbuild/cli/compile_reuse/main/_framed_section.py delete mode 100644 src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py delete mode 100644 src/sqlbuild/compiler/compile/_helpers/render/reuse.py delete mode 100644 src/sqlbuild/compiler/compile/classes/render_reuse_registry.py delete mode 100644 src/sqlbuild/compiler/compile/classes/render_reuse_session.py delete mode 100644 src/sqlbuild/compiler/compile/classes/stored_model_analyses.py delete mode 100644 src/sqlbuild/compiler/fact_cache/__init__.py delete mode 100644 src/sqlbuild/compiler/fact_cache/_helpers/__init__.py delete mode 100644 src/sqlbuild/compiler/fact_cache/_helpers/publication.py delete mode 100644 src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py delete mode 100644 src/sqlbuild/compiler/fact_cache/classes/__init__.py delete mode 100644 src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py delete mode 100644 src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py delete mode 100644 src/sqlbuild/compiler/fact_cache/constants.py delete mode 100644 src/sqlbuild/compiler/fact_cache/exceptions.py delete mode 100644 src/sqlbuild/compiler/fact_cache/main/__init__.py delete mode 100644 src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py delete mode 100644 src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py delete mode 100644 src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py delete mode 100644 src/sqlbuild/compiler/fact_cache/main/code_identity.py rename src/sqlbuild/{compiler/fact_cache/_helpers => rule_engine/_helpers/run}/code_identity.py (87%) delete mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/classes/__init__.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py delete mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py delete mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py delete mode 100644 tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py delete mode 100644 tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py delete mode 100644 tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py delete mode 100644 tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py delete mode 100644 tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py diff --git a/fensu.toml b/fensu.toml index 8f64aec491..a10fe5c399 100644 --- a/fensu.toml +++ b/fensu.toml @@ -177,12 +177,6 @@ path = "src/sqlbuild/runtime/event_exporting/main/command_output_sink.py" symbols = ["command_output_sink"] reason = "Bare command-output sink decorators are evaluated as decorator(function), requiring positional function calling." -[[targets.python.rule_exceptions]] -rule = "FFS120" -path = "src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py" -symbols = ["_RestrictedUnpickler.find_class"] -reason = "This override preserves pickle.Unpickler's positional find_class(module, name) protocol." - [[targets.python.rule_exceptions]] rule = "FFR402" path = "src/sqlbuild/integrations/dagster/__init__.py" diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index bfe4eee700..03da9883f1 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -432,7 +432,6 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "external_sql_reference_resolver", "scope_index", "declaration_scope", - "analysis_reuse", } ) RENDER_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index c2e24834a3..cfb0291d65 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -122,36 +122,19 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data, the project has very many files, or it records -thousands of new model renders, compile prints `Recording compile for reuse...` and a completion -line to stderr before the report. - -### Compiling after model-only edits - -When the only changes since the stored compile are edits to existing model files, compile runs in -full but reuses the stored render of every unchanged model, together with the diagnostics and -environment reads that rendering reported. Only the edited models are rendered again. Declaration -files other than models are not read again, and an unchanged model reuses its stored column -analysis only when the analyses of its upstream models in this compile match the ones it was stored -with. Contracts, semantic validation, and rules still run for the whole project, and a model whose -upstream analyses changed is analyzed again, so downstream effects of an edit, such as a changed -column type, are reported exactly as in an uncached compile, even when another command such as -`sqb build` ran in between. Compiled -files that still hold the bytes the stored compile wrote are not read or rewritten. The output is -byte-identical to `sqb compile --no-cache`. - -Every other change renders the whole project: adding, removing, or renaming a model, and editing -macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope -folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings -is always rendered again. - -Renders are stored only when a compile result for the same project and target is already stored, -so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. -The next full compile stores them, and edits after that reuse them. Stored renders are compressed -and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints -`Loading stored renders...` and a completion line to stderr before it starts. While renders are -reused, the Python modules recorded by the previous compile stay tracked even if this compile did -not import them, so editing such a module runs the next compile in full. +report is ready. When it has to hash a lot of data or the project has very many files, compile +prints `Recording compile for reuse...` and a completion line to stderr before the report. + +### Compiling after an edit + +Any change runs the whole compile again: every model, test, and audit is rendered again, and +contracts and semantic validation run for the whole project. No render is carried over from the +previous compile. A few caches under `target/cache/` still apply: an unchanged model takes its +column analysis from the analysis cache when its query and the analyses of its upstream models are +unchanged, rule results are reused for unchanged inputs, and macro call results may be reused as +described in [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). +Until the native compiler ships, editing one model therefore costs close to a full compile. The +output is byte-identical to `sqb compile --no-cache`. Known limits: @@ -171,8 +154,7 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or -rendered again after model-only edits and `analysis_reuse_hits` for stored model analyses reused. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 808371a6c9..07ccca54b1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -222,7 +222,7 @@ files or environment variables that the call does not receive. SQLBuild relies on this contract to reuse macro results: -- [Compile reuse](../cli/compile.md) replays stored renders without running macros again. +- [Compile reuse](../cli/compile.md) replays an unchanged compile without running macros again. - A compile may run each distinct macro call once and reuse its result for identical calls elsewhere in the project. - Macro call results may be kept between compiles under `target/cache/`. They are recomputed when diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index 45cbc47e1b..1f21ac3d72 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -34,7 +34,6 @@ from sqlbuild.cli.output.models import ( WrittenTarget, ) -from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestReferenceError from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, @@ -122,7 +121,6 @@ def _analyze_compile_project( sql_analysis_enabled_override=( False if profile_flags.skip_discovery_sql_analysis else None ), - declaration_reuse=COMPILE_RENDER_REUSE.claim_discovery(), ) reject_unit_test_selectors(select=select, exclude=exclude, discovered_inputs=discovered_inputs) discover_ms: int = elapsed_ms(discover_start) @@ -328,7 +326,7 @@ def write_compile_dag_artifact( python_graph=python_graph, ) _ = resolved_dag_path.write_text(dag_json, encoding="utf-8") - _ = COMPILE_ARTIFACT_WRITES.written( + COMPILE_ARTIFACT_WRITES.written( path=resolved_dag_path, contents=dag_json.replace("\n", os.linesep).encode("utf-8") ) _ = complete_compile_phase( diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index da8b849949..8fda15011b 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -688,7 +688,7 @@ def _write_sql(*, path: Path, sql: str, check_existing: bool = True) -> None: def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): - _ = COMPILE_ARTIFACT_WRITES.written( + COMPILE_ARTIFACT_WRITES.written( path=path, contents=contents.replace(_POSIX_LINE_SEPARATOR, os.linesep).encode("utf-8"), ) @@ -703,9 +703,7 @@ def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): - unchanged: bool = COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) - if check_existing and unchanged: - return + COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) if check_existing: existing: bytes | None = _read_existing_file(path=path) if existing == contents: diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 3902e30069..7edbb6d3e1 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -28,11 +28,7 @@ CompileCommandResult, CompileWriteResult, ) -from sqlbuild.cli.compile_render_reuse.main._activated_render_reuse import activated_render_reuse -from sqlbuild.cli.compile_render_reuse.main._render_reuse_session import render_reuse_session -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile @@ -68,7 +64,6 @@ def run_compile( else reuse_attempt ) output: RecordedCompileOutput = RecordedCompileOutput() - render_reuse: CompileRenderReuse | None = render_reuse_session(attempt=attempt) status: TransientStatusReporter | None = start_compile_status( json_output=request.json_output, no_color=request.no_color, @@ -78,12 +73,7 @@ def run_compile( with ( collect_compile_timings() as detailed_timings, COMPILE_INPUT_READS.recording() as input_reads, - COMPILE_ARTIFACT_WRITES.recording( - stored=StoredArtifacts( - stamps=attempt.prior_target_files, digests=attempt.prior_target_digests - ) - ) as artifact_writes, - activated_render_reuse(render_reuse=render_reuse), + COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, PreparedCompileArtifacts( enabled=not request.profile_flags.skip_write ) as prepared_artifacts, @@ -111,7 +101,6 @@ def run_compile( artifacts_written=result.artifacts_written, dag_artifact_path=result.dag_artifact_path, json_output=request.json_output, - render_reuse=render_reuse, ) finally: output.print_stdout() diff --git a/src/sqlbuild/cli/compile_render_reuse/__init__.py b/src/sqlbuild/cli/compile_render_reuse/__init__.py deleted file mode 100644 index 1b6139a171..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Reuse unaffected renders of the stored compile in the next full compile.""" diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py deleted file mode 100644 index b89b7a1c87..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py +++ /dev/null @@ -1,49 +0,0 @@ -"""Lifecycle lines for loading a large render store before the compile status starts.""" - -from __future__ import annotations - -import sys -import time -from pathlib import Path - -from sqlbuild.cli.compile_render_reuse.constants import ( - RENDER_LOAD_DONE_MESSAGE, - RENDER_LOAD_FAILED_MESSAGE, - RENDER_LOAD_NOTICE_BYTES, - RENDER_LOAD_START_MESSAGE, -) -from sqlbuild.cli.compile_reuse.constants import BYTES_PER_MEBIBYTE, REUSE_RENDER_STATE_SUFFIX - - -def start_load_notice(*, entry_path: Path) -> float | None: - """Announce loading on stderr when the slot's stored render files are large.""" - - stored_bytes: int = sum( - _file_size(path=path) - for path in entry_path.parent.glob(f"{entry_path.stem}-*{REUSE_RENDER_STATE_SUFFIX}") - ) - if stored_bytes < RENDER_LOAD_NOTICE_BYTES: - return None - print( - RENDER_LOAD_START_MESSAGE.format(mebibytes=stored_bytes / BYTES_PER_MEBIBYTE), - file=sys.stderr, - ) - return time.monotonic() - - -def _file_size(*, path: Path) -> int: - """Size a stored render file, treating one removed by a concurrent compile as empty.""" - - try: - return path.stat().st_size - except OSError: - return 0 - - -def finish_load_notice(*, started: float | None, loaded: bool) -> None: - """Report whether an announced load produced usable renders.""" - - if started is None: - return - message: str = RENDER_LOAD_DONE_MESSAGE if loaded else RENDER_LOAD_FAILED_MESSAGE - print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py deleted file mode 100644 index f68ef0b514..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py +++ /dev/null @@ -1,230 +0,0 @@ -"""Checksummed, layered storage for the renders of the compile one stored entry describes.""" - -from __future__ import annotations - -import json -import zlib -from collections.abc import Iterable -from pathlib import Path -from typing import cast - -from sqlbuild.cli.compile_render_reuse.constants import ( - RENDER_COMPRESSION_LEVEL, - RENDER_INDEX_FIELD_COUNT, - RENDER_OVERLAY_MAX_SHARE, - RENDER_STATE_MAGIC, -) -from sqlbuild.cli.compile_render_reuse.models import ( - StoredRenderLayer, - StoredRenderLayers, - StoredRenderState, -) -from sqlbuild.cli.compile_reuse.constants import ( - REUSE_ENTRY_BYTE_ORDER, - REUSE_ENTRY_CHECKSUM_BYTES, - REUSE_ENTRY_LENGTH_BYTES, - REUSE_MAX_ENTRY_BYTES, - REUSE_RENDER_STATE_SUFFIX, -) -from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError -from sqlbuild.cli.compile_reuse.main._framed_section import framed_stored_section -from sqlbuild.cli.compile_reuse.main._read_framed_section import read_stored_section -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState - - -def read_render_state(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderState | None: - """Read verified renders when every changed path is a stored model; any fault means none.""" - - try: - newest: StoredRenderLayer | None = _read_layer(path=path, changed_paths=changed_paths) - if newest is None: - return None - if newest.base_file is None: - return _stored_state(base_file=path.name, base=newest, overlay=None) - base: StoredRenderLayer | None = _read_layer( - path=path.parent / _layer_name(newest.base_file), changed_paths=frozenset() - ) - if base is None or base.base_file is not None: - return None - return _stored_state(base_file=newest.base_file, base=base, overlay=newest) - except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError, zlib.error): - return None - - -def render_state_layer( - *, state: RenderReuseState, stored: StoredRenderLayers | None -) -> RenderStateLayer | None: - """Return an overlay on the stored base, a new base, or None when released bytes are needed.""" - - if ( - stored is not None - and state.model_paths == stored.model_paths - and stored.model_keys <= state.model_payloads.keys() - and stored.group_keys <= state.group_payloads.keys() - ): - models: list[tuple[str, memoryview]] = _held( - (path, state.model_payloads.get(path)) for path in state.model_paths - ) - groups: list[tuple[str, memoryview]] = _held(sorted(state.group_payloads.items())) - if ( - sum(len(item) for _, item in (*models, *groups)) - <= stored.base_bytes * RENDER_OVERLAY_MAX_SHARE - ): - return _layer( - model_paths=state.model_paths, - base_file=stored.base_file, - models=models, - groups=groups, - ) - if None in state.model_payloads.values() or None in state.group_payloads.values(): - return None - return _layer( - model_paths=state.model_paths, - base_file=None, - models=_held((path, state.model_payloads.get(path)) for path in state.model_paths), - groups=_held(sorted(state.group_payloads.items())), - ) - - -def _held(items: Iterable[tuple[str, memoryview | None]]) -> list[tuple[str, memoryview]]: - return [(key, payload) for key, payload in items if payload is not None] - - -def _layer( - *, - model_paths: tuple[str, ...], - base_file: str | None, - models: list[tuple[str, memoryview]], - groups: list[tuple[str, memoryview]], -) -> RenderStateLayer: - index: bytes = json.dumps( - { - "model_paths": list(model_paths), - "base": base_file, - "models": [[path, len(payload)] for path, payload in models], - "groups": [[name, len(payload)] for name, payload in groups], - }, - separators=(",", ":"), - ).encode("utf-8", "surrogateescape") - compressor: zlib._Compress = zlib.compressobj(RENDER_COMPRESSION_LEVEL) - compressed: list[bytes] = [compressor.compress(item) for _, item in (*models, *groups)] - compressed.append(compressor.flush()) - return RenderStateLayer( - chunks=( - RENDER_STATE_MAGIC, - framed_stored_section(data=index), - _section_header(payloads=compressed), - *compressed, - ), - base_file=base_file, - ) - - -def _section_header(*, payloads: Iterable[bytes]) -> bytes: - length: int = 0 - checksum: int = 0 - for payload in payloads: - length += len(payload) - checksum = zlib.crc32(payload, checksum) - return length.to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER) + checksum.to_bytes( - REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER - ) - - -def _read_layer(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderLayer | None: - with open(path, "rb") as handle: - if handle.read(len(RENDER_STATE_MAGIC)) != RENDER_STATE_MAGIC: - return None - index: object = json.loads(read_stored_section(handle=handle)) - if not isinstance(index, dict): - return None - fields: dict[str, object] = cast(dict[str, object], index) - model_paths: tuple[str, ...] = tuple(_string(item) for item in _list(fields["model_paths"])) - if not changed_paths.issubset(model_paths): - return None - compressed: bytes = read_stored_section(handle=handle) - if handle.read(1) or len(compressed) > REUSE_MAX_ENTRY_BYTES: - return None - base: object = fields["base"] - model_rows: list[tuple[str, int]] = _rows(fields["models"]) - group_rows: list[tuple[str, int]] = _rows(fields["groups"]) - size: int = sum(length for _, length in (*model_rows, *group_rows)) - if size > REUSE_MAX_ENTRY_BYTES: - return None - decompressor: zlib._Decompress = zlib.decompressobj() - payload: bytes = decompressor.decompress(compressed, size + 1) - del compressed - if len(payload) != size or not decompressor.eof or decompressor.unconsumed_tail: - return None - view: memoryview = memoryview(payload) - offset: int = 0 - models: dict[str, memoryview] = {} - for name, length in model_rows: - models[name] = view[offset : offset + length] - offset += length - groups: dict[str, memoryview] = {} - for name, length in group_rows: - groups[name] = view[offset : offset + length] - offset += length - return StoredRenderLayer( - model_paths=model_paths, - base_file=None if base is None else _string(base), - models=models, - groups=groups, - size=len(payload), - ) - - -def _stored_state( - *, base_file: str, base: StoredRenderLayer, overlay: StoredRenderLayer | None -) -> StoredRenderState: - newest: StoredRenderLayer = base if overlay is None else overlay - state: RenderReuseState = RenderReuseState( - model_paths=newest.model_paths, - model_payloads=dict(base.models) if overlay is None else {**base.models, **overlay.models}, - group_payloads=dict(base.groups) if overlay is None else {**base.groups, **overlay.groups}, - ) - return StoredRenderState( - state=state, - layers=StoredRenderLayers( - model_paths=newest.model_paths, - model_keys=frozenset(state.model_payloads), - group_keys=frozenset(state.group_payloads), - base_file=base_file, - base_bytes=base.size, - overlay_models=frozenset() if overlay is None else frozenset(overlay.models), - overlay_groups=frozenset() if overlay is None else frozenset(overlay.groups), - ), - ) - - -def _layer_name(name: str) -> str: - if Path(name).name != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): - raise CompileReuseEntryError("invalid stored render base name") - return name - - -def _rows(value: object) -> list[tuple[str, int]]: - rows: list[tuple[str, int]] = [] - for row in _list(value): - items: list[object] = _list(row) - if len(items) != RENDER_INDEX_FIELD_COUNT: - raise CompileReuseEntryError("invalid stored render index") - length: object = items[1] - if not isinstance(length, int) or isinstance(length, bool) or length < 0: - raise CompileReuseEntryError("invalid stored render length") - rows.append((_string(items[0]), length)) - return rows - - -def _list(value: object) -> list[object]: - if not isinstance(value, list): - raise CompileReuseEntryError("invalid stored render list") - return cast(list[object], value) - - -def _string(value: object) -> str: - if not isinstance(value, str): - raise CompileReuseEntryError("invalid stored render string") - return value diff --git a/src/sqlbuild/cli/compile_render_reuse/constants.py b/src/sqlbuild/cli/compile_render_reuse/constants.py deleted file mode 100644 index be5e190f0a..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/constants.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Render storage format constants.""" - -from __future__ import annotations - -RENDER_STATE_MAGIC: bytes = b"SQBRENDER2\n" -RENDER_COMPRESSION_LEVEL: int = 1 -RENDER_OVERLAY_MAX_SHARE: float = 0.25 -RENDER_INDEX_FIELD_COUNT: int = 2 -RENDER_LOAD_NOTICE_BYTES: int = 8 * 1024 * 1024 -RENDER_LOAD_START_MESSAGE: str = "Loading stored renders ({mebibytes:.0f} MiB)..." -RENDER_LOAD_DONE_MESSAGE: str = "Loaded stored renders ({seconds:.1f} s)" -RENDER_LOAD_FAILED_MESSAGE: str = ( - "Stored renders not reused; rendering every model again ({seconds:.1f} s)" -) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/__init__.py b/src/sqlbuild/cli/compile_render_reuse/main/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py b/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py deleted file mode 100644 index a40d737c78..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Public entry for exposing one compile's render reuse to the rendering it runs.""" - -from __future__ import annotations - -from contextlib import AbstractContextManager, nullcontext - -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse -from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE - - -def activated_render_reuse( - *, render_reuse: CompileRenderReuse | None -) -> AbstractContextManager[object]: - """Activate the render reuse session for the compile block, or do nothing without one.""" - - return ( - nullcontext() - if render_reuse is None - else COMPILE_RENDER_REUSE.activated(render_reuse.session) - ) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py deleted file mode 100644 index 399a33b9e5..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Public entry for the render reuse one full compile runs with.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.compile_render_reuse._helpers.load_notice import ( - finish_load_notice, - start_load_notice, -) -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import read_render_state -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse, StoredRenderState -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt -from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession - - -def render_reuse_session(*, attempt: CompileReuseAttempt) -> CompileRenderReuse | None: - """Record renders once a compile is stored, reusing those the changed paths cannot affect.""" - - if ( - attempt.outcome is not CompileReuseOutcome.MISS - or attempt.entry_path is None - or not attempt.prior_entry - ): - return None - stored: StoredRenderState | None = ( - None - if attempt.changed_paths is None or attempt.render_state_path is None - else _loaded_render_state( - entry_path=attempt.entry_path, - path=attempt.render_state_path, - changed_paths=attempt.changed_paths, - ) - ) - return CompileRenderReuse( - session=CompileRenderReuseSession( - prior=None if stored is None else stored.state, - changed_paths=attempt.changed_paths, - retained_models=frozenset() if stored is None else stored.layers.overlay_models, - retained_groups=frozenset() if stored is None else stored.layers.overlay_groups, - ), - stored=None if stored is None else stored.layers, - ) - - -def _loaded_render_state( - *, entry_path: Path, path: Path, changed_paths: frozenset[str] -) -> StoredRenderState | None: - started: float | None = start_load_notice(entry_path=entry_path) - stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) - _ = finish_load_notice(started=started, loaded=stored is not None) - return stored diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py deleted file mode 100644 index 83467f8b70..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Public entry for the render layer a finished compile stores with its entry.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import render_state_layer -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState - - -def stored_render_layer(*, render_reuse: CompileRenderReuse | None) -> RenderStateLayer | None: - """Return this compile's renders as an overlay on the stored ones, or as a new base.""" - - state: RenderReuseState | None = ( - None if render_reuse is None else render_reuse.session.stored_state() - ) - if render_reuse is None or state is None: - return None - layer: RenderStateLayer | None = render_state_layer(state=state, stored=render_reuse.stored) - if layer is not None: - return layer - complete: RenderReuseState | None = render_reuse.session.stored_state(complete=True) - return None if complete is None else render_state_layer(state=complete, stored=None) diff --git a/src/sqlbuild/cli/compile_render_reuse/models.py b/src/sqlbuild/cli/compile_render_reuse/models.py deleted file mode 100644 index a7595e63c1..0000000000 --- a/src/sqlbuild/cli/compile_render_reuse/models.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Stored renders of a previous compile and the render reuse of one full compile.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.models import RenderReuseState - - -@dataclass(frozen=True) -class StoredRenderLayer: - """One decoded render file: its index and its payloads by key.""" - - model_paths: tuple[str, ...] - base_file: str | None - models: dict[str, memoryview] - groups: dict[str, memoryview] - size: int - - -@dataclass(frozen=True) -class StoredRenderLayers: - """How a stored compile's renders are split between a base file and one overlay.""" - - model_paths: tuple[str, ...] - model_keys: frozenset[str] - group_keys: frozenset[str] - base_file: str - base_bytes: int - overlay_models: frozenset[str] - overlay_groups: frozenset[str] - - -@dataclass(frozen=True) -class StoredRenderState: - """Renders a stored compile recorded, merged across layers, with their layout.""" - - state: RenderReuseState - layers: StoredRenderLayers - - -@dataclass(frozen=True) -class CompileRenderReuse: - """One full compile's render reuse session with the stored renders it started from.""" - - session: CompileRenderReuseSession - stored: StoredRenderLayers | None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index dac7203b29..5b9872a59f 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -9,14 +9,12 @@ from pathlib import Path from sqlbuild.cli.compile_reuse._helpers.entry_file import ( - entry_render_state_path, read_entry_header, read_entry_stdout, rewrite_entry_inputs, ) from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, - changed_project_paths, compare_project_files, needs_refresh, restamped_paths, @@ -106,16 +104,7 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ) attempt = replace( attempt, - prior_entry=True, - prior_modules=header.inputs.modules, - prior_target_files=header.inputs.target_files, - prior_target_digests=header.inputs.target_digests, restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), - render_state_path=( - None - if attempt.changed_paths is None or attempt.entry_path is None - else entry_render_state_path(path=attempt.entry_path, header=header) - ), ) return replace( attempt, @@ -168,22 +157,12 @@ def _checked_attempt( unchanged: bool = comparison.unchanged and target_files_unchanged( stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree ) - changed_paths: frozenset[str] | None = None - verified: dict[str, str] = comparison.verified - if identity_unchanged and not unchanged: - changed_paths, verified = changed_project_paths( - project_dir=project_dir, - stored=inputs.project_files, - current=attempt.snapshot, - verified=comparison.verified, - ) return replace( attempt, outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, digests=carried_forward_digests( - stored=inputs.project_files, current=attempt.snapshot, verified=verified + stored=inputs.project_files, current=attempt.snapshot, verified=comparison.verified ), - changed_paths=changed_paths, ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py index aceecdc384..37f8c3f597 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py @@ -20,15 +20,12 @@ REUSE_ENTRY_SUFFIX, REUSE_FORMAT_VERSION, REUSE_MAX_ENTRY_BYTES, - REUSE_MAX_RENDER_STATES, REUSE_MAX_STORED_ENTRIES, - REUSE_RENDER_STATE_SUFFIX, REUSE_STDOUT_SEPARATOR, REUSE_STDOUT_SUFFIX, ) from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError from sqlbuild.cli.compile_reuse.models import ( - RenderStateLayer, SettingsEnvironmentInputs, StoredCompileHeader, StoredCompileInputs, @@ -51,7 +48,7 @@ def read_entry_header(*, path: Path) -> StoredCompileHeader | None: with open(path, "rb") as handle: if handle.read(len(REUSE_ENTRY_MAGIC)) != REUSE_ENTRY_MAGIC: return None - metadata: bytes = read_framed_section(handle=handle) + metadata: bytes = _read_section(handle=handle) if handle.read(1): return None payload: object = json.loads(metadata) @@ -83,24 +80,13 @@ def write_entry( inputs: StoredCompileInputs, output: StoredCompileOutput, stdout: str, - render_state: RenderStateLayer | None = None, ) -> None: - """Publish one entry, its stdout, and its renders atomically, naming them in the entry.""" + """Publish one entry and its stdout atomically, filling in the stdout file and checksum.""" encoded_stdout: bytes = stdout.encode("utf-8", "surrogateescape") - stamp: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}" - stdout_file: str = f"{stamp}{REUSE_STDOUT_SUFFIX}" - render_state_file: str | None = ( - None - if render_state is None - or sum(len(chunk) for chunk in render_state.chunks) > REUSE_MAX_ENTRY_BYTES - else f"{stamp}{REUSE_RENDER_STATE_SUFFIX}" - ) + stdout_file: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}{REUSE_STDOUT_SUFFIX}" stored_output: StoredCompileOutput = replace( - output, - stdout_file=stdout_file, - stdout_checksum=zlib.crc32(encoded_stdout), - render_state_file=render_state_file, + output, stdout_file=stdout_file, stdout_checksum=zlib.crc32(encoded_stdout) ) metadata: bytes = _metadata(inputs=inputs, output=stored_output) if len(metadata) + len(encoded_stdout) > REUSE_MAX_ENTRY_BYTES: @@ -108,31 +94,14 @@ def write_entry( return try: path.parent.mkdir(parents=True, exist_ok=True) - _publish_file(path=path.parent / stdout_file, contents=(encoded_stdout,)) - if render_state is not None and render_state_file is not None: - _publish_file(path=path.parent / render_state_file, contents=render_state.chunks) - _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) - _remove_stdout_files(entry_path=path, keep=(stdout_file,)) - _remove_render_files( - entry_path=path, - keep=( - () - if render_state is None or render_state_file is None - else (render_state_file, render_state.base_file) - ), - ) + _publish_file(path=path.parent / stdout_file, contents=encoded_stdout) + _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) + _remove_side_files(entry_path=path, keep=stdout_file) _prune_entries(directory=path.parent, keep=path) except OSError: remove_entry(path=path) -def entry_render_state_path(*, path: Path, header: StoredCompileHeader) -> Path | None: - """Return where the entry's renders are stored, when it stored any.""" - - name: str | None = header.output.render_state_file - return None if name is None else path.parent / name - - def rewrite_entry_inputs( *, path: Path, header: StoredCompileHeader, inputs: StoredCompileInputs ) -> None: @@ -140,7 +109,7 @@ def rewrite_entry_inputs( metadata: bytes = _metadata(inputs=inputs, output=header.output) with contextlib.suppress(OSError): - _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) + _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) def remove_entry(*, path: Path) -> None: @@ -148,18 +117,16 @@ def remove_entry(*, path: Path) -> None: with contextlib.suppress(OSError): path.unlink(missing_ok=True) - _remove_stdout_files(entry_path=path, keep=()) - _remove_render_files(entry_path=path, keep=()) + _remove_side_files(entry_path=path, keep=None) -def _publish_file(*, path: Path, contents: tuple[bytes | memoryview, ...]) -> None: +def _publish_file(*, path: Path, contents: bytes) -> None: descriptor, temporary_path = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent ) try: with os.fdopen(descriptor, "wb") as handle: - for chunk in contents: - _ = handle.write(chunk) + _ = handle.write(contents) os.replace(temporary_path, path) except BaseException: with contextlib.suppress(OSError): @@ -174,48 +141,31 @@ def _metadata(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> by def _framed_metadata(*, metadata: bytes) -> bytes: - return REUSE_ENTRY_MAGIC + framed_section(data=metadata) - - -def framed_section(*, data: bytes) -> bytes: - """Prefix one section with its length and checksum so a reader can verify it.""" - return b"".join( ( - len(data).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), - zlib.crc32(data).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), - data, + REUSE_ENTRY_MAGIC, + len(metadata).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), + zlib.crc32(metadata).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), + metadata, ) ) -def _remove_stdout_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: - _remove_entry_files(entry_path=entry_path, suffix=REUSE_STDOUT_SUFFIX, keep=keep) - - -def _remove_render_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: - _remove_entry_files(entry_path=entry_path, suffix=REUSE_RENDER_STATE_SUFFIX, keep=keep) +def _remove_side_files(*, entry_path: Path, keep: str | None) -> None: + """Remove every file stored beside an entry except keep, including older releases' files.""" - -def _remove_entry_files(*, entry_path: Path, suffix: str, keep: tuple[str | None, ...]) -> None: prefix: str = f"{entry_path.stem}{REUSE_STDOUT_SEPARATOR}" try: candidates: list[Path] = list(entry_path.parent.iterdir()) except OSError: return for candidate in candidates: - if ( - candidate.name.startswith(prefix) - and candidate.suffix == suffix - and candidate.name not in keep - ): + if candidate.name.startswith(prefix) and candidate.name != keep: with contextlib.suppress(OSError): candidate.unlink() -def read_framed_section(*, handle: BinaryIO) -> bytes: - """Read one length- and checksum-framed section, raising when it is damaged.""" - +def _read_section(*, handle: BinaryIO) -> bytes: length: int = int.from_bytes(handle.read(REUSE_ENTRY_LENGTH_BYTES), REUSE_ENTRY_BYTE_ORDER) checksum: int = int.from_bytes(handle.read(REUSE_ENTRY_CHECKSUM_BYTES), REUSE_ENTRY_BYTE_ORDER) if length > REUSE_MAX_ENTRY_BYTES: @@ -236,8 +186,6 @@ def _prune_entries(*, directory: Path, keep: Path) -> None: entries.sort(reverse=True) for _, stale in entries[REUSE_MAX_STORED_ENTRIES - 1 :]: remove_entry(path=stale) - for _, older in entries[REUSE_MAX_RENDER_STATES - 1 : REUSE_MAX_STORED_ENTRIES - 1]: - _remove_render_files(entry_path=older, keep=()) def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dict[str, object]: @@ -254,14 +202,12 @@ def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dic for path, item in inputs.project_files.items() }, "target_files": {path: list(stamp) for path, stamp in inputs.target_files.items()}, - "target_digests": inputs.target_digests, "stderr_lines": list(output.stderr_lines), "exit_code": output.exit_code, "timings_span": None if output.timings_span is None else list(output.timings_span), "stdout_length": output.stdout_length, "stdout_checksum": output.stdout_checksum, "stdout_file": output.stdout_file, - "render_state_file": output.render_state_file, "target_tree": inputs.target_tree, "settings_inputs": [ { @@ -310,10 +256,6 @@ def _header(*, payload: object) -> StoredCompileHeader: _settings_inputs(item) for item in _list(fields["settings_inputs"]) ), settings_digest=_string(fields["settings_digest"]), - target_digests={ - _string(path): _string(digest) - for path, digest in _mapping(fields.get("target_digests", {})).items() - }, ), output=StoredCompileOutput( stderr_lines=tuple(_string(line) for line in _list(fields["stderr_lines"])), @@ -322,7 +264,6 @@ def _header(*, payload: object) -> StoredCompileHeader: stdout_length=_integer(fields["stdout_length"]), stdout_checksum=_integer(fields["stdout_checksum"]), stdout_file=_stdout_file(fields["stdout_file"]), - render_state_file=_render_state_file(fields.get("render_state_file")), ), ) @@ -345,15 +286,6 @@ def _stdout_file(value: object) -> str: return name -def _render_state_file(value: object) -> str | None: - if value is None: - return None - name: str = _string(value) - if os.path.basename(name) != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): - raise CompileReuseEntryError("invalid stored render file name") - return name - - def _timings_span(value: object) -> tuple[int, int] | None: if value is None: return None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py index e68d78330c..5c0472b33c 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py @@ -101,37 +101,6 @@ def compare_project_files( return ProjectFilesComparison(unchanged=True, verified=verified) -def changed_project_paths( - *, - project_dir: str, - stored: dict[str, StoredProjectFile], - current: dict[str, FileStamp], - verified: dict[str, str], -) -> tuple[frozenset[str], dict[str, str]]: - """Return every path added, removed, or changed since the stored compile, and new digests.""" - - digests: dict[str, str] = dict(verified) - changed: set[str] = {path for path in stored if path not in current} - for relative_path, stamp in current.items(): - previous: StoredProjectFile | None = stored.get(relative_path) - if previous is None or not _same_identity(stamp=stamp, recorded=previous.stamp): - changed.add(relative_path) - continue - if stamp.kind in _PRESENCE_ONLY_KINDS or (stamp == previous.stamp and not previous.racy): - continue - if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: - changed.add(relative_path) - continue - digest: str | None = digests.get(relative_path) or file_digest( - path=os.path.join(project_dir, relative_path) - ) - if digest is None or digest != previous.digest: - changed.add(relative_path) - else: - digests[relative_path] = digest - return frozenset(changed), digests - - def carried_forward_digests( *, stored: dict[str, StoredProjectFile], diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index 3c792b9ba5..a676b33f60 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -71,19 +71,6 @@ def loaded_module_stamps(*, covered_paths: frozenset[str]) -> tuple[tuple[str, i return tuple(sorted(stamps.values())) -def with_carried_module_stamps( - *, - current: tuple[tuple[str, int, int], ...], - carried: tuple[tuple[str, int, int], ...], - covered_paths: frozenset[str], -) -> tuple[tuple[str, int, int], ...]: - """Add stored stamps of modules a reused render imported but this process did not load.""" - - stamps: dict[str, tuple[str, int, int]] = {stamp[0]: stamp for stamp in carried} - stamps.update((stamp[0], stamp) for stamp in current) - return tuple(sorted(stamp for path, stamp in stamps.items() if path not in covered_paths)) - - def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool: """Return whether every recorded module file still has the same stat identity.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index 853013c52f..7bed444a1d 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -8,8 +8,6 @@ import time from pathlib import Path -from sqlbuild.cli.compile_render_reuse.main._render_state_layer import stored_render_layer -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.entry_file import remove_entry, write_entry from sqlbuild.cli.compile_reuse._helpers.project_files import ( pending_digest_bytes, @@ -23,7 +21,6 @@ loaded_module_stamps, settings_inputs_digest, tracked_environment_names, - with_carried_module_stamps, ) from sqlbuild.cli.compile_reuse._helpers.target_files import verified_target_files from sqlbuild.cli.compile_reuse._helpers.timings import compile_timings_span @@ -35,7 +32,6 @@ REUSE_STORE_DONE_MESSAGE, REUSE_STORE_NOTICE_BYTES, REUSE_STORE_NOTICE_PATHS, - REUSE_STORE_NOTICE_RENDERS, REUSE_STORE_SKIPPED_MESSAGE, REUSE_STORE_START_MESSAGE, ) @@ -62,7 +58,6 @@ def write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, - render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" @@ -71,6 +66,7 @@ def write_compile_entry( try: _write_compile_entry( attempt=attempt, + entry_path=attempt.entry_path, output=output, exit_code=exit_code, input_reads=input_reads, @@ -79,7 +75,6 @@ def write_compile_entry( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, - render_reuse=render_reuse, ) except Exception: _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) @@ -89,6 +84,7 @@ def write_compile_entry( def _write_compile_entry( *, attempt: CompileReuseAttempt, + entry_path: Path, output: RecordedCompileOutput, exit_code: int, input_reads: CompileInputReads, @@ -97,11 +93,7 @@ def _write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, - render_reuse: CompileRenderReuse | None, ) -> None: - if attempt.entry_path is None: - return - entry_path: Path = attempt.entry_path stdout: str | None = output.stdout settings: SettingsInputsResult = provider_settings_inputs( settings_classes=input_reads.settings_classes @@ -117,10 +109,7 @@ def _write_compile_entry( remove_entry(path=entry_path) return project_dir: str = str(attempt.project_dir) - notice_started: float | None = _start_notice( - attempt=attempt, - pending_renders=0 if render_reuse is None else render_reuse.session.pending_renders(), - ) + notice_started: float | None = _start_notice(attempt=attempt) digests: dict[str, str] = with_missing_digests( project_dir=project_dir, snapshot=attempt.snapshot, @@ -154,8 +143,10 @@ def _write_compile_entry( environment_names=environment_names, environment_digest=environment_digest(names=environment_names), search_path=attempt.search_path, - modules=_module_stamps( - attempt=attempt, project_dir=project_dir, render_reuse=render_reuse + modules=loaded_module_stamps( + covered_paths=frozenset( + os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot + ) ), project_files=stored_project_files( snapshot=attempt.snapshot, digests=digests, snapshot_ns=attempt.snapshot_ns @@ -164,11 +155,6 @@ def _write_compile_entry( target_tree=artifacts_written, settings_inputs=settings.inputs, settings_digest=settings_inputs_digest(inputs=settings.inputs), - target_digests={ - path: artifact.digest - for path, artifact in artifact_writes.artifacts.items() - if artifact.digest is not None and path in target_files - }, ), output=StoredCompileOutput( stderr_lines=output.stderr_lines, @@ -179,40 +165,21 @@ def _write_compile_entry( stdout_file="", ), stdout=stdout, - render_state=stored_render_layer(render_reuse=render_reuse), ) _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) -def _module_stamps( - *, attempt: CompileReuseAttempt, project_dir: str, render_reuse: CompileRenderReuse | None -) -> tuple[tuple[str, int, int], ...]: - covered_paths: frozenset[str] = frozenset( - os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot - ) - current: tuple[tuple[str, int, int], ...] = loaded_module_stamps(covered_paths=covered_paths) - if render_reuse is None or not render_reuse.session.reused_any(): - return current - return with_carried_module_stamps( - current=current, carried=attempt.prior_modules, covered_paths=covered_paths - ) - - -def _start_notice(*, attempt: CompileReuseAttempt, pending_renders: int) -> float | None: +def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: pending_bytes: int = pending_digest_bytes( snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped ) - if ( - pending_bytes < REUSE_STORE_NOTICE_BYTES - and len(attempt.snapshot) < REUSE_STORE_NOTICE_PATHS - and pending_renders < REUSE_STORE_NOTICE_RENDERS + if pending_bytes < REUSE_STORE_NOTICE_BYTES and len(attempt.snapshot) < ( + REUSE_STORE_NOTICE_PATHS ): return None print( REUSE_STORE_START_MESSAGE.format( - paths=len(attempt.snapshot), - mebibytes=pending_bytes / BYTES_PER_MEBIBYTE, - renders=pending_renders, + paths=len(attempt.snapshot), mebibytes=pending_bytes / BYTES_PER_MEBIBYTE ), file=sys.stderr, ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py index 4cbddde38a..72d1fd4ecf 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py @@ -32,11 +32,11 @@ def snapshot_target_files( if entry.is_dir(follow_symlinks=False): pending.append(entry.path) continue - stamp: FileStamp | None = path_stamp(path=entry.path) + stamp: FileStamp | None = _path_stamp(path=entry.path) if stamp is not None: stamps[entry.path] = stamp for path in extra_paths: - extra: FileStamp | None = path_stamp(path=path) + extra: FileStamp | None = _path_stamp(path=path) if extra is not None: stamps[path] = extra return stamps @@ -81,7 +81,7 @@ def _target_stamps(*, project_dir: str, paths: tuple[str, ...], tree: bool) -> d """Stamp the whole compiled tree plus extra paths, or only the given paths.""" if not tree: - return {path: stamp for path in paths if (stamp := path_stamp(path=path)) is not None} + return {path: stamp for path in paths if (stamp := _path_stamp(path=path)) is not None} compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep extra_paths: tuple[str, ...] = tuple( path for path in paths if not path.startswith(compiled_prefix) @@ -103,9 +103,7 @@ def _compiled_root(*, project_dir: str) -> str: return os.path.join(project_dir, TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME) -def path_stamp(*, path: str) -> FileStamp | None: - """Return the stat identity of one path, or None when it cannot be stated.""" - +def _path_stamp(*, path: str) -> FileStamp | None: try: status: os.stat_result = os.lstat(path) except OSError: diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py index cba8640bd6..069c0f24a9 100644 --- a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py +++ b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py @@ -7,14 +7,10 @@ import threading from collections.abc import Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.models import RecordedArtifact -if TYPE_CHECKING: - from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts - class CompileArtifactWriteRegistry: """Deliver artifact writes from every thread, including background staging threads.""" @@ -23,41 +19,31 @@ def __init__(self, *, digest_size: int) -> None: self._digest_size: int = digest_size self._lock: threading.Lock = threading.Lock() self._recorders: tuple[CompileArtifactWrites, ...] = () - self._stored: StoredArtifacts | None = None @contextmanager - def recording( - self, *, stored: StoredArtifacts | None = None - ) -> Iterator[CompileArtifactWrites]: - """Record artifact writes until the block exits, trusting unchanged stored artifacts.""" + def recording(self) -> Iterator[CompileArtifactWrites]: + """Record artifact writes until the block exits.""" recorder: CompileArtifactWrites = CompileArtifactWrites() with self._lock: self._recorders = (*self._recorders, recorder) - if stored is not None: - self._stored = stored try: yield recorder finally: with self._lock: self._recorders = tuple(item for item in self._recorders if item is not recorder) - if stored is not None: - self._stored = None - def written(self, *, path: str | os.PathLike[str], contents: bytes) -> bool: - """Note the exact bytes an artifact holds; return whether it already holds them.""" + def written(self, *, path: str | os.PathLike[str], contents: bytes) -> None: + """Note the exact bytes an artifact holds after a write or an unchanged-file check.""" recorders: tuple[CompileArtifactWrites, ...] = self._recorders if not recorders: - return False - absolute_path: str = os.path.abspath(path) + return artifact: RecordedArtifact = RecordedArtifact( digest=hashlib.blake2b(contents, digest_size=self._digest_size).hexdigest() ) for recorder in recorders: - recorder.record(path=absolute_path, artifact=artifact) - stored: StoredArtifacts | None = self._stored - return stored is not None and stored.holds(path=absolute_path, digest=artifact.digest) + recorder.record(path=os.path.abspath(path), artifact=artifact) def kept(self, *, path: str | os.PathLike[str], size: int, mtime_ns: int) -> None: """Note an earlier artifact kept as-is after its size and mtime were validated.""" diff --git a/src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py b/src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py deleted file mode 100644 index 0fc105cfbb..0000000000 --- a/src/sqlbuild/cli/compile_reuse/classes/stored_artifacts.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Artifacts a stored compile left in target/, trusted while their stat identity is unchanged.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.target_files import path_stamp -from sqlbuild.cli.compile_reuse.types import FileStamp - - -class StoredArtifacts: - """Answer whether an artifact still holds the bytes a stored compile recorded for it.""" - - def __init__(self, *, stamps: dict[str, FileStamp], digests: dict[str, str]) -> None: - self._stamps: dict[str, FileStamp] = stamps - self._digests: dict[str, str] = digests - - def holds(self, *, path: str, digest: str | None) -> bool: - """Return whether the file at path is unchanged since it was stored with this digest.""" - - return ( - digest is not None - and self._digests.get(path) == digest - and self.unchanged_since_stored(path=path) - ) - - def unchanged_since_stored(self, *, path: str) -> bool: - """Return whether the file at path still has the stat identity it was stored with.""" - - stamp: FileStamp | None = self._stamps.get(path) - return stamp is not None and path_stamp(path=path) == stamp diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 2bd4a76bac..1fb957fa41 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -20,8 +20,6 @@ REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" REUSE_STDOUT_SEPARATOR: str = "-" -REUSE_RENDER_STATE_SUFFIX: str = ".render" -REUSE_MAX_RENDER_STATES: int = 2 REUSE_ENTRY_LENGTH_BYTES: int = 8 REUSE_ENTRY_CHECKSUM_BYTES: int = 4 REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" @@ -95,10 +93,9 @@ MILLISECONDS_PER_SECOND: int = 1000 REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 REUSE_STORE_NOTICE_PATHS: int = 100_000 -REUSE_STORE_NOTICE_RENDERS: int = 4_000 BYTES_PER_MEBIBYTE: int = 1024 * 1024 REUSE_STORE_START_MESSAGE: str = ( - "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash, {renders} renders)..." + "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." ) REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" REUSE_STORE_SKIPPED_MESSAGE: str = ( diff --git a/src/sqlbuild/cli/compile_reuse/main/_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_framed_section.py deleted file mode 100644 index 9259128048..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_framed_section.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for framing one stored section with its length and checksum.""" - -from __future__ import annotations - -from sqlbuild.cli.compile_reuse._helpers.entry_file import framed_section - - -def framed_stored_section(*, data: bytes) -> bytes: - """Prefix one section with its length and checksum so a reader can verify it.""" - - return framed_section(data=data) diff --git a/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py deleted file mode 100644 index 08583cd0f7..0000000000 --- a/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Public entry for reading one framed stored section.""" - -from __future__ import annotations - -from typing import BinaryIO - -from sqlbuild.cli.compile_reuse._helpers.entry_file import read_framed_section - - -def read_stored_section(*, handle: BinaryIO) -> bytes: - """Read one length- and checksum-framed section, raising when it is damaged.""" - - return read_framed_section(handle=handle) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py index 0fbf395025..0736318972 100644 --- a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py +++ b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py @@ -4,7 +4,6 @@ from pathlib import Path -from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput @@ -23,7 +22,6 @@ def write_reusable_compile( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, - render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable full compile, or drop the stored one when it cannot be reused.""" @@ -37,5 +35,4 @@ def write_reusable_compile( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, - render_reuse=render_reuse, ) diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index 16f143dfb5..cc2bd55c4b 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -81,7 +81,6 @@ class StoredCompileInputs: target_tree: bool settings_inputs: tuple[SettingsEnvironmentInputs, ...] settings_digest: str - target_digests: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) @@ -94,7 +93,6 @@ class StoredCompileOutput: stdout_length: int stdout_checksum: int stdout_file: str - render_state_file: str | None = None @dataclass(frozen=True) @@ -130,17 +128,3 @@ class CompileReuseAttempt: snapshot_ns: int = 0 digests: dict[str, str] = field(default_factory=dict) restamped: frozenset[str] = frozenset() - changed_paths: frozenset[str] | None = None - render_state_path: Path | None = None - prior_entry: bool = False - prior_modules: tuple[tuple[str, int, int], ...] = () - prior_target_files: dict[str, FileStamp] = field(default_factory=dict) - prior_target_digests: dict[str, str] = field(default_factory=dict) - - -@dataclass(frozen=True) -class RenderStateLayer: - """One render layer to publish: its framed chunks, and the base layer it extends.""" - - chunks: tuple[bytes | memoryview, ...] - base_file: str | None diff --git a/src/sqlbuild/compiler/analysis_session/models.py b/src/sqlbuild/compiler/analysis_session/models.py index fedb337ce2..5323f6d344 100644 --- a/src/sqlbuild/compiler/analysis_session/models.py +++ b/src/sqlbuild/compiler/analysis_session/models.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from sqlbuild.adapter.contract.models import ExpressionInferenceProfile -from sqlbuild.compiler.compile.models import CompileModelInput, ModelAnalysisCaching +from sqlbuild.compiler.compile.models import AnalysisCacheContext, CompileModelInput from sqlbuild.compiler.lineage.types import InferredNullability from sqlbuild.spec.contracts.models import SchemaDynamicColumnFamily @@ -22,6 +22,6 @@ class NativeModelAnalysisRequest: inference_profile: ExpressionInferenceProfile allow_compact_analysis: bool rich_type_inference: bool - analysis_caching: ModelAnalysisCaching | None + analysis_cache: AnalysisCacheContext | None complete_binding_schemas: dict[str, dict[str, str]] dynamic_families_by_table: dict[str, tuple[SchemaDynamicColumnFamily, ...]] diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py index 337da09d48..b084a8d591 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/binding_waves.py @@ -7,11 +7,7 @@ from sqlbuild.adapter.contract.models import ExpressionInferenceProfile from sqlbuild.compiler.compile.classes.binding_dataflow import BindingDataflow -from sqlbuild.compiler.compile.models import ( - CompileModelInput, - DataflowReuse, - PolyglotAnalysisResult, -) +from sqlbuild.compiler.compile.models import CompileModelInput, PolyglotAnalysisResult from sqlbuild.compiler.compile.models import ( ModelSqlAnalysis as _ModelSqlAnalysis, ) @@ -30,7 +26,8 @@ def analyze_binding_waves( *, requests: tuple[_ModelSqlAnalysisRequest, ...], names: tuple[str, ...], - reuse: DataflowReuse, + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], shapes: dict[str, dict[str, str]], types: dict[str, dict[str, str]], nullability: dict[str, dict[str, InferredNullability]], @@ -43,7 +40,8 @@ def analyze_binding_waves( return BindingDataflow( requests=requests, names=names, - reuse=reuse, + cached=cached, + previous_signatures=previous_signatures, shapes=shapes, types=types, nullability=nullability, @@ -57,7 +55,8 @@ def analyze_binding_dataflow( *, requests: tuple[_ModelSqlAnalysisRequest, ...], names: tuple[str, ...], - reuse: DataflowReuse, + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], shapes: dict[str, dict[str, str]], types: dict[str, dict[str, str]], nullability: dict[str, dict[str, InferredNullability]], @@ -71,7 +70,8 @@ def analyze_binding_dataflow( BindingDataflow, requests=requests, names=names, - reuse=reuse, + cached=cached, + previous_signatures=previous_signatures, shapes=shapes, types=types, nullability=nullability, diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py index fabfe7f530..b4360ccdd3 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/project.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/project.py @@ -19,7 +19,9 @@ model_analysis_cache_key, model_analysis_output_signature, read_compact_analysis_cache_candidate, + read_model_analyses, record_analysis_cache_metrics, + write_model_analyses, ) from sqlbuild.compiler.compile._helpers.analysis.columns import substitute_placeholder_defaults from sqlbuild.compiler.compile._helpers.analysis.compact import ( @@ -54,6 +56,7 @@ build_column_nullability_by_table, build_complete_binding_schemas, published_model_shape, + upstream_signatures, ) from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import ( build_declared_column_types as _build_column_types_by_table, @@ -103,7 +106,6 @@ report_mocks_reading_referencing_helpers, ) from sqlbuild.compiler.compile._helpers.sql_tests.identity import build_sql_test_case_fingerprint -from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses from sqlbuild.compiler.compile.main._scope_index_with_compile_usages import ( scope_index_with_compile_usages, ) @@ -139,11 +141,9 @@ CompileSqlFunctionInput, CompileSqlScenarioInput, CompileSqlTestInput, - DataflowReuse, DynamicColumnContractProof, InferredColumn, MacroContext, - ModelAnalysisCaching, PolyglotAnalysisResult, ) from sqlbuild.compiler.compile.models import ( @@ -353,14 +353,7 @@ def assemble_compiled_project( inference_profile=profile, allow_compact_analysis=allow_compact_analysis, rich_type_inference=rich_type_inference, - analysis_caching=( - None - if analysis_cache is None - else ModelAnalysisCaching( - cache=analysis_cache, - reuse=inputs.analysis_reuse if analysis_model_names is None else None, - ) - ), + analysis_cache=analysis_cache, complete_binding_schemas=complete_binding_schemas, ) model_sql_analysis_by_name = analyze_model_sql_by_engine( @@ -662,14 +655,11 @@ def _analyze_model_sql_in_parallel( inference_profile: ExpressionInferenceProfile, allow_compact_analysis: bool, rich_type_inference: bool, - analysis_caching: ModelAnalysisCaching | None, + analysis_cache: AnalysisCacheContext | None, complete_binding_schemas: dict[str, dict[str, str]], ) -> dict[str, _ModelSqlAnalysis]: if not model_inputs: return {} - analysis_cache: AnalysisCacheContext | None = ( - None if analysis_caching is None else analysis_caching.cache - ) analyzed_model_names: frozenset[str] = frozenset( _model_name(model_input) for model_input in model_inputs ) @@ -683,13 +673,6 @@ def _analyze_model_sql_in_parallel( ) for model_input in model_inputs ) - stored: StoredModelAnalyses = StoredModelAnalyses( - caching=analysis_caching, - requests=requests, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) request_cache_keys: tuple[str, ...] = tuple( request.cache_key for request in requests if request.cache_key is not None ) @@ -698,7 +681,7 @@ def _analyze_model_sql_in_parallel( for request in requests ) compact_batch_plan: CompactAnalysisCachePlan | None = None - if analysis_cache is not None and not stored.served: + if analysis_cache is not None: compact_batch_plan = build_compact_analysis_cache_plan( context=analysis_cache, models=tuple( @@ -726,24 +709,32 @@ def _analyze_model_sql_in_parallel( else None ) if compact_candidate is not None: - completed_analyses: dict[str, PolyglotAnalysisResult] = ( - {} - if analysis_cache is None - else stored.read_every_model(context=analysis_cache, requests=requests) - ) - if analysis_cache is not None and all( - key in completed_analyses for key in request_cache_keys - ): - record_analysis_cache_metrics( - batch_hits=len(requests), entry_hits=0, misses=0, bypasses=0 + completed_analyses: dict[str, PolyglotAnalysisResult] = {} + if analysis_cache is not None: + completed_analyses, _, _ = read_model_analyses( + context=analysis_cache, + cache_keys=request_cache_keys, + model_names=tuple(_model_name(request.model_input) for request in requests), + upstream_model_names_by_key={ + request.cache_key: _referenced_model_names( + model_input=request.model_input, + available_names=analyzed_model_names, + ) + for request in requests + if request.cache_key is not None + }, ) - return { - _model_name(request.model_input): _ModelSqlAnalysis( - polyglot_analysis=completed_analyses[cache_key], - placeholders=request.placeholders, + if all(key in completed_analyses for key in request_cache_keys): + record_analysis_cache_metrics( + batch_hits=len(requests), entry_hits=0, misses=0, bypasses=0 ) - for request, cache_key in zip(requests, request_cache_keys, strict=True) - } + return { + _model_name(request.model_input): _ModelSqlAnalysis( + polyglot_analysis=completed_analyses[cache_key], + placeholders=request.placeholders, + ) + for request, cache_key in zip(requests, request_cache_keys, strict=True) + } try: compact_analyses: tuple[_ModelSqlAnalysis, ...] = _analyze_model_sql_requests( requests=requests, @@ -781,7 +772,6 @@ def _analyze_model_sql_in_parallel( complete_binding_schemas=complete_binding_schemas, inference_profile=inference_profile, ) - stored.record_none() return { _model_name(model_input): analysis for model_input, analysis in zip(model_inputs, compact_analyses, strict=True) @@ -793,7 +783,7 @@ def _analyze_model_sql_in_parallel( previous_signatures: dict[str, str] cached_output_signatures_by_key: dict[str, str] cached_analyses, previous_signatures, cached_output_signatures_by_key = ( - stored.read( + read_model_analyses( context=analysis_cache, cache_keys=entry_cache_keys, model_names=tuple(_model_name(request.model_input) for request in requests), @@ -835,11 +825,8 @@ def _analyze_model_sql_in_parallel( analyses, cached_analyses = analyze_binding_dataflow( requests=requests, names=tuple(_model_name(request.model_input) for request in requests), - reuse=DataflowReuse( - cached=cached_analyses, - previous_signatures=previous_signatures, - served={name: analysis.dependencies for name, analysis in stored.served.items()}, - ), + cached=cached_analyses, + previous_signatures=previous_signatures, shapes=complete_binding_schemas, types=column_types_by_table, nullability=column_nullability_by_table, @@ -938,11 +925,6 @@ def _analyze_model_sql_in_parallel( invalidated_names: set[str] = _downstream_model_names( model_inputs=model_inputs, changed_names=changed_signature_names, - ) | _with_downstream( - model_inputs=model_inputs, - names=stored.stale_names( - requests=requests, cached=cached_analyses, current_signatures=current_signatures_by_name - ), ) if invalidated_names and not dependency_ordered: invalidated_requests: tuple[_ModelSqlAnalysisRequest, ...] = tuple( @@ -1004,17 +986,33 @@ def _analyze_model_sql_in_parallel( for model_name, invalidated_analysis in invalidated_analyses_by_name.items() } ) - stored.publish( - context=analysis_cache, - requests=requests, - results=tuple(analysis.polyglot_analysis for analysis in analyses), - cached=cached_analyses, - cached_output_signatures=cached_output_signatures_by_key, - invalidated_names=invalidated_names, - latest_by_model=analyses_to_record_by_name, - previous_signatures=previous_signatures, - current_signatures=current_signatures_by_name, - ) + with record_compile_timing("cache_publication_ms"): + write_model_analyses( + context=analysis_cache, + analyses_by_key={ + request.cache_key: analysis.polyglot_analysis + for request, analysis in zip(requests, analyses, strict=True) + if request.cache_key is not None + and ( + request.cache_key not in cached_analyses + or _model_name(request.model_input) in invalidated_names + ) + }, + latest_analyses_by_model=analyses_to_record_by_name, + dependency_signatures_by_key={ + request.cache_key: upstream_signatures( + model_input=request.model_input, + signatures=current_signatures_by_name, + available_names=analyzed_model_names, + ) + for request in requests + if request.cache_key is not None + and ( + request.cache_key not in cached_analyses + or _model_name(request.model_input) in invalidated_names + ) + }, + ) return { _model_name(model_input): analysis for model_input, analysis in zip(model_inputs, analyses, strict=True) @@ -1384,10 +1382,6 @@ def _should_recover_cte_facts(model_input: CompileModelInput) -> bool: ) -def _with_downstream(*, model_inputs: tuple[CompileModelInput, ...], names: set[str]) -> set[str]: - return names | _downstream_model_names(model_inputs=model_inputs, changed_names=names) - - def _model_name(model_input: CompileModelInput) -> str: return model_input.model_file.file_path.stem diff --git a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py index 85f3713d60..0513354e15 100644 --- a/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py +++ b/src/sqlbuild/compiler/compile/_helpers/assembly/semantic_shapes.py @@ -390,3 +390,15 @@ def referenced_model_names( } ) ) + + +def upstream_signatures( + *, model_input: CompileModelInput, signatures: dict[str, str], available_names: frozenset[str] +) -> dict[str, str]: + """Return the output signatures of the analyzed models this model references.""" + + return { + name: signatures[name] + for name in referenced_model_names(model_input=model_input, available_names=available_names) + if name in signatures + } diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index 0ace8f95ed..17e155d9d1 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -2,9 +2,7 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import dataclass, field -from functools import partial from pathlib import Path from sqlbuild.compiler.attachments.constants import DEFAULT_POLICY, INSTANCE_POLICY @@ -42,14 +40,10 @@ render_parameterized_sql, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics -from sqlbuild.compiler.compile._helpers.render.reuse import reused_or_rendered from sqlbuild.compiler.compile._helpers.render.sql_vars import ( expand_authored_sql_result, ) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.constants import ( - RENDER_REUSE_ATTACHED_AUDITS_GROUP, - RENDER_REUSE_SINGULAR_AUDITS_GROUP, SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, ) from sqlbuild.compiler.compile.exceptions import CompileInputError @@ -143,7 +137,6 @@ def build_project_audit_inputs( model_inputs: tuple[CompileModelInput, ...], source_inputs: tuple[CompileSourceInput, ...], seed_inputs: tuple[CompileSeedInput, ...], - render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: """Resolve built-in-aware generic audits and build every project audit input.""" @@ -159,7 +152,6 @@ def build_project_audit_inputs( source_inputs=source_inputs, generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, - render_reuse=render_reuse, ) return audit_inputs, diagnostics @@ -173,7 +165,6 @@ def build_audit_inputs( generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] | None = None, seed_inputs: tuple[CompileSeedInput, ...] = (), - render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileAuditInput, ...]: """Build compile-time audit inputs from discovered SQL audit blocks.""" @@ -204,38 +195,20 @@ def build_audit_inputs( sql_lexical_syntax=sql_lexical_syntax, ) audit_inputs: list[CompileAuditInput] = list( - reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SINGULAR_AUDITS_GROUP, - render=partial( - _build_singular_audit_inputs, - discovered_inputs=discovered_inputs, - context=attachment_context, - ), + _build_singular_audit_inputs( + discovered_inputs=discovered_inputs, context=attachment_context ) ) model_input: CompileModelInput for model_input in model_inputs: if model_input.schema_entry is None: continue - render: Callable[[], tuple[CompileAuditInput, ...]] = partial( - build_model_attached_audit_inputs, model_input=model_input, context=attachment_context - ) audit_inputs.extend( - render() - if render_reuse is None - else render_reuse.model_audits(model_input=model_input, render=render) + build_model_attached_audit_inputs(model_input=model_input, context=attachment_context) ) audit_inputs.extend( - reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_ATTACHED_AUDITS_GROUP, - render=partial( - _build_source_and_seed_audit_inputs, - source_inputs=source_inputs, - seed_inputs=seed_inputs, - context=attachment_context, - ), + _build_source_and_seed_audit_inputs( + source_inputs=source_inputs, seed_inputs=seed_inputs, context=attachment_context ) ) return tuple(audit_inputs) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index 78564c72cb..0e900cedab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -4,9 +4,7 @@ import inspect import re -from collections.abc import Callable from dataclasses import dataclass, field, fields, replace -from functools import partial from inspect import Parameter, Signature from pathlib import Path from typing import Any, cast @@ -91,7 +89,6 @@ expand_effective_vars, expand_template_data, ) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_FULL_REFRESH_CONFIG_KEY, @@ -446,7 +443,6 @@ def build_model_inputs( defer_model_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, reference_cache_dir: Path | None = None, - render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileModelInput, ...]: """Attach schema metadata to discovered model files.""" @@ -464,7 +460,6 @@ def build_model_inputs( external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, legacy_schema_files=legacy_schema_files, - render_reuse=render_reuse, ) @@ -477,7 +472,6 @@ def _build_model_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, extract_references: SqlReferenceExtractor, legacy_schema_files: tuple[DiscoveredSchemaFile, ...], - render_reuse: CompileRenderReuseSession | None, ) -> tuple[CompileModelInput, ...]: effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings @@ -537,11 +531,7 @@ def _build_model_inputs( sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( discovered_inputs.sql_hook_files ) - render_files: tuple[DiscoveredSqlModelFile, ...] = tuple( - model_file - for model_file in discovered_inputs.model_files - if render_reuse is None or not render_reuse.has_reusable_model(model_file=model_file) - ) + render_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files prepared_var_substituted_sqls: dict[Path, str | None] = dict( zip( (model_file.file_path for model_file in render_files), @@ -594,22 +584,14 @@ def _build_model_inputs( model_inputs: list[CompileModelInput] = [] model_file: DiscoveredSqlModelFile for model_file in discovered_inputs.model_files: - reused: CompileModelInput | None = ( - None if render_reuse is None else render_reuse.reused_model(model_file=model_file) - ) - if reused is not None: - model_inputs.append(reused) - continue - render: Callable[[], CompileModelInput] = partial( - _build_model_input, - loop=loop, - model_file=model_file, - prepared_var_substituted_sql=prepared_var_substituted_sqls.get(model_file.file_path), - ) model_inputs.append( - render() - if render_reuse is None - else render_reuse.rendered_model(model_file=model_file, render=render) + _build_model_input( + loop=loop, + model_file=model_file, + prepared_var_substituted_sql=prepared_var_substituted_sqls.get( + model_file.file_path + ), + ) ) validate_declared_schema_models_are_attached( diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 84b8dbc1cc..02862cc016 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -5,7 +5,6 @@ import inspect from collections.abc import Mapping from dataclasses import replace -from pathlib import Path from types import CodeType from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( @@ -42,7 +41,6 @@ def build_declaration_scope( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" @@ -54,7 +52,6 @@ def build_declaration_scope( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, ) if native_scope is not None: return native_scope @@ -76,7 +73,6 @@ def build_declaration_scope( discovered_inputs=discovered_inputs, index=index, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, ) if has_scoped_relationship_declarations and (discovered_inputs.test_files or discovered_inputs.scenario_files) @@ -105,7 +101,6 @@ def _build_native_declaration_scope( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None, ) -> DeclarationScopeBuild | None: """Build the scope natively with Python's errors and order, or None for the Python stage.""" @@ -126,7 +121,6 @@ def _build_native_declaration_scope( facts, fault = extract_scope_relationship_facts( discovered_inputs=discovered_inputs, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, ) if fault is not None: raise CompileInputError(fault) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index 46758adddc..ebc6f13278 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -4,7 +4,6 @@ from collections.abc import Sequence from pathlib import Path -from typing import cast from sqlbuild.compiler.compile._helpers.render.macros import ( find_macro_call_names, @@ -17,11 +16,7 @@ extract_sql_test_expected_model_names, extract_unclassified_sql_test_ctes, ) -from sqlbuild.compiler.compile.constants import ( - MACRO_ACTUAL_TEST_CTE_NAME, - SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, - SQL_TEST_FACT_CACHE_NAMESPACE, -) +from sqlbuild.compiler.compile.constants import MACRO_ACTUAL_TEST_CTE_NAME from sqlbuild.compiler.compile.models import ( CompileSqlTestCte, ScopeRelationshipBuild, @@ -29,7 +24,6 @@ ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlTestFile -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.scopes.main._native_expected_model_names import ( native_expected_model_names, ) @@ -62,24 +56,17 @@ def build_scope_relationship_grants( discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex, sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" lookup: ScopeLookup = build_scope_lookup(index=index) shared_declarations: _SharedDeclarations = _shared_declarations_by_directory(lookup=lookup) - with FactCacheStore( - root=compile_cache_dir, - namespace=SQL_TEST_FACT_CACHE_NAMESPACE, - algorithm=SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, - ) as fact_cache: - test_grants, test_faults = _test_relationship_grants( - discovered_inputs=discovered_inputs, - lookup=lookup, - shared_declarations=shared_declarations, - fact_cache=fact_cache, - syntax=sql_lexical_syntax, - ) + test_grants, test_faults = _test_relationship_grants( + discovered_inputs=discovered_inputs, + lookup=lookup, + shared_declarations=shared_declarations, + syntax=sql_lexical_syntax, + ) scenario_grants, scenario_faults = _scenario_relationship_grants( discovered_inputs=discovered_inputs, lookup=lookup, @@ -96,98 +83,49 @@ def extract_scope_relationship_facts( *, discovered_inputs: DiscoveredProjectInputs, sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, ) -> tuple[tuple[RelationshipFact, ...], str | None]: """Return the relationship names grants are resolved from, and the first extraction fault.""" facts: list[RelationshipFact] = [] faults: list[ScopeRelationshipFault] = [] - with FactCacheStore( - root=compile_cache_dir, - namespace=SQL_TEST_FACT_CACHE_NAMESPACE, - algorithm=SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM, - ) as fact_cache: - cache_keys: dict[int, str] = ( - { - file_index: _expected_models_fact_key( - test_file=test_file, fact_cache=fact_cache, syntax=sql_lexical_syntax - ) - for file_index, test_file in enumerate(discovered_inputs.test_files) - } - if fact_cache.enabled - else {} - ) - cached_names: dict[str, object] = fact_cache.read_many( - tuple( - (_expected_models_fact_slot(discovered_inputs.test_files[file_index]), cache_key) - for file_index, cache_key in cache_keys.items() - ) - ) - cached_by_file: list[tuple[tuple[str, ...], ...] | None] = [ - _cached_expected_names( - value=cached_names.get(cache_keys[file_index]) - if file_index in cache_keys - else None, - block_count=len(test_file.blocks), - ) - for file_index, test_file in enumerate(discovered_inputs.test_files) - ] - scanned_names: dict[tuple[int, int], tuple[str, ...]] = _scanned_expected_names( - test_files=discovered_inputs.test_files, - cached_by_file=cached_by_file, - syntax=sql_lexical_syntax, - ) - for file_index, test_file in enumerate(discovered_inputs.test_files): - cache_key: str | None = cache_keys.get(file_index) - cached_file_names: tuple[tuple[str, ...], ...] | None = cached_by_file[file_index] - extracted_names: list[tuple[str, ...]] = [] - for block_index, block in enumerate(test_file.blocks): - scanned: tuple[str, ...] | None = scanned_names.get((file_index, block_index)) - try: - expected_names: tuple[str, ...] = ( - cached_file_names[block_index] - if cached_file_names is not None - else scanned - if scanned is not None - else extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=sql_lexical_syntax, - mode=block.mode, - ) + scanned_names: dict[tuple[int, int], tuple[str, ...]] = _scanned_expected_names( + test_files=discovered_inputs.test_files, syntax=sql_lexical_syntax + ) + for file_index, test_file in enumerate(discovered_inputs.test_files): + for block_index, block in enumerate(test_file.blocks): + scanned: tuple[str, ...] | None = scanned_names.get((file_index, block_index)) + try: + expected_names: tuple[str, ...] = ( + scanned + if scanned is not None + else extract_sql_test_expected_model_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=sql_lexical_syntax, + mode=block.mode, ) - extracted_names.append(expected_names) - called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) - facts.append( - RelationshipFact( - resource=ResourceIdentity( - ResourceKind.TEST, block.name or test_file.relative_path.stem - ), - expected_models=expected_names, - called_macros=called_macros, - tested_macros=( - _tested_macro_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=sql_lexical_syntax, - ) - if block.mode is SqlTestMode.MACRO - else () - ), - ) + ) + called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) + facts.append( + RelationshipFact( + resource=ResourceIdentity( + ResourceKind.TEST, block.name or test_file.relative_path.stem + ), + expected_models=expected_names, + called_macros=called_macros, + tested_macros=( + _tested_macro_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=sql_lexical_syntax, + ) + if block.mode is SqlTestMode.MACRO + else () + ), ) - except Exception as error: - faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - if ( - cache_key is not None - and cached_file_names is None - and len(extracted_names) == len(test_file.blocks) - ): - fact_cache.stage( - key=cache_key, - slot=_expected_models_fact_slot(test_file), - value=tuple(extracted_names), ) + except Exception as error: + faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) scenario_names: list[tuple[str, ...] | None] = native_expected_model_names( sqls=[scenario.sql_body for scenario in discovered_inputs.scenario_files], syntax=sql_lexical_syntax, @@ -214,15 +152,12 @@ def extract_scope_relationship_facts( def _scanned_expected_names( *, test_files: Sequence[DiscoveredSqlTestFile], - cached_by_file: Sequence[tuple[tuple[str, ...], ...] | None], syntax: SqlLexicalSyntax, ) -> dict[tuple[int, int], tuple[str, ...]]: - """Scan uncached model-mode blocks natively, keeping only the names the scan reproduces.""" + """Scan model-mode blocks natively, keeping only the names the scan reproduces.""" pending: list[tuple[int, int]] = [] for file_index, test_file in enumerate(test_files): - if cached_by_file[file_index] is not None: - continue pending.extend( (file_index, block_index) for block_index, block in enumerate(test_file.blocks) @@ -257,47 +192,19 @@ def _test_relationship_grants( discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup, shared_declarations: _SharedDeclarations, - fact_cache: FactCacheStore, syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] - cache_keys: dict[int, str] = ( - { - file_index: _expected_models_fact_key( - test_file=test_file, fact_cache=fact_cache, syntax=syntax - ) - for file_index, test_file in enumerate(discovered_inputs.test_files) - } - if fact_cache.enabled - else {} - ) - cached_names: dict[str, object] = fact_cache.read_many( - tuple( - (_expected_models_fact_slot(discovered_inputs.test_files[file_index]), cache_key) - for file_index, cache_key in cache_keys.items() - ) - ) - for file_index, test_file in enumerate(discovered_inputs.test_files): - cache_key: str | None = cache_keys.get(file_index) - cached_file_names: tuple[tuple[str, ...], ...] | None = _cached_expected_names( - value=None if cache_key is None else cached_names.get(cache_key), - block_count=len(test_file.blocks), - ) - extracted_names: list[tuple[str, ...]] = [] - for block_index, block in enumerate(test_file.blocks): + for test_file in discovered_inputs.test_files: + for block in test_file.blocks: try: - expected_names: tuple[str, ...] = ( - cached_file_names[block_index] - if cached_file_names is not None - else extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=syntax, - mode=block.mode, - ) + expected_names: tuple[str, ...] = extract_sql_test_expected_model_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=syntax, + mode=block.mode, ) - extracted_names.append(expected_names) grants.extend( _expected_model_grants( lookup=lookup, @@ -323,43 +230,9 @@ def _test_relationship_grants( ) except Exception as error: faults.append(ScopeRelationshipFault(test_file.relative_path, str(error))) - if ( - cache_key is not None - and cached_file_names is None - and len(extracted_names) == len(test_file.blocks) - ): - fact_cache.stage( - key=cache_key, - slot=_expected_models_fact_slot(test_file), - value=tuple(extracted_names), - ) return tuple(grants), tuple(faults) -def _expected_models_fact_key( - *, test_file: DiscoveredSqlTestFile, fact_cache: FactCacheStore, syntax: SqlLexicalSyntax -) -> str: - parts: list[str] = [str(test_file.relative_path), syntax.cache_key] - for block in test_file.blocks: - parts.extend((block.sql_body, block.mode.value)) - return fact_cache.key(*parts) - - -def _cached_expected_names( - *, value: object, block_count: int -) -> tuple[tuple[str, ...], ...] | None: - if not isinstance(value, tuple) or len(value) != block_count: - return None - for names in value: - if not isinstance(names, tuple) or not all(isinstance(name, str) for name in names): - return None - return cast(tuple[tuple[str, ...], ...], value) - - -def _expected_models_fact_slot(test_file: DiscoveredSqlTestFile) -> str: - return f"expected:{test_file.relative_path}" - - def _scenario_relationship_grants( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index b143019c3b..f33069446d 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -46,7 +46,7 @@ sql_test_reads, ) from sqlbuild.compiler.compile._helpers.sql_tests.native import ( - extract_expanded_sql_tests_cached, + extract_expanded_sql_tests, extract_unexpanded_sql_test, ) from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestExtractionError @@ -118,7 +118,6 @@ def build_test_inputs_with_cache( declaration_expansion: DeclarationExpansionContext, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...], - compile_cache_dir: Path | None, sql_lexical_syntax: SqlLexicalSyntax, ) -> tuple[CompileSqlTestInput, ...]: """Build SQL test inputs inside the compile timing boundary.""" @@ -132,7 +131,6 @@ def build_test_inputs_with_cache( declaration_expansion=declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, - compile_cache_dir=compile_cache_dir, sql_lexical_syntax=sql_lexical_syntax, ) @@ -147,7 +145,6 @@ def build_test_inputs( sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = (), - compile_cache_dir: Path | None = None, ) -> tuple[CompileSqlTestInput, ...]: """Build compile-time test inputs from discovered SQL-native test blocks.""" @@ -308,12 +305,11 @@ def build_test_inputs( ) ) try: - test_ctes_batch: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests_cached( + test_ctes_batch: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( tests=tuple( (test.sql_body, str(test.test_file.relative_path), test.mode) for test in expanded_tests ), - cache_root=compile_cache_dir, syntax=sql_lexical_syntax, ) except SqlTestExtractionError as error: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py index 3d7b954235..335e83bec4 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from pathlib import Path from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars @@ -9,8 +10,12 @@ resolve_early_model_templates, resolve_run_id, ) +from sqlbuild.compiler.compile.constants import ( + COMPILE_CACHE_DISABLE_ENV_VAR, + COMPILE_CACHE_DISABLE_VALUE, +) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name from sqlbuild.spec.contracts.models import TargetConfig @@ -46,6 +51,21 @@ def build_compile_target_context( return target_name, target_config, cache_dir +def compile_cache_root( + *, project_dir: Path | None, target_config: TargetConfig | None, no_cache: bool +) -> Path | None: + """Return the shared compile-cache directory, or None when caching is disabled.""" + + if ( + no_cache + or project_dir is None + or (target_config is not None and target_config.compile_cache is False) + or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE + ): + return None + return compiler_cache_directory(project_dir) + + def build_effective_target_namespace( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index 9a7ee0f036..ae6080acb7 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -21,23 +21,6 @@ _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None ) -_ACTIVE_TAPS: ContextVar[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = ContextVar( - "sqlbuild_tapped_compile_diagnostics", default=() -) - - -@contextmanager -def tapped_compile_diagnostics() -> Iterator[list[tuple[tuple[str, ...], CompilerDiagnostic]]]: - """Also note every diagnostic reported inside the block, in report order, for replay.""" - - tap: list[tuple[tuple[str, ...], CompilerDiagnostic]] = [] - token: Token[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = _ACTIVE_TAPS.set( - (*_ACTIVE_TAPS.get(), tap) - ) - try: - yield tap - finally: - _ACTIVE_TAPS.reset(token) @contextmanager @@ -71,8 +54,6 @@ def collecting(*args: P.args, **kwargs: P.kwargs) -> R: def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: """Record a diagnostic in the active collector, or raise it when none is active.""" - for tap in _ACTIVE_TAPS.get(): - tap.append((key, diagnostic)) collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() if collector is not None: collector.add(key=key, diagnostic=diagnostic) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/reuse.py b/src/sqlbuild/compiler/compile/_helpers/render/reuse.py deleted file mode 100644 index 70132c44da..0000000000 --- a/src/sqlbuild/compiler/compile/_helpers/render/reuse.py +++ /dev/null @@ -1,111 +0,0 @@ -"""Route project-wide renders through the active render reuse session, if any.""" - -from __future__ import annotations - -from collections.abc import Callable -from functools import partial -from pathlib import Path - -from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs -from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( - build_scenario_inputs, - build_test_inputs_with_cache, -) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import ( - COMPILE_RENDER_REUSE, - RENDER_REUSE_SCENARIOS_GROUP, - RENDER_REUSE_SOURCES_GROUP, - RENDER_REUSE_TESTS_GROUP, -) -from sqlbuild.compiler.compile.models import ( - CompileSourceInput, - CompileSqlFunctionInput, - CompileSqlScenarioInput, - CompileSqlTestInput, - ModelInputBuildContext, -) -from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver - - -def reused_or_rendered[T]( - *, render_reuse: CompileRenderReuseSession | None, name: str, render: Callable[[], T] -) -> T: - """Return the stored render of one project-wide group, or render and record it.""" - - return render() if render_reuse is None else render_reuse.group(name=name, render=render) - - -def claimed_render_reuse( - *, discovered_inputs: DiscoveredProjectInputs -) -> CompileRenderReuseSession | None: - """Claim the active render reuse session, if any, and plan it for the discovered models.""" - - render_reuse: CompileRenderReuseSession | None = COMPILE_RENDER_REUSE.claim() - if render_reuse is not None: - render_reuse.plan_models(model_files=discovered_inputs.model_files) - return render_reuse - - -def build_sql_resource_inputs( - *, - discovered_inputs: DiscoveredProjectInputs, - context: ModelInputBuildContext, - sql_function_inputs: tuple[CompileSqlFunctionInput, ...], - no_sql_validation: bool, - external_sql_reference_resolver: ExternalSqlReferenceResolver | None, - compile_cache_dir: Path | None, - render_reuse: CompileRenderReuseSession | None, -) -> tuple[ - tuple[CompileSourceInput, ...], - tuple[CompileSqlTestInput, ...], - tuple[CompileSqlScenarioInput, ...], -]: - """Render sources, tests, and scenarios, each reused whole after model-only edits.""" - - source_inputs: tuple[CompileSourceInput, ...] = reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SOURCES_GROUP, - render=partial( - build_source_inputs, - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - effective_settings=context.effective_settings, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=context.declaration_expansion, - no_sql_validation=no_sql_validation, - ), - ) - test_inputs: tuple[CompileSqlTestInput, ...] = reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_TESTS_GROUP, - render=partial( - build_test_inputs_with_cache, - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, - compile_cache_dir=compile_cache_dir, - sql_lexical_syntax=context.sql_lexical_syntax, - ), - ) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SCENARIOS_GROUP, - render=partial( - build_scenario_inputs, - discovered_inputs=discovered_inputs, - effective_vars=context.effective_vars, - macro_context=context.macro_context, - loaded_macros=context.loaded_macros, - declaration_expansion=context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=context.sql_lexical_syntax, - ), - ) - return source_inputs, test_inputs, scenario_inputs diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 8c2a4aa13f..45f2985970 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,16 +2,11 @@ from __future__ import annotations -from pathlib import Path from typing import Protocol, cast import orjson import sqlbuild._native as _native -from sqlbuild.compiler.compile.constants import ( - SQL_TEST_FACT_CACHE_ALGORITHM, - SQL_TEST_FACT_CACHE_NAMESPACE, -) from sqlbuild.compiler.compile.exceptions import ( CompileInputError, NativeSqlTestResponseError, @@ -24,7 +19,6 @@ CompileSqlTestCtes, ) from sqlbuild.compiler.compile.types import SqlTestMode -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -46,88 +40,6 @@ def extract_sql_tests_json(self, request_json: str) -> str: ... _AUTHORED_CTE_LENGTH: int = 3 -def extract_expanded_sql_tests_cached( - *, - tests: tuple[tuple[str, str, SqlTestMode], ...], - cache_root: Path | None, - syntax: SqlLexicalSyntax, -) -> tuple[CompileSqlTestCtes, ...]: - """Reuse exact per-file extraction facts and extract only tests of changed files natively.""" - - with FactCacheStore( - root=cache_root, - namespace=SQL_TEST_FACT_CACHE_NAMESPACE, - algorithm=SQL_TEST_FACT_CACHE_ALGORITHM, - ) as fact_cache: - if not fact_cache.enabled: - return extract_expanded_sql_tests(tests=tests, syntax=syntax) - indexes_by_file: dict[str, list[int]] = {} - for index, (_sql, file_label, _mode) in enumerate(tests): - indexes_by_file.setdefault(file_label, []).append(index) - keys_by_file: dict[str, str] = { - file_label: fact_cache.key( - file_label, - syntax.cache_key, - *_file_test_key_parts(tests=tests, indexes=indexes), - ) - for file_label, indexes in indexes_by_file.items() - } - cached: dict[str, object] = fact_cache.read_many(tuple(keys_by_file.items())) - results: list[CompileSqlTestCtes | None] = [None] * len(tests) - missing_files: list[str] = [] - for file_label, indexes in indexes_by_file.items(): - file_results: tuple[CompileSqlTestCtes, ...] | None = _cached_file_tests( - value=cached.get(keys_by_file[file_label]), - modes=tuple(tests[index][2] for index in indexes), - ) - if file_results is None: - missing_files.append(file_label) - continue - for index, test_ctes in zip(indexes, file_results, strict=True): - results[index] = test_ctes - missing_indexes: list[int] = [] - for file_label in missing_files: - missing_indexes.extend(indexes_by_file[file_label]) - if missing_indexes: - try: - extracted: tuple[CompileSqlTestCtes, ...] = extract_expanded_sql_tests( - tests=tuple(tests[index] for index in missing_indexes), syntax=syntax - ) - except SqlTestExtractionError as error: - error.test_index = missing_indexes[error.test_index] - raise - for index, test_ctes in zip(missing_indexes, extracted, strict=True): - results[index] = test_ctes - for file_label in missing_files: - fact_cache.stage( - key=keys_by_file[file_label], - slot=file_label, - value=tuple(results[index] for index in indexes_by_file[file_label]), - ) - return tuple(result for result in results if result is not None) - - -def _file_test_key_parts( - *, tests: tuple[tuple[str, str, SqlTestMode], ...], indexes: list[int] -) -> list[str]: - parts: list[str] = [] - for index in indexes: - sql, _file_label, mode = tests[index] - parts.extend((sql, mode.value)) - return parts - - -def _cached_file_tests( - *, value: object, modes: tuple[SqlTestMode, ...] -) -> tuple[CompileSqlTestCtes, ...] | None: - if not isinstance(value, tuple) or len(value) != len(modes): - return None - for test_ctes, mode in zip(value, modes, strict=True): - if not isinstance(test_ctes, CompileSqlTestCtes) or test_ctes.mode is not mode: - return None - return cast(tuple[CompileSqlTestCtes, ...], value) - - def extract_expanded_sql_tests( *, tests: tuple[tuple[str, str, SqlTestMode], ...], diff --git a/src/sqlbuild/compiler/compile/classes/binding_dataflow.py b/src/sqlbuild/compiler/compile/classes/binding_dataflow.py index 638f39f370..0909ba58c7 100644 --- a/src/sqlbuild/compiler/compile/classes/binding_dataflow.py +++ b/src/sqlbuild/compiler/compile/classes/binding_dataflow.py @@ -19,9 +19,7 @@ published_model_shape, referenced_model_names, ) -from sqlbuild.compiler.compile.classes.stored_model_analyses import dependencies_current from sqlbuild.compiler.compile.models import ( - DataflowReuse, ModelSqlAnalysis, ModelSqlAnalysisRequest, PolyglotAnalysisResult, @@ -44,7 +42,8 @@ def __init__( *, requests: tuple[ModelSqlAnalysisRequest, ...], names: tuple[str, ...], - reuse: DataflowReuse, + cached: dict[str, PolyglotAnalysisResult], + previous_signatures: dict[str, str], shapes: dict[str, dict[str, str]], types: dict[str, dict[str, str]], nullability: dict[str, dict[str, InferredNullability]], @@ -54,15 +53,8 @@ def __init__( ) -> None: self._requests: tuple[ModelSqlAnalysisRequest, ...] = requests self._names: tuple[str, ...] = names + self._cached: dict[str, PolyglotAnalysisResult] = cached self._by_name: dict[str, ModelSqlAnalysisRequest] = dict(zip(names, requests, strict=True)) - served_keys: frozenset[str | None] = frozenset( - self._by_name[name].cache_key for name in reuse.served if name in self._by_name - ) - self._cached: dict[str, PolyglotAnalysisResult] = { - key: analysis for key, analysis in reuse.cached.items() if key not in served_keys - } - self._served: dict[str, dict[str, str]] = reuse.served - self._signatures: dict[str, str] = {} available_names: frozenset[str] = frozenset(names) self._dependencies: dict[str, tuple[str, ...]] = { name: referenced_model_names( @@ -70,11 +62,11 @@ def __init__( ) for name, request in self._by_name.items() } - self._previous_signatures: dict[str, str] = reuse.previous_signatures + self._previous_signatures: dict[str, str] = previous_signatures self._complete_shapes: dict[str, dict[str, str]] = dict(shapes) self._available_types: dict[str, dict[str, str]] = dict(types) self._available_nullability: dict[str, dict[str, InferredNullability]] = dict(nullability) - self._reusable: dict[str, PolyglotAnalysisResult] = dict(reuse.cached) + self._reusable: dict[str, PolyglotAnalysisResult] = dict(cached) self._profile: ExpressionInferenceProfile = profile self._analyze: Callable[..., tuple[ModelSqlAnalysis, ...]] = analyze self._complete: Callable[..., tuple[ModelSqlAnalysis, ...]] = complete @@ -259,8 +251,6 @@ def _analyze_ready(self, ready: tuple[str, ...]) -> None: self._changed.add(name) if request.cache_key is not None: self._reusable.pop(request.cache_key, None) - if not self._served_parents_final(name) and request.cache_key is not None: - self._reusable.pop(request.cache_key, None) bindings: dict[str, dict[str, str]] | None = ( None if request.binding_schema is None @@ -298,24 +288,11 @@ def _analyze_ready(self, ready: tuple[str, ...]) -> None: self._results[name] = result self._publish(name=name, request=request, result=result) - def _served_parents_final(self, name: str) -> bool: - """Return whether a stored analysis, if served, was made from the parents' final state.""" - - dependencies: dict[str, str] | None = self._served.get(name) - return dependencies is None or dependencies_current( - dependencies=dependencies, - parent_signatures={ - parent: self._signatures.get(parent) for parent in self._dependencies[name] - }, - ) - def _publish( self, *, name: str, request: ModelSqlAnalysisRequest, result: ModelSqlAnalysis ) -> None: analysis: PolyglotAnalysisResult = result.polyglot_analysis - signature: str = model_analysis_output_signature(analysis) - self._signatures[name] = signature - if signature != self._previous_signatures.get(name): + if model_analysis_output_signature(analysis) != self._previous_signatures.get(name): self._changed.add(name) required: frozenset[str] = binding_relation_names(request.model_input.references) star_known: bool = not analysis.has_star or ( diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py b/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py deleted file mode 100644 index ad40b52907..0000000000 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Process-wide access to the render reuse session of the running compile command.""" - -from __future__ import annotations - -import threading -from collections.abc import Iterator -from contextlib import contextmanager -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession - - -class CompileRenderReuseRegistry: - """Expose one compile command's render reuse session to the rendering code it runs.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._session: CompileRenderReuseSession | None = None - - @contextmanager - def activated(self, session: CompileRenderReuseSession) -> Iterator[CompileRenderReuseSession]: - """Make the session available to rendering until the block exits.""" - - with self._lock: - previous: CompileRenderReuseSession | None = self._session - self._session = session - try: - yield session - finally: - with self._lock: - self._session = previous - - def claim(self) -> CompileRenderReuseSession | None: - """Hand the active session to the first project render only, so it is used once.""" - - with self._lock: - session: CompileRenderReuseSession | None = self._session - if session is None or not session.claim(): - return None - return session - - def claim_discovery(self) -> CompileRenderReuseSession | None: - """Hand the active session to the first project discovery only, so it is used once.""" - - with self._lock: - session: CompileRenderReuseSession | None = self._session - if session is None or not session.claim_discovery(): - return None - return session diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py deleted file mode 100644 index f0aca97da8..0000000000 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Reuse unaffected rendered inputs from the previous compile and record this compile's.""" - -from __future__ import annotations - -import pickle -import threading -from collections.abc import Callable -from dataclasses import replace -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - report_compile_diagnostic, - tapped_compile_diagnostics, -) -from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads -from sqlbuild.compiler.compile.constants import ( - COMPILE_INPUT_READS, - RENDER_REUSE_ANALYSIS_PREFIX, - RENDER_REUSE_DECLARATIONS_PREFIX, - RENDER_REUSE_MODEL_AUDITS_PREFIX, -) -from sqlbuild.compiler.compile.models import ( - CompileAuditInput, - CompileModelInput, - CompilerDiagnostic, - RenderReuseState, - StoredModelAnalysis, - StoredRender, -) -from sqlbuild.compiler.discovery.models import DiscoveredDeclarationFiles, DiscoveredSqlModelFile -from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError -from sqlbuild.compiler.fact_cache.main._dump_fact_payload import dumped_fact_payload -from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload -from sqlbuild.compiler.profiling.main._metric import record_compile_metric - -_LOAD_ERRORS: tuple[type[Exception], ...] = ( - FactCachePayloadError, - pickle.UnpicklingError, - AttributeError, - EOFError, - ImportError, - IndexError, - KeyError, - TypeError, - ValueError, - RecursionError, - MemoryError, - OverflowError, -) - - -class CompileRenderReuseSession: - """Serve stored renders of inputs a change set cannot affect, and record every render.""" - - def __init__( - self, - *, - prior: RenderReuseState | None, - changed_paths: frozenset[str] | None, - retained_models: frozenset[str] = frozenset(), - retained_groups: frozenset[str] = frozenset(), - ) -> None: - self._lock: threading.Lock = threading.Lock() - self._claimed: bool = False - self._discovery_claimed: bool = False - self._prior: RenderReuseState | None = prior if changed_paths is not None else None - self._changed_paths: frozenset[str] = changed_paths or frozenset() - self._reusable: bool = False - self._model_paths: tuple[str, ...] | None = None - self._retained_models: frozenset[str] = retained_models - self._retained_groups: frozenset[str] = retained_groups - self._model_records: dict[str, memoryview | None] = {} - self._group_records: dict[str, memoryview | None] = {} - self._decoded_models: dict[str, StoredRender] = {} - self._decoded_groups: dict[str, StoredRender] = {} - self._pending_models: list[tuple[str, StoredRender]] = [] - self._pending_groups: list[tuple[str, StoredRender]] = [] - self._serialized_models: dict[str, memoryview] = {} - self._serialized_groups: dict[str, memoryview] = {} - self._reused_paths: set[str] = set() - self._stored_analyses: dict[str, StoredModelAnalysis] = {} - self._stored_analysis_payloads: dict[str, memoryview] = {} - - def claim(self) -> bool: - """Return True for the first claim only; later project renders neither reuse nor record.""" - - with self._lock: - claimed: bool = self._claimed - self._claimed = True - return not claimed - - def claim_discovery(self) -> bool: - """Return True for the first discovery claim only; later discoveries run in full.""" - - with self._lock: - claimed: bool = self._discovery_claimed - self._discovery_claimed = True - return not claimed - - def declaration_files( - self, - *, - variant: str, - discover: Callable[[], DiscoveredDeclarationFiles], - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], - ) -> DiscoveredDeclarationFiles: - """Reuse stored declaration files after model-only edits, or discover and record them.""" - - name: str = f"{RENDER_REUSE_DECLARATIONS_PREFIX}{variant}" - prior: RenderReuseState | None = self._prior - payload: memoryview | None = ( - prior.group_payloads.get(name) - if prior is not None - and edits_only_models(changed_paths=self._changed_paths, model_paths=prior.model_paths) - else None - ) - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if ( - stored is not None - and payload is not None - and isinstance(stored.value, DiscoveredDeclarationFiles) - ): - model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() - _replay(stored=stored) - self._group_records[name] = payload - self._decoded_groups[name] = stored - return replace(stored.value, model_files=model_files) - with ( - COMPILE_INPUT_READS.recording() as reads, - tapped_compile_diagnostics() as reported, - ): - declarations: DiscoveredDeclarationFiles = discover() - recorded: StoredRender | None = _stored_render( - reads=reads, - value=replace(declarations, model_files=(), native_session=None), - reported=reported, - ) - if recorded is not None: - self._pending_groups.append((name, recorded)) - return declarations - - def plan_models(self, *, model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: - """Reuse stored renders only when the change set touches nothing but existing models.""" - - model_paths: tuple[str, ...] = tuple(str(item.relative_path) for item in model_files) - self._model_paths = model_paths - prior: RenderReuseState | None = self._prior - self._reusable = ( - prior is not None - and prior.model_paths == model_paths - and self._changed_paths.issubset(model_paths) - ) - - def has_reusable_model(self, *, model_file: DiscoveredSqlModelFile) -> bool: - """Return whether a stored render of this model may stand in for rendering it.""" - - path: str = str(model_file.relative_path) - return ( - self._reusable - and self._prior is not None - and path not in self._changed_paths - and path in self._prior.model_payloads - ) - - def reused_model(self, *, model_file: DiscoveredSqlModelFile) -> CompileModelInput | None: - """Return the stored render of an unchanged model, replaying what rendering reported.""" - - if not self.has_reusable_model(model_file=model_file) or self._prior is None: - return None - path: str = str(model_file.relative_path) - payload: memoryview | None = self._prior.model_payloads[path] - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if stored is None or not isinstance(stored.value, CompileModelInput): - return None - _replay(stored=stored) - self._model_records[path] = payload - self._decoded_models[path] = stored - self._reused_paths.add(path) - record_compile_metric(metric="render_reuse_hits", value=1) - return replace(stored.value, model_file=model_file) - - def rendered_model( - self, - *, - model_file: DiscoveredSqlModelFile, - render: Callable[[], CompileModelInput], - ) -> CompileModelInput: - """Render one model and record its render with what rendering reported and read.""" - - with ( - COMPILE_INPUT_READS.recording() as reads, - tapped_compile_diagnostics() as reported, - ): - model_input: CompileModelInput = render() - record_compile_metric(metric="render_reuse_misses", value=1) - stored: StoredRender | None = _stored_render( - reads=reads, value=replace(model_input, model_file=None), reported=reported - ) - if stored is not None: - self._pending_models.append((str(model_file.relative_path), stored)) - return model_input - - def model_audits( - self, - *, - model_input: CompileModelInput, - render: Callable[[], tuple[CompileAuditInput, ...]], - ) -> tuple[CompileAuditInput, ...]: - """Return a reused model's stored attached audits, or render and record them.""" - - path: str = str(model_input.model_file.relative_path) - return self.group( - name=f"{RENDER_REUSE_MODEL_AUDITS_PREFIX}{path}", - render=render, - reusable=path in self._reused_paths, - ) - - def group[T](self, *, name: str, render: Callable[[], T], reusable: bool = True) -> T: - """Return a stored project-wide render no model edit can affect, or render and record it.""" - - payload: memoryview | None = ( - self._prior.group_payloads.get(name) - if reusable and self._reusable and self._prior is not None - else None - ) - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if stored is not None and payload is not None: - _replay(stored=stored) - self._group_records[name] = payload - self._decoded_groups[name] = stored - return cast(T, stored.value) - with ( - COMPILE_INPUT_READS.recording() as reads, - tapped_compile_diagnostics() as reported, - ): - value: T = render() - rendered: StoredRender | None = _stored_render(reads=reads, value=value, reported=reported) - if rendered is not None: - self._pending_groups.append((name, rendered)) - return value - - def pending_renders(self) -> int: - """Return how many new renders storing this compile still has to serialize.""" - - return len(self._pending_models) + len(self._pending_groups) - - def reused_any(self) -> bool: - """Return whether any render was reused, so imports it made were skipped this run.""" - - return bool(self._decoded_models or self._decoded_groups) - - def reused_model_names(self) -> frozenset[str]: - """Return the names of models whose stored renders this compile reused.""" - - return frozenset(Path(path).stem for path in self._reused_paths) - - def stored_analyses(self) -> dict[str, StoredModelAnalysis]: - """Return stored analyses of reused models by model name.""" - - return self._stored_analyses - - def record_analyses(self, *, analyses: dict[str, StoredModelAnalysis | None]) -> None: - """Record every model's analysis for the next compile; absent or None drops it.""" - - for path in self._model_paths or (): - name: str = f"{RENDER_REUSE_ANALYSIS_PREFIX}{path}" - model_name: str = Path(path).stem - analysis: StoredModelAnalysis | None = analyses.get(model_name) - stored: StoredRender = StoredRender( - value=analysis, diagnostics=(), environment_names=() - ) - if analysis is not None and analysis is self._stored_analyses.get(model_name): - self._group_records[name] = self._stored_analysis_payloads.get(name) - self._decoded_groups[name] = stored - else: - self._pending_groups.append((name, stored)) - - def release_stored(self) -> None: - """Drop stored base bytes once every render is decoded, recording reused renders as None.""" - - self._keep_stored_analyses() - self._prior = None - self._reusable = False - for records, retained in ( - (self._model_records, self._retained_models), - (self._group_records, self._retained_groups), - ): - for key in records.keys() - retained: - records[key] = None - - def _keep_stored_analyses(self) -> None: - prior: RenderReuseState | None = self._prior - if prior is None or not self._reusable: - return - for path in self._reused_paths: - name: str = f"{RENDER_REUSE_ANALYSIS_PREFIX}{path}" - payload: memoryview | None = prior.group_payloads.get(name) - stored: StoredRender | None = None if payload is None else _loaded_render(payload) - if payload is None or stored is None: - continue - if not isinstance(stored.value, StoredModelAnalysis): - continue - self._stored_analyses[Path(path).stem] = stored.value - if name in self._retained_groups: - self._stored_analysis_payloads[name] = memoryview(bytes(payload)) - - def stored_state(self, *, complete: bool = False) -> RenderReuseState | None: - """Serialize this compile's renders; with complete, re-serialize released reused ones.""" - - if self._model_paths is None: - return None - self.release_stored() - self._serialize_pending() - if complete: - self._model_records = _reserialized( - records=self._model_records, decoded=self._decoded_models - ) - self._group_records = _reserialized( - records=self._group_records, decoded=self._decoded_groups - ) - return RenderReuseState( - model_paths=self._model_paths, - model_payloads={**self._model_records, **self._serialized_models}, - group_payloads={**self._group_records, **self._serialized_groups}, - ) - - def _serialize_pending(self) -> None: - for pending, serialized in ( - (self._pending_models, self._serialized_models), - (self._pending_groups, self._serialized_groups), - ): - for key, stored in pending: - payload: bytes | None = _dumped(stored) - if payload is not None: - serialized[key] = memoryview(payload) - pending.clear() - - -def edits_only_models(*, changed_paths: frozenset[str], model_paths: tuple[str, ...]) -> bool: - """Return whether every changed path is a model file of the stored compile.""" - - return changed_paths.issubset(model_paths) - - -def _reserialized( - *, records: dict[str, memoryview | None], decoded: dict[str, StoredRender] -) -> dict[str, memoryview | None]: - complete: dict[str, memoryview | None] = {} - key: str - payload: memoryview | None - for key, payload in records.items(): - if payload is None: - dumped: bytes | None = _dumped(decoded[key]) - if dumped is None: - continue - payload = memoryview(dumped) - complete[key] = payload - return complete - - -def _stored_render( - *, - reads: CompileInputReads, - value: object, - reported: list[tuple[tuple[str, ...], CompilerDiagnostic]], -) -> StoredRender | None: - """Keep one render for reuse unless it read the run identity or provider settings.""" - - if reads.read_run_id or reads.settings_classes: - return None - return StoredRender( - value=value, diagnostics=tuple(reported), environment_names=reads.environment_names - ) - - -def _dumped(stored: StoredRender) -> bytes | None: - try: - return dumped_fact_payload(stored) - except (pickle.PicklingError, TypeError, AttributeError, RecursionError): - return None - - -def _replay(*, stored: StoredRender) -> None: - for name in stored.environment_names: - COMPILE_INPUT_READS.environment_read(name) - for key, diagnostic in stored.diagnostics: - report_compile_diagnostic(key=key, diagnostic=diagnostic) - - -def _loaded_render(payload: memoryview) -> StoredRender | None: - try: - loaded: object = loaded_fact_payload(payload) - except _LOAD_ERRORS: - return None - return loaded if isinstance(loaded, StoredRender) else None diff --git a/src/sqlbuild/compiler/compile/classes/stored_model_analyses.py b/src/sqlbuild/compiler/compile/classes/stored_model_analyses.py deleted file mode 100644 index 1204f1e697..0000000000 --- a/src/sqlbuild/compiler/compile/classes/stored_model_analyses.py +++ /dev/null @@ -1,452 +0,0 @@ -"""Serve unchanged models' analyses from the previous compile instead of the analysis cache.""" - -from __future__ import annotations - -import hashlib -from collections import Counter - -import orjson - -from sqlbuild.compiler.compile._helpers.analysis.cache import ( - model_analysis_output_signature, - read_model_analyses, - write_model_analyses, -) -from sqlbuild.compiler.compile._helpers.assembly.semantic_shapes import referenced_model_names -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompileModelInput, - ModelAnalysisCaching, - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, - StoredAnalysisScope, - StoredModelAnalysis, -) -from sqlbuild.compiler.compile.types import ModelAnalysisReuse -from sqlbuild.compiler.lineage.types import InferredNullability -from sqlbuild.compiler.profiling.main._metric import record_compile_metric -from sqlbuild.compiler.profiling.main.record import record_compile_timing - - -class StoredModelAnalyses: - """Analyses a compile may serve from the previous compile, and what it records for the next.""" - - def __init__( - self, - *, - caching: ModelAnalysisCaching | None, - requests: tuple[ModelSqlAnalysisRequest, ...], - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], - ) -> None: - self._scope: StoredAnalysisScope | None = _stored_analysis_scope( - caching=caching, - requests=requests, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) - - @property - def served(self) -> dict[str, StoredModelAnalysis]: - """Return stored analyses that may serve their own model, whose cache key no other uses.""" - - return {} if self._scope is None else self._scope.stored - - def stale_names( - self, - *, - requests: tuple[ModelSqlAnalysisRequest, ...], - cached: dict[str, PolyglotAnalysisResult], - current_signatures: dict[str, str], - ) -> set[str]: - """Return served models whose parents' final signatures differ from their stored ones.""" - - analyzed_names: frozenset[str] = frozenset( - _name(request.model_input) for request in requests - ) - served: dict[str, StoredModelAnalysis] = self.served - return { - name - for request in requests - if (name := _name(request.model_input)) in served - and request.cache_key is not None - and cached.get(request.cache_key) is served[name].analysis - and not dependencies_current( - dependencies=served[name].dependencies, - parent_signatures=_parent_signatures( - request=request, - signatures=current_signatures, - analyzed_names=analyzed_names, - ), - ) - } - - def read( - self, - *, - context: AnalysisCacheContext, - cache_keys: tuple[str, ...], - model_names: tuple[str, ...], - upstream_model_names_by_key: dict[str, tuple[str, ...]], - ) -> tuple[dict[str, PolyglotAnalysisResult], dict[str, str], dict[str, str]]: - """Read the analysis cache for keys without a stored analysis; serve the rest.""" - - stored: dict[str, StoredModelAnalysis] = self.served - stored_by_key: dict[str, StoredModelAnalysis] = { - analysis.cache_key: analysis for analysis in stored.values() - } - analyses, signatures, output_signatures = read_model_analyses( - context=context, - cache_keys=tuple(key for key in cache_keys if key not in stored_by_key), - model_names=model_names, - upstream_model_names_by_key={ - key: names - for key, names in upstream_model_names_by_key.items() - if key not in stored_by_key - }, - ) - for key in cache_keys: - served: StoredModelAnalysis | None = stored_by_key.get(key) - if served is not None: - analyses[key] = served.analysis - output_signatures[key] = served.output_signature - return analyses, signatures, output_signatures - - def read_every_model( - self, *, context: AnalysisCacheContext, requests: tuple[ModelSqlAnalysisRequest, ...] - ) -> dict[str, PolyglotAnalysisResult]: - """Read every model's analysis from the cache; record them when the cache held them all.""" - - analyzed_names: frozenset[str] = frozenset( - _name(request.model_input) for request in requests - ) - cache_keys: tuple[str, ...] = tuple( - request.cache_key for request in requests if request.cache_key is not None - ) - analyses, signatures, output_signatures = read_model_analyses( - context=context, - cache_keys=cache_keys, - model_names=tuple(_name(request.model_input) for request in requests), - upstream_model_names_by_key={ - request.cache_key: referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - for request in requests - if request.cache_key is not None - }, - ) - if all(key in analyses for key in cache_keys): - self._record( - requests=requests, - cached=analyses, - cached_output_signatures=output_signatures, - written={}, - written_dependencies={}, - previous_signatures=signatures, - current_signatures={ - _name(request.model_input): output_signatures[request.cache_key] - for request in requests - if request.cache_key is not None - }, - ) - return analyses - - def record_none(self) -> None: - """Record no analyses, so the next compile analyzes or reads every model from the cache.""" - - if self._scope is not None: - self._scope.reuse.record_analyses(analyses={}) - - def publish( - self, - *, - context: AnalysisCacheContext, - requests: tuple[ModelSqlAnalysisRequest, ...], - results: tuple[PolyglotAnalysisResult, ...], - cached: dict[str, PolyglotAnalysisResult], - cached_output_signatures: dict[str, str], - invalidated_names: set[str], - latest_by_model: dict[str, PolyglotAnalysisResult], - previous_signatures: dict[str, str], - current_signatures: dict[str, str], - ) -> None: - """Write new and invalidated analyses to the cache, then record them for the next run.""" - - analyzed_names: frozenset[str] = frozenset( - _name(request.model_input) for request in requests - ) - written: dict[str, PolyglotAnalysisResult] = { - request.cache_key: result - for request, result in zip(requests, results, strict=True) - if request.cache_key is not None - and (request.cache_key not in cached or _name(request.model_input) in invalidated_names) - } - written_dependencies: dict[str, dict[str, str]] = { - key: _dependency_signatures( - request=request, - current_signatures=current_signatures, - analyzed_names=analyzed_names, - ) - for request in requests - if (key := request.cache_key) is not None and key in written - } - record_compile_metric( - metric="analysis_reuse_hits", - value=sum( - request.cache_key is not None - and (served := self.served.get(_name(request.model_input))) is not None - and cached.get(request.cache_key) is served.analysis - and _name(request.model_input) not in invalidated_names - for request in requests - ), - ) - with record_compile_timing("cache_publication_ms"): - write_model_analyses( - context=context, - analyses_by_key=written, - latest_analyses_by_model=latest_by_model, - dependency_signatures_by_key=written_dependencies, - ) - self._record( - requests=requests, - cached=cached, - cached_output_signatures=cached_output_signatures, - written=written, - written_dependencies=written_dependencies, - previous_signatures=previous_signatures, - current_signatures=current_signatures, - ) - - def _record( - self, - *, - requests: tuple[ModelSqlAnalysisRequest, ...], - cached: dict[str, PolyglotAnalysisResult], - cached_output_signatures: dict[str, str], - written: dict[str, PolyglotAnalysisResult], - written_dependencies: dict[str, dict[str, str]], - previous_signatures: dict[str, str], - current_signatures: dict[str, str], - ) -> None: - if self._scope is None: - return - self._scope.reuse.record_analyses( - analyses=_recorded_analyses( - scope=self._scope, - requests=requests, - cached=cached, - cached_output_signatures=cached_output_signatures, - written=written, - written_dependencies=written_dependencies, - previous_signatures=previous_signatures, - current_signatures=current_signatures, - ) - ) - - -def dependencies_current( - *, dependencies: dict[str, str], parent_signatures: dict[str, str | None] -) -> bool: - """Return whether a stored analysis was made from parents with these final signatures.""" - - return dependencies == parent_signatures - - -def _stored_analysis_scope( - *, - caching: ModelAnalysisCaching | None, - requests: tuple[ModelSqlAnalysisRequest, ...], - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], -) -> StoredAnalysisScope | None: - if caching is None or caching.reuse is None: - return None - context: str = analysis_reuse_context( - cache=caching.cache, - model_names=frozenset(_name(request.model_input) for request in requests), - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) - stored: dict[str, StoredModelAnalysis] = _reusable_stored_analyses( - reuse=caching.reuse, context=context, requests=requests - ) - record_compile_metric(metric="analysis_reuse_hits", value=0) - return StoredAnalysisScope(reuse=caching.reuse, context=context, stored=stored) - - -def analysis_reuse_context( - *, - cache: AnalysisCacheContext, - model_names: frozenset[str], - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], -) -> str: - """Digest every analysis input shared by all models, including non-model relations.""" - - relations: dict[str, list[object]] = { - name: _relation_inputs( - name=name, - column_types_by_table=column_types_by_table, - column_nullability_by_table=column_nullability_by_table, - complete_binding_schemas=complete_binding_schemas, - ) - for name in ( - column_types_by_table.keys() - | column_nullability_by_table.keys() - | complete_binding_schemas.keys() - ) - - model_names - } - return hashlib.sha256( - orjson.dumps( - { - "shared_fingerprint": cache.shared_fingerprint, - "signature_namespace": cache.signature_namespace, - "relations": relations, - }, - option=orjson.OPT_SORT_KEYS, - ) - ).hexdigest() - - -def _relation_inputs( - *, - name: str, - column_types_by_table: dict[str, dict[str, str]], - column_nullability_by_table: dict[str, dict[str, InferredNullability]], - complete_binding_schemas: dict[str, dict[str, str]], -) -> list[object]: - nullability: dict[str, InferredNullability] = column_nullability_by_table.get(name, {}) - return [ - column_types_by_table.get(name), - {column: value.value for column, value in nullability.items()}, - complete_binding_schemas.get(name), - ] - - -def _reusable_stored_analyses( - *, - reuse: ModelAnalysisReuse, - context: str, - requests: tuple[ModelSqlAnalysisRequest, ...], -) -> dict[str, StoredModelAnalysis]: - reused_names: frozenset[str] = reuse.reused_model_names() - stored: dict[str, StoredModelAnalysis] = reuse.stored_analyses() - key_counts: Counter[str | None] = Counter(request.cache_key for request in requests) - reusable: dict[str, StoredModelAnalysis] = {} - for request in requests: - name: str = _name(request.model_input) - analysis: StoredModelAnalysis | None = stored.get(name) - if ( - analysis is not None - and analysis.context == context - and name in reused_names - and request.cache_key is not None - and analysis.cache_key == request.cache_key - and key_counts[request.cache_key] == 1 - ): - reusable[name] = analysis - return reusable - - -def _parent_signatures( - *, - request: ModelSqlAnalysisRequest, - signatures: dict[str, str], - analyzed_names: frozenset[str], -) -> dict[str, str | None]: - return { - upstream_name: signatures.get(upstream_name) - for upstream_name in referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - } - - -def _dependency_signatures( - *, - request: ModelSqlAnalysisRequest, - current_signatures: dict[str, str], - analyzed_names: frozenset[str], -) -> dict[str, str]: - return { - upstream_name: current_signatures[upstream_name] - for upstream_name in referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - if upstream_name in current_signatures - } - - -def _recorded_analyses( - *, - scope: StoredAnalysisScope, - requests: tuple[ModelSqlAnalysisRequest, ...], - cached: dict[str, PolyglotAnalysisResult], - cached_output_signatures: dict[str, str], - written: dict[str, PolyglotAnalysisResult], - written_dependencies: dict[str, dict[str, str]], - previous_signatures: dict[str, str], - current_signatures: dict[str, str], -) -> dict[str, StoredModelAnalysis | None]: - analyzed_names: frozenset[str] = frozenset(_name(request.model_input) for request in requests) - recorded: dict[str, StoredModelAnalysis | None] = {} - for request in requests: - name: str = _name(request.model_input) - key: str | None = request.cache_key - written_analysis: PolyglotAnalysisResult | None = None if key is None else written.get(key) - cached_analysis: PolyglotAnalysisResult | None = None if key is None else cached.get(key) - previous: StoredModelAnalysis | None = scope.stored.get(name) - upstream: tuple[str, ...] = referenced_model_names( - model_input=request.model_input, available_names=analyzed_names - ) - parents: dict[str, str | None] = _parent_signatures( - request=request, signatures=current_signatures, analyzed_names=analyzed_names - ) - if key is None or (written_analysis is None and cached_analysis is None): - recorded[name] = None - elif written_analysis is not None: - recorded[name] = StoredModelAnalysis( - context=scope.context, - cache_key=key, - analysis=written_analysis, - output_signature=model_analysis_output_signature(written_analysis), - dependencies=written_dependencies.get(key, {}), - signature=current_signatures.get(name), - ) - elif ( - previous is not None - and previous.cache_key == key - and previous.analysis is cached_analysis - and previous.signature == current_signatures.get(name) - and dependencies_current(dependencies=previous.dependencies, parent_signatures=parents) - ): - recorded[name] = previous - elif cached_analysis is not None and all( - upstream_name in previous_signatures - and previous_signatures[upstream_name] == parents[upstream_name] - for upstream_name in upstream - ): - recorded[name] = StoredModelAnalysis( - context=scope.context, - cache_key=key, - analysis=cached_analysis, - output_signature=cached_output_signatures.get(key) - or model_analysis_output_signature(cached_analysis), - dependencies={ - upstream_name: previous_signatures[upstream_name] for upstream_name in upstream - }, - signature=current_signatures.get(name), - ) - else: - recorded[name] = None - return recorded - - -def _name(model_input: CompileModelInput) -> str: - return model_input.model_file.file_path.stem diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 75ac536a3f..4f3bfc2c17 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -7,7 +7,6 @@ from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( CompileInputReadRegistry, ) -from sqlbuild.compiler.compile.classes.render_reuse_registry import CompileRenderReuseRegistry from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind @@ -177,18 +176,6 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() -COMPILE_RENDER_REUSE: CompileRenderReuseRegistry = CompileRenderReuseRegistry() -RENDER_REUSE_SOURCES_GROUP: str = "sources" -RENDER_REUSE_TESTS_GROUP: str = "tests" -RENDER_REUSE_SCENARIOS_GROUP: str = "scenarios" -RENDER_REUSE_SINGULAR_AUDITS_GROUP: str = "singular_audits" -RENDER_REUSE_ATTACHED_AUDITS_GROUP: str = "source_and_seed_audits" -RENDER_REUSE_MODEL_AUDITS_PREFIX: str = "model_audits:" -RENDER_REUSE_DECLARATIONS_PREFIX: str = "declarations:" -RENDER_REUSE_ANALYSIS_PREFIX: str = "analysis:" -SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" -SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v3" -SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" COMPACT_ANALYSIS_RESPONSE_LENGTH: int = 2 COMPACT_RELATION_STUB_PREFIX: str = "__sqlbuild_project_input_" COMPACT_REFERENCE_MARKER_PATTERN: re.Pattern[str] = re.compile( diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 2b857fdbdb..720032c6d2 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -18,6 +18,11 @@ from sqlbuild.compiler.compile._helpers.attachment.references import ( validate_table_function_call_arities, ) +from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs +from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( + build_scenario_inputs, + build_test_inputs_with_cache, +) from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, ) @@ -39,11 +44,6 @@ build_public_model_schema_index, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile._helpers.render.reuse import ( - build_sql_resource_inputs, - claimed_render_reuse, -) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAuditInput, @@ -126,14 +126,10 @@ def build_compile_inputs( _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), ) resolved_run_id: str = resolve_run_id(selected_run_id=run_id) - render_reuse: CompileRenderReuseSession | None = claimed_render_reuse( - discovered_inputs=discovered_inputs - ) loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) declaration_scope: DeclarationScopeBuild = build_declaration_scope( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, - compile_cache_dir=compile_cache_dir, sql_lexical_syntax=adapter_context.sql_lexical_syntax, ) model_context: ModelInputBuildContext = ModelInputBuildContext( @@ -156,7 +152,6 @@ def build_compile_inputs( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=compile_cache_dir, - render_reuse=render_reuse, ) model_context = model_build.context seed_inputs: tuple[CompileSeedInput, ...] = build_seed_inputs(discovered_inputs) @@ -170,17 +165,33 @@ def build_compile_inputs( adapter_context.python_functions_inherit_default_namespace ), ) - source_inputs: tuple[CompileSourceInput, ...] - test_inputs: tuple[CompileSqlTestInput, ...] - scenario_inputs: tuple[CompileSqlScenarioInput, ...] - source_inputs, test_inputs, scenario_inputs = build_sql_resource_inputs( + source_inputs: tuple[CompileSourceInput, ...] = build_source_inputs( discovered_inputs=discovered_inputs, - context=model_context, - sql_function_inputs=sql_function_inputs, + effective_vars=model_context.effective_vars, + effective_settings=model_context.effective_settings, + macro_context=model_context.macro_context, + loaded_macros=model_context.loaded_macros, + declaration_expansion=model_context.declaration_expansion, no_sql_validation=no_sql_validation, + ) + test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs_with_cache( + discovered_inputs=discovered_inputs, + effective_vars=model_context.effective_vars, + macro_context=model_context.macro_context, + loaded_macros=model_context.loaded_macros, + declaration_expansion=model_context.declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, - compile_cache_dir=compile_cache_dir, - render_reuse=render_reuse, + sql_function_inputs=sql_function_inputs, + sql_lexical_syntax=model_context.sql_lexical_syntax, + ) + scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( + discovered_inputs=discovered_inputs, + effective_vars=model_context.effective_vars, + macro_context=model_context.macro_context, + loaded_macros=model_context.loaded_macros, + declaration_expansion=model_context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=model_context.sql_lexical_syntax, ) audit_inputs: tuple[CompileAuditInput, ...] audit_diagnostics: tuple[CompilerDiagnostic, ...] @@ -190,10 +201,7 @@ def build_compile_inputs( model_inputs=model_build.inputs, source_inputs=source_inputs, seed_inputs=seed_inputs, - render_reuse=render_reuse, ) - if render_reuse is not None: - render_reuse.release_stored() diagnostics: tuple[CompilerDiagnostic, ...] = ( *audit_diagnostics, *model_build.diagnostics, @@ -243,7 +251,6 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, scope_index=declaration_scope.index, declaration_scope=declaration_scope, - analysis_reuse=render_reuse, ) @@ -283,7 +290,6 @@ def _build_models_with_declarations( defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, reference_cache_dir: Path | None, - render_reuse: CompileRenderReuseSession | None, ) -> ModelInputScopeBuild: public_enums: dict[str, EnumDeclaration] public_constants: dict[str, ConstantDeclaration] @@ -309,7 +315,6 @@ def _build_models_with_declarations( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=reference_cache_dir, - render_reuse=render_reuse, ) return ModelInputScopeBuild( inputs=model_inputs, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 62a07a1fcc..b3ddc912f2 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -23,7 +23,6 @@ DiagnosticPhase, DiagnosticSeverity, FunctionLanguage, - ModelAnalysisReuse, SqlTestMode, TypedSqlValueRenderer, ) @@ -826,44 +825,6 @@ class ModelSqlAnalysis: dynamic_column_contract: DynamicColumnContractProof | None = None -@dataclass(frozen=True) -class DataflowReuse: - """Analyses a binding dataflow may reuse, and the signatures that decide whether it may.""" - - cached: dict[str, PolyglotAnalysisResult] - previous_signatures: dict[str, str] - served: dict[str, dict[str, str]] - - -@dataclass(frozen=True) -class StoredModelAnalysis: - """One model's analysis as the analysis cache held it after a compile, kept for reuse.""" - - context: str - cache_key: str - analysis: PolyglotAnalysisResult - output_signature: str - dependencies: dict[str, str] - signature: str | None - - -@dataclass(frozen=True) -class StoredAnalysisScope: - """Stored analyses one compile may serve, and the session recording its own analyses.""" - - reuse: ModelAnalysisReuse - context: str - stored: dict[str, StoredModelAnalysis] - - -@dataclass(frozen=True) -class ModelAnalysisCaching: - """The analysis cache one compile reads and writes, and its stored analysis reuse.""" - - cache: AnalysisCacheContext - reuse: ModelAnalysisReuse | None = None - - @dataclass(frozen=True) class ModelSqlAnalysisRequest: """Inputs to one model's SQL analysis.""" @@ -978,7 +939,6 @@ class CompileProjectInputs: external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None scope_index: ScopeIndex = field(default_factory=ScopeIndex) declaration_scope: DeclarationScopeBuild | None = field(default=None, repr=False, compare=False) - analysis_reuse: ModelAnalysisReuse | None = field(default=None, repr=False, compare=False) @dataclass(frozen=True) @@ -1748,21 +1708,3 @@ class SqlAnalysisOptOutRequest: query_sql: str placeholders: dict[str, str] | None project_config_path: Path - - -@dataclass(frozen=True) -class StoredRender: - """One stored render with the diagnostics it reported and environment variables it read.""" - - value: object - diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] - environment_names: tuple[str, ...] - - -@dataclass(frozen=True) -class RenderReuseState: - """Serialized renders of one compile by model path and group; None marks released bytes.""" - - model_paths: tuple[str, ...] - model_payloads: dict[str, memoryview | None] - group_payloads: dict[str, memoryview | None] diff --git a/src/sqlbuild/compiler/compile/types.py b/src/sqlbuild/compiler/compile/types.py index d3faf5c129..b8e69b9a86 100644 --- a/src/sqlbuild/compiler/compile/types.py +++ b/src/sqlbuild/compiler/compile/types.py @@ -13,7 +13,6 @@ CompileProjectInputs, SqlReferenceOrigin, SqlReferenceScan, - StoredModelAnalysis, ) from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -51,22 +50,6 @@ class CompileInputsReadyCallback(Protocol): def __call__(self, *, inputs: CompileProjectInputs) -> None: ... -class ModelAnalysisReuse(Protocol): - """Analyses a previous compile stored for models whose renders this compile reused.""" - - def reused_model_names(self) -> frozenset[str]: - """Return the names of models whose stored renders this compile reused.""" - ... - - def stored_analyses(self) -> dict[str, StoredModelAnalysis]: - """Return stored analyses of reused models by model name.""" - ... - - def record_analyses(self, *, analyses: dict[str, StoredModelAnalysis | None]) -> None: - """Record every model's analysis for the next compile; absent or None drops it.""" - ... - - class AttachedAuditTargetKind(StrEnum): MODEL = "model" SOURCE = "source" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py index c8cff23b66..0a91af1c9b 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/aggregation.py @@ -3,7 +3,6 @@ from __future__ import annotations from collections.abc import Callable -from functools import partial from pathlib import Path from sqlbuild.compiler.discovery._helpers.filesystem.command_output_sinks import ( @@ -70,7 +69,6 @@ DiscoveryFileFault, TolerantScopeDiscovery, ) -from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -82,33 +80,17 @@ def build_discovered_project_inputs( local_config: LocalConfig, sql_analysis_enabled: bool, extract_output_column_locations: bool = True, - declaration_reuse: DeclarationFilesReuse | None = None, ) -> DiscoveredProjectInputs: """Discover all project files and functions into one inputs bundle.""" with OperationLifecycle( operation_kind="project", operation_name="discovery_declaration_parse" ) as declaration_lifecycle: - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]] = partial( - _discover_model_files, + declarations: DiscoveredDeclarationFiles = _discover_declaration_files( project_dir=project_dir, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, ) - discover: Callable[[], DiscoveredDeclarationFiles] = partial( - _discover_declaration_files, - project_dir=project_dir, - discover_models=discover_models, - ) - declarations: DiscoveredDeclarationFiles = ( - discover() - if declaration_reuse is None - else declaration_reuse.declaration_files( - variant=f"{int(sql_analysis_enabled)}{int(extract_output_column_locations)}", - discover=discover, - discover_models=discover_models, - ) - ) declaration_lifecycle.completed( metadata={ "item_count": sum( @@ -216,13 +198,18 @@ def build_discovered_project_inputs( def _discover_declaration_files( *, project_dir: Path, - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], + sql_analysis_enabled: bool, + extract_output_column_locations: bool, ) -> DiscoveredDeclarationFiles: with DirectorySnapshot.scope(project_dir=project_dir): source_files: tuple[DiscoveredSourceFile, ...] = discover_native_source_files( project_dir=project_dir ) - model_files: tuple[DiscoveredSqlModelFile, ...] = discover_models() + model_files: tuple[DiscoveredSqlModelFile, ...] = discover_native_model_files( + project_dir=project_dir, + extract_implicit_alias_columns=sql_analysis_enabled, + extract_output_column_locations=extract_output_column_locations, + ) return DiscoveredDeclarationFiles( source_files=source_files, model_files=model_files, @@ -243,19 +230,6 @@ def _discover_declaration_files( ) -def _discover_model_files( - *, - project_dir: Path, - sql_analysis_enabled: bool, - extract_output_column_locations: bool, -) -> tuple[DiscoveredSqlModelFile, ...]: - return discover_native_model_files( - project_dir=project_dir, - extract_implicit_alias_columns=sql_analysis_enabled, - extract_output_column_locations=extract_output_column_locations, - ) - - def build_tolerant_scope_discovery(*, project_dir: Path) -> TolerantScopeDiscovery: """Aggregate bounded scope inputs while retaining independent authored faults.""" diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index 560e3ed11b..5b387ef28a 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -23,7 +23,6 @@ ) from sqlbuild.compiler.discovery.constants import SQL_ANALYSIS_SETTING_KEY from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs -from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle @@ -35,7 +34,6 @@ def discover_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None = None, extract_output_column_locations: bool = True, - declaration_reuse: DeclarationFilesReuse | None = None, ) -> DiscoveredProjectInputs: """Load all raw project inputs from disk before semantic resolution.""" @@ -47,7 +45,6 @@ def discover_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=sql_analysis_enabled_override, extract_output_column_locations=extract_output_column_locations, - declaration_reuse=declaration_reuse, ), ) @@ -57,14 +54,12 @@ def _discover_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None, extract_output_column_locations: bool, - declaration_reuse: DeclarationFilesReuse | None, ) -> DiscoveredProjectInputs: with OperationLifecycle(operation_kind="project", operation_name="discovery_project_assembly"): return _assemble_discovered_project_inputs( project_dir=project_dir, sql_analysis_enabled_override=sql_analysis_enabled_override, extract_output_column_locations=extract_output_column_locations, - declaration_reuse=declaration_reuse, ) @@ -73,7 +68,6 @@ def _assemble_discovered_project_inputs( project_dir: Path, sql_analysis_enabled_override: bool | None, extract_output_column_locations: bool, - declaration_reuse: DeclarationFilesReuse | None, ) -> DiscoveredProjectInputs: project_config: ProjectConfig = load_project_config(project_dir=project_dir) local_config: LocalConfig = load_local_config(project_dir=project_dir) @@ -102,7 +96,6 @@ def _assemble_discovered_project_inputs( local_config=local_config, sql_analysis_enabled=sql_analysis_enabled, extract_output_column_locations=extract_output_column_locations, - declaration_reuse=declaration_reuse, ) validate_discovered_inputs(discovered_inputs) from sqlbuild.runtime.event_exporting.main.configure_discovered_event_exporters import ( diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index 1dcdded57a..daff13b0a5 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -2,7 +2,6 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import Field from enum import StrEnum from pathlib import Path @@ -11,10 +10,6 @@ from sqlbuild.compiler.scopes.types import ScopeKind if TYPE_CHECKING: - from sqlbuild.compiler.discovery.models import ( - DiscoveredDeclarationFiles, - DiscoveredSqlModelFile, - ) from sqlbuild.provider.classes.provider import Provider type ProjectProvider = Provider @@ -57,17 +52,3 @@ class DirectorySnapshotEntry(NamedTuple): name: str is_dir: bool is_walkable_dir: bool - - -class DeclarationFilesReuse(Protocol): - """Reuse the non-model declaration files a previous compile parsed when only models changed.""" - - def declaration_files( - self, - *, - variant: str, - discover: Callable[[], DiscoveredDeclarationFiles], - discover_models: Callable[[], tuple[DiscoveredSqlModelFile, ...]], - ) -> DiscoveredDeclarationFiles: - """Return stored declaration files with freshly discovered models, or discover all.""" - ... diff --git a/src/sqlbuild/compiler/fact_cache/__init__.py b/src/sqlbuild/compiler/fact_cache/__init__.py deleted file mode 100644 index 996c10d790..0000000000 --- a/src/sqlbuild/compiler/fact_cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Content-addressed local cache for deterministic per-file compile facts.""" diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/__init__.py b/src/sqlbuild/compiler/fact_cache/_helpers/__init__.py deleted file mode 100644 index e3ff32475e..0000000000 --- a/src/sqlbuild/compiler/fact_cache/_helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache helpers.""" diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/publication.py b/src/sqlbuild/compiler/fact_cache/_helpers/publication.py deleted file mode 100644 index 53cabf2787..0000000000 --- a/src/sqlbuild/compiler/fact_cache/_helpers/publication.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Background publication of fact rows in multi-row statements so writes rarely take the GIL.""" - -from __future__ import annotations - -import sqlite3 -import threading -import zlib -from contextlib import closing -from pathlib import Path - -from sqlbuild.compiler.fact_cache.classes.fact_publication_registry import ( - FactPublicationRegistry, -) -from sqlbuild.compiler.fact_cache.constants import ( - FACT_CACHE_CREATE_TABLE_SQL, - FACT_CACHE_INSERT_CHUNK_ROWS, - FACT_CACHE_INSERT_ROW_SQL, - FACT_CACHE_INSERT_SQL, - FACT_CACHE_SQLITE_TIMEOUT_SECONDS, - FACT_CACHE_WRITE_PRAGMAS, - FACT_CACHE_WRITER_THREAD_NAME, -) - -_PUBLICATIONS: FactPublicationRegistry = FactPublicationRegistry() - - -def publish_fact_rows( - *, database_path: Path, rows: tuple[tuple[str, str, str, bytes], ...] -) -> None: - """Publish (slot, key, digest, payload) rows after earlier publications to the same file.""" - - def write(previous: threading.Thread | None) -> None: - _write_rows(database_path=database_path, rows=rows, previous=previous) - - _PUBLICATIONS.start( - database_path=database_path, write=write, thread_name=FACT_CACHE_WRITER_THREAD_NAME - ) - - -def await_fact_publication(*, database_path: Path | None = None) -> None: - """Wait until pending publications to one database, or to every database, are durable.""" - - _PUBLICATIONS.wait(database_path=database_path) - - -def entry_digest(*, cache_key: str, payload: bytes) -> str: - """Return the CRC-32 checksum binding one payload to its exact cache key.""" - - return f"{zlib.crc32(payload, zlib.crc32(cache_key.encode() + b'\0')):08x}" - - -def _write_rows( - *, - database_path: Path, - rows: tuple[tuple[str, str, str, bytes], ...], - previous: threading.Thread | None, -) -> None: - if previous is not None: - previous.join() - try: - _insert_records(database_path=database_path, records=rows) - except sqlite3.DatabaseError as error: - if type(error) is sqlite3.DatabaseError: - _replace_unreadable_database(database_path=database_path, records=rows) - except (sqlite3.Error, OSError): - return - - -def _replace_unreadable_database( - *, database_path: Path, records: tuple[tuple[str, str, str, bytes], ...] -) -> None: - try: - database_path.unlink(missing_ok=True) - _insert_records(database_path=database_path, records=records) - except (OSError, sqlite3.Error): - return - - -def _insert_records( - *, database_path: Path, records: tuple[tuple[str, str, str, bytes], ...] -) -> None: - database_path.parent.mkdir(parents=True, exist_ok=True) - with ( - closing( - sqlite3.connect(database_path, timeout=FACT_CACHE_SQLITE_TIMEOUT_SECONDS) - ) as connection, - connection, - ): - for pragma in FACT_CACHE_WRITE_PRAGMAS: - _ = connection.execute(pragma) - _ = connection.execute(FACT_CACHE_CREATE_TABLE_SQL) - for start in range(0, len(records), FACT_CACHE_INSERT_CHUNK_ROWS): - chunk: tuple[tuple[str, str, str, bytes], ...] = records[ - start : start + FACT_CACHE_INSERT_CHUNK_ROWS - ] - parameters: list[str | bytes] = [] - for record in chunk: - parameters.extend(record) - _ = connection.execute( - FACT_CACHE_INSERT_SQL + ",".join([FACT_CACHE_INSERT_ROW_SQL] * len(chunk)), - parameters, - ) diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py b/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py deleted file mode 100644 index 757e20e635..0000000000 --- a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Pickle codec restricted to SQLBuild value types for local fact cache payloads.""" - -from __future__ import annotations - -import dataclasses -import enum -import io -import pickle -from typing import override - -from sqlbuild.compiler.fact_cache.constants import ( - FACT_CACHE_ALLOWED_GLOBALS, - FACT_CACHE_ALLOWED_PACKAGE, - FACT_CACHE_PICKLE_PROTOCOL, -) -from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError - - -class _RestrictedUnpickler(pickle.Unpickler): - """Resolve only immutable-value classes, never functions or arbitrary callables.""" - - @override - def find_class(self, module: str, name: str) -> type: - if (module, name) in FACT_CACHE_ALLOWED_GLOBALS: - resolved: object = super().find_class(module, name) - if isinstance(resolved, type): - return resolved - if module == FACT_CACHE_ALLOWED_PACKAGE or module.startswith( - f"{FACT_CACHE_ALLOWED_PACKAGE}." - ): - resolved = super().find_class(module, name) - if isinstance(resolved, type) and ( - dataclasses.is_dataclass(resolved) - or issubclass(resolved, enum.Enum) - or (issubclass(resolved, tuple) and hasattr(resolved, "_fields")) - ): - return resolved - raise FactCachePayloadError(f"fact cache payload references disallowed {module}.{name}") - - -def dump_fact_payload(value: object) -> bytes: - """Serialize one cache value with the fixed fact-cache protocol.""" - - return pickle.dumps(value, protocol=FACT_CACHE_PICKLE_PROTOCOL) - - -def load_fact_payload(payload: bytes | memoryview) -> object: - """Deserialize one verified cache payload through the restricted class allowlist.""" - - return _RestrictedUnpickler(io.BytesIO(payload)).load() diff --git a/src/sqlbuild/compiler/fact_cache/classes/__init__.py b/src/sqlbuild/compiler/fact_cache/classes/__init__.py deleted file mode 100644 index 595a1c6848..0000000000 --- a/src/sqlbuild/compiler/fact_cache/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache classes.""" diff --git a/src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py b/src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py deleted file mode 100644 index cd0809f5ca..0000000000 --- a/src/sqlbuild/compiler/fact_cache/classes/fact_cache_store.py +++ /dev/null @@ -1,189 +0,0 @@ -"""SQLite-backed content-addressed store for deterministic per-file compile facts.""" - -from __future__ import annotations - -import hashlib -import hmac -import pickle -import sqlite3 -from collections.abc import Sequence -from contextlib import closing -from pathlib import Path -from types import TracebackType -from typing import Any - -from sqlbuild.compiler.fact_cache._helpers.code_identity import installed_code_identity -from sqlbuild.compiler.fact_cache._helpers.publication import ( - await_fact_publication, - entry_digest, - publish_fact_rows, -) -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import ( - dump_fact_payload, - load_fact_payload, -) -from sqlbuild.compiler.fact_cache.constants import ( - FACT_CACHE_DATABASE_SUFFIX, - FACT_CACHE_DIGEST_BYTES, - FACT_CACHE_DIRECTORY_PREFIX, - FACT_CACHE_MAX_ENTRY_BYTES, - FACT_CACHE_QUERY_CHUNK_SIZE, - FACT_CACHE_SQLITE_TIMEOUT_SECONDS, - FACT_CACHE_VERSION, -) -from sqlbuild.compiler.profiling.main._metric import record_compile_metric - -_READ_ERRORS: tuple[type[BaseException], ...] = ( - OSError, - sqlite3.DatabaseError, - pickle.UnpicklingError, - AttributeError, - EOFError, - ImportError, - IndexError, - KeyError, - TypeError, - ValueError, - RecursionError, - MemoryError, -) - - -class FactCacheStore: - """Reuse exact facts keyed by every input and the producing code; faults are misses.""" - - def __init__(self, *, root: Path | None, namespace: str, algorithm: str) -> None: - self._database_path: Path | None = ( - None - if root is None - else root - / f"{FACT_CACHE_DIRECTORY_PREFIX}{FACT_CACHE_VERSION}" - / f"{namespace}{FACT_CACHE_DATABASE_SUFFIX}" - ) - self._key_prefix: bytes = ( - f"{FACT_CACHE_VERSION}\0{namespace}\0{algorithm}\0".encode() - + (installed_code_identity().encode() if root is not None else b"") - + b"\0" - ) - self._key_hasher: Any = hashlib.blake2b( - self._key_prefix, digest_size=FACT_CACHE_DIGEST_BYTES - ) - self._pending: dict[str, tuple[str, str, bytes]] = {} - self._hits: int = 0 - self._misses: int = 0 - - @property - def enabled(self) -> bool: - """Return whether this invocation reads and publishes cached facts.""" - - return self._database_path is not None - - def __enter__(self) -> FactCacheStore: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - del exc_value, traceback - try: - if exc_type is None: - self._write_pending() - finally: - self._pending.clear() - if self._database_path is not None: - record_compile_metric(metric="fact_cache_hits", value=self._hits) - record_compile_metric(metric="fact_cache_misses", value=self._misses) - - def key(self, *parts: str | bytes) -> str: - """Return the exact identity of one fact computed from the given ordered inputs.""" - - digest: Any = self._key_hasher.copy() - for part in parts: - encoded: bytes = ( - part if isinstance(part, bytes) else part.encode("utf-8", "surrogatepass") - ) - digest.update(len(encoded).to_bytes(8, "little")) - digest.update(encoded) - return str(digest.hexdigest()) - - def read_many(self, entries: Sequence[tuple[str, str]]) -> dict[str, object]: - """Return verified cached values for requested (slot, key) pairs; faults are misses.""" - - found: dict[str, object] = {} - if self._database_path is None or not entries: - return found - expected_keys: dict[str, str] = dict(entries) - await_fact_publication(database_path=self._database_path) - if self._database_path.is_file(): - try: - with closing( - sqlite3.connect( - f"file:{self._database_path}?mode=ro", - uri=True, - timeout=FACT_CACHE_SQLITE_TIMEOUT_SECONDS, - ) - ) as connection: - slots: tuple[str, ...] = tuple(expected_keys) - for start in range(0, len(slots), FACT_CACHE_QUERY_CHUNK_SIZE): - chunk: tuple[str, ...] = slots[start : start + FACT_CACHE_QUERY_CHUNK_SIZE] - placeholders: str = ",".join("?" for _ in chunk) - rows: list[tuple[object, object, object, object]] = connection.execute( - f"SELECT slot, cache_key, digest, payload FROM fact " - f"WHERE slot IN ({placeholders})", - chunk, - ).fetchall() - for slot, cache_key, digest, payload in rows: - if not isinstance(slot, str) or expected_keys.get(slot) != cache_key: - continue - value: object | None = _verified_value( - cache_key=cache_key, digest=digest, payload=payload - ) - if value is not None: - found[expected_keys[slot]] = value - except _READ_ERRORS: - found = {} - self._hits += sum(1 for key in expected_keys.values() if key in found) - self._misses += sum(1 for key in expected_keys.values() if key not in found) - return found - - def stage(self, *, key: str, slot: str, value: object) -> None: - """Queue one freshly computed fact for publication after a successful invocation.""" - - if self._database_path is None: - return - try: - payload: bytes = dump_fact_payload(value) - except (pickle.PicklingError, TypeError, AttributeError, RecursionError, ValueError): - return - if len(payload) > FACT_CACHE_MAX_ENTRY_BYTES: - return - self._pending[slot] = (key, entry_digest(cache_key=key, payload=payload), payload) - - def _write_pending(self) -> None: - if self._database_path is None or not self._pending: - return - publish_fact_rows( - database_path=self._database_path, - rows=tuple( - (slot, key, digest, payload) - for slot, (key, digest, payload) in self._pending.items() - ), - ) - - -def _verified_value(*, cache_key: object, digest: object, payload: object) -> object | None: - if ( - not isinstance(cache_key, str) - or not isinstance(digest, str) - or not isinstance(payload, bytes) - or len(payload) > FACT_CACHE_MAX_ENTRY_BYTES - or not hmac.compare_digest(digest, entry_digest(cache_key=cache_key, payload=payload)) - ): - return None - try: - return load_fact_payload(payload) - except _READ_ERRORS: - return None diff --git a/src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py b/src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py deleted file mode 100644 index 13824c2e4a..0000000000 --- a/src/sqlbuild/compiler/fact_cache/classes/fact_publication_registry.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Process-wide ordering of background fact publications per cache database.""" - -from __future__ import annotations - -import threading -from collections.abc import Callable -from pathlib import Path - - -class FactPublicationRegistry: - """Serialize publications per database and let readers wait for pending writes.""" - - def __init__(self) -> None: - self._lock: threading.Lock = threading.Lock() - self._writers: dict[Path, threading.Thread] = {} - - def start( - self, - *, - database_path: Path, - write: Callable[[threading.Thread | None], None], - thread_name: str, - ) -> None: - """Start one writer that first waits for the previous writer of the same database.""" - - with self._lock: - previous: threading.Thread | None = self._writers.get(database_path) - writer: threading.Thread = threading.Thread( - target=write, args=(previous,), name=thread_name - ) - self._writers[database_path] = writer - writer.start() - - def wait(self, *, database_path: Path | None) -> None: - """Wait for the latest writer of one database, or of every database when None.""" - - with self._lock: - writers: tuple[threading.Thread, ...] = tuple( - writer - for path, writer in self._writers.items() - if database_path is None or path == database_path - ) - for writer in writers: - writer.join() diff --git a/src/sqlbuild/compiler/fact_cache/constants.py b/src/sqlbuild/compiler/fact_cache/constants.py deleted file mode 100644 index 4bcf1840f9..0000000000 --- a/src/sqlbuild/compiler/fact_cache/constants.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Stable constants for the local compile fact cache.""" - -from __future__ import annotations - -FACT_CACHE_VERSION: int = 1 -FACT_CACHE_DIRECTORY_PREFIX: str = "facts-v" -FACT_CACHE_DATABASE_SUFFIX: str = ".sqlite3" -FACT_CACHE_DIGEST_BYTES: int = 32 -FACT_CACHE_MAX_ENTRY_BYTES: int = 16 * 1024 * 1024 -FACT_CACHE_SQLITE_TIMEOUT_SECONDS: float = 0.1 -FACT_CACHE_QUERY_CHUNK_SIZE: int = 500 -FACT_CACHE_PICKLE_PROTOCOL: int = 5 -FACT_CACHE_CREATE_TABLE_SQL: str = """ -CREATE TABLE IF NOT EXISTS fact ( - slot TEXT PRIMARY KEY, - cache_key TEXT NOT NULL, - digest TEXT NOT NULL, - payload BLOB NOT NULL -) -""" -FACT_CACHE_INSERT_SQL: str = ( - "INSERT OR REPLACE INTO fact (slot, cache_key, digest, payload) VALUES " -) -FACT_CACHE_INSERT_ROW_SQL: str = "(?, ?, ?, ?)" -FACT_CACHE_PARAMETERS_PER_ROW: int = 4 -SQLITE_PRE_3_32_MAX_BOUND_PARAMETERS: int = 999 -FACT_CACHE_INSERT_CHUNK_ROWS: int = ( - SQLITE_PRE_3_32_MAX_BOUND_PARAMETERS // FACT_CACHE_PARAMETERS_PER_ROW -) -FACT_CACHE_WRITE_PRAGMAS: tuple[str, ...] = ( - "PRAGMA synchronous = OFF", - "PRAGMA journal_mode = MEMORY", -) -FACT_CACHE_WRITER_THREAD_NAME: str = "sqlbuild-fact-cache-writer" -FACT_CACHE_ALLOWED_PACKAGE: str = "sqlbuild" -FACT_CACHE_ALLOWED_GLOBALS: frozenset[tuple[str, str]] = frozenset( - { - ("pathlib", "Path"), - ("pathlib", "PosixPath"), - ("pathlib", "PurePath"), - ("pathlib", "PurePosixPath"), - ("pathlib", "PureWindowsPath"), - ("pathlib", "WindowsPath"), - ("pathlib._local", "Path"), - ("pathlib._local", "PosixPath"), - ("pathlib._local", "PurePath"), - ("pathlib._local", "PurePosixPath"), - ("pathlib._local", "PureWindowsPath"), - ("pathlib._local", "WindowsPath"), - ("decimal", "Decimal"), - ("datetime", "date"), - ("datetime", "datetime"), - ("datetime", "time"), - ("datetime", "timedelta"), - ("datetime", "timezone"), - ("builtins", "frozenset"), - ("builtins", "set"), - ("builtins", "complex"), - ("collections", "OrderedDict"), - } -) diff --git a/src/sqlbuild/compiler/fact_cache/exceptions.py b/src/sqlbuild/compiler/fact_cache/exceptions.py deleted file mode 100644 index 37bf4d202f..0000000000 --- a/src/sqlbuild/compiler/fact_cache/exceptions.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Fact cache exceptions.""" - -from __future__ import annotations - - -class FactCachePayloadError(ValueError): - """Raised when a stored fact payload is not a trusted value graph.""" diff --git a/src/sqlbuild/compiler/fact_cache/main/__init__.py b/src/sqlbuild/compiler/fact_cache/main/__init__.py deleted file mode 100644 index 56c61b674b..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache public entry modules.""" diff --git a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py b/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py deleted file mode 100644 index 27ffbcebd1..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Project-local compile-cache root resolution.""" - -from __future__ import annotations - -import os -from pathlib import Path - -from sqlbuild.compiler.compile.constants import ( - COMPILE_CACHE_DISABLE_ENV_VAR, - COMPILE_CACHE_DISABLE_VALUE, -) -from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory -from sqlbuild.spec.contracts.models import TargetConfig - - -def compile_cache_root( - *, project_dir: Path | None, target_config: TargetConfig | None, no_cache: bool -) -> Path | None: - """Return the shared compile-cache directory, or None when caching is disabled.""" - - if ( - no_cache - or project_dir is None - or (target_config is not None and target_config.compile_cache is False) - or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE - ): - return None - return compiler_cache_directory(project_dir) diff --git a/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py deleted file mode 100644 index 35f3cdd661..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for serializing one local cache value with the fact-cache codec.""" - -from __future__ import annotations - -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import dump_fact_payload - - -def dumped_fact_payload(value: object) -> bytes: - """Serialize one cache value with the fixed fact-cache protocol.""" - - return dump_fact_payload(value) diff --git a/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py deleted file mode 100644 index 1687bed1ed..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Public entry for reading one local cache value through the restricted fact-cache codec.""" - -from __future__ import annotations - -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import load_fact_payload - - -def loaded_fact_payload(payload: bytes | memoryview) -> object: - """Deserialize one verified cache payload through the restricted class allowlist.""" - - return load_fact_payload(payload) diff --git a/src/sqlbuild/compiler/fact_cache/main/code_identity.py b/src/sqlbuild/compiler/fact_cache/main/code_identity.py deleted file mode 100644 index afdc7f4ea1..0000000000 --- a/src/sqlbuild/compiler/fact_cache/main/code_identity.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Public identity of the installed compiler code for derived caches.""" - -import hashlib - -import sqlbuild._native as _native -from sqlbuild.compiler.fact_cache._helpers.code_identity import installed_code_identity - - -def compiled_code_identity() -> str: - """Return the digest that changes with every released, locally edited, or rebuilt version.""" - - return hashlib.sha256( - f"{installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() - ).hexdigest() diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 8d8a0a4009..2794341f76 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -22,9 +22,4 @@ "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", - "analysis_reuse_hits", - "fact_cache_hits", - "fact_cache_misses", - "render_reuse_hits", - "render_reuse_misses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index a1bddfd128..64d8b0dd73 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -22,9 +22,4 @@ "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", - "analysis_reuse_hits", - "fact_cache_hits", - "fact_cache_misses", - "render_reuse_hits", - "render_reuse_misses", ] diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 60699505be..0b88c31df3 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -14,7 +14,6 @@ import orjson from sqlbuild.compiler.compile.models import CompiledProject -from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, @@ -27,6 +26,7 @@ run_custom_hosts, ) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path +from sqlbuild.rule_engine._helpers.run.code_identity import compiled_code_identity from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/code_identity.py b/src/sqlbuild/rule_engine/_helpers/run/code_identity.py similarity index 87% rename from src/sqlbuild/compiler/fact_cache/_helpers/code_identity.py rename to src/sqlbuild/rule_engine/_helpers/run/code_identity.py index 9d5616bf4b..1514af90b1 100644 --- a/src/sqlbuild/compiler/fact_cache/_helpers/code_identity.py +++ b/src/sqlbuild/rule_engine/_helpers/run/code_identity.py @@ -1,4 +1,4 @@ -"""Identity of the installed code that produces cached compile facts.""" +"""Identity of the installed code that produces cached rule results.""" from __future__ import annotations @@ -11,6 +11,8 @@ from pathlib import Path from typing import Any +import sqlbuild._native as _native + _PACKAGE_NAME: str = "sqlbuild" _POLYGLOT_PACKAGE_NAME: str = "polyglot-sql-chio" _YAML_PACKAGE_NAME: str = "PyYAML" @@ -20,8 +22,16 @@ _PACKAGE_ROOT_PARENT_DEPTH: int = 3 +def compiled_code_identity() -> str: + """Return the digest that changes with every released, locally edited, or rebuilt version.""" + + return hashlib.sha256( + f"{_installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() + ).hexdigest() + + @cache -def installed_code_identity() -> str: +def _installed_code_identity() -> str: """Return a digest that changes with every released or locally edited code version.""" digest: Any = hashlib.sha256() diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py index af631e553e..93964edf80 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -10,8 +10,8 @@ import orjson -from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path +from sqlbuild.rule_engine._helpers.run.code_identity import compiled_code_identity from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 9e8f5cfd02..b53eb4afc5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -88,11 +88,6 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", - "analysis_reuse_hits", - "fact_cache_hits", - "fact_cache_misses", - "render_reuse_hits", - "render_reuse_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index b0402ffab7..43c1e58221 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -389,7 +389,7 @@ class CompileCacheInvalidationTestCase: edited_env: dict[str, str] = field(default_factory=dict) expected_failure: bool = False expected_output_change: bool = True - expected_rewarmed_fact_cache_misses: int = 0 + expected_rewarmed_analysis_cache_misses: int = 0 @dataclass(frozen=True) @@ -400,8 +400,8 @@ class CompileCacheDisabledTestCase: compile_args: tuple[str, ...] = () env: dict[str, str] = field(default_factory=dict) edit: Callable[[Path], None] = lambda _root: None - expected_fact_cache_counts: tuple[int, int] = (0, 0) - expected_fact_databases: tuple[Path, ...] = () + expected_analysis_cache_counts: tuple[int, int] = (0, 0) + expected_analysis_databases: tuple[Path, ...] = () @dataclass(frozen=True) @@ -693,12 +693,12 @@ class CompileReuseStoreFailureTestCase: @dataclass(frozen=True) class IncrementalEditStep: - """One authored edit, and whether its compile may reuse unaffected renders.""" + """One authored edit, the command run over it before its compile, and whether it replays.""" description: str edit: Callable[[Path], None] - expected_render_reuse: bool between: Callable[[Path], None] + expected_replayed: bool = False @dataclass(frozen=True) @@ -723,48 +723,17 @@ class RandomEditChainTestCase: @dataclass(frozen=True) class BrokenEditInvalidationTestCase: - """An incremental invalidation deliberately broken, and edits whose output it must change.""" + """A cache invalidation deliberately broken, and an edit whose output it must change.""" description: str - compiled_edits: tuple[Callable[[Path], None], ...] edit: Callable[[Path], None] sabotage: Callable[[pytest.MonkeyPatch], None] expected_matches_uncached: bool - after_edit: Callable[[Path, pytest.MonkeyPatch], None] - - -@dataclass(frozen=True) -class BrokenChangeDetectionTestCase: - """A model edit compiled while change detection reports nothing changed.""" - - description: str - edit: Callable[[Path], None] - expected_matches_uncached: bool - - -@dataclass(frozen=True) -class RenderStoreNoticeTestCase: - """A render-count notice threshold and whether storing renders announces itself.""" - - description: str - notice_renders: int - expected_notice: bool - - -@dataclass(frozen=True) -class RenderSavePolicyTestCase: - """Compiles after a cold compile, and the render files and reuse each one must leave.""" - - description: str - edits: tuple[Callable[[Path], None], ...] - expected_cold_render_files: int - expected_render_files: tuple[int, ...] - expected_reused: tuple[bool, ...] @dataclass(frozen=True) class ExternalModuleEditTestCase: - """An outside module imported only while rendering, rewritten after its render is reused.""" + """An outside module a macro imports while rendering, rewritten in place after a compile.""" description: str initial_value: str @@ -773,15 +742,6 @@ class ExternalModuleEditTestCase: expected_compiled_value: str -@dataclass(frozen=True) -class RenderLoadNoticeTestCase: - """A stored-render size threshold and whether loading the store announces itself.""" - - description: str - notice_bytes: int - expected_notice: bool - - @dataclass(frozen=True) class DenseBatchedPreparationTestCase: description: str @@ -794,7 +754,7 @@ class DenseBatchedPreparationTestCase: @dataclass(frozen=True) class SharedCacheKeyTestCase: - """A model analysis schedule under which a model shares a served model's cache key.""" + """A model analysis schedule under which two models share one analysis cache key.""" description: str schedule: Callable[[pytest.MonkeyPatch], None] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 8a1c29d7b4..913ed35b1a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -31,8 +31,6 @@ import sqlbuild.adapter.type_system._helpers.type_normalization as type_normalization import sqlbuild.cli.commands._helpers.compile.target_writer as target_writer import sqlbuild.cli.commands.main.project._compile as compile_command_module -import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice -import sqlbuild.cli.compile_render_reuse.main._render_reuse_session as render_reuse_main import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files import sqlbuild.cli.compile_reuse._helpers.store as reuse_store @@ -42,9 +40,6 @@ import sqlbuild.compiler.compile._helpers.macro_bridge.call_store as call_store_module import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage -import sqlbuild.compiler.compile.classes.binding_dataflow as binding_dataflow -import sqlbuild.compiler.compile.classes.render_reuse_session as render_reuse_session -import sqlbuild.compiler.compile.classes.stored_model_analyses as stored_model_analyses import sqlbuild.compiler.contracts.main.validate as contract_validation import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class @@ -58,13 +53,15 @@ read_entry_stdout, write_entry, ) -from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts from sqlbuild.cli.compile_reuse.constants import ( REUSE_DISABLE_ENV_VAR, REUSE_ENTRY_DIRECTORY_NAME, - REUSE_RENDER_STATE_SUFFIX, ) -from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs +from sqlbuild.cli.compile_reuse.models import ( + ProjectFilesComparison, + StoredCompileHeader, + StoredCompileInputs, +) from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME @@ -2540,8 +2537,8 @@ class CompileCacheOutcome(NamedTuple): returncode: int diagnostics: tuple[str, ...] fingerprint: str - fact_cache_hits: int - fact_cache_misses: int + analysis_cache_hits: int + analysis_cache_misses: int def run_installed_sqb( @@ -2567,7 +2564,7 @@ def run_installed_sqb( def compile_cache_outcome( *, project_dir: Path, env: dict[str, str], compile_args: tuple[str, ...] = () ) -> CompileCacheOutcome: - """Compile in a fresh process and return its semantic fingerprint and fact-cache counts.""" + """Compile in a fresh process and return its semantic fingerprint and analysis-cache counts.""" result: subprocess.CompletedProcess[str] = run_installed_sqb( project_dir=project_dir, args=("compile", "--json", *compile_args), env=env @@ -2580,8 +2577,9 @@ def compile_cache_outcome( fingerprint=semantic_compile_fingerprint( payload=values, compiled_dir=project_dir / "target" / "compiled" ), - fact_cache_hits=timings.get("fact_cache_hits", 0), - fact_cache_misses=timings.get("fact_cache_misses", 0), + analysis_cache_hits=timings.get("analysis_batch_cache_hits", 0) + + timings.get("analysis_entry_cache_hits", 0), + analysis_cache_misses=timings.get("analysis_cache_misses", 0), ) @@ -3378,10 +3376,6 @@ def compile_edit_without_reuse(project_dir: Path, monkeypatch: pytest.MonkeyPatc monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, "0") -def leave_edit_uncompiled(_project_dir: Path, _monkeypatch: pytest.MonkeyPatch) -> None: - """Leave the edit for the compared compile to see first.""" - - def write_before_reuse_store( *, monkeypatch: pytest.MonkeyPatch, project_dir: Path, write: Callable[[Path], None] ) -> None: @@ -3500,12 +3494,6 @@ class IncrementalEditComparison(NamedTuple): incremental: CompileReuseRun reference: CompileReuseRun - @property - def reused_renders(self) -> int: - """Return how many model renders the incremental compile reused.""" - - return self.incremental.timings.get("render_reuse_hits", 0) - @property def matches(self) -> bool: """Return whether exit code, report, and every compiled artifact are byte-identical.""" @@ -3577,18 +3565,6 @@ def in_process_reuse_run( "introduce_error": ("introduce_error", "fix_error"), "drop_column": ("drop_column", "comment", "restore_column"), } -_MODEL_ONLY_EDIT_KINDS: frozenset[str] = frozenset( - { - "comment", - "add_column", - "type_change", - "header_change", - "introduce_error", - "fix_error", - "drop_column", - "restore_column", - } -) _DROPPED_COLUMN: str = " customer_id,\n" _KEPT_COLUMN: str = " order_id,\n" _EDIT_ANCHORS: dict[str, str] = {"type_change": " quantity", "drop_column": _DROPPED_COLUMN} @@ -3651,12 +3627,6 @@ def random_edit_plan(*, seed: int, step_count: int) -> tuple[str, ...]: ) -def is_model_only_edit(kind: str) -> bool: - """Return whether an edit kind touches model files only, so renders may be reused.""" - - return kind in _MODEL_ONLY_EDIT_KINDS - - def _swapped(contents: str, first: str, second: str) -> str: return ( contents.replace(first, _SWAP_PLACEHOLDER) @@ -3772,26 +3742,25 @@ def no_intervening_command(_root: Path) -> None: """Compile the edit directly after making it.""" -def model_edit_step( +def edit_step( description: str, edit: Callable[[Path], None], between: Callable[[Path], None] = no_intervening_command, ) -> IncrementalEditStep: - """Return an edit to model files only, whose compile must reuse unaffected renders.""" + """Return one edit, optionally followed by another command before the compared compile.""" - return IncrementalEditStep( - description=description, edit=edit, expected_render_reuse=True, between=between - ) + return IncrementalEditStep(description=description, edit=edit, between=between) -def full_edit_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: - """Return an edit beyond model contents, whose compile must render everything again.""" +def star_chain_steps(*, between: Callable[[Path], None]) -> tuple[IncrementalEditStep, ...]: + """Add a star chain, then add and remove an upstream column with a command before each.""" - return IncrementalEditStep( - description=description, - edit=edit, - expected_render_reuse=False, - between=no_intervening_command, + return ( + edit_step("star_chain_added", star_chain_added), + edit_step("upstream_column_added", staging_extra_flag, between), + edit_step("edit_after_added_column", fact_comment), + edit_step("upstream_column_removed", staging_extra_flag_removed, between), + edit_step("edit_after_removed_column", payments_comment), ) @@ -3855,7 +3824,7 @@ def star_chain_with_twin_added(root: Path) -> None: def twin_header_changed(root: Path) -> None: - """Change the twin's header only, so its render is not reused while its SQL is unchanged.""" + """Change the twin's header only, leaving its SQL and so its analysis cache key unchanged.""" replace_project_text( root, @@ -4087,85 +4056,36 @@ def stg_orders_test_edit(root: Path) -> None: def keep_invalidation(_monkeypatch: pytest.MonkeyPatch) -> None: - """Leave every incremental invalidation intact.""" - + """Leave every cache invalidation intact.""" -def skip_upstream_analysis_checks(monkeypatch: pytest.MonkeyPatch) -> None: - """Serve stored analyses even when an upstream model's final signature changed.""" - for module in (stored_model_analyses, binding_dataflow): - monkeypatch.setattr(module, "dependencies_current", lambda **_kwargs: True) - - -def reuse_declarations_after_any_edit(monkeypatch: pytest.MonkeyPatch) -> None: - """Load stored renders and reuse stored declaration files even after a non-model edit.""" +def ignore_project_changes(monkeypatch: pytest.MonkeyPatch) -> None: + """Make compile reuse believe no project file changed since the stored compile.""" - read_render_state: Callable[..., object] = render_reuse_main.read_render_state monkeypatch.setattr( - render_reuse_main, - "read_render_state", - lambda *, path, changed_paths: read_render_state(path=path, changed_paths=frozenset()), + reuse_attempt, + "compare_project_files", + lambda **_kwargs: ProjectFilesComparison(unchanged=True, verified={}), ) - monkeypatch.setattr(render_reuse_session, "edits_only_models", lambda **_kwargs: True) -def trust_changed_artifacts(monkeypatch: pytest.MonkeyPatch) -> None: - """Trust stored artifact digests even when the artifact changed since it was stored.""" - - monkeypatch.setattr(StoredArtifacts, "unchanged_since_stored", lambda _self, **_kwargs: True) - - -def disable_change_detection(monkeypatch: pytest.MonkeyPatch) -> None: - """Make compile reuse believe no project file changed since the stored compile.""" - - monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: (frozenset(), {})) +def ignore_query_in_analysis_key(monkeypatch: pytest.MonkeyPatch) -> None: + """Key cached model analyses without their query, so an edited query reads a stale analysis.""" + analysis_key: Callable[..., str] = project_assembly.model_analysis_cache_key -def set_store_notice_renders(monkeypatch: pytest.MonkeyPatch, renders: int) -> None: - """Announce storing a compile once it has at least this many new renders to record.""" + def query_blind_key(**kwargs: Any) -> str: + kwargs["query_sql"] = "" + return analysis_key(**kwargs) - monkeypatch.setattr(reuse_store, "REUSE_STORE_NOTICE_RENDERS", renders) - - -def prime_render_store(project_dir: Path) -> CompileReuseRun: - """After a cold compile, compile one leaf edit so the stored compile also holds renders.""" - - fact_comment(project_dir) - return run_reuse_compile(project_dir=project_dir) - - -def prime_render_store_in_process(project_dir: Path) -> int: - """In this process: compile cold, then compile one leaf edit so renders are stored.""" - - cold: int = compile_in_process(project_dir=project_dir) - fact_comment(project_dir) - return max(cold, compile_in_process(project_dir=project_dir)) - - -def render_store_files(project_dir: Path) -> int: - """Return how many stored render files the project's compile reuse folder holds.""" - - return len( - list( - (compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME).glob( - f"*{REUSE_RENDER_STATE_SUFFIX}" - ) - ) - ) - - -def edit_and_compile(*, project_dir: Path, edit: Callable[[Path], None]) -> CompileReuseRun: - """Apply one edit, compile with reuse enabled, and return the run.""" - - edit(project_dir) - return run_reuse_compile(project_dir=project_dir) + monkeypatch.setattr(project_assembly, "model_analysis_cache_key", query_blind_key) EXTERNAL_FLAVOR_MODULE: str = "extflavor" def write_external_flavor(extlib: Path, value: str) -> None: - """Write, or rewrite in place, an outside module that a macro imports only while rendering.""" + """Write, or rewrite in place, an outside module that a macro imports while rendering.""" extlib.mkdir(parents=True, exist_ok=True) (extlib / f"{EXTERNAL_FLAVOR_MODULE}.py").write_text(f"VALUE = {value!r}\n", encoding="utf-8") @@ -4197,18 +4117,6 @@ def compiled_text(*, run: CompileReuseRun, suffix: str) -> str: return run.compiled[path].decode("utf-8") -def set_render_load_notice_bytes(monkeypatch: pytest.MonkeyPatch, stored_bytes: int) -> None: - """Announce loading stored renders once their files hold at least this many bytes.""" - - monkeypatch.setattr(render_load_notice, "RENDER_LOAD_NOTICE_BYTES", stored_bytes) - - -def json_report_keys(stdout: str) -> tuple[str, ...]: - """Parse a JSON compile report and return its top-level keys.""" - - return tuple(cast(dict[str, object], json.loads(stdout))) - - _COMPILER_ENGINE_LINE: re.Pattern[str] = re.compile(r'\n "compiler_engine": "([a-z-]+)",') diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py index 38d5db9526..7d27d5d81a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py @@ -194,13 +194,13 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d cold: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) warm: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) assert cold.returncode == 0 - assert cold.fact_cache_misses > 0 + assert cold.analysis_cache_misses > 0 assert warm == CompileCacheOutcome( returncode=0, diagnostics=cold.diagnostics, fingerprint=cold.fingerprint, - fact_cache_hits=cold.fact_cache_misses, - fact_cache_misses=0, + analysis_cache_hits=cold.analysis_cache_misses, + analysis_cache_misses=0, ) test_case.edit(project_dir) @@ -217,7 +217,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d ) is test_case.expected_output_change assert edited[:3] == reference[:3] assert rewarmed[:3] == reference[:3] - assert rewarmed.fact_cache_misses == test_case.expected_rewarmed_fact_cache_misses + assert rewarmed.analysis_cache_misses == test_case.expected_rewarmed_analysis_cache_misses @pytest.mark.parametrize( @@ -240,7 +240,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d ], ids=lambda case: case.description, ) -def test_given_disabled_compile_cache_when_compiling_then_no_facts_are_read_or_written( +def test_given_disabled_compile_cache_when_compiling_then_no_analyses_are_read_or_written( cache_invalidation_project: Path, test_case: CompileCacheDisabledTestCase ) -> None: project_dir: Path = cache_invalidation_project @@ -258,11 +258,14 @@ def test_given_disabled_compile_cache_when_compiling_then_no_facts_are_read_or_w first_timings: dict[str, int] = json.loads(first.stdout)["compile_timings"] second_timings: dict[str, int] = json.loads(second.stdout)["compile_timings"] assert ( - first_timings["fact_cache_hits"], - first_timings["fact_cache_misses"], - ) == test_case.expected_fact_cache_counts + first_timings["analysis_batch_cache_hits"] + first_timings["analysis_entry_cache_hits"], + first_timings["analysis_cache_misses"], + ) == test_case.expected_analysis_cache_counts assert ( - second_timings["fact_cache_hits"], - second_timings["fact_cache_misses"], - ) == test_case.expected_fact_cache_counts - assert tuple((project_dir / "target").rglob("facts-v*")) == test_case.expected_fact_databases + second_timings["analysis_batch_cache_hits"] + second_timings["analysis_entry_cache_hits"], + second_timings["analysis_cache_misses"], + ) == test_case.expected_analysis_cache_counts + assert ( + tuple((project_dir / "target").rglob("model-analysis.sqlite3")) + == test_case.expected_analysis_databases + ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py deleted file mode 100644 index f2520c73c2..0000000000 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edit_phases.py +++ /dev/null @@ -1,305 +0,0 @@ -"""Edits that reuse stored declarations, analyses, and artifacts must match an uncached compile.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - BrokenEditInvalidationTestCase, - IncrementalEditSequenceTestCase, - SharedCacheKeyTestCase, -) -from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( - CompileReuseRun, - IncrementalEditComparison, - analyze_in_one_batch, - analyze_one_model_at_a_time, - build_between, - compare_incremental_compile, - compile_edit_without_reuse, - compile_in_process, - compile_without_reuse_between, - compiled_artifact_tampered, - enable_compile_reuse, - fact_comment, - fact_quantity_type_declared, - fact_quantity_type_removed, - full_edit_step, - in_process_reuse_run, - keep_invalidation, - leave_edit_uncompiled, - model_edit_step, - other_target_build_between, - payments_comment, - plan_between, - prime_render_store, - prime_render_store_in_process, - replace_project_text, - reuse_declarations_after_any_edit, - run_reuse_compile, - skip_upstream_analysis_checks, - staging_column_removed, - staging_column_renamed, - staging_column_restored, - staging_comment, - staging_extra_flag, - staging_extra_flag_removed, - staging_quantity_restored, - staging_quantity_text, - staging_rename_reverted, - star_chain_added, - star_chain_with_twin_added, - stg_orders_test_edit, - trust_changed_artifacts, - twin_header_changed, -) - - -@pytest.mark.parametrize( - "test_case", - [ - IncrementalEditSequenceTestCase( - description="upstream_shape_changes", - steps=( - model_edit_step("upstream_column_removed", staging_column_removed), - model_edit_step("edit_while_downstream_broken", payments_comment), - model_edit_step("upstream_column_restored", staging_column_restored), - model_edit_step("upstream_column_renamed", staging_column_renamed), - model_edit_step("upstream_rename_reverted", staging_rename_reverted), - ), - ), - IncrementalEditSequenceTestCase( - description="contract_changes", - steps=( - model_edit_step("downstream_type_declared", fact_quantity_type_declared), - model_edit_step("upstream_type_breaks_contract", staging_quantity_text), - model_edit_step("edit_after_contract_error", fact_comment), - model_edit_step("upstream_type_restored", staging_quantity_restored), - model_edit_step("downstream_type_removed", fact_quantity_type_removed), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_plan", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step("upstream_column_added", staging_extra_flag, plan_between), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", staging_extra_flag_removed, plan_between - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_build", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step("upstream_column_added", staging_extra_flag, build_between), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", staging_extra_flag_removed, build_between - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_uncached_compile", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step( - "upstream_column_added", staging_extra_flag, compile_without_reuse_between - ), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", - staging_extra_flag_removed, - compile_without_reuse_between, - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="star_chain_after_other_target_build", - steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step( - "upstream_column_added", staging_extra_flag, other_target_build_between - ), - model_edit_step("edit_after_added_column", fact_comment), - model_edit_step( - "upstream_column_removed", - staging_extra_flag_removed, - other_target_build_between, - ), - model_edit_step("edit_after_removed_column", payments_comment), - ), - ), - IncrementalEditSequenceTestCase( - description="declarations_and_artifacts", - steps=( - full_edit_step( - "source_type_edit", - lambda root: replace_project_text( - root, - "sources/raw.yml", - " - name: quantity\n type: INTEGER", - " - name: quantity\n type: BIGINT", - ), - ), - model_edit_step("edit_after_source", staging_comment), - model_edit_step("compiled_artifact_tampered", compiled_artifact_tampered), - full_edit_step("test_edit", stg_orders_test_edit), - model_edit_step("edit_after_test", fact_comment), - ), - ), - ], - ids=lambda case: case.description, -) -def test_given_edit_sequence_when_reusing_stored_phases_then_each_step_matches_uncached_compile( - compile_reuse_project: Path, test_case: IncrementalEditSequenceTestCase -) -> None: - project_dir: Path = compile_reuse_project - cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - primed: CompileReuseRun = prime_render_store(project_dir) - assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr - - for step in test_case.steps: - step.edit(project_dir) - step.between(project_dir) - comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) - - assert comparison.matches is test_case.expected_matches_uncached, ( - step.description, - comparison.mismatched_artifacts, - comparison.incremental.stderr, - ) - assert (comparison.reused_renders > 0) is step.expected_render_reuse, step.description - assert ( - comparison.incremental.timings.get("analysis_reuse_hits", 0) > 0 - ) is step.expected_render_reuse, step.description - - -@pytest.mark.parametrize( - "test_case", - [ - BrokenEditInvalidationTestCase( - description="intact_invalidation_after_upstream_shape_change", - compiled_edits=(), - edit=staging_column_removed, - sabotage=keep_invalidation, - expected_matches_uncached=True, - after_edit=compile_edit_without_reuse, - ), - BrokenEditInvalidationTestCase( - description="analysis_served_despite_upstream_shape_change", - compiled_edits=(), - edit=staging_column_removed, - sabotage=skip_upstream_analysis_checks, - expected_matches_uncached=False, - after_edit=compile_edit_without_reuse, - ), - BrokenEditInvalidationTestCase( - description="intact_invalidation_after_test_edit", - compiled_edits=(), - edit=stg_orders_test_edit, - sabotage=keep_invalidation, - expected_matches_uncached=True, - after_edit=leave_edit_uncompiled, - ), - BrokenEditInvalidationTestCase( - description="declarations_reused_despite_test_edit", - compiled_edits=(), - edit=stg_orders_test_edit, - sabotage=reuse_declarations_after_any_edit, - expected_matches_uncached=False, - after_edit=leave_edit_uncompiled, - ), - BrokenEditInvalidationTestCase( - description="intact_invalidation_after_tampering", - compiled_edits=(), - edit=compiled_artifact_tampered, - sabotage=keep_invalidation, - expected_matches_uncached=True, - after_edit=leave_edit_uncompiled, - ), - BrokenEditInvalidationTestCase( - description="artifact_trusted_despite_tampering", - compiled_edits=(), - edit=compiled_artifact_tampered, - sabotage=trust_changed_artifacts, - expected_matches_uncached=False, - after_edit=leave_edit_uncompiled, - ), - ], - ids=lambda case: case.description, -) -def test_given_broken_edit_invalidation_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( - compile_reuse_project: Path, - test_case: BrokenEditInvalidationTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - project_dir: Path = compile_reuse_project - enable_compile_reuse(monkeypatch) - assert prime_render_store_in_process(project_dir) == 0 - for edit in test_case.compiled_edits: - edit(project_dir) - assert compile_in_process(project_dir=project_dir) == 0 - test_case.edit(project_dir) - test_case.after_edit(project_dir, monkeypatch) - test_case.sabotage(monkeypatch) - - broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) - reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) - comparison: IncrementalEditComparison = IncrementalEditComparison( - incremental=broken, reference=reference - ) - - assert comparison.matches is test_case.expected_matches_uncached, ( - comparison.mismatched_artifacts - ) - - -@pytest.mark.parametrize( - "test_case", - [ - SharedCacheKeyTestCase( - description="dataflow_one_model_at_a_time", schedule=analyze_one_model_at_a_time - ), - SharedCacheKeyTestCase(description="one_batch", schedule=analyze_in_one_batch), - ], - ids=lambda case: case.description, -) -def test_given_model_sharing_a_served_cache_key_when_upstream_changes_then_it_matches_uncached( - compile_reuse_project: Path, - test_case: SharedCacheKeyTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - project_dir: Path = compile_reuse_project - enable_compile_reuse(monkeypatch) - test_case.schedule(monkeypatch) - star_chain_with_twin_added(project_dir) - assert prime_render_store_in_process(project_dir) == 0 - staging_extra_flag(project_dir) - twin_header_changed(project_dir) - compile_edit_without_reuse(project_dir, monkeypatch) - - incremental: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) - reference: CompileReuseRun = in_process_reuse_run( - project_dir=project_dir, capsys=capsys, args=("--no-cache",) - ) - comparison: IncrementalEditComparison = IncrementalEditComparison( - incremental=incremental, reference=reference - ) - - assert comparison.matches is test_case.expected_matches_uncached, ( - comparison.mismatched_artifacts - ) - assert incremental.timings.get("analysis_reuse_hits", 0) > 0 - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index 90bd684ca7..1e99f2c30c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -1,4 +1,4 @@ -"""Incremental edit compiles must match an uncached compile of the same edited project.""" +"""Compiles after an edit must match an uncached compile of the same edited project.""" from __future__ import annotations @@ -7,53 +7,65 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( - BrokenChangeDetectionTestCase, + BrokenEditInvalidationTestCase, ExternalModuleEditTestCase, IncrementalEditSequenceTestCase, RandomEditChainTestCase, - RenderLoadNoticeTestCase, - RenderSavePolicyTestCase, - RenderStoreNoticeTestCase, + SharedCacheKeyTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( CompileReuseRun, IncrementalEditComparison, RandomEditChain, add_external_flavor_macro, + analyze_in_one_batch, + analyze_one_model_at_a_time, + build_between, compare_incremental_compile, + compile_edit_without_reuse, compile_in_process, - compile_in_process_output, + compile_without_reuse_between, + compiled_artifact_tampered, compiled_text, completed_order_udf_signature_change, - disable_change_detection, - edit_and_compile, + edit_step, enable_compile_reuse, fact_audit_added, fact_comment, fact_error_fixed, fact_error_introduced, - full_edit_step, + fact_quantity_type_declared, + fact_quantity_type_removed, + ignore_project_changes, + ignore_query_in_analysis_key, in_process_reuse_run, - is_model_only_edit, - json_report_keys, - model_edit_step, + keep_invalidation, move_project_file, order_status_nullability_change, + other_target_build_between, payment_expression_type_change, - prime_render_store, - prime_render_store_in_process, + payments_comment, + plan_between, random_edit_plan, - render_store_files, replace_project_text, run_reuse_compile, - set_render_load_notice_bytes, - set_store_notice_renders, + staging_column_removed, + staging_column_renamed, + staging_column_restored, staging_comment, staging_contract_change, + staging_extra_flag, staging_header_change, staging_new_column, + staging_quantity_restored, + staging_quantity_text, + staging_rename_reverted, staging_type_change, star_chain_added, + star_chain_steps, + star_chain_with_twin_added, + stg_orders_test_edit, + twin_header_changed, write_external_flavor, write_generated_edit_models, write_project_file, @@ -66,20 +78,20 @@ IncrementalEditSequenceTestCase( description="model_edits", steps=( - model_edit_step("comment_without_shape_change", staging_comment), - model_edit_step("type_change_propagates_downstream", staging_type_change), - model_edit_step("new_output_column", staging_new_column), - model_edit_step("header_config_change", staging_header_change), - model_edit_step("column_contract_change", staging_contract_change), - model_edit_step("attached_audit_added", fact_audit_added), - model_edit_step("error_introduced", fact_error_introduced), - model_edit_step("error_fixed", fact_error_fixed), + edit_step("comment_without_shape_change", staging_comment), + edit_step("type_change_propagates_downstream", staging_type_change), + edit_step("new_output_column", staging_new_column), + edit_step("header_config_change", staging_header_change), + edit_step("column_contract_change", staging_contract_change), + edit_step("attached_audit_added", fact_audit_added), + edit_step("error_introduced", fact_error_introduced), + edit_step("error_fixed", fact_error_fixed), ), ), IncrementalEditSequenceTestCase( description="model_set_changes", steps=( - full_edit_step( + edit_step( "model_added", lambda root: write_project_file( root, @@ -88,8 +100,8 @@ 'SELECT order_id, quantity FROM __ref("stg_orders")\n', ), ), - model_edit_step("edit_after_add", staging_comment), - full_edit_step( + edit_step("edit_after_add", staging_comment), + edit_step( "model_renamed", lambda root: move_project_file( root, @@ -97,17 +109,17 @@ "models/marts/order_quantity_totals.sql", ), ), - full_edit_step( + edit_step( "model_removed", lambda root: (root / "models/marts/order_quantity_totals.sql").unlink(), ), - model_edit_step("edit_after_remove", fact_comment), + edit_step("edit_after_remove", fact_comment), ), ), IncrementalEditSequenceTestCase( description="declaration_edits", steps=( - full_edit_step( + edit_step( "macro_edit", lambda root: replace_project_text( root, @@ -116,14 +128,14 @@ "{quantity} * {price_cents}", ), ), - model_edit_step("edit_after_macro", fact_comment), - full_edit_step( + edit_step("edit_after_macro", fact_comment), + edit_step( "module_imported_by_macro_edit", lambda root: replace_project_text( root, "macros/_rounding.py", "_SCALE: int = 2", "_SCALE: int = 3" ), ), - full_edit_step( + edit_step( "scoped_macro_edit", lambda root: replace_project_text( root, @@ -132,7 +144,7 @@ "/ 100.0, 3)", ), ), - full_edit_step( + edit_step( "path_defaults_edit", lambda root: replace_project_text( root, @@ -141,7 +153,7 @@ '[path_defaults.staging]\nmaterialized = "table"', ), ), - full_edit_step( + edit_step( "enum_edit", lambda root: replace_project_text( root, @@ -150,7 +162,7 @@ 'WEB "online"', ), ), - full_edit_step( + edit_step( "constant_edit", lambda root: replace_project_text( root, @@ -159,7 +171,7 @@ "value 2)", ), ), - full_edit_step( + edit_step( "new_scope_folder", lambda root: write_project_file( root, @@ -169,19 +181,19 @@ ' return f"({price_cents}) * ({quantity})"\n', ), ), - model_edit_step("edit_after_declarations", staging_comment), + edit_step("edit_after_declarations", staging_comment), ), ), IncrementalEditSequenceTestCase( description="resource_edits", steps=( - full_edit_step( + edit_step( "seed_edit", lambda root: replace_project_text( root, "seeds/waffle_types.csv", "Classic Belgian", "Classic Brussels" ), ), - full_edit_step( + edit_step( "source_edit", lambda root: replace_project_text( root, @@ -190,17 +202,9 @@ " - name: quantity\n type: BIGINT", ), ), - model_edit_step("edit_after_source", staging_type_change), - full_edit_step( - "test_edit", - lambda root: replace_project_text( - root, - "tests/unit/test_stg_orders.sql", - "100 AS customer_id", - "101 AS customer_id", - ), - ), - full_edit_step( + edit_step("edit_after_source", staging_type_change), + edit_step("test_edit", stg_orders_test_edit), + edit_step( "audit_edit", lambda root: replace_project_text( root, @@ -209,44 +213,80 @@ "WHERE NOT (@expression) AND 1 = 1", ), ), - model_edit_step("edit_after_resources", fact_comment), + edit_step("edit_after_resources", fact_comment), + edit_step("compiled_artifact_tampered", compiled_artifact_tampered), ), ), IncrementalEditSequenceTestCase( description="analysis_input_edits", steps=( - full_edit_step("star_chain_added", star_chain_added), - model_edit_step("upstream_type_flips_downstream_shapes", staging_type_change), - model_edit_step("contract_type_edit", staging_contract_change), - full_edit_step("source_expression_edit", payment_expression_type_change), - full_edit_step("udf_signature_edit", completed_order_udf_signature_change), - full_edit_step("schema_entry_nullability_edit", order_status_nullability_change), - model_edit_step("edit_after_analysis_inputs", fact_comment), + edit_step("star_chain_added", star_chain_added), + edit_step("upstream_type_flips_downstream_shapes", staging_type_change), + edit_step("contract_type_edit", staging_contract_change), + edit_step("source_expression_edit", payment_expression_type_change), + edit_step("udf_signature_edit", completed_order_udf_signature_change), + edit_step("schema_entry_nullability_edit", order_status_nullability_change), + edit_step("edit_after_analysis_inputs", fact_comment), + ), + ), + IncrementalEditSequenceTestCase( + description="upstream_shape_changes", + steps=( + edit_step("upstream_column_removed", staging_column_removed), + edit_step("edit_while_downstream_broken", payments_comment), + edit_step("upstream_column_restored", staging_column_restored), + edit_step("upstream_column_renamed", staging_column_renamed), + edit_step("upstream_rename_reverted", staging_rename_reverted), + ), + ), + IncrementalEditSequenceTestCase( + description="contract_changes", + steps=( + edit_step("downstream_type_declared", fact_quantity_type_declared), + edit_step("upstream_type_breaks_contract", staging_quantity_text), + edit_step("edit_after_contract_error", fact_comment), + edit_step("upstream_type_restored", staging_quantity_restored), + edit_step("downstream_type_removed", fact_quantity_type_removed), ), ), + IncrementalEditSequenceTestCase( + description="star_chain_after_plan", + steps=star_chain_steps(between=plan_between), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_build", + steps=star_chain_steps(between=build_between), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_uncached_compile", + steps=star_chain_steps(between=compile_without_reuse_between), + ), + IncrementalEditSequenceTestCase( + description="star_chain_after_other_target_build", + steps=star_chain_steps(between=other_target_build_between), + ), ], ids=lambda case: case.description, ) -def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches_uncached_compile( +def test_given_edit_sequence_when_compiling_with_caches_then_each_step_matches_uncached_compile( compile_reuse_project: Path, test_case: IncrementalEditSequenceTestCase ) -> None: project_dir: Path = compile_reuse_project cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - primed: CompileReuseRun = prime_render_store(project_dir) - assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr + warm: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + assert (cold.returncode, warm.returncode, warm.reused) == (0, 0, True), warm.stderr for step in test_case.steps: step.edit(project_dir) step.between(project_dir) comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) - assert comparison.incremental.reused is False, step.description + assert comparison.incremental.reused is step.expected_replayed, step.description assert comparison.matches is test_case.expected_matches_uncached, ( step.description, comparison.mismatched_artifacts, comparison.incremental.stderr, ) - assert (comparison.reused_renders > 0) is step.expected_render_reuse, step.description @pytest.mark.parametrize( @@ -257,7 +297,7 @@ def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches ], ids=lambda case: case.description, ) -def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_matches_uncached_compile( +def test_given_random_edit_chain_when_compiling_with_caches_then_each_step_matches_uncached_compile( compile_reuse_project: Path, test_case: RandomEditChainTestCase ) -> None: project_dir: Path = compile_reuse_project @@ -265,8 +305,7 @@ def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_mat project_dir=project_dir, model_count=test_case.model_count, seed=test_case.seed ) cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - primed: CompileReuseRun = prime_render_store(project_dir) - assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr + assert cold.returncode == 0, cold.stderr chain: RandomEditChain = RandomEditChain(project_dir=project_dir, seed=test_case.seed) for step, kind in enumerate( @@ -282,34 +321,49 @@ def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_mat comparison.mismatched_artifacts, comparison.incremental.stderr, ) - assert (comparison.reused_renders > 0) is is_model_only_edit(kind), (step, kind) - assert (comparison.incremental.timings.get("analysis_reuse_hits", 0) > 0) is ( - is_model_only_edit(kind) - ), (step, kind) @pytest.mark.parametrize( "test_case", [ - BrokenChangeDetectionTestCase( - description="new_output_column", + BrokenEditInvalidationTestCase( + description="intact_invalidation_after_new_output_column", edit=staging_new_column, + sabotage=keep_invalidation, + expected_matches_uncached=True, + ), + BrokenEditInvalidationTestCase( + description="compile_replayed_despite_model_edit", + edit=staging_new_column, + sabotage=ignore_project_changes, + expected_matches_uncached=False, + ), + BrokenEditInvalidationTestCase( + description="intact_invalidation_after_error_introduced", + edit=fact_error_introduced, + sabotage=keep_invalidation, + expected_matches_uncached=True, + ), + BrokenEditInvalidationTestCase( + description="stale_analysis_despite_query_edit", + edit=fact_error_introduced, + sabotage=ignore_query_in_analysis_key, expected_matches_uncached=False, ), ], ids=lambda case: case.description, ) -def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( +def test_given_broken_cache_invalidation_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( compile_reuse_project: Path, - test_case: BrokenChangeDetectionTestCase, + test_case: BrokenEditInvalidationTestCase, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: project_dir: Path = compile_reuse_project enable_compile_reuse(monkeypatch) - assert prime_render_store_in_process(project_dir) == 0 + test_case.sabotage(monkeypatch) + assert compile_in_process(project_dir=project_dir) == 0 test_case.edit(project_dir) - disable_change_detection(monkeypatch) broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) @@ -317,78 +371,54 @@ def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_re incremental=broken, reference=reference ) - assert comparison.reused_renders > 0 - assert comparison.matches is test_case.expected_matches_uncached + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) @pytest.mark.parametrize( "test_case", [ - RenderStoreNoticeTestCase( - description="few_renders_store_quietly", notice_renders=100_000, expected_notice=False - ), - RenderStoreNoticeTestCase( - description="many_renders_announce_recording", notice_renders=1, expected_notice=True + SharedCacheKeyTestCase( + description="dataflow_one_model_at_a_time", schedule=analyze_one_model_at_a_time ), + SharedCacheKeyTestCase(description="one_batch", schedule=analyze_in_one_batch), ], ids=lambda case: case.description, ) -def test_given_new_renders_when_storing_after_an_edit_then_slow_recording_is_announced( +def test_given_models_sharing_an_analysis_cache_key_when_upstream_changes_then_it_matches_uncached( compile_reuse_project: Path, - test_case: RenderStoreNoticeTestCase, + test_case: SharedCacheKeyTestCase, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: + project_dir: Path = compile_reuse_project enable_compile_reuse(monkeypatch) - set_store_notice_renders(monkeypatch, test_case.notice_renders) - assert compile_in_process(project_dir=compile_reuse_project) == 0 - fact_comment(compile_reuse_project) - - code, _, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) - - assert code == 0 - assert ("renders)..." in err) is test_case.expected_notice - assert ("Recorded compile for reuse" in err) is test_case.expected_notice - - -@pytest.mark.parametrize( - "test_case", - [ - RenderSavePolicyTestCase( - description="cold_then_two_model_edits", - edits=(fact_comment, staging_comment), - expected_cold_render_files=0, - expected_render_files=(1, 2), - expected_reused=(False, True), - ), - ], - ids=lambda case: case.description, -) -def test_given_cold_compile_when_editing_then_renders_are_stored_only_once_a_compile_exists( - compile_reuse_project: Path, test_case: RenderSavePolicyTestCase -) -> None: - cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project) - cold_render_files: int = render_store_files(compile_reuse_project) - render_files: list[int] = [] - reused: list[bool] = [] - - for edit in test_case.edits: - run: CompileReuseRun = edit_and_compile(project_dir=compile_reuse_project, edit=edit) - assert run.returncode == 0, run.stderr - render_files.append(render_store_files(compile_reuse_project)) - reused.append(run.timings.get("render_reuse_hits", 0) > 0) + test_case.schedule(monkeypatch) + star_chain_with_twin_added(project_dir) + assert compile_in_process(project_dir=project_dir) == 0 + staging_extra_flag(project_dir) + twin_header_changed(project_dir) + compile_edit_without_reuse(project_dir, monkeypatch) + + incremental: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = in_process_reuse_run( + project_dir=project_dir, capsys=capsys, args=("--no-cache",) + ) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=incremental, reference=reference + ) - assert cold.returncode == 0, cold.stderr - assert cold_render_files == test_case.expected_cold_render_files - assert tuple(render_files) == test_case.expected_render_files - assert tuple(reused) == test_case.expected_reused + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) @pytest.mark.parametrize( "test_case", [ ExternalModuleEditTestCase( - description="module_rewritten_in_place_after_reuse", + description="module_rewritten_in_place_after_a_compile", initial_value="'vanilla'", edited_value="'choco'", expected_matches_uncached=True, @@ -397,7 +427,7 @@ def test_given_cold_compile_when_editing_then_renders_are_stored_only_once_a_com ], ids=lambda case: case.description, ) -def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_then_output_is_fresh( +def test_given_module_imported_while_rendering_when_it_changes_after_a_compile_then_output_is_fresh( compile_reuse_project: Path, tmp_path: Path, test_case: ExternalModuleEditTestCase ) -> None: extlib: Path = tmp_path / "extlib" @@ -406,9 +436,7 @@ def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_ ) cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) staging_comment(compile_reuse_project) - recorded: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) - staging_comment(compile_reuse_project) - reused: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + edited: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) write_external_flavor(extlib, test_case.edited_value) comparison: IncrementalEditComparison = IncrementalEditComparison( @@ -418,8 +446,8 @@ def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_ ), ) - assert (cold.returncode, recorded.returncode, reused.returncode) == (0, 0, 0), reused.stderr - assert reused.timings.get("render_reuse_hits", 0) > 0 + assert (cold.returncode, edited.returncode) == (0, 0), edited.stderr + assert comparison.incremental.reused is False assert comparison.matches is test_case.expected_matches_uncached, ( comparison.mismatched_artifacts ) @@ -428,32 +456,5 @@ def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_ ) -@pytest.mark.parametrize( - "test_case", - [ - RenderLoadNoticeTestCase( - description="small_store_loads_quietly", notice_bytes=1 << 40, expected_notice=False - ), - RenderLoadNoticeTestCase( - description="large_store_announces_loading", notice_bytes=1, expected_notice=True - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_renders_when_loading_them_then_large_loads_are_announced_on_stderr( - compile_reuse_project: Path, - test_case: RenderLoadNoticeTestCase, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - enable_compile_reuse(monkeypatch) - set_render_load_notice_bytes(monkeypatch, test_case.notice_bytes) - assert prime_render_store_in_process(compile_reuse_project) == 0 - staging_comment(compile_reuse_project) - - code, out, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) - - assert code == 0 - assert "compile_timings" in json_report_keys(out) - assert ("Loading stored renders" in err) is test_case.expected_notice - assert ("Loaded stored renders" in err) is test_case.expected_notice +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py index 5e1f98e710..11565105d4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_macro_call_store.py @@ -188,7 +188,7 @@ description="full_render_each_step", project_reuse=False, steps=_EDIT_STEPS ), MacroCallStoreEditSequenceTestCase( - description="with_project_and_render_reuse", project_reuse=True, steps=_EDIT_STEPS + description="with_project_reuse", project_reuse=True, steps=_EDIT_STEPS ), ], ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index d449de76ef..7cdafa3983 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -791,14 +791,12 @@ def counting_build( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, ) monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) @@ -853,14 +851,12 @@ def counting_build( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, - compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, - compile_cache_dir=compile_cache_dir, ) def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, LoadedMacro]: diff --git a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py index 272473d7e4..0898ca5204 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/_test_types.py @@ -221,17 +221,6 @@ class TolerantDeclarationFilesTestCase: expected_declaration_faults: int -@dataclass(frozen=True) -class DeclarationReuseTestCase: - """A preview compile followed by an edit that only touches one model file.""" - - description: str - files: tuple[tuple[str, bytes], ...] - edited_path: str - edited_contents: bytes - expected_reused_session: object = None - - @dataclass(frozen=True) class FailureTextTestCase: """A project whose discovery failure message must keep its authored text unchanged.""" diff --git a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py index 2c997d3792..72eb243627 100644 --- a/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py +++ b/tests/integration/src/sqlbuild/compiler/discovery/test_native_declaration_files.py @@ -9,17 +9,12 @@ import pytest from sqlbuild import _native -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import RENDER_REUSE_DECLARATIONS_PREFIX -from sqlbuild.compiler.compile.models import RenderReuseState from sqlbuild.compiler.discovery._helpers.filesystem import core from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryFileFault -from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from tests.integration.src.sqlbuild.compiler.discovery._test_types import ( DeclarationFilesParityTestCase, - DeclarationReuseTestCase, DeepDeclarationHeaderTestCase, GeneratedDeclarationFileTestCase, NativeDeclarationFailureTestCase, @@ -504,47 +499,5 @@ def test_given_engine_when_discovering_then_native_session_is_kept_only_for_prev ) -@pytest.mark.parametrize( - "test_case", - [ - DeclarationReuseTestCase( - description="a model edit reuses every stored declaration file", - files=_EVERY_KIND, - edited_path="models/marts/orders.sql", - edited_contents=_MODEL + b" -- edited\n", - ) - ], - ids=lambda case: case.description, -) -def test_given_preview_compile_when_only_a_model_changes_then_declaration_files_are_reused( - test_case: DeclarationReuseTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - write_project(project_dir=tmp_path, files=test_case.files) - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, _PREVIEW) - first_session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=None, changed_paths=None - ) - first: DiscoveredProjectInputs = discover_project_inputs( - project_dir=tmp_path, declaration_reuse=first_session - ) - first_session.plan_models(model_files=first.model_files) - state: RenderReuseState | None = first_session.stored_state() - assert state is not None - _ = (tmp_path / test_case.edited_path).write_bytes(test_case.edited_contents) - second_session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=state, changed_paths=frozenset({test_case.edited_path}) - ) - - second: DiscoveredProjectInputs = discover_project_inputs( - project_dir=tmp_path, declaration_reuse=second_session - ) - monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, "python") - fresh: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) - - assert any(name.startswith(RENDER_REUSE_DECLARATIONS_PREFIX) for name in state.group_payloads) - assert second.native_session is test_case.expected_reused_session - assert render_stage_capture(second) == render_stage_capture(fresh) - - if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py deleted file mode 100644 index 913e647485..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Test case types for layered render storage.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class RenderLayerTestCase: - """Renders changed after a stored compile, and whether an overlay can hold them.""" - - description: str - changed_models: tuple[str, ...] - removed_models: tuple[str, ...] - expected_overlay: bool - - -@dataclass(frozen=True) -class RenderEditChainTestCase: - """Successive one-model edits, each stored on top of the previous compile.""" - - description: str - edits: tuple[str, ...] - expected_overlay_models: frozenset[str] - - -@dataclass(frozen=True) -class StoredRenderReadTestCase: - """A stored render file, possibly damaged, and whether reading it yields renders.""" - - description: str - prepare: Callable[[Path], tuple[Path, frozenset[str]]] - expected_readable: bool - - -@dataclass(frozen=True) -class RenderLoadNoticeTestCase: - """Stored render files, some removed while being sized, and the expected notice.""" - - description: str - file_sizes: tuple[int, ...] - removed_while_sizing: int - notice_bytes: int - expected_notice: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py deleted file mode 100644 index c215b98d84..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Shared builders for layered render storage unit tests.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( - read_render_state, - render_state_layer, -) -from sqlbuild.cli.compile_render_reuse.models import StoredRenderState -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState - -MODEL_PATHS: tuple[str, ...] = tuple(f"models/orders_{index:02d}.sql" for index in range(20)) - - -def publish_render_state(*, directory: Path, name: str, layer: RenderStateLayer | None) -> Path: - """Write one render layer as the store would publish it.""" - - assert layer is not None - path: Path = directory / name - path.write_bytes(b"".join(bytes(chunk) for chunk in layer.chunks)) - return path - - -def read_stored(path: Path) -> StoredRenderState: - """Read a render file that must decode.""" - - stored: StoredRenderState | None = read_render_state(path=path, changed_paths=frozenset()) - assert stored is not None - return stored - - -def stored_base(directory: Path) -> tuple[Path, StoredRenderState]: - """Publish a base render file holding every model and one group, and read it back.""" - - state: RenderReuseState = RenderReuseState( - model_paths=MODEL_PATHS, - model_payloads={path: memoryview(f"render {path}".encode() * 20) for path in MODEL_PATHS}, - group_payloads={"sources": memoryview(b"source renders" * 20)}, - ) - path: Path = publish_render_state( - directory=directory, - name="orders-base.render", - layer=render_state_layer(state=state, stored=None), - ) - return path, read_stored(path) - - -def next_render_state( - *, stored: StoredRenderState, edited: tuple[str, ...], removed: tuple[str, ...] = () -) -> RenderReuseState: - """Return the next compile's renders as a session stores them after releasing reused bytes. - - Renders reused from the stored base are None; reused overlay renders keep their bytes. - """ - - kept: dict[str, memoryview | None] = { - path: retained_payload(stored=stored, path=path, payload=payload) - for path, payload in filter( - lambda item: item[0] not in removed, stored.state.model_payloads.items() - ) - } - return RenderReuseState( - model_paths=MODEL_PATHS, - model_payloads={ - **kept, - **{path: memoryview(f"edited {path}".encode() * 20) for path in edited}, - }, - group_payloads=dict.fromkeys(stored.state.group_payloads), - ) - - -def retained_payload( - *, stored: StoredRenderState, path: str, payload: memoryview | None -) -> memoryview | None: - """Keep a reused render's bytes only when it lives in the stored overlay.""" - - return {True: payload, False: None}[path in stored.layers.overlay_models] - - -def payload_bytes(state: RenderReuseState) -> dict[str, bytes]: - """Return every model payload as bytes for comparison.""" - - return {path: bytes(payload or b"") for path, payload in state.model_payloads.items()} - - -def store_edit_chain(*, directory: Path, edits: tuple[str, ...]) -> StoredRenderState: - """Store one compile per edited model, each layered on the previous one, and read the last.""" - - stored: StoredRenderState - _, stored = stored_base(directory) - for index, edited in enumerate(edits): - path: Path = publish_render_state( - directory=directory, - name=f"orders-edit-{index}.render", - layer=render_state_layer( - state=next_render_state(stored=stored, edited=(edited,)), stored=stored.layers - ), - ) - stored = read_stored(path) - return stored - - -def chain_payload_bytes(edits: tuple[str, ...]) -> dict[str, bytes]: - """Return the model payloads an edit chain must leave readable.""" - - return { - **{path: f"render {path}".encode() * 20 for path in MODEL_PATHS}, - **{path: f"edited {path}".encode() * 20 for path in edits}, - } - - -def flipped_last_byte(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and corrupt its last payload byte.""" - - path, _ = stored_base(directory) - data: bytearray = bytearray(path.read_bytes()) - data[-1] ^= 0xFF - path.write_bytes(bytes(data)) - return path, frozenset() - - -def truncated(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and cut off its tail.""" - - path, _ = stored_base(directory) - path.write_bytes(path.read_bytes()[:-7]) - return path, frozenset() - - -def emptied(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and empty it.""" - - path, _ = stored_base(directory) - path.write_bytes(b"") - return path, frozenset() - - -def overlay_without_base(directory: Path) -> tuple[Path, frozenset[str]]: - """Store an overlay and delete the base file it layers on.""" - - base_path, stored = stored_base(directory) - overlay_path: Path = publish_render_state( - directory=directory, - name="orders-overlay.render", - layer=render_state_layer( - state=next_render_state(stored=stored, edited=(MODEL_PATHS[0],)), stored=stored.layers - ), - ) - base_path.unlink() - return overlay_path, frozenset() - - -def changed_macro(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and report a changed path that is not a stored model.""" - - path, _ = stored_base(directory) - return path, frozenset({"macros/currency.py"}) - - -def intact(directory: Path) -> tuple[Path, frozenset[str]]: - """Store a base render file and report one stored model as changed.""" - - path, _ = stored_base(directory) - return path, frozenset({MODEL_PATHS[4]}) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py deleted file mode 100644 index 3feb8dc648..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Sizing the render store for the load notice must tolerate files removed by a concurrent compile.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_render_reuse._helpers import load_notice -from sqlbuild.cli.compile_reuse.constants import REUSE_RENDER_STATE_SUFFIX -from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( - RenderLoadNoticeTestCase, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderLoadNoticeTestCase( - description="large_store_announced", - file_sizes=(64, 64), - removed_while_sizing=0, - notice_bytes=100, - expected_notice=True, - ), - RenderLoadNoticeTestCase( - description="small_store_silent", - file_sizes=(16, 16), - removed_while_sizing=0, - notice_bytes=100, - expected_notice=False, - ), - RenderLoadNoticeTestCase( - description="file_removed_while_sizing_counts_as_empty", - file_sizes=(64, 64), - removed_while_sizing=1, - notice_bytes=100, - expected_notice=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_render_files_when_starting_load_notice_then_sizes_without_failing( - test_case: RenderLoadNoticeTestCase, - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - # Arrange - entry_path: Path = tmp_path / "dev.entry" - stored_paths: list[Path] = [ - tmp_path / f"dev-{index}{REUSE_RENDER_STATE_SUFFIX}" - for index in range(len(test_case.file_sizes)) - ] - for path, size in zip(stored_paths, test_case.file_sizes, strict=True): - path.write_bytes(b"x" * size) - listed: list[Path] = sorted(stored_paths) - monkeypatch.setattr(load_notice, "RENDER_LOAD_NOTICE_BYTES", test_case.notice_bytes) - monkeypatch.setattr(Path, "glob", lambda self, pattern: iter(listed)) - for path in listed[: test_case.removed_while_sizing]: - path.unlink() - - # Act - started: float | None = load_notice.start_load_notice(entry_path=entry_path) - - # Assert - assert (started is not None) is test_case.expected_notice - assert capsys.readouterr().out == "" diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py deleted file mode 100644 index bca44fbe3e..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Layered render storage must round-trip exactly and treat every fault as no stored renders.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( - read_render_state, - render_state_layer, -) -from sqlbuild.cli.compile_render_reuse.models import StoredRenderState -from sqlbuild.cli.compile_reuse.models import RenderStateLayer -from sqlbuild.compiler.compile.models import RenderReuseState -from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( - RenderEditChainTestCase, - RenderLayerTestCase, - StoredRenderReadTestCase, -) -from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers.helpers import ( - MODEL_PATHS, - chain_payload_bytes, - changed_macro, - emptied, - flipped_last_byte, - intact, - next_render_state, - overlay_without_base, - payload_bytes, - store_edit_chain, - stored_base, - truncated, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderLayerTestCase( - description="one_model_changed", - changed_models=(MODEL_PATHS[3],), - removed_models=(), - expected_overlay=True, - ), - RenderLayerTestCase( - description="most_models_changed", - changed_models=MODEL_PATHS[:15], - removed_models=(), - expected_overlay=False, - ), - RenderLayerTestCase( - description="stored_render_dropped", - changed_models=(MODEL_PATHS[3],), - removed_models=(MODEL_PATHS[5],), - expected_overlay=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_released_reused_renders_when_storing_then_only_a_small_overlay_needs_no_rewrite( - test_case: RenderLayerTestCase, tmp_path: Path -) -> None: - _, stored = stored_base(tmp_path) - current: RenderReuseState = next_render_state( - stored=stored, edited=test_case.changed_models, removed=test_case.removed_models - ) - - layer: RenderStateLayer | None = render_state_layer(state=current, stored=stored.layers) - - assert (layer is not None) is test_case.expected_overlay - - -@pytest.mark.parametrize( - "test_case", - [ - RenderEditChainTestCase( - description="different_models_edited", - edits=(MODEL_PATHS[1], MODEL_PATHS[2]), - expected_overlay_models=frozenset({MODEL_PATHS[1], MODEL_PATHS[2]}), - ), - RenderEditChainTestCase( - description="same_model_edited_twice", - edits=(MODEL_PATHS[1], MODEL_PATHS[1]), - expected_overlay_models=frozenset({MODEL_PATHS[1]}), - ), - ], - ids=lambda case: case.description, -) -def test_given_overlay_when_storing_again_then_earlier_overlay_renders_are_kept( - test_case: RenderEditChainTestCase, tmp_path: Path -) -> None: - stored: StoredRenderState = store_edit_chain(directory=tmp_path, edits=test_case.edits) - - assert stored.layers.overlay_models == test_case.expected_overlay_models - assert payload_bytes(stored.state) == chain_payload_bytes(test_case.edits) - - -@pytest.mark.parametrize( - "test_case", - [ - StoredRenderReadTestCase( - description="intact_with_changed_model", prepare=intact, expected_readable=True - ), - StoredRenderReadTestCase( - description="flipped_payload_byte", prepare=flipped_last_byte, expected_readable=False - ), - StoredRenderReadTestCase( - description="truncated", prepare=truncated, expected_readable=False - ), - StoredRenderReadTestCase(description="empty", prepare=emptied, expected_readable=False), - StoredRenderReadTestCase( - description="overlay_whose_base_is_missing", - prepare=overlay_without_base, - expected_readable=False, - ), - StoredRenderReadTestCase( - description="changed_path_is_not_a_stored_model", - prepare=changed_macro, - expected_readable=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_render_file_when_reading_then_only_intact_relevant_renders_are_returned( - test_case: StoredRenderReadTestCase, tmp_path: Path -) -> None: - path, changed_paths = test_case.prepare(tmp_path) - - stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) - - assert (stored is not None) is test_case.expected_readable - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index 90ea98eb84..fd9b5d9c9c 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -8,9 +8,6 @@ import pytest -from sqlbuild.cli.compile_reuse.models import StoredProjectFile -from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import digested_project_files - @dataclass(frozen=True) class ProjectFingerprintTestCase: @@ -83,16 +80,6 @@ class SettingsDigestTestCase: expected_changed: bool -@dataclass(frozen=True) -class ChangedPathsTestCase: - """One project change and every project path a change set must name for it.""" - - description: str - change: Callable[[Path], object] - expected_paths: tuple[str, ...] - store: Callable[[Path], dict[str, StoredProjectFile]] = digested_project_files - - @dataclass(frozen=True) class EngineReuseDigestTestCase: """Two engine environments for one invocation and whether their reuse keys may match.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index dd03ca35e2..766bd626ac 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -131,12 +131,6 @@ def break_link(project_dir: Path) -> None: (project_dir.parent / "shared/customers.sql").unlink() -def digested_project_files(project_dir: Path) -> dict[str, StoredProjectFile]: - """Record the project with digests and a snapshot time that leaves no file racy.""" - - return stored_project_files(project_dir=project_dir, snapshot_ns=later_snapshot_ns()) - - def stat_only_project_files(*, project_dir: Path) -> dict[str, StoredProjectFile]: """Record the project as a stored compile does by default: stat stamps without digests.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py index cb38c6869c..1de9e2e1a5 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py @@ -10,7 +10,6 @@ from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, - changed_project_paths, compare_project_files, restamped_paths, snapshot_project_files, @@ -18,7 +17,6 @@ from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile from sqlbuild.cli.compile_reuse.types import FileStamp from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( - ChangedPathsTestCase, DigestCarryForwardTestCase, LinkCycleTestCase, ProjectFingerprintTestCase, @@ -149,68 +147,13 @@ def test_given_stored_fingerprint_when_project_changes_then_change_is_detected_e ) test_case.change(project_dir) - current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) comparison: ProjectFilesComparison = compare_project_files( - project_dir=str(project_dir), stored=stored, current=current - ) - changed, _ = changed_project_paths( - project_dir=str(project_dir), stored=stored, current=current, verified={} - ) - - assert comparison.unchanged is test_case.expected_unchanged - assert (not changed) is test_case.expected_unchanged - - -@pytest.mark.parametrize( - "test_case", - [ - ChangedPathsTestCase( - description="touched_without_change", - change=touch_without_change, - expected_paths=(), - ), - ChangedPathsTestCase( - description="model_edited", - change=lambda root: write_file(root / "models/orders.sql", "SELECT 10 AS order_id\n"), - expected_paths=("models/orders.sql",), - ), - ChangedPathsTestCase( - description="same_size_and_mtime_new_content", - change=rewrite_same_size_keeping_mtime, - expected_paths=("models/orders.sql",), - ), - ChangedPathsTestCase( - description="file_added_and_removed", - change=lambda root: ( - write_file(root / "models/returns.sql", "SELECT 1\n"), - (root / "macros/currency.py").unlink(), - ), - expected_paths=("macros/currency.py", "models/returns.sql"), - ), - ChangedPathsTestCase( - description="touched_without_stored_digest", - change=touch_without_change, - expected_paths=("models/orders.sql",), - store=lambda root: stat_only_project_files(project_dir=root), - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_fingerprint_when_listing_changes_then_returns_every_changed_path( - test_case: ChangedPathsTestCase, tmp_path: Path -) -> None: - project_dir: Path = write_fingerprint_project(tmp_path) - stored: dict[str, StoredProjectFile] = test_case.store(project_dir) - - test_case.change(project_dir) - changed, _ = changed_project_paths( project_dir=str(project_dir), stored=stored, current=snapshot_project_files(project_dir=str(project_dir)), - verified={}, ) - assert changed == frozenset(test_case.expected_paths) + assert comparison.unchanged is test_case.expected_unchanged @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/__init__.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py deleted file mode 100644 index 7ed01cf136..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/_test_types.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Test case types for trusting artifacts a stored compile left.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class StoredArtifactTestCase: - """A change to a stored artifact and whether its stored digest may still be trusted.""" - - description: str - change: Callable[[Path], None] - written_contents: bytes - expected_holds: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py deleted file mode 100644 index d7998b411c..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/helpers.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Builders for stored compile artifact unit tests.""" - -from __future__ import annotations - -import hashlib -from pathlib import Path - -from sqlbuild.cli.compile_reuse.constants import DIGEST_SIZE_BYTES - -STORED_SQL: bytes = b"SELECT order_id FROM orders\n" - - -def artifact_unchanged(_path: Path) -> None: - """Leave the stored artifact as it is.""" - - -def artifact_rewritten(path: Path) -> None: - """Rewrite the stored artifact with other contents.""" - - path.write_bytes(b"SELECT order_id FROM returns\n") - - -def artifact_removed(path: Path) -> None: - """Remove the stored artifact.""" - - path.unlink() - - -def artifact_digest(contents: bytes) -> str: - """Return the digest compile reuse records for artifact contents.""" - - return hashlib.blake2b(contents, digest_size=DIGEST_SIZE_BYTES).hexdigest() diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py b/tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py deleted file mode 100644 index cac7162a49..0000000000 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/classes/test_stored_artifacts.py +++ /dev/null @@ -1,73 +0,0 @@ -"""A stored artifact is trusted only while its stat identity and digest both still match.""" - -from __future__ import annotations - -import os -from pathlib import Path - -import pytest - -from sqlbuild.cli.compile_reuse._helpers.target_files import path_stamp -from sqlbuild.cli.compile_reuse.classes.stored_artifacts import StoredArtifacts -from sqlbuild.cli.compile_reuse.types import FileStamp -from tests.unit.src.sqlbuild.cli.compile_reuse.classes._test_types import StoredArtifactTestCase -from tests.unit.src.sqlbuild.cli.compile_reuse.classes.helpers import ( - STORED_SQL, - artifact_digest, - artifact_removed, - artifact_rewritten, - artifact_unchanged, -) - - -@pytest.mark.parametrize( - "test_case", - [ - StoredArtifactTestCase( - description="unchanged_same_contents", - change=artifact_unchanged, - written_contents=STORED_SQL, - expected_holds=True, - ), - StoredArtifactTestCase( - description="unchanged_new_contents", - change=artifact_unchanged, - written_contents=b"SELECT 1 AS order_id\n", - expected_holds=False, - ), - StoredArtifactTestCase( - description="rewritten_since_stored", - change=artifact_rewritten, - written_contents=STORED_SQL, - expected_holds=False, - ), - StoredArtifactTestCase( - description="removed_since_stored", - change=artifact_removed, - written_contents=STORED_SQL, - expected_holds=False, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_artifact_when_writing_then_trusted_only_while_stamp_and_digest_match( - tmp_path: Path, test_case: StoredArtifactTestCase -) -> None: - artifact: Path = tmp_path / "compiled" / "orders.sql" - artifact.parent.mkdir() - artifact.write_bytes(STORED_SQL) - path: str = os.fspath(artifact) - stamp: FileStamp | None = path_stamp(path=path) - assert stamp is not None - stored: StoredArtifacts = StoredArtifacts( - stamps={path: stamp}, digests={path: artifact_digest(STORED_SQL)} - ) - test_case.change(artifact) - - holds: bool = stored.holds(path=path, digest=artifact_digest(test_case.written_contents)) - - assert holds is test_case.expected_holds - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py deleted file mode 100644 index 82cd55965b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py +++ /dev/null @@ -1,79 +0,0 @@ -"""Test case types for the compile render reuse session and stored model analyses.""" - -from __future__ import annotations - -from dataclasses import dataclass - -from sqlbuild.compiler.compile.models import CompilerDiagnostic - - -@dataclass(frozen=True) -class RenderReplayTestCase: - """A reused render and what reusing it must report again.""" - - description: str - model_name: str - expected_diagnostics: tuple[CompilerDiagnostic, ...] - expected_environment_names: tuple[str, ...] - - -@dataclass(frozen=True) -class ReusableModelsTestCase: - """Stored and current models, and which current models may reuse their stored render.""" - - description: str - stored_models: tuple[str, ...] - current_models: tuple[str, ...] - changed_paths: frozenset[str] - run_id_readers: frozenset[str] - expected_reusable: dict[str, bool] - - -@dataclass(frozen=True) -class CorruptRenderTestCase: - """A stored render payload that cannot be decoded.""" - - description: str - model_name: str - expected_rendered_again: bool - - -@dataclass(frozen=True) -class ReleasedRenderTestCase: - """A reused and an edited render, and which stored bytes storing the compile releases.""" - - description: str - retained_models: frozenset[str] - expected_released: dict[str, bool] - expected_query_sqls: dict[str, str] - - -@dataclass(frozen=True) -class DeclarationReuseTestCase: - """Paths changed since a stored compile, and whether its declaration files are reused.""" - - description: str - changed_paths: frozenset[str] | None - recorded_variant: str - expected_full_discoveries: int - - -@dataclass(frozen=True) -class StoredAnalysisScopeTestCase: - """Which renders a compile reused, how analyses were stored, and which may be served.""" - - description: str - reused_models: frozenset[str] - rekeyed_models: frozenset[str] - twinned_models: frozenset[str] - stored_column_types: dict[str, dict[str, str]] - expected_served: frozenset[str] - - -@dataclass(frozen=True) -class StaleServedAnalysisTestCase: - """Final parent signatures in this compile, and which served analyses they make stale.""" - - description: str - current_signatures: dict[str, str] - expected_stale: set[str] diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py deleted file mode 100644 index 9482548c3a..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py +++ /dev/null @@ -1,342 +0,0 @@ -"""Shared builders for render reuse session and stored model analysis unit tests.""" - -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import replace -from pathlib import Path -from typing import cast - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( - collect_compile_diagnostics, - report_compile_diagnostic, -) -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.classes.stored_model_analyses import ( - StoredModelAnalyses, - analysis_reuse_context, -) -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.models import ( - AnalysisCacheContext, - CompileModelInput, - CompilerDiagnostic, - CompileSqlReference, - ModelAnalysisCaching, - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, - RenderReuseState, - StoredModelAnalysis, - StoredRender, -) -from sqlbuild.compiler.compile.types import CompileContextKey, DiagnosticPhase, DiagnosticSeverity -from sqlbuild.compiler.discovery.models import ( - DiscoveredDeclarationFiles, - DiscoveredMacroFile, - DiscoveredSqlModelFile, -) -from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload -from sqlbuild.compiler.references.types import SqlReferenceKind - -REGION_ENV_VAR: str = "ORDERS_REGION" -ORDERS_CACHE: AnalysisCacheContext = AnalysisCacheContext( - root=Path("/orders/target/cache"), shared_fingerprint="orders-fingerprint" -) -RAW_ORDERS_TYPES: dict[str, dict[str, str]] = {"raw_orders": {"order_id": "INTEGER"}} - - -def model_file(name: str, sql: str = "SELECT 1 AS order_id") -> DiscoveredSqlModelFile: - """Return one discovered SQL model.""" - - return DiscoveredSqlModelFile( - file_path=Path("/orders") / "models" / f"{name}.sql", - relative_path=Path("models") / f"{name}.sql", - contents=sql, - header_values={}, - header_column_locations={}, - output_column_locations={}, - query_sql=sql, - ) - - -def missing_description(name: str) -> CompilerDiagnostic: - """Return the warning a render of the named model reports.""" - - return CompilerDiagnostic( - phase=DiagnosticPhase.COMPILE, - severity=DiagnosticSeverity.WARNING, - code="P010", - message=f"Model '{name}' has no description", - ) - - -def _render(item: DiscoveredSqlModelFile) -> CompileModelInput: - report_compile_diagnostic( - key=(item.file_path.stem,), diagnostic=missing_description(item.file_path.stem) - ) - COMPILE_INPUT_READS.environment_read(REGION_ENV_VAR) - return CompileModelInput(model_file=item, query_sql=item.query_sql) - - -def _render_reading_run_id(item: DiscoveredSqlModelFile) -> CompileModelInput: - COMPILE_INPUT_READS.context_read(CompileContextKey.RUN_ID) - return _render(item) - - -def recorded_state( - model_files: tuple[DiscoveredSqlModelFile, ...], - *, - run_id_readers: frozenset[str] = frozenset(), -) -> RenderReuseState: - """Render every model through a fresh session and return what it stores.""" - - session: CompileRenderReuseSession = CompileRenderReuseSession(prior=None, changed_paths=None) - session.plan_models(model_files=model_files) - renderers: dict[bool, Callable[[DiscoveredSqlModelFile], CompileModelInput]] = { - False: _render, - True: _render_reading_run_id, - } - with collect_compile_diagnostics(): - for item in model_files: - render: Callable[[DiscoveredSqlModelFile], CompileModelInput] = renderers[ - item.file_path.stem in run_id_readers - ] - _ = session.rendered_model(model_file=item, render=lambda item=item, r=render: r(item)) - state: RenderReuseState | None = session.stored_state() - assert state is not None - return state - - -def planned_session( - *, - stored: tuple[DiscoveredSqlModelFile, ...], - current: tuple[DiscoveredSqlModelFile, ...], - changed_paths: frozenset[str] = frozenset(), - run_id_readers: frozenset[str] = frozenset(), -) -> CompileRenderReuseSession: - """Record renders for the stored models, then plan a session for the current models.""" - - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=recorded_state(stored, run_id_readers=run_id_readers), changed_paths=changed_paths - ) - session.plan_models(model_files=current) - return session - - -def corrupted_session(item: DiscoveredSqlModelFile) -> CompileRenderReuseSession: - """Plan a session whose only stored render payload cannot be decoded.""" - - state: RenderReuseState = recorded_state((item,)) - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=replace( - state, - model_payloads={str(item.relative_path): memoryview(b"not a render")}, - ), - changed_paths=frozenset(), - ) - session.plan_models(model_files=(item,)) - return session - - -def edited_session(*, retained_models: frozenset[str]) -> CompileRenderReuseSession: - """Reuse the stored orders render and render an edited customers model.""" - - orders: DiscoveredSqlModelFile = model_file("orders") - customers: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=recorded_state((orders, model_file("customers"))), - changed_paths=frozenset({str(customers.relative_path)}), - retained_models=retained_models, - ) - session.plan_models(model_files=(orders, customers)) - with collect_compile_diagnostics(): - _ = session.reused_model(model_file=orders) - _ = session.rendered_model(model_file=customers, render=lambda: _render(customers)) - return session - - -def released_paths(state: RenderReuseState | None) -> dict[str, bool]: - """Return, per stored model, whether its bytes were released.""" - - assert state is not None - return {path: payload is None for path, payload in state.model_payloads.items()} - - -def stored_query_sqls(state: RenderReuseState | None) -> dict[str, str]: - """Decode every stored model render and return its query SQL.""" - - assert state is not None - return { - path: _stored_query_sql(cast(memoryview, payload)) - for path, payload in filter(lambda item: item[1] is not None, state.model_payloads.items()) - } - - -def _stored_query_sql(payload: memoryview) -> str: - stored: StoredRender = cast(StoredRender, loaded_fact_payload(payload)) - return cast(CompileModelInput, stored.value).query_sql - - -DECLARATIONS_VARIANT: str = "10" -OTHER_DECLARATIONS_VARIANT: str = "01" - - -def declaration_files( - model_files: tuple[DiscoveredSqlModelFile, ...], -) -> DiscoveredDeclarationFiles: - """Return discovered declaration files holding one macro file and the given models.""" - - return DiscoveredDeclarationFiles( - source_files=(), - model_files=model_files, - enum_files=(), - constant_files=(), - model_schema_files=(), - sql_function_files=(), - sql_hook_files=(), - python_function_files=(), - schema_files=(), - seed_files=(), - test_files=(), - scenario_files=(), - audit_files=(), - macro_files=( - DiscoveredMacroFile( - file_path=Path("/orders/macros/currency.py"), - relative_path=Path("macros/currency.py"), - contents="def total(price: str) -> str:\n return price\n", - ), - ), - adapter_file=None, - ) - - -class CountingDiscovery: - """Discover fixed declaration files and count full and model-only discoveries.""" - - def __init__(self, model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: - self.model_files: tuple[DiscoveredSqlModelFile, ...] = model_files - self.full: int = 0 - - def discover(self) -> DiscoveredDeclarationFiles: - """Discover every declaration file.""" - - self.full += 1 - return declaration_files(self.model_files) - - def discover_models(self) -> tuple[DiscoveredSqlModelFile, ...]: - """Discover model files only.""" - - return self.model_files - - -def declarations_state(*, recorded_variant: str) -> RenderReuseState: - """Return stored renders of two models, with declaration files discovered for one variant.""" - - stored: tuple[DiscoveredSqlModelFile, ...] = (model_file("orders"), model_file("customers")) - session: CompileRenderReuseSession = CompileRenderReuseSession(prior=None, changed_paths=None) - _ = session.declaration_files( - variant=recorded_variant, - discover=CountingDiscovery(stored).discover, - discover_models=CountingDiscovery(stored).discover_models, - ) - session.plan_models(model_files=stored) - with collect_compile_diagnostics(): - for item in stored: - _ = session.rendered_model(model_file=item, render=lambda item=item: _render(item)) - state: RenderReuseState | None = session.stored_state() - assert state is not None - return state - - -def analyzed_model(name: str, *upstream: str) -> CompileModelInput: - """Return one model input reading the given upstream models.""" - - model: DiscoveredSqlModelFile = model_file(name) - return CompileModelInput( - model_file=model, - query_sql=model.contents, - references=tuple( - CompileSqlReference(ref_kind=SqlReferenceKind.REF, ref_name=upstream_name) - for upstream_name in upstream - ), - ) - - -def stored_analysis( - *, name: str, context: str, dependencies: dict[str, str] -) -> StoredModelAnalysis: - """Return the analysis a previous compile stored for one model.""" - - return StoredModelAnalysis( - context=context, - cache_key=f"{name}-key", - analysis=PolyglotAnalysisResult(analysis_succeeded=True), - output_signature=f"{name}-signature", - dependencies=dependencies, - signature=f"{name}-signature", - ) - - -def analysis_request(model_input: CompileModelInput, cache_key: str) -> ModelSqlAnalysisRequest: - """Return the analysis request of one model with the given cache key.""" - - return ModelSqlAnalysisRequest( - model_input=model_input, - query_sql=model_input.query_sql, - placeholders=None, - cache_key=cache_key, - binding_schema=None, - ) - - -def orders_analysis_context( - *, requests: tuple[ModelSqlAnalysisRequest, ...], column_types: dict[str, dict[str, str]] -) -> str: - """Return the shared analysis inputs digest of the orders project with these source types.""" - - return analysis_reuse_context( - cache=ORDERS_CACHE, - model_names=frozenset(item.model_input.model_file.file_path.stem for item in requests), - column_types_by_table=column_types, - column_nullability_by_table={}, - complete_binding_schemas={}, - ) - - -def orders_stored_analyses( - *, requests: tuple[ModelSqlAnalysisRequest, ...], reuse: RecordingAnalysisReuse -) -> StoredModelAnalyses: - """Return the stored analyses a compile of the orders project may serve.""" - - return StoredModelAnalyses( - caching=ModelAnalysisCaching(cache=ORDERS_CACHE, reuse=reuse), - requests=requests, - column_types_by_table=RAW_ORDERS_TYPES, - column_nullability_by_table={}, - complete_binding_schemas={}, - ) - - -class RecordingAnalysisReuse: - """Stored analyses and reused renders of a previous compile, recording new analyses.""" - - def __init__(self, *, reused: frozenset[str], stored: dict[str, StoredModelAnalysis]) -> None: - self._reused: frozenset[str] = reused - self._stored: dict[str, StoredModelAnalysis] = stored - self.recorded: dict[str, StoredModelAnalysis | None] | None = None - - def reused_model_names(self) -> frozenset[str]: - """Return the names of models whose renders were reused.""" - - return self._reused - - def stored_analyses(self) -> dict[str, StoredModelAnalysis]: - """Return the stored analyses by model name.""" - - return self._stored - - def record_analyses(self, *, analyses: dict[str, StoredModelAnalysis | None]) -> None: - """Keep what the compile records.""" - - self.recorded = analyses diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py deleted file mode 100644 index 1fd8865668..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py +++ /dev/null @@ -1,254 +0,0 @@ -"""Render reuse must replay what an unchanged render reported and never serve a changed one.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS -from sqlbuild.compiler.compile.models import CompileModelInput -from sqlbuild.compiler.discovery.models import DiscoveredDeclarationFiles, DiscoveredSqlModelFile -from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( - CorruptRenderTestCase, - DeclarationReuseTestCase, - ReleasedRenderTestCase, - RenderReplayTestCase, - ReusableModelsTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( - DECLARATIONS_VARIANT, - OTHER_DECLARATIONS_VARIANT, - REGION_ENV_VAR, - CountingDiscovery, - corrupted_session, - declaration_files, - declarations_state, - edited_session, - missing_description, - model_file, - planned_session, - released_paths, - stored_query_sqls, -) - - -@pytest.mark.parametrize( - "test_case", - [ - RenderReplayTestCase( - description="unchanged_model", - model_name="orders", - expected_diagnostics=(missing_description("orders"),), - expected_environment_names=(REGION_ENV_VAR,), - ), - ], - ids=lambda case: case.description, -) -def test_given_unchanged_model_when_reusing_then_render_and_reports_are_replayed( - test_case: RenderReplayTestCase, -) -> None: - unchanged: DiscoveredSqlModelFile = model_file(test_case.model_name) - edited: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") - session: CompileRenderReuseSession = planned_session( - stored=(unchanged, model_file("customers")), - current=(unchanged, edited), - changed_paths=frozenset({str(edited.relative_path)}), - ) - - with ( - COMPILE_INPUT_READS.recording() as reads, - collect_compile_diagnostics() as collected, - ): - reused: CompileModelInput | None = session.reused_model(model_file=unchanged) - - assert reused == CompileModelInput(model_file=unchanged, query_sql=unchanged.query_sql) - assert reused is not None - assert reused.model_file is unchanged - assert collected.diagnostics == test_case.expected_diagnostics - assert reads.environment_names == test_case.expected_environment_names - - -@pytest.mark.parametrize( - "test_case", - [ - ReusableModelsTestCase( - description="changed_model", - stored_models=("orders", "customers"), - current_models=("orders", "customers"), - changed_paths=frozenset({"models/customers.sql"}), - run_id_readers=frozenset(), - expected_reusable={"orders": True, "customers": False}, - ), - ReusableModelsTestCase( - description="render_read_run_identity", - stored_models=("orders", "customers"), - current_models=("orders", "customers"), - changed_paths=frozenset(), - run_id_readers=frozenset({"orders"}), - expected_reusable={"orders": False, "customers": True}, - ), - ReusableModelsTestCase( - description="model_added", - stored_models=("orders",), - current_models=("orders", "customers"), - changed_paths=frozenset({"models/customers.sql"}), - run_id_readers=frozenset(), - expected_reusable={"orders": False, "customers": False}, - ), - ReusableModelsTestCase( - description="model_removed", - stored_models=("orders", "customers", "returns"), - current_models=("orders", "customers"), - changed_paths=frozenset(), - run_id_readers=frozenset(), - expected_reusable={"orders": False, "customers": False}, - ), - ReusableModelsTestCase( - description="non_model_input_changed", - stored_models=("orders", "customers"), - current_models=("orders", "customers"), - changed_paths=frozenset({"macros/currency.py"}), - run_id_readers=frozenset(), - expected_reusable={"orders": False, "customers": False}, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_renders_when_planning_then_only_unchanged_identical_model_sets_reuse( - test_case: ReusableModelsTestCase, -) -> None: - current: tuple[DiscoveredSqlModelFile, ...] = tuple(map(model_file, test_case.current_models)) - session: CompileRenderReuseSession = planned_session( - stored=tuple(map(model_file, test_case.stored_models)), - current=current, - changed_paths=test_case.changed_paths, - run_id_readers=test_case.run_id_readers, - ) - - reusable: dict[str, bool] = { - item.file_path.stem: session.has_reusable_model(model_file=item) for item in current - } - - assert reusable == test_case.expected_reusable - - -@pytest.mark.parametrize( - "test_case", - [ - CorruptRenderTestCase( - description="undecodable_payload", model_name="orders", expected_rendered_again=True - ), - ], - ids=lambda case: case.description, -) -def test_given_corrupt_stored_render_when_reusing_then_the_model_is_rendered_again( - test_case: CorruptRenderTestCase, -) -> None: - item: DiscoveredSqlModelFile = model_file(test_case.model_name) - session: CompileRenderReuseSession = corrupted_session(item) - - with collect_compile_diagnostics(): - reused: CompileModelInput | None = session.reused_model(model_file=item) - - assert (reused is None) is test_case.expected_rendered_again - - -@pytest.mark.parametrize( - "test_case", - [ - ReleasedRenderTestCase( - description="reused_from_base", - retained_models=frozenset(), - expected_released={"models/orders.sql": True, "models/customers.sql": False}, - expected_query_sqls={ - "models/orders.sql": "SELECT 1 AS order_id", - "models/customers.sql": "SELECT 2 AS customer_id", - }, - ), - ReleasedRenderTestCase( - description="reused_from_overlay", - retained_models=frozenset({"models/orders.sql"}), - expected_released={"models/orders.sql": False, "models/customers.sql": False}, - expected_query_sqls={ - "models/orders.sql": "SELECT 1 AS order_id", - "models/customers.sql": "SELECT 2 AS customer_id", - }, - ), - ], - ids=lambda case: case.description, -) -def test_given_reused_render_when_storing_then_base_bytes_are_released_and_recoverable( - test_case: ReleasedRenderTestCase, -) -> None: - session: CompileRenderReuseSession = edited_session(retained_models=test_case.retained_models) - - released: dict[str, bool] = released_paths(session.stored_state()) - complete: dict[str, str] = stored_query_sqls(session.stored_state(complete=True)) - - assert released == test_case.expected_released - assert complete == test_case.expected_query_sqls - - -@pytest.mark.parametrize( - "test_case", - [ - DeclarationReuseTestCase( - description="model_edited", - changed_paths=frozenset({"models/customers.sql"}), - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=0, - ), - DeclarationReuseTestCase( - description="model_added", - changed_paths=frozenset({"models/returns.sql"}), - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - DeclarationReuseTestCase( - description="macro_edited", - changed_paths=frozenset({"macros/currency.py"}), - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - DeclarationReuseTestCase( - description="changes_unknown", - changed_paths=None, - recorded_variant=DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - DeclarationReuseTestCase( - description="declarations_stored_for_other_discovery", - changed_paths=frozenset({"models/customers.sql"}), - recorded_variant=OTHER_DECLARATIONS_VARIANT, - expected_full_discoveries=1, - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_declarations_when_discovering_then_only_model_only_edits_reuse_them( - test_case: DeclarationReuseTestCase, -) -> None: - current: tuple[DiscoveredSqlModelFile, ...] = ( - model_file("orders"), - model_file("customers", "SELECT 2 AS customer_id"), - ) - discovery: CountingDiscovery = CountingDiscovery(current) - session: CompileRenderReuseSession = CompileRenderReuseSession( - prior=declarations_state(recorded_variant=test_case.recorded_variant), - changed_paths=test_case.changed_paths, - ) - - with collect_compile_diagnostics(): - discovered: DiscoveredDeclarationFiles = session.declaration_files( - variant=DECLARATIONS_VARIANT, - discover=discovery.discover, - discover_models=discovery.discover_models, - ) - - assert discovered == declaration_files(current) - assert discovery.full == test_case.expected_full_discoveries - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py deleted file mode 100644 index 79d647a281..0000000000 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/test_stored_model_analyses.py +++ /dev/null @@ -1,166 +0,0 @@ -"""A stored analysis is served only while its inputs and its parents' final signatures match.""" - -from __future__ import annotations - -import pytest - -from sqlbuild.compiler.compile.classes.stored_model_analyses import StoredModelAnalyses -from sqlbuild.compiler.compile.models import ( - ModelSqlAnalysisRequest, - PolyglotAnalysisResult, - StoredModelAnalysis, -) -from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( - StaleServedAnalysisTestCase, - StoredAnalysisScopeTestCase, -) -from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( - RAW_ORDERS_TYPES, - RecordingAnalysisReuse, - analysis_request, - analyzed_model, - orders_analysis_context, - orders_stored_analyses, - stored_analysis, -) - -_CHAIN: tuple[ModelSqlAnalysisRequest, ...] = ( - analysis_request(analyzed_model("stg_orders"), "stg_orders-key"), - analysis_request(analyzed_model("int_orders", "stg_orders"), "int_orders-key"), - analysis_request(analyzed_model("fct_orders", "int_orders"), "fct_orders-key"), -) -_CHAIN_DEPENDENCIES: dict[str, dict[str, str]] = { - "stg_orders": {}, - "int_orders": {"stg_orders": "stg_orders-signature"}, - "fct_orders": {"int_orders": "int_orders-signature"}, -} -_STORED_SIGNATURES: dict[str, str] = { - "stg_orders": "stg_orders-signature", - "int_orders": "int_orders-signature", - "fct_orders": "fct_orders-signature", -} -_ALL_MODELS: frozenset[str] = frozenset(_CHAIN_DEPENDENCIES) - - -@pytest.mark.parametrize( - "test_case", - [ - StoredAnalysisScopeTestCase( - description="nothing_changed", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset(), - twinned_models=frozenset(), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=_ALL_MODELS, - ), - StoredAnalysisScopeTestCase( - description="root_edited", - reused_models=frozenset({"int_orders", "fct_orders"}), - rekeyed_models=frozenset(), - twinned_models=frozenset(), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=frozenset({"int_orders", "fct_orders"}), - ), - StoredAnalysisScopeTestCase( - description="cache_key_changed", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset({"int_orders"}), - twinned_models=frozenset(), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=frozenset({"stg_orders", "fct_orders"}), - ), - StoredAnalysisScopeTestCase( - description="cache_key_shared_with_another_model", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset(), - twinned_models=frozenset({"fct_orders"}), - stored_column_types=RAW_ORDERS_TYPES, - expected_served=frozenset({"stg_orders", "int_orders"}), - ), - StoredAnalysisScopeTestCase( - description="source_types_changed", - reused_models=_ALL_MODELS, - rekeyed_models=frozenset(), - twinned_models=frozenset(), - stored_column_types={"raw_orders": {"order_id": "BIGINT"}}, - expected_served=frozenset(), - ), - ], - ids=lambda case: case.description, -) -def test_given_stored_analyses_when_scoping_reuse_then_only_identical_inputs_are_served( - test_case: StoredAnalysisScopeTestCase, -) -> None: - context: str = orders_analysis_context( - requests=_CHAIN, column_types=test_case.stored_column_types - ) - stored: dict[str, StoredModelAnalysis] = { - name: stored_analysis(name=name, context=context, dependencies=dependencies) - for name, dependencies in _CHAIN_DEPENDENCIES.items() - } - requests: tuple[ModelSqlAnalysisRequest, ...] = tuple( - analysis_request( - request.model_input, - f"{request.model_input.model_file.file_path.stem}-key" - + "-edited" - * (request.model_input.model_file.file_path.stem in test_case.rekeyed_models), - ) - for request in _CHAIN - ) + tuple( - analysis_request(analyzed_model(f"{name}_twin", "int_orders"), f"{name}-key") - for name in sorted(test_case.twinned_models) - ) - - served: StoredModelAnalyses = orders_stored_analyses( - requests=requests, - reuse=RecordingAnalysisReuse(reused=test_case.reused_models, stored=stored), - ) - - assert frozenset(served.served) == test_case.expected_served - - -@pytest.mark.parametrize( - "test_case", - [ - StaleServedAnalysisTestCase( - description="parents_unchanged", - current_signatures=_STORED_SIGNATURES, - expected_stale=set(), - ), - StaleServedAnalysisTestCase( - description="root_signature_changed", - current_signatures={**_STORED_SIGNATURES, "stg_orders": "stg_orders-widened"}, - expected_stale={"int_orders"}, - ), - StaleServedAnalysisTestCase( - description="middle_signature_changed", - current_signatures={**_STORED_SIGNATURES, "int_orders": "int_orders-widened"}, - expected_stale={"fct_orders"}, - ), - ], - ids=lambda case: case.description, -) -def test_given_served_analyses_when_parents_are_final_then_changed_parents_make_children_stale( - test_case: StaleServedAnalysisTestCase, -) -> None: - context: str = orders_analysis_context(requests=_CHAIN, column_types=RAW_ORDERS_TYPES) - stored: dict[str, StoredModelAnalysis] = { - name: stored_analysis(name=name, context=context, dependencies=dependencies) - for name, dependencies in _CHAIN_DEPENDENCIES.items() - } - served: StoredModelAnalyses = orders_stored_analyses( - requests=_CHAIN, reuse=RecordingAnalysisReuse(reused=_ALL_MODELS, stored=stored) - ) - cached: dict[str, PolyglotAnalysisResult] = { - analysis.cache_key: analysis.analysis for analysis in stored.values() - } - - stale: set[str] = served.stale_names( - requests=_CHAIN, cached=cached, current_signatures=test_case.current_signatures - ) - - assert stale == test_case.expected_stale - - -if __name__ == "__main__": - raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py b/tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py deleted file mode 100644 index 89bbeb8c34..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache tests.""" diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py deleted file mode 100644 index 611c1f64d4..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Fact cache class tests.""" diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py deleted file mode 100644 index 78a760c91b..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/_test_types.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path - - -@dataclass(frozen=True) -class FactCacheReadTestCase: - """One read of a fact published for the ("orders", "SELECT 1") inputs.""" - - description: str - read_key_parts: tuple[str, ...] - read_algorithm: str - expected_hit: bool - expected_metrics: tuple[int, int] - - -@dataclass(frozen=True) -class FactCacheCodeIdentityTestCase: - """One change to the installed code that produced a published fact.""" - - description: str - next_code_identity: str - expected_found_count: int - - -@dataclass(frozen=True) -class FactCacheCorruptionTestCase: - """One way a persisted fact row can become untrustworthy.""" - - description: str - corrupt: Callable[[Path], None] - expected_found_count: int - expected_republished_hit: bool - - -@dataclass(frozen=True) -class FactCacheRetentionTestCase: - """A sequence of (key parts, slot) publications and the rows that must remain.""" - - description: str - publications: tuple[tuple[tuple[str, ...], str], ...] - stale_key_parts: tuple[str, ...] - expected_slots: list[str] - expected_stale_found_count: int - - -@dataclass(frozen=True) -class FactCacheNoPersistenceTestCase: - """One invocation shape that must leave no fact database behind.""" - - description: str - expected_enabled: bool - expected_database_files: list[Path] - - -@dataclass(frozen=True) -class FactCacheWriterErrorTestCase: - """One SQLite error raised while publishing after an earlier fact was stored.""" - - description: str - error: Exception - expected_earlier_found_count: int - - -@dataclass(frozen=True) -class FactCacheParameterLimitTestCase: - """One SQLite bound-parameter limit and a publication larger than one statement allows.""" - - description: str - max_bound_parameters: int - fact_count: int - expected_stored_count: int diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py deleted file mode 100644 index 853b5dc9b3..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/helpers.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Helpers for fact cache store unit tests.""" - -from __future__ import annotations - -import pickle -import sqlite3 -from contextlib import closing -from pathlib import Path - -from sqlbuild.compiler.fact_cache._helpers.publication import ( - await_fact_publication, - entry_digest, -) -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore -from sqlbuild.spec.contracts.models import SourceLocation - -FACT_NAMESPACE: str = "unit" -FACT_ALGORITHM: str = "unit-facts-v1" -FACT_VALUE: tuple[SourceLocation, ...] = ( - SourceLocation(path=Path("models/orders.sql"), line=3, column=5, end_line=3, end_column=12), -) - - -def fact_database(root: Path) -> Path: - return next(root.rglob(f"{FACT_NAMESPACE}.sqlite3")) - - -def publish_fact( - root: Path, *, key_parts: tuple[str, ...], slot: str, value: object = FACT_VALUE -) -> str: - with FactCacheStore(root=root, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store: - key: str = store.key(*key_parts) - store.stage(key=key, slot=slot, value=value) - await_fact_publication() - return key - - -def read_fact( - root: Path | None, - *, - key_parts: tuple[str, ...], - slot: str = "orders", - algorithm: str = FACT_ALGORITHM, -) -> dict[str, object]: - with FactCacheStore(root=root, namespace=FACT_NAMESPACE, algorithm=algorithm) as store: - return store.read_many(((slot, store.key(*key_parts)),)) - - -def stored_fact_slots(root: Path) -> list[str]: - with closing(sqlite3.connect(fact_database(root))) as connection: - rows: list[tuple[str]] = connection.execute( - "SELECT slot FROM fact ORDER BY slot" - ).fetchall() - return [slot for (slot,) in rows] - - -def _overwrite_verified_payload(database: Path, payload: bytes) -> None: - with closing(sqlite3.connect(database)) as connection, connection: - cache_key: str = connection.execute("SELECT cache_key FROM fact").fetchone()[0] - digest: str = entry_digest(cache_key=cache_key, payload=payload) - _ = connection.execute("UPDATE fact SET payload = ?, digest = ?", (payload, digest)) - - -def flip_payload_byte(root: Path) -> None: - database: Path = fact_database(root) - with closing(sqlite3.connect(database)) as connection, connection: - payload: bytes = connection.execute("SELECT payload FROM fact").fetchone()[0] - _ = connection.execute("UPDATE fact SET payload = ?", (payload[:-1] + b"\x00",)) - - -def write_untrusted_global(root: Path) -> None: - _overwrite_verified_payload(fact_database(root), pickle.dumps(Path.home, protocol=5)) - - -def truncate_verified_payload(root: Path) -> None: - _overwrite_verified_payload(fact_database(root), b"\x80\x05") - - -def replace_database_with_garbage(root: Path) -> None: - fact_database(root).write_bytes(b"not a sqlite database") - - -def publish_facts(root: Path, *, slots: tuple[str, ...]) -> None: - with FactCacheStore(root=root, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store: - for slot in slots: - store.stage(key=store.key(slot), slot=slot, value=FACT_VALUE) - await_fact_publication() diff --git a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py b/tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py deleted file mode 100644 index 69d738f151..0000000000 --- a/tests/unit/src/sqlbuild/compiler/fact_cache/classes/test_fact_cache_store.py +++ /dev/null @@ -1,314 +0,0 @@ -from __future__ import annotations - -import sqlite3 -import threading -from collections.abc import Callable -from pathlib import Path - -import pytest - -from sqlbuild.compiler.fact_cache._helpers import publication -from sqlbuild.compiler.fact_cache.classes import fact_cache_store -from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore -from sqlbuild.compiler.profiling.main.collect import collect_compile_timings -from tests.unit.src.sqlbuild.compiler.fact_cache.classes._test_types import ( - FactCacheCodeIdentityTestCase, - FactCacheCorruptionTestCase, - FactCacheNoPersistenceTestCase, - FactCacheParameterLimitTestCase, - FactCacheReadTestCase, - FactCacheRetentionTestCase, - FactCacheWriterErrorTestCase, -) -from tests.unit.src.sqlbuild.compiler.fact_cache.classes.helpers import ( - FACT_ALGORITHM, - FACT_NAMESPACE, - FACT_VALUE, - flip_payload_byte, - publish_fact, - publish_facts, - read_fact, - replace_database_with_garbage, - stored_fact_slots, - truncate_verified_payload, - write_untrusted_global, -) - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheReadTestCase( - description="same_inputs_hit", - read_key_parts=("orders", "SELECT 1"), - read_algorithm=FACT_ALGORITHM, - expected_hit=True, - expected_metrics=(1, 0), - ), - FactCacheReadTestCase( - description="changed_input_misses", - read_key_parts=("orders", "SELECT 2"), - read_algorithm=FACT_ALGORITHM, - expected_hit=False, - expected_metrics=(0, 1), - ), - FactCacheReadTestCase( - description="extra_empty_input_misses", - read_key_parts=("orders", "SELECT 1", ""), - read_algorithm=FACT_ALGORITHM, - expected_hit=False, - expected_metrics=(0, 1), - ), - FactCacheReadTestCase( - description="concatenated_inputs_miss", - read_key_parts=("ordersSELECT 1",), - read_algorithm=FACT_ALGORITHM, - expected_hit=False, - expected_metrics=(0, 1), - ), - FactCacheReadTestCase( - description="changed_algorithm_misses", - read_key_parts=("orders", "SELECT 1"), - read_algorithm="unit-facts-v2", - expected_hit=False, - expected_metrics=(0, 1), - ), - ), - ids=lambda case: case.description, -) -def test_given_published_fact_when_reading_then_hits_only_for_identical_inputs( - tmp_path: Path, test_case: FactCacheReadTestCase -) -> None: - key: str = publish_fact(tmp_path, key_parts=("orders", "SELECT 1"), slot="orders") - - with collect_compile_timings() as timings: - found: dict[str, object] = read_fact( - tmp_path, key_parts=test_case.read_key_parts, algorithm=test_case.read_algorithm - ) - - assert (found == {key: FACT_VALUE}) is test_case.expected_hit - assert ( - timings.metrics["fact_cache_hits"], - timings.metrics["fact_cache_misses"], - ) == test_case.expected_metrics - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheCodeIdentityTestCase( - description="next_release", next_code_identity="next-release", expected_found_count=0 - ), - ), - ids=lambda case: case.description, -) -def test_given_published_fact_when_producing_code_changes_then_misses( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: FactCacheCodeIdentityTestCase -) -> None: - _ = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - monkeypatch.setattr( - fact_cache_store, "installed_code_identity", lambda: test_case.next_code_identity - ) - - assert len(read_fact(tmp_path, key_parts=("orders",))) == test_case.expected_found_count - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheCorruptionTestCase( - description="payload_bytes_changed", - corrupt=flip_payload_byte, - expected_found_count=0, - expected_republished_hit=True, - ), - FactCacheCorruptionTestCase( - description="verified_payload_references_function", - corrupt=write_untrusted_global, - expected_found_count=0, - expected_republished_hit=True, - ), - FactCacheCorruptionTestCase( - description="verified_payload_truncated", - corrupt=truncate_verified_payload, - expected_found_count=0, - expected_republished_hit=True, - ), - FactCacheCorruptionTestCase( - description="database_file_replaced", - corrupt=replace_database_with_garbage, - expected_found_count=0, - expected_republished_hit=True, - ), - ), - ids=lambda case: case.description, -) -def test_given_corrupted_fact_when_reading_then_misses_and_next_publication_repairs_it( - tmp_path: Path, test_case: FactCacheCorruptionTestCase -) -> None: - _ = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - test_case.corrupt(tmp_path) - - assert len(read_fact(tmp_path, key_parts=("orders",))) == test_case.expected_found_count - - key: str = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - assert ( - read_fact(tmp_path, key_parts=("orders",)) == {key: FACT_VALUE} - ) is test_case.expected_republished_hit - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheRetentionTestCase( - description="same_slot_replaced", - publications=( - (("orders", "v1"), "orders"), - (("orders", "v2"), "orders"), - (("customers", "v1"), "customers"), - ), - stale_key_parts=("orders", "v1"), - expected_slots=["customers", "orders"], - expected_stale_found_count=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_new_fact_for_same_slot_when_publishing_then_previous_fact_is_pruned( - tmp_path: Path, test_case: FactCacheRetentionTestCase -) -> None: - for key_parts, slot in test_case.publications: - _ = publish_fact(tmp_path, key_parts=key_parts, slot=slot) - - assert stored_fact_slots(tmp_path) == test_case.expected_slots - assert ( - len(read_fact(tmp_path, key_parts=test_case.stale_key_parts)) - == test_case.expected_stale_found_count - ) - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheNoPersistenceTestCase( - description="disabled_root", expected_enabled=False, expected_database_files=[] - ), - ), - ids=lambda case: case.description, -) -def test_given_disabled_root_when_staging_then_nothing_is_persisted( - tmp_path: Path, test_case: FactCacheNoPersistenceTestCase -) -> None: - with FactCacheStore(root=None, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store: - store.stage(key=store.key("orders"), slot="orders", value=FACT_VALUE) - assert store.enabled is test_case.expected_enabled - assert store.read_many((("orders", store.key("orders")),)) == {} - - assert list(tmp_path.iterdir()) == test_case.expected_database_files - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheNoPersistenceTestCase( - description="failed_invocation", expected_enabled=True, expected_database_files=[] - ), - ), - ids=lambda case: case.description, -) -def test_given_failed_invocation_when_exiting_then_staged_facts_are_discarded( - tmp_path: Path, test_case: FactCacheNoPersistenceTestCase -) -> None: - with ( - pytest.raises(RuntimeError), - FactCacheStore(root=tmp_path, namespace=FACT_NAMESPACE, algorithm=FACT_ALGORITHM) as store, - ): - assert store.enabled is test_case.expected_enabled - store.stage(key=store.key("orders"), slot="orders", value=FACT_VALUE) - raise RuntimeError("compile failed") - - assert list(tmp_path.rglob("*.sqlite3")) == test_case.expected_database_files - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheWriterErrorTestCase( - description="busy_database_keeps_rows", - error=sqlite3.OperationalError("database is locked"), - expected_earlier_found_count=1, - ), - FactCacheWriterErrorTestCase( - description="integrity_error_keeps_rows", - error=sqlite3.IntegrityError("constraint failed"), - expected_earlier_found_count=1, - ), - FactCacheWriterErrorTestCase( - description="interface_error_keeps_rows", - error=sqlite3.InterfaceError("bad parameter"), - expected_earlier_found_count=1, - ), - FactCacheWriterErrorTestCase( - description="unreadable_database_is_replaced", - error=sqlite3.DatabaseError("file is not a database"), - expected_earlier_found_count=0, - ), - ), - ids=lambda case: case.description, -) -def test_given_writer_error_when_publishing_then_only_corruption_discards_rows_and_nothing_escapes( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: FactCacheWriterErrorTestCase -) -> None: - _ = publish_fact(tmp_path, key_parts=("orders",), slot="orders") - escaped: list[threading.ExceptHookArgs] = [] - monkeypatch.setattr(threading, "excepthook", escaped.append) - - def failing_insert(**_: object) -> None: - raise test_case.error - - monkeypatch.setattr(publication, "_insert_records", failing_insert) - _ = publish_fact(tmp_path, key_parts=("customers",), slot="customers") - monkeypatch.undo() - - assert escaped == [] - assert len(read_fact(tmp_path, key_parts=("orders",))) == ( - test_case.expected_earlier_found_count - ) - - -@pytest.mark.parametrize( - "test_case", - ( - FactCacheParameterLimitTestCase( - description="sqlite_before_3_32", - max_bound_parameters=999, - fact_count=600, - expected_stored_count=600, - ), - ), - ids=lambda case: case.description, -) -def test_given_sqlite_parameter_limit_when_publishing_many_facts_then_all_are_stored( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, test_case: FactCacheParameterLimitTestCase -) -> None: - connect: Callable[..., sqlite3.Connection] = sqlite3.connect - - def limited_connect(database: Path, timeout: float = 5.0) -> sqlite3.Connection: - connection: sqlite3.Connection = connect(database, timeout=timeout) - _ = connection.setlimit( - sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, test_case.max_bound_parameters - ) - return connection - - monkeypatch.setattr(sqlite3, "connect", limited_connect) - slots: tuple[str, ...] = tuple(f"order_{index:04d}" for index in range(test_case.fact_count)) - - publish_facts(tmp_path, slots=slots) - - stored: list[str] = stored_fact_slots(tmp_path) - assert len(stored) == test_case.expected_stored_count - assert stored == list(slots) - - -if __name__ == "__main__": - pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py index 0507124821..4c0da94975 100644 --- a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -6,8 +6,8 @@ from enum import StrEnum from pathlib import Path, PurePosixPath +from sqlbuild.compiler.compile._helpers.attachment.target import compile_cache_root from sqlbuild.compiler.compile.models import CompactLineageFacts -from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME from sqlbuild.compiler.sql_analysis.classes.binding_catalog import BindingCatalog diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index b07bbd8c1c..de8e40f8c8 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -113,36 +113,19 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler-native-v1/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data, the project has very many files, or it records -thousands of new model renders, compile prints `Recording compile for reuse...` and a completion -line to stderr before the report. - -### Compiling after model-only edits - -When the only changes since the stored compile are edits to existing model files, compile runs in -full but reuses the stored render of every unchanged model, together with the diagnostics and -environment reads that rendering reported. Only the edited models are rendered again. Declaration -files other than models are not read again, and an unchanged model reuses its stored column -analysis only when the analyses of its upstream models in this compile match the ones it was stored -with. Contracts, semantic validation, and rules still run for the whole project, and a model whose -upstream analyses changed is analyzed again, so downstream effects of an edit, such as a changed -column type, are reported exactly as in an uncached compile, even when another command such as -`sqb build` ran in between. Compiled -files that still hold the bytes the stored compile wrote are not read or rewritten. The output is -byte-identical to `sqb compile --no-cache`. - -Every other change renders the whole project: adding, removing, or renaming a model, and editing -macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope -folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings -is always rendered again. - -Renders are stored only when a compile result for the same project and target is already stored, -so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. -The next full compile stores them, and edits after that reuse them. Stored renders are compressed -and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints -`Loading stored renders...` and a completion line to stderr before it starts. While renders are -reused, the Python modules recorded by the previous compile stay tracked even if this compile did -not import them, so editing such a module runs the next compile in full. +report is ready. When it has to hash a lot of data or the project has very many files, compile +prints `Recording compile for reuse...` and a completion line to stderr before the report. + +### Compiling after an edit + +Any change runs the whole compile again: every model, test, and audit is rendered again, and +contracts and semantic validation run for the whole project. No render is carried over from the +previous compile. A few caches under `target/cache/` still apply: an unchanged model takes its +column analysis from the analysis cache when its query and the analyses of its upstream models are +unchanged, rule results are reused for unchanged inputs, and macro call results may be reused as +described in [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). +Until the native compiler ships, editing one model therefore costs close to a full compile. The +output is byte-identical to `sqb compile --no-cache`. Known limits: @@ -163,8 +146,7 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or -rendered again after model-only edits and `analysis_reuse_hits` for stored model analyses reused. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 8e3fda3c32..a79d1fcfff 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -212,7 +212,7 @@ files or environment variables that the call does not receive. SQLBuild relies on this contract to reuse macro results: -- [Compile reuse](/docs/cli/compile) replays stored renders without running macros again. +- [Compile reuse](/docs/cli/compile) replays an unchanged compile without running macros again. - A compile may run each distinct macro call once and reuse its result for identical calls elsewhere in the project. - Macro call results may be kept between compiles under `target/cache/`. They are recomputed when From 68791abd39d0e7fc4b543a7f164ab4b4b626b5a4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:48:31 +0100 Subject: [PATCH 2/5] test(compile): expect one fewer compiler cache database without the fact cache --- .../commands/main/build/test_runtime_artifact_preservation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 789e841de6..5c800ff775 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -33,7 +33,7 @@ "target/run/functions/python/is_completed_order_py.sql", ), expected_exit_code=0, - expected_compiler_cache_count=4, + expected_compiler_cache_count=3, expected_local_history=False, expected_compiled_paths=( "target/compiled/models/marts/fact_orders.sql", From 448ae51ccb72cfe9a8bfec1f0ccb9fbb43c95eb2 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:12:21 +0100 Subject: [PATCH 3/5] perf(compile): keep per-file SQL-test scans in the shared native store Store whole per-file expanded SQL-test CTE extraction and expected-model scans as a new kind of the shared native store, keyed by the scanned text, adapter lexical rules, native build and scan version. Failed scans are not stored and damaged or foreign entries are rescanned. Digest installed code before the custom-rules worker starts so its file walk no longer waits on the GIL beside built-in rules. Give the same-runner one-model edit mode its own ratio limit until native rendering is the default. --- .github/workflows/verify.yml | 6 + .../src/bindings/_helpers/cache/mod.rs | 2 + .../bindings/_helpers/cache/native_store.rs | 34 +++- .../bindings/_helpers/cache/registration.rs | 11 ++ .../_helpers/cache/sql_test_scan_store.rs | 67 ++++++++ .../_helpers/render/macro_call_memo.rs | 18 +- .../src/bindings/main/register.rs | 2 +- .../_helpers/report.py | 5 +- .../_helpers/verdict.py | 7 +- .../compile_performance_ratio/main/compare.py | 12 ++ .../sqlbuild/references/docs/cli/compile.md | 23 ++- src/sqlbuild/_native.pyi | 6 + .../_helpers/attachment/declaration_scope.py | 12 ++ .../attachment/scope_relationships.py | 155 +++++++++++++++--- .../compile/_helpers/attachment/sql_tests.py | 11 +- .../compile/_helpers/sql_tests/native.py | 94 +++++++++-- .../compile/classes/sql_test_scan_cache.py | 84 ++++++++++ src/sqlbuild/compiler/compile/constants.py | 4 + .../compile/main/_build_compile_inputs.py | 2 + src/sqlbuild/compiler/compile/models.py | 2 + src/sqlbuild/compiler/frontier/constants.py | 1 + src/sqlbuild/compiler/profiling/constants.py | 2 + src/sqlbuild/compiler/profiling/types.py | 2 + .../rule_engine/_helpers/engine/native.py | 5 +- .../main/build/test_compile_json_behavior.py | 2 + .../cli/commands/main/compile/_test_types.py | 11 ++ .../cli/commands/main/compile/helpers.py | 126 +++++++++++++- .../compile/test_compile_incremental_edits.py | 112 +++++++++++++ .../cli/commands/main/rules/_test_types.py | 13 ++ .../cli/commands/main/rules/helpers.py | 44 ++++- .../test_custom_rules_incremental_sequence.py | 52 ++++++ .../sqlbuild/cli/commands/main/test_rules.py | 4 + .../_helpers/_test_types.py | 3 +- .../_helpers/test_verdict.py | 20 +++ website/src/content/docs/docs/cli/compile.mdx | 23 ++- 35 files changed, 900 insertions(+), 77 deletions(-) create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs create mode 100644 crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs create mode 100644 src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 27e8cb57cc..973adcb5f6 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -561,11 +561,15 @@ jobs: strategy: fail-fast: false matrix: + # Temporary: a one-model edit recompiles the edited model uncached until native + # rendering becomes the default; tighten edit_max_ratio back to max_ratio then. include: - kind: dense max_ratio: "1.10" + edit_max_ratio: "1.45" - kind: fresh max_ratio: "1.10" + edit_max_ratio: "1.25" steps: - uses: actions/checkout@v4 - name: Check out the comparison base @@ -596,11 +600,13 @@ jobs: env: KIND: ${{ matrix.kind }} MAX_RATIO: ${{ matrix.max_ratio }} + EDIT_MAX_RATIO: ${{ matrix.edit_max_ratio }} run: | uv run python -m scripts.compare_compile_performance \ --kind "${KIND}" \ --modes cold warm edit \ --max-ratio "${MAX_RATIO}" \ + --edit-max-ratio "${EDIT_MAX_RATIO}" \ --models 3000 \ --base-root base \ --base-python base/.venv/bin/python \ diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs index 31c5e16613..1928fa5ace 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/mod.rs @@ -1 +1,3 @@ pub(crate) mod native_store; +pub(crate) mod registration; +pub(crate) mod sql_test_scan_store; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs index 0cd6e1c9a0..fd2e81a45c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/native_store.rs @@ -1,4 +1,4 @@ -//! Digests and project fingerprints of the shared native store, with the GIL released. +//! Digests, project fingerprints and store files of the shared native store, with the GIL released. use pyo3::exceptions::{PyOSError, PyRuntimeError}; use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; @@ -8,8 +8,10 @@ use sqlbuild_cache::digest::main::digest_files::digest_files as digest_each; use sqlbuild_cache::digest::main::fingerprint_project_files::fingerprint_project_files as fingerprint; use sqlbuild_cache::digest::main::hex_digest::hex_digest; use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::main::open_native_store::open_native_store; +use sqlbuild_cache::store::models::NativeStore; use std::collections::HashSet; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use crate::bindings::types::CompilerDetach; @@ -50,6 +52,34 @@ fn readable_digest(digest: &std::io::Result) -> Option { } } +/// Open one cache kind's store with the GIL released; a failed read is an `OSError`. +pub(crate) fn open_store( + py: Python<'_>, + path: &Path, + kind: &str, + environment: &str, +) -> PyResult { + py.compiler_detach(|| Ok(open_native_store(path, kind, environment))) + .map_err(PyRuntimeError::new_err)? + .map_err(|error| PyOSError::new_err(error.to_string())) +} + +/// Atomically save a store holding unsaved values with the GIL released; returns entries written. +pub(crate) fn save_store( + py: Python<'_>, + store: &NativeStore, + path: &Path, + metadata: &[u8], +) -> PyResult> { + if !store.needs_save() { + return Ok(None); + } + py.compiler_detach(|| Ok(store.save(path, metadata))) + .map_err(PyRuntimeError::new_err)? + .map(Some) + .map_err(|error| PyOSError::new_err(error.to_string())) +} + pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(content_digest, module)?)?; module.add_function(wrap_pyfunction!(fingerprint_project_files, module)?)?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs new file mode 100644 index 0000000000..c9b123dbc1 --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/registration.rs @@ -0,0 +1,11 @@ +//! Register the shared native store functions and classes. + +use pyo3::prelude::{Bound, PyModule, PyResult}; + +use crate::bindings::_helpers::cache::{native_store, sql_test_scan_store}; + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + native_store::register(module)?; + sql_test_scan_store::register(module)?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs new file mode 100644 index 0000000000..d53b762eea --- /dev/null +++ b/crates/sqlbuild-python/src/bindings/_helpers/cache/sql_test_scan_store.rs @@ -0,0 +1,67 @@ +//! Whole per-file SQL-test scan results kept as one kind of the shared native store. + +use pyo3::exceptions::PyValueError; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, Python}; +use pyo3::{pyclass, pymethods}; +use sqlbuild_cache::digest::main::content_digest::content_digest; +use sqlbuild_cache::digest::types::ContentDigest; +use sqlbuild_cache::store::models::NativeStore; +use std::path::PathBuf; +use std::sync::Mutex; + +use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; + +/// The native store kind holding SQL-test scan results. +const SQL_TEST_SCAN_STORE_KIND: &str = "sql-test-scans"; + +/// One compile's view of the stored scan results, keyed by the digest of the caller's key parts. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub(crate) struct SqlTestScanStore { + store: Mutex, +} + +fn poisoned(_: T) -> pyo3::PyErr { + PyValueError::new_err("SQL test scan store is poisoned") +} + +#[pymethods] +impl SqlTestScanStore { + /// Open the store at `path` for `environment`; missing, damaged or foreign files open empty. + #[new] + fn new(py: Python<'_>, path: PathBuf, environment: &str) -> PyResult { + let store: NativeStore = open_store(py, &path, SQL_TEST_SCAN_STORE_KIND, environment)?; + Ok(Self { + store: Mutex::new(store), + }) + } + + /// The value stored under the digest of `key_parts`, or `None`. + fn get(&self, key_parts: Vec) -> PyResult>> { + let key: ContentDigest = content_digest(&key_parts); + Ok(self + .store + .lock() + .map_err(poisoned)? + .get(&key) + .map(<[u8]>::to_vec)) + } + + /// Store `value` under the digest of `key_parts`. + fn put(&self, key_parts: Vec, value: Vec) -> PyResult<()> { + let key: ContentDigest = content_digest(&key_parts); + self.store.lock().map_err(poisoned)?.put(key, value); + Ok(()) + } + + /// Atomically save the store when this compile stored a value; returns entries written. + fn save(&self, py: Python<'_>, path: PathBuf) -> PyResult> { + let store = self.store.lock().map_err(poisoned)?; + save_store(py, &store, &path, &[]) + } +} + +pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_class::()?; + Ok(()) +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs index 718dcff75f..63bec19bab 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/render/macro_call_memo.rs @@ -1,15 +1,14 @@ //! The macro call memo class: one compile's recorded calls and its optional native store. -use pyo3::exceptions::{PyOSError, PyRuntimeError, PyValueError}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::Python; use pyo3::{PyResult, pyclass, pymethods}; -use sqlbuild_cache::store::main::open_native_store::open_native_store; use sqlbuild_cache::store::models::NativeStore; use sqlbuild_render::macro_calls::models::MacroCallEntry; use std::path::PathBuf; use std::sync::Mutex; -use crate::bindings::types::CompilerDetach; +use crate::bindings::_helpers::cache::native_store::{open_store, save_store}; use crate::bindings::_helpers::render::macro_call_rows::{EntryRow, event_from_row, event_row}; @@ -80,10 +79,7 @@ impl MacroCallMemo { path: PathBuf, environment: &str, ) -> PyResult<(Vec, usize)> { - let store: NativeStore = py - .compiler_detach(|| Ok(open_native_store(&path, MACRO_CALL_STORE_KIND, environment))) - .map_err(PyRuntimeError::new_err)? - .map_err(|error| PyOSError::new_err(error.to_string()))?; + let store: NativeStore = open_store(py, &path, MACRO_CALL_STORE_KIND, environment)?; let opened = (store.metadata().to_vec(), store.loaded_entries()); self.inner.lock().map_err(poisoned)?.attach_store(store); Ok(opened) @@ -119,13 +115,9 @@ impl MacroCallMemo { metadata: &[u8], ) -> PyResult> { let mut memo = self.inner.lock().map_err(poisoned)?; - let Some(store) = memo.store_mut().filter(|store| store.needs_save()) else { + let Some(store) = memo.store_mut() else { return Ok(None); }; - let store: &NativeStore = store; - py.compiler_detach(|| Ok(store.save(&path, metadata))) - .map_err(PyRuntimeError::new_err)? - .map(Some) - .map_err(|error| PyOSError::new_err(error.to_string())) + save_store(py, store, &path, metadata) } } diff --git a/crates/sqlbuild-python/src/bindings/main/register.rs b/crates/sqlbuild-python/src/bindings/main/register.rs index 37e0e09255..9e1dce2fcb 100644 --- a/crates/sqlbuild-python/src/bindings/main/register.rs +++ b/crates/sqlbuild-python/src/bindings/main/register.rs @@ -32,7 +32,7 @@ pub fn _native(module: &Bound<'_, PyModule>) -> PyResult<()> { scopes::registration::register(module)?; model_config::registration::register(module)?; render::macro_calls::register(module)?; - cache::native_store::register(module)?; + cache::registration::register(module)?; attachments::registration::register(module)?; register_analysis_stages(module)?; module.add("API_VERSION", API_VERSION)?; diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index a92535a050..84cdd7589e 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -3,6 +3,7 @@ from __future__ import annotations import platform +from collections.abc import Mapping from pathlib import Path from scripts.compile_performance_ratio.constants import MODE_TITLES, REPORTED_PHASES @@ -16,6 +17,7 @@ def comparison_markdown( max_ratio: float, per_side_projects: bool, failures: tuple[str, ...], + mode_max_ratios: Mapping[str, float] | None = None, ) -> str: """Summarize medians, ratios and per-phase timings of every mode as Markdown tables.""" @@ -31,7 +33,8 @@ def comparison_markdown( f"### {comparison.kind} {comparison.models} models, " f"{MODE_TITLES[comparison.mode]}: head vs base (same runner)", "", - f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", + f"Runner CPU: {cpu_model()}; {runs} alternating runs each; " + f"limit {(mode_max_ratios or {}).get(comparison.mode, max_ratio):.2f}x.", "", generation, "", diff --git a/scripts/compile_performance_ratio/_helpers/verdict.py b/scripts/compile_performance_ratio/_helpers/verdict.py index 6a85f2f240..cca469c30b 100644 --- a/scripts/compile_performance_ratio/_helpers/verdict.py +++ b/scripts/compile_performance_ratio/_helpers/verdict.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +from collections.abc import Mapping from scripts.compile_performance_ratio.models import CompileComparison @@ -13,6 +14,7 @@ def ratio_failures( modes: tuple[str, ...], max_ratio: float, noise_floor_seconds: float, + mode_max_ratios: Mapping[str, float] | None = None, ) -> tuple[str, ...]: """Describe every requested mode that is unmeasured or slower than ratio plus floor allow.""" @@ -26,6 +28,7 @@ def ratio_failures( failures.append(f"{mode}: no measurement") continue prefix: str = f"{comparison.kind} {comparison.models} {mode}" + limit: float = (mode_max_ratios or {}).get(mode, max_ratio) for metric, base, head, ratio in ( ( "wall", @@ -39,8 +42,8 @@ def ratio_failures( failures.append( f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)" ) - elif head > base * max_ratio + noise_floor_seconds: - failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}") + elif head > base * limit + noise_floor_seconds: + failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {limit:.2f}") return tuple(failures) diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index fec5e8d32c..330bc2704d 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -19,6 +19,7 @@ COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, + EDIT_MODE, NOISE_FLOOR_SECONDS, PROJECT_KINDS, ) @@ -59,11 +60,15 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: runs=args.runs, modes=modes, ) + mode_max_ratios: dict[str, float] = ( + {} if args.edit_max_ratio is None else {EDIT_MODE: args.edit_max_ratio} + ) failures: tuple[str, ...] = ratio_failures( comparisons=comparisons, modes=modes, max_ratio=args.max_ratio, noise_floor_seconds=args.noise_floor_seconds, + mode_max_ratios=mode_max_ratios, ) markdown: str = comparison_markdown( comparisons=comparisons, @@ -71,6 +76,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: max_ratio=args.max_ratio, per_side_projects=per_side_projects, failures=failures, + mode_max_ratios=mode_max_ratios, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) @@ -104,6 +110,12 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) + parser.add_argument( + "--edit-max-ratio", + type=float, + default=None, + help="Ratio limit of the one-model edit mode only; defaults to --max-ratio.", + ) parser.add_argument( "--noise-floor-seconds", type=float, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index cfb0291d65..aac63d480d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -129,12 +129,20 @@ prints `Recording compile for reuse...` and a completion line to stderr before t Any change runs the whole compile again: every model, test, and audit is rendered again, and contracts and semantic validation run for the whole project. No render is carried over from the -previous compile. A few caches under `target/cache/` still apply: an unchanged model takes its -column analysis from the analysis cache when its query and the analyses of its upstream models are -unchanged, rule results are reused for unchanged inputs, and macro call results may be reused as -described in [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). -Until the native compiler ships, editing one model therefore costs close to a full compile. The -output is byte-identical to `sqb compile --no-cache`. +previous compile, so until the native compiler ships, an edited model is recompiled without any +cache. A few caches under `target/cache/` shorten the rest: + +- An unchanged model takes its column analysis from the analysis cache when its query and the + analyses of its upstream models are unchanged. +- An unchanged SQL test file reuses its stored test scan, the CTEs and expected models read from + it, from `sql-test-scans.bin`. Entries are keyed by the file's expanded text, the adapter's SQL + lexical rules, and the SQLBuild build; a damaged or unreadable entry is scanned again. +- Rule results are reused for unchanged inputs. +- Macro call results may be reused as described in + [Macros must be deterministic](../concepts/macros.md#macros-must-be-deterministic). + +An edit therefore costs a full render of the project plus the analysis, test scans, and rules of +what changed. The output is byte-identical to `sqb compile --no-cache`. Known limits: @@ -154,7 +162,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings and cache counters such as `sql_test_scan_cache_hits` and +`sql_test_scan_cache_misses`. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/src/sqlbuild/_native.pyi b/src/sqlbuild/_native.pyi index 251e98b73f..af327366cc 100644 --- a/src/sqlbuild/_native.pyi +++ b/src/sqlbuild/_native.pyi @@ -293,6 +293,12 @@ class MacroCallMemo: def store_stats(self) -> tuple[int, int]: ... def save_store(self, path: str, metadata: bytes) -> int | None: ... +class SqlTestScanStore: + def __init__(self, path: str, environment: str) -> None: ... + def get(self, key_parts: list[str]) -> bytes | None: ... + def put(self, key_parts: list[str], value: bytes) -> None: ... + def save(self, path: str) -> int | None: ... + def content_digest(parts: list[str]) -> str: ... def fingerprint_project_files(root: str, excluded_files: list[str]) -> str: ... def digest_files(paths: list[str]) -> list[str | None]: ... diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py index 02862cc016..12a8478da9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/declaration_scope.py @@ -5,6 +5,7 @@ import inspect from collections.abc import Mapping from dataclasses import replace +from pathlib import Path from types import CodeType from sqlbuild.compiler.compile._helpers.attachment.scope_relationships import ( @@ -17,6 +18,7 @@ from sqlbuild.compiler.compile._helpers.sql_tests.extraction_errors import ( validate_authored_test_cte_names, ) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( DeclarationScopeBuild, @@ -41,9 +43,12 @@ def build_declaration_scope( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: """Build one canonical index and validate it before SQL expansion.""" + scan_cache: SqlTestScanCache = SqlTestScanCache(cache_dir=compile_cache_dir) + validate_authored_test_cte_names( test_files=discovered_inputs.test_files, syntax=sql_lexical_syntax ) @@ -52,6 +57,7 @@ def build_declaration_scope( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, + scan_cache=scan_cache, ) if native_scope is not None: return native_scope @@ -73,6 +79,7 @@ def build_declaration_scope( discovered_inputs=discovered_inputs, index=index, sql_lexical_syntax=sql_lexical_syntax, + scan_cache=scan_cache, ) if has_scoped_relationship_declarations and (discovered_inputs.test_files or discovered_inputs.scenario_files) @@ -93,6 +100,7 @@ def build_declaration_scope( scope_index=index, loaded_macros=loaded_macros, ), + sql_test_scans=scan_cache, ) @@ -101,6 +109,7 @@ def _build_native_declaration_scope( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache, ) -> DeclarationScopeBuild | None: """Build the scope natively with Python's errors and order, or None for the Python stage.""" @@ -121,6 +130,7 @@ def _build_native_declaration_scope( facts, fault = extract_scope_relationship_facts( discovered_inputs=discovered_inputs, sql_lexical_syntax=sql_lexical_syntax, + scan_cache=scan_cache, ) if fault is not None: raise CompileInputError(fault) @@ -139,6 +149,7 @@ def _build_native_declaration_scope( loaded_macros=loaded_macros, lookup=lookup, ), + sql_test_scans=scan_cache, ) @@ -160,6 +171,7 @@ def rebind_declaration_scope( scope_index=scope.index, loaded_macros=loaded_macros, ), + sql_test_scans=scope.sql_test_scans, ) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py index ebc6f13278..f722b7a28c 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/scope_relationships.py @@ -2,8 +2,11 @@ from __future__ import annotations -from collections.abc import Sequence +from collections.abc import Callable, Sequence from pathlib import Path +from typing import cast + +import orjson from sqlbuild.compiler.compile._helpers.render.macros import ( find_macro_call_names, @@ -16,7 +19,11 @@ extract_sql_test_expected_model_names, extract_unclassified_sql_test_ctes, ) -from sqlbuild.compiler.compile.constants import MACRO_ACTUAL_TEST_CTE_NAME +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import ( + MACRO_ACTUAL_TEST_CTE_NAME, + SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, +) from sqlbuild.compiler.compile.models import ( CompileSqlTestCte, ScopeRelationshipBuild, @@ -49,6 +56,7 @@ from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax type _SharedDeclarations = dict[tuple[str, str], tuple[DeclarationRecord, ...]] +type _BlockNames = tuple[str, ...] | Exception def build_scope_relationship_grants( @@ -56,6 +64,7 @@ def build_scope_relationship_grants( discovered_inputs: DiscoveredProjectInputs, index: ScopeIndex, sql_lexical_syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache | None = None, ) -> ScopeRelationshipBuild: """Return expected-model grants while retaining independent extraction faults.""" @@ -65,6 +74,7 @@ def build_scope_relationship_grants( discovered_inputs=discovered_inputs, lookup=lookup, shared_declarations=shared_declarations, + scan_cache=scan_cache or SqlTestScanCache(cache_dir=None), syntax=sql_lexical_syntax, ) scenario_grants, scenario_faults = _scenario_relationship_grants( @@ -83,28 +93,22 @@ def extract_scope_relationship_facts( *, discovered_inputs: DiscoveredProjectInputs, sql_lexical_syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache | None = None, ) -> tuple[tuple[RelationshipFact, ...], str | None]: """Return the relationship names grants are resolved from, and the first extraction fault.""" facts: list[RelationshipFact] = [] faults: list[ScopeRelationshipFault] = [] - scanned_names: dict[tuple[int, int], tuple[str, ...]] = _scanned_expected_names( - test_files=discovered_inputs.test_files, syntax=sql_lexical_syntax + names_by_file: list[list[_BlockNames]] = _expected_names_by_file( + test_files=discovered_inputs.test_files, + scan_cache=scan_cache or SqlTestScanCache(cache_dir=None), + syntax=sql_lexical_syntax, + scan_natively=True, ) - for file_index, test_file in enumerate(discovered_inputs.test_files): - for block_index, block in enumerate(test_file.blocks): - scanned: tuple[str, ...] | None = scanned_names.get((file_index, block_index)) + for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): + for block, block_names in zip(test_file.blocks, file_names, strict=True): try: - expected_names: tuple[str, ...] = ( - scanned - if scanned is not None - else extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=sql_lexical_syntax, - mode=block.mode, - ) - ) + expected_names: tuple[str, ...] = _names_or_raise(block_names) called_macros: tuple[str, ...] = find_nested_macro_call_names(block.sql_body) facts.append( RelationshipFact( @@ -149,15 +153,112 @@ def extract_scope_relationship_facts( return tuple(facts), faults[0].message if faults else None +def _expected_names_by_file( + *, + test_files: Sequence[DiscoveredSqlTestFile], + scan_cache: SqlTestScanCache, + syntax: SqlLexicalSyntax, + scan_natively: bool, +) -> list[list[_BlockNames]]: + """Return each block's expected-model names or extraction error, reusing whole stored files.""" + + stored: list[tuple[tuple[str, ...], ...] | None] = [ + scan_cache.read( + algorithm=SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, + syntax=syntax, + parts=_expected_names_parts(test_file), + decode=_stored_expected_names(block_count=len(test_file.blocks)), + ) + for test_file in test_files + ] + scanned: dict[tuple[int, int], tuple[str, ...]] = ( + _scanned_expected_names(test_files=test_files, stored=stored, syntax=syntax) + if scan_natively + else {} + ) + names_by_file: list[list[_BlockNames]] = [] + for file_index, test_file in enumerate(test_files): + stored_names: tuple[tuple[str, ...], ...] | None = stored[file_index] + if stored_names is not None: + names_by_file.append(list(stored_names)) + continue + file_names: list[_BlockNames] = [] + for block_index, block in enumerate(test_file.blocks): + native_names: tuple[str, ...] | None = scanned.get((file_index, block_index)) + if native_names is not None: + file_names.append(native_names) + continue + try: + file_names.append( + extract_sql_test_expected_model_names( + sql=block.sql_body, + file_label=str(test_file.relative_path), + syntax=syntax, + mode=block.mode, + ) + ) + except Exception as error: + file_names.append(error) + if not any(isinstance(names, Exception) for names in file_names): + scan_cache.write( + algorithm=SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM, + syntax=syntax, + parts=_expected_names_parts(test_file), + value=orjson.dumps(file_names), + ) + names_by_file.append(file_names) + return names_by_file + + +def _expected_names_parts(test_file: DiscoveredSqlTestFile) -> list[str]: + parts: list[str] = [str(test_file.relative_path)] + for block in test_file.blocks: + parts.extend((block.sql_body, block.mode.value)) + return parts + + +def _stored_expected_names( + *, block_count: int +) -> Callable[[bytes], tuple[tuple[str, ...], ...] | None]: + """Decode one file's stored names, rejecting anything that is not their exact shape.""" + + def decode(value: bytes) -> tuple[tuple[str, ...], ...] | None: + try: + decoded: object = orjson.loads(value) + except orjson.JSONDecodeError: + return None + if not isinstance(decoded, list) or len(cast(list[object], decoded)) != block_count: + return None + blocks: list[tuple[str, ...]] = [] + for names in cast(list[object], decoded): + if not isinstance(names, list) or not all( + isinstance(name, str) for name in cast(list[object], names) + ): + return None + blocks.append(tuple(cast(list[str], names))) + return tuple(blocks) + + return decode + + +def _names_or_raise(names: _BlockNames) -> tuple[str, ...]: + if isinstance(names, Exception): + raise names + return names + + def _scanned_expected_names( *, test_files: Sequence[DiscoveredSqlTestFile], + stored: Sequence[tuple[tuple[str, ...], ...] | None], syntax: SqlLexicalSyntax, ) -> dict[tuple[int, int], tuple[str, ...]]: - """Scan model-mode blocks natively, keeping only the names the scan reproduces.""" + """Scan model-mode blocks of unstored files natively, keeping only names the scan reproduces.""" pending: list[tuple[int, int]] = [] for file_index, test_file in enumerate(test_files): + if stored[file_index] is not None: + continue pending.extend( (file_index, block_index) for block_index, block in enumerate(test_file.blocks) @@ -192,19 +293,21 @@ def _test_relationship_grants( discovered_inputs: DiscoveredProjectInputs, lookup: ScopeLookup, shared_declarations: _SharedDeclarations, + scan_cache: SqlTestScanCache, syntax: SqlLexicalSyntax, ) -> tuple[tuple[GrantRecord, ...], tuple[ScopeRelationshipFault, ...]]: grants: list[GrantRecord] = [] faults: list[ScopeRelationshipFault] = [] - for test_file in discovered_inputs.test_files: - for block in test_file.blocks: + names_by_file: list[list[_BlockNames]] = _expected_names_by_file( + test_files=discovered_inputs.test_files, + scan_cache=scan_cache, + syntax=syntax, + scan_natively=False, + ) + for test_file, file_names in zip(discovered_inputs.test_files, names_by_file, strict=True): + for block, block_names in zip(test_file.blocks, file_names, strict=True): try: - expected_names: tuple[str, ...] = extract_sql_test_expected_model_names( - sql=block.sql_body, - file_label=str(test_file.relative_path), - syntax=syntax, - mode=block.mode, - ) + expected_names: tuple[str, ...] = _names_or_raise(block_names) grants.extend( _expected_model_grants( lookup=lookup, diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py index f33069446d..17b278dce8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/sql_tests.py @@ -49,6 +49,7 @@ extract_expanded_sql_tests, extract_unexpanded_sql_test, ) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.exceptions import CompileInputError, SqlTestExtractionError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -119,11 +120,12 @@ def build_test_inputs_with_cache( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...], sql_lexical_syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache, ) -> tuple[CompileSqlTestInput, ...]: - """Build SQL test inputs inside the compile timing boundary.""" + """Build SQL test inputs inside the compile timing boundary, then save the scan store.""" with record_compile_timing("test_input_compile_ms"): - return build_test_inputs( + test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs( discovered_inputs=discovered_inputs, effective_vars=effective_vars, macro_context=macro_context, @@ -132,7 +134,10 @@ def build_test_inputs_with_cache( external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, sql_lexical_syntax=sql_lexical_syntax, + scan_cache=scan_cache, ) + scan_cache.save() + return test_inputs def build_test_inputs( @@ -145,6 +150,7 @@ def build_test_inputs( sql_lexical_syntax: SqlLexicalSyntax, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, sql_function_inputs: tuple[CompileSqlFunctionInput, ...] = (), + scan_cache: SqlTestScanCache | None = None, ) -> tuple[CompileSqlTestInput, ...]: """Build compile-time test inputs from discovered SQL-native test blocks.""" @@ -310,6 +316,7 @@ def build_test_inputs( (test.sql_body, str(test.test_file.relative_path), test.mode) for test in expanded_tests ), + scan_cache=scan_cache, syntax=sql_lexical_syntax, ) except SqlTestExtractionError as error: diff --git a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py index 45f2985970..1577857686 100644 --- a/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py +++ b/src/sqlbuild/compiler/compile/_helpers/sql_tests/native.py @@ -2,11 +2,14 @@ from __future__ import annotations +from collections.abc import Callable from typing import Protocol, cast import orjson import sqlbuild._native as _native +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import SQL_TEST_CTE_SCAN_ALGORITHM from sqlbuild.compiler.compile.exceptions import ( CompileInputError, NativeSqlTestResponseError, @@ -44,18 +47,89 @@ def extract_expanded_sql_tests( *, tests: tuple[tuple[str, str, SqlTestMode], ...], syntax: SqlLexicalSyntax, + scan_cache: SqlTestScanCache | None = None, ) -> tuple[CompileSqlTestCtes, ...]: - """Extract and classify expanded tests natively under the adapter's lexical rules.""" + """Extract expanded tests natively, reusing whole stored results of unchanged files.""" + + store: SqlTestScanCache = scan_cache or SqlTestScanCache(cache_dir=None) + indexes_by_file: dict[str, list[int]] = {} + for index, (_sql, file_label, _mode) in enumerate(tests): + indexes_by_file.setdefault(file_label, []).append(index) + results: list[CompileSqlTestCtes | None] = [None] * len(tests) + parts_by_file: dict[str, list[str]] = {} + for file_label, indexes in indexes_by_file.items(): + parts: list[str] = [file_label] + for index in indexes: + parts.extend((tests[index][0], tests[index][2].value)) + modes: tuple[SqlTestMode, ...] = tuple(tests[index][2] for index in indexes) + stored: tuple[CompileSqlTestCtes, ...] | None = store.read( + algorithm=SQL_TEST_CTE_SCAN_ALGORITHM, + syntax=syntax, + parts=parts, + decode=_stored_file_tests(modes=modes), + ) + if stored is None: + parts_by_file[file_label] = parts + continue + for index, test_ctes in zip(indexes, stored, strict=True): + results[index] = test_ctes + missing: list[int] = [] + for file_label in parts_by_file: + missing.extend(indexes_by_file[file_label]) + missing.sort() + if missing: + try: + items: list[object] = _native_batch( + tests=[ + { + "sql": tests[index][0], + "fileLabel": tests[index][1], + "mode": tests[index][2].value, + "raw": False, + } + for index in missing + ], + syntax=syntax, + ) + except SqlTestExtractionError as error: + error.test_index = missing[error.test_index] + raise + items_by_index: dict[int, object] = dict(zip(missing, items, strict=True)) + for file_label, parts in parts_by_file.items(): + file_items: list[object] = [ + items_by_index[index] for index in indexes_by_file[file_label] + ] + for index, item in zip(indexes_by_file[file_label], file_items, strict=True): + results[index] = _test_ctes_from_native(item) + store.write( + algorithm=SQL_TEST_CTE_SCAN_ALGORITHM, + syntax=syntax, + parts=parts, + value=orjson.dumps(file_items), + ) + return tuple(result for result in results if result is not None) - if not tests: - return () - expanded: tuple[tuple[str, str, SqlTestMode, bool], ...] = tuple( - (sql, file_label, mode, False) for sql, file_label, mode in tests - ) - extracted: tuple[tuple[CompileSqlTestCtes, bool], ...] = _extract_natively( - tests=expanded, syntax=syntax - ) - return tuple(test_ctes for test_ctes, _invalid_calls in extracted) + +def _stored_file_tests( + *, modes: tuple[SqlTestMode, ...] +) -> Callable[[bytes], tuple[CompileSqlTestCtes, ...] | None]: + """Decode one file's stored tests, rejecting anything that is not their exact shape.""" + + def decode(value: bytes) -> tuple[CompileSqlTestCtes, ...] | None: + try: + items: object = orjson.loads(value) + if not isinstance(items, list) or len(cast(list[object], items)) != len(modes): + return None + decoded: tuple[CompileSqlTestCtes, ...] = tuple( + _test_ctes_from_native(item) for item in cast(list[object], items) + ) + except (orjson.JSONDecodeError, CompileInputError): + return None + if any(test_ctes.mode is not mode for test_ctes, mode in zip(decoded, modes, strict=True)): + return None + return decoded + + return decode def extract_unexpanded_sql_test( diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py new file mode 100644 index 0000000000..306157d1c7 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py @@ -0,0 +1,84 @@ +"""Per-file SQL-test scan results reused across compiles from the shared native store.""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Sequence +from pathlib import Path + +import sqlbuild._native as _native +from sqlbuild.compiler.compile.constants import ( + SQL_TEST_SCAN_STORE_FILE_NAME, + SQL_TEST_SCAN_STORE_VERSION, +) +from sqlbuild.compiler.profiling.main._metric import record_compile_metric +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax + + +class SqlTestScanCache: + """Whole per-file scan results keyed by scan algorithm, lexical rules and scanned text.""" + + def __init__(self, *, cache_dir: Path | None) -> None: + self._path: Path | None = ( + None if cache_dir is None else cache_dir / SQL_TEST_SCAN_STORE_FILE_NAME + ) + self._store: _native.SqlTestScanStore | None = None + self._hits: int = 0 + self._misses: int = 0 + + def read[T]( + self, + *, + algorithm: str, + syntax: SqlLexicalSyntax, + parts: Sequence[str], + decode: Callable[[bytes], T | None], + ) -> T | None: + """Return one file's decoded stored result, or None when it must be scanned again.""" + + store: _native.SqlTestScanStore | None = self._opened() + if store is None: + return None + found: bytes | None = store.get([algorithm, syntax.cache_key, *parts]) + decoded: T | None = None if found is None else decode(found) + if decoded is None: + self._misses += 1 + else: + self._hits += 1 + return decoded + + def write( + self, *, algorithm: str, syntax: SqlLexicalSyntax, parts: Sequence[str], value: bytes + ) -> None: + """Store one file's complete scan result; callers never store failed scans.""" + + store: _native.SqlTestScanStore | None = self._opened() + if store is not None: + store.put([algorithm, syntax.cache_key, *parts], value) + + def save(self) -> None: + """Persist results stored by this compile and report hit and miss counts.""" + + record_compile_metric(metric="sql_test_scan_cache_hits", value=self._hits) + record_compile_metric(metric="sql_test_scan_cache_misses", value=self._misses) + if self._store is None or self._path is None: + return + try: + _ = self._store.save(str(self._path)) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("SQL test scan store not saved: %s", error) + + def _opened(self) -> _native.SqlTestScanStore | None: + if self._path is None: + return None + if self._store is None: + environment: str = _native.content_digest( + [SQL_TEST_SCAN_STORE_VERSION, _native.BUILD_IDENTITY] + ) + try: + self._store = _native.SqlTestScanStore(str(self._path), environment) + except (OSError, RuntimeError) as error: + logging.getLogger(__name__).debug("SQL test scan store not loaded: %s", error) + self._path = None + return None + return self._store diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 4f3bfc2c17..5b51523ff2 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -187,3 +187,7 @@ COMPACT_ANALYSIS_FACT_LENGTH: int = 6 COMPACT_ANALYSIS_SOURCE_LENGTH: int = 3 MODEL_SCHEMA_CONFIG_KEY: str = "schema" +SQL_TEST_SCAN_STORE_FILE_NAME: str = "sql-test-scans.bin" +SQL_TEST_SCAN_STORE_VERSION: str = "sql-test-scan-store-v1" +SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" +SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: str = "sql-test-expected-models-v1" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 720032c6d2..819da6a2e5 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -131,6 +131,7 @@ def build_compile_inputs( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=adapter_context.sql_lexical_syntax, + compile_cache_dir=compile_cache_dir, ) model_context: ModelInputBuildContext = ModelInputBuildContext( effective_vars=effective_vars, @@ -183,6 +184,7 @@ def build_compile_inputs( external_sql_reference_resolver=external_sql_reference_resolver, sql_function_inputs=sql_function_inputs, sql_lexical_syntax=model_context.sql_lexical_syntax, + scan_cache=declaration_scope.sql_test_scans, ) scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( discovered_inputs=discovered_inputs, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index b3ddc912f2..e7f4fd938c 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -13,6 +13,7 @@ from sqlbuild.compiler.auditing.models import MeasurementContract, MeasurementThresholds from sqlbuild.compiler.auditing.types import AuditEvaluationMode, AuditSeverity from sqlbuild.compiler.compile.classes.macro_expansion_facts import MacroExpansionFacts +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache from sqlbuild.compiler.compile.constants import DEFAULT_SQL_TEST_MODE, MACRO_CONTEXT_PARAMETER_NAME from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.types import ( @@ -333,6 +334,7 @@ class DeclarationScopeBuild: loaded_macros: dict[str, LoadedMacro] index: ScopeIndex resolver: DeclarationScopeResolver + sql_test_scans: SqlTestScanCache @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index 6da274d310..6594b32e77 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -66,6 +66,7 @@ ), "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), + "sqlbuild.compiler.compile.models:DeclarationScopeBuild": frozenset({"sql_test_scans"}), "sqlbuild.compiler.compile.models:CompiledProject": frozenset({"native_session"}), } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 2794341f76..1ff96dd869 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -22,4 +22,6 @@ "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", + "sql_test_scan_cache_hits", + "sql_test_scan_cache_misses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index 64d8b0dd73..22f51e54cc 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -22,4 +22,6 @@ "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", + "sql_test_scan_cache_hits", + "sql_test_scan_cache_misses", ] diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index a2a4ff7470..75f1599fdb 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -35,6 +35,7 @@ ) from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path +from sqlbuild.rule_engine._helpers.run.code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.run.native_memo import ( native_payload_digests, native_request_identity, @@ -244,13 +245,15 @@ def start_custom_rules( dialect: str, verify_determinism: bool = False, ) -> Future[CustomRulesOutcome] | None: - """Submit selected custom rules, reusing the implementation fingerprints of their payloads.""" + """Submit selected custom rules after digesting installed code here, clear of GIL contention.""" custom_rules: tuple[Rule, ...] = _selected_custom_rules( config=config, catalogue=catalogue, custom_payloads=custom_payloads ) if not custom_rules: return None + if config.cache.enabled: + _ = compiled_code_identity() return executor.submit( evaluate_custom_rules_cached, project=project, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index b53eb4afc5..1bf00f83ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -88,6 +88,8 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_entry_cache_hits", "analysis_cache_misses", "analysis_cache_bypasses", + "sql_test_scan_cache_hits", + "sql_test_scan_cache_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 43c1e58221..ca7d026a0c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -742,6 +742,17 @@ class ExternalModuleEditTestCase: expected_compiled_value: str +@dataclass(frozen=True) +class SqlTestScanStoreTestCase: + """A change between two compiles, and how many stored SQL-test scans it must invalidate.""" + + description: str + change: Callable[[Path, pytest.MonkeyPatch], None] + expected_rescans: int + arrange: Callable[[pytest.MonkeyPatch], None] = lambda _monkeypatch: None + expected_matches_uncached: bool = True + + @dataclass(frozen=True) class DenseBatchedPreparationTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 913ed35b1a..9d45e57f88 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -16,7 +16,7 @@ import time import zipfile from bisect import bisect_left -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager, redirect_stdout from dataclasses import replace from io import StringIO @@ -47,6 +47,7 @@ from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, ) +from sqlbuild.adapter.contract.classes.duckdb_backed_adapter import DuckDbBackedAdapter from sqlbuild.cli.commands.main.entrypoint.entry import main from sqlbuild.cli.compile_reuse._helpers.entry_file import ( read_entry_header, @@ -62,9 +63,12 @@ StoredCompileHeader, StoredCompileInputs, ) +from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache +from sqlbuild.compiler.compile.constants import SQL_TEST_SCAN_STORE_FILE_NAME from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME +from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax from sqlbuild.observability import EventDispatcher, LifecycleEvent from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, @@ -4055,6 +4059,126 @@ def stg_orders_test_edit(root: Path) -> None: ) +def stg_orders_test_edited_again(root: Path) -> None: + """Change the staging orders SQL test's edited expected value once more.""" + + replace_project_text( + root, "tests/unit/test_stg_orders.sql", "101 AS customer_id", "102 AS customer_id" + ) + + +def adapter_switched(*, before: str, after: str) -> Callable[[Path], None]: + """Return an edit that switches the project adapter.""" + + def switch(root: Path) -> None: + replace_project_text( + root, "sqlbuild_project.toml", f'adapter = "{before}"', f'adapter = "{after}"' + ) + + return switch + + +def disable_project_reuse(monkeypatch: pytest.MonkeyPatch) -> None: + """Set the fixture environment but compile every time, so the finer caches are exercised.""" + + enable_compile_reuse(monkeypatch) + monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, "1") + + +def sql_test_scan_counts(run: CompileReuseRun) -> tuple[int, int]: + """Return the SQL-test scan store hits and misses of one compile.""" + + return run.timings["sql_test_scan_cache_hits"], run.timings["sql_test_scan_cache_misses"] + + +def edit_sql_test_scan_input(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Edit one SQL test file between compiles.""" + + stg_orders_test_edit(root) + + +def edit_unrelated_model(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Edit a model no SQL test text contains.""" + + fact_comment(root) + + +def change_adapter_lexical_rules(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Give the project adapter different lexical rules that leave this project's SQL unchanged.""" + + monkeypatch.setattr( + DuckDbBackedAdapter, + "sql_lexical_syntax", + replace(DuckDbBackedAdapter.sql_lexical_syntax, triple_quoted_strings=True), + ) + + +def upgrade_native_build(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Pretend the native extension was rebuilt from different source.""" + + monkeypatch.setattr(native_module, "BUILD_IDENTITY", f"{native_module.BUILD_IDENTITY}-rebuilt") + + +def corrupt_sql_test_scan_store(root: Path, _monkeypatch: pytest.MonkeyPatch) -> None: + """Overwrite the stored SQL-test scans with bytes that are not a store file.""" + + path: Path = compiler_cache_directory(root) / SQL_TEST_SCAN_STORE_FILE_NAME + assert path.is_file() + _ = path.write_bytes(b"not a native store") + + +_ORIGINAL_SCAN_WRITE: Callable[..., None] = SqlTestScanCache.write + + +def _stored_scans_replaced(replacement: bytes) -> Callable[[pytest.MonkeyPatch], None]: + def arrange(monkeypatch: pytest.MonkeyPatch) -> None: + def write( + self: SqlTestScanCache, + *, + algorithm: str, + syntax: SqlLexicalSyntax, + parts: Sequence[str], + value: bytes, + ) -> None: + del value + _ORIGINAL_SCAN_WRITE( + self, algorithm=algorithm, syntax=syntax, parts=parts, value=replacement + ) + + monkeypatch.setattr(SqlTestScanCache, "write", write) + + return arrange + + +store_undecodable_scans: Callable[[pytest.MonkeyPatch], None] = _stored_scans_replaced(b"{]") +store_misshapen_scans: Callable[[pytest.MonkeyPatch], None] = _stored_scans_replaced( + b'[{"mode": "model"}]' +) + + +def ignore_test_text_in_scan_key(monkeypatch: pytest.MonkeyPatch) -> None: + """Key stored SQL-test scans by file path only, so an edited test reads a stale scan.""" + + read: Callable[..., object] = SqlTestScanCache.read + write: Callable[..., None] = SqlTestScanCache.write + + def path_only(kwargs: dict[str, Any]) -> dict[str, Any]: + return {**kwargs, "parts": kwargs["parts"][:1]} + + monkeypatch.setattr( + SqlTestScanCache, "read", lambda self, **kwargs: read(self, **path_only(kwargs)) + ) + monkeypatch.setattr( + SqlTestScanCache, "write", lambda self, **kwargs: write(self, **path_only(kwargs)) + ) + + +def restore_sql_test_scan_writes(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Store real scan results again after an arrangement replaced them.""" + + monkeypatch.setattr(SqlTestScanCache, "write", _ORIGINAL_SCAN_WRITE) + + def keep_invalidation(_monkeypatch: pytest.MonkeyPatch) -> None: """Leave every cache invalidation intact.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index 1e99f2c30c..864e326272 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -12,15 +12,18 @@ IncrementalEditSequenceTestCase, RandomEditChainTestCase, SharedCacheKeyTestCase, + SqlTestScanStoreTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( CompileReuseRun, IncrementalEditComparison, RandomEditChain, + adapter_switched, add_external_flavor_macro, analyze_in_one_batch, analyze_one_model_at_a_time, build_between, + change_adapter_lexical_rules, compare_incremental_compile, compile_edit_without_reuse, compile_in_process, @@ -28,7 +31,11 @@ compiled_artifact_tampered, compiled_text, completed_order_udf_signature_change, + corrupt_sql_test_scan_store, + disable_project_reuse, + edit_sql_test_scan_input, edit_step, + edit_unrelated_model, enable_compile_reuse, fact_audit_added, fact_comment, @@ -38,6 +45,7 @@ fact_quantity_type_removed, ignore_project_changes, ignore_query_in_analysis_key, + ignore_test_text_in_scan_key, in_process_reuse_run, keep_invalidation, move_project_file, @@ -48,7 +56,9 @@ plan_between, random_edit_plan, replace_project_text, + restore_sql_test_scan_writes, run_reuse_compile, + sql_test_scan_counts, staging_column_removed, staging_column_renamed, staging_column_restored, @@ -65,7 +75,11 @@ star_chain_steps, star_chain_with_twin_added, stg_orders_test_edit, + stg_orders_test_edited_again, + store_misshapen_scans, + store_undecodable_scans, twin_header_changed, + upgrade_native_build, write_external_flavor, write_generated_edit_models, write_project_file, @@ -249,6 +263,16 @@ edit_step("downstream_type_removed", fact_quantity_type_removed), ), ), + IncrementalEditSequenceTestCase( + description="sql_test_scan_inputs", + steps=( + edit_step("test_edited", stg_orders_test_edit), + edit_step("unrelated_model_edited", fact_comment), + edit_step("adapter_switched", adapter_switched(before="duckdb", after="postgres")), + edit_step("test_edited_on_other_adapter", stg_orders_test_edited_again), + edit_step("adapter_restored", adapter_switched(before="postgres", after="duckdb")), + ), + ), IncrementalEditSequenceTestCase( description="star_chain_after_plan", steps=star_chain_steps(between=plan_between), @@ -350,6 +374,18 @@ def test_given_random_edit_chain_when_compiling_with_caches_then_each_step_match sabotage=ignore_query_in_analysis_key, expected_matches_uncached=False, ), + BrokenEditInvalidationTestCase( + description="intact_invalidation_after_test_edit", + edit=stg_orders_test_edit, + sabotage=keep_invalidation, + expected_matches_uncached=True, + ), + BrokenEditInvalidationTestCase( + description="stale_sql_test_scan_despite_test_edit", + edit=stg_orders_test_edit, + sabotage=ignore_test_text_in_scan_key, + expected_matches_uncached=False, + ), ], ids=lambda case: case.description, ) @@ -414,6 +450,82 @@ def test_given_models_sharing_an_analysis_cache_key_when_upstream_changes_then_i ) +@pytest.mark.parametrize( + "test_case", + [ + SqlTestScanStoreTestCase( + description="test_file_edited", + change=edit_sql_test_scan_input, + expected_rescans=2, + ), + SqlTestScanStoreTestCase( + description="unrelated_model_edited", + change=edit_unrelated_model, + expected_rescans=0, + ), + SqlTestScanStoreTestCase( + description="adapter_lexical_rules_changed", + change=change_adapter_lexical_rules, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="native_build_upgraded", + change=upgrade_native_build, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="store_file_corrupted", + change=corrupt_sql_test_scan_store, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="stored_entries_undecodable", + arrange=store_undecodable_scans, + change=restore_sql_test_scan_writes, + expected_rescans=12, + ), + SqlTestScanStoreTestCase( + description="stored_entries_misshapen", + arrange=store_misshapen_scans, + change=restore_sql_test_scan_writes, + expected_rescans=12, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_sql_test_scans_when_inputs_change_then_only_changed_files_rescan( + compile_reuse_project: Path, + test_case: SqlTestScanStoreTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = compile_reuse_project + disable_project_reuse(monkeypatch) + test_case.arrange(monkeypatch) + cold: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + test_case.change(project_dir, monkeypatch) + + incremental: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = in_process_reuse_run( + project_dir=project_dir, capsys=capsys, args=("--no-cache",) + ) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=incremental, reference=reference + ) + stored: int = sum(sql_test_scan_counts(cold)) + + assert cold.returncode == 0, cold.stderr + assert sql_test_scan_counts(cold) == (0, stored) + assert sql_test_scan_counts(incremental) == ( + stored - test_case.expected_rescans, + test_case.expected_rescans, + ) + assert sql_test_scan_counts(reference) == (0, 0) + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py index 4ca693ff57..cd14c2f000 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/_test_types.py @@ -224,6 +224,19 @@ class RulesEditChainCase: expected_exit_code: int = 1 +@dataclass(frozen=True) +class OneModelCustomRuleEditCase: + """One model edited after a warm compile of a model-subject custom rule.""" + + description: str + edited_model: str + before: str + after: str + expected_warm_counts: tuple[int, int] + expected_edit_counts: tuple[int, int] + expected_edit_exit_code: int + + @dataclass(frozen=True) class BrokenInvalidationChainCase: """An edit chain compiled with deliberately stale fact digests against a cache-free oracle.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py index c4013f5e32..94aa44f9a3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/helpers.py @@ -507,6 +507,32 @@ def _stale_sources(self, key): """ +ONE_MODEL_EDIT_MODEL_COUNT: int = 6 +# One cached result per model for the custom rule, plus the cached built-in rules request. +ONE_MODEL_EDIT_CACHED_RESULTS: int = ONE_MODEL_EDIT_MODEL_COUNT + 1 +ONE_MODEL_EDIT_PROJECT: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n\n' + '[rules]\nselect = ["XSQBRONE001"]\n\n' + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" + ), + "rules/orders.py": ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n\n" + '@rule(code="XSQBRONE001", slug="literal-order", message="Order 3 is a literal", ' + 'remediation="Select order 3 from a source.")\n' + "def literal_order(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + " source: str = ctx.sql.for_model(model).expanded.source\n" + ' return [ctx.finding(subject=model)] if "SELECT 3 " in source else []\n' + ), + **{ + f"models/orders_{index:03d}.sql": ( + f'MODEL (description "Orders {index}");\n\nSELECT {index} AS order_id\n' + ) + for index in range(ONE_MODEL_EDIT_MODEL_COUNT) + }, +} + + def custom_rules_edit_chain( *, root: Path, edits: tuple[Any, ...], environment: tuple[tuple[str, str], ...] = () ) -> tuple[ @@ -542,6 +568,14 @@ def rules_compile_outcome( ) -> tuple[int, str, str]: """Compile in a fresh process and return its exit code, diagnostics, and artifacts digest.""" + return rules_compile_run(project_dir, *arguments, environment=environment)[0] + + +def rules_compile_run( + project_dir: Path, *arguments: str, environment: tuple[tuple[str, str], ...] = () +) -> tuple[tuple[int, str, str], tuple[int, int]]: + """Compile in a fresh process; return its comparable outcome and rule cache hits and misses.""" + result: subprocess.CompletedProcess[str] = subprocess.run( [ str(Path(sys.executable).with_name("sqb")), @@ -563,10 +597,14 @@ def rules_compile_outcome( for path in sorted(filter(Path.is_file, compiled.rglob("*"))): artifacts.update(path.relative_to(compiled).as_posix().encode()) artifacts.update(path.read_bytes()) + timings: dict[str, int] = payload.get("compile_timings", {}) return ( - result.returncode, - json.dumps(payload.get("diagnostics"), sort_keys=True), - artifacts.hexdigest(), + ( + result.returncode, + json.dumps(payload.get("diagnostics"), sort_keys=True), + artifacts.hexdigest(), + ), + (timings.get("rule_cache_hits", 0), timings.get("rule_cache_misses", 0)), ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py index d0c9caa44d..72661eccf3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/rules/test_custom_rules_incremental_sequence.py @@ -1,5 +1,6 @@ """Incremental custom Rules across split hosts must equal a cache-free compile after every edit.""" +import shutil from pathlib import Path import pytest @@ -10,14 +11,19 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.rules._test_types import ( BrokenInvalidationChainCase, + OneModelCustomRuleEditCase, RulesCacheEditCase, RulesEditChainCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.rules.helpers import ( + ONE_MODEL_EDIT_CACHED_RESULTS, + ONE_MODEL_EDIT_PROJECT, STALE_SOURCE_FACTS_SITECUSTOMIZE, custom_rules_edit_chain, rule_cache_counts, rules_compile_outcome, + rules_compile_run, + write_project_files, ) CONFIG: str = "sqlbuild_project.toml" @@ -132,5 +138,51 @@ def test_given_broken_invalidation_when_applying_edit_chain_then_oracle_detects_ ) +@pytest.mark.parametrize( + "test_case", + ( + OneModelCustomRuleEditCase( + description="edit_adds_a_finding", + edited_model="models/orders_002.sql", + before="SELECT 2 AS", + after="SELECT 3 AS", + expected_warm_counts=(ONE_MODEL_EDIT_CACHED_RESULTS, 0), + expected_edit_counts=(ONE_MODEL_EDIT_CACHED_RESULTS - 1, 1), + expected_edit_exit_code=1, + ), + OneModelCustomRuleEditCase( + description="edit_removes_the_finding", + edited_model="models/orders_003.sql", + before="SELECT 3 AS", + after="SELECT 30 AS", + expected_warm_counts=(ONE_MODEL_EDIT_CACHED_RESULTS, 0), + expected_edit_counts=(ONE_MODEL_EDIT_CACHED_RESULTS - 1, 1), + expected_edit_exit_code=0, + ), + ), + ids=lambda case: case.description, +) +def test_given_model_custom_rule_when_one_model_is_edited_then_only_that_model_reruns( + tmp_path: Path, test_case: OneModelCustomRuleEditCase +) -> None: + project_dir: Path = tmp_path / "orders" + write_project_files(project_dir=project_dir, files=ONE_MODEL_EDIT_PROJECT) + cold, cold_counts = rules_compile_run(project_dir) + _, warm_counts = rules_compile_run(project_dir) + replace_project_text(project_dir, test_case.edited_model, test_case.before, test_case.after) + + edited, edit_counts = rules_compile_run(project_dir) + oracle_dir: Path = tmp_path / "oracle" + shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) + oracle: tuple[int, str, str] = rules_compile_outcome(oracle_dir, "--no-cache") + + assert cold[0] == 1, cold[1] + assert cold_counts == (0, ONE_MODEL_EDIT_CACHED_RESULTS) + assert warm_counts == test_case.expected_warm_counts + assert edit_counts == test_case.expected_edit_counts + assert edited[0] == test_case.expected_edit_exit_code, edited[1] + assert edited == oracle + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 7cdafa3983..d449de76ef 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -791,12 +791,14 @@ def counting_build( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, + compile_cache_dir=compile_cache_dir, ) monkeypatch.setattr(compile_inputs_module, "build_declaration_scope", counting_build) @@ -851,12 +853,14 @@ def counting_build( discovered_inputs: DiscoveredProjectInputs, loaded_macros: dict[str, LoadedMacro], sql_lexical_syntax: SqlLexicalSyntax, + compile_cache_dir: Path | None = None, ) -> DeclarationScopeBuild: scope_builds.append(tmp_path) return original_build( discovered_inputs=discovered_inputs, loaded_macros=loaded_macros, sql_lexical_syntax=sql_lexical_syntax, + compile_cache_dir=compile_cache_dir, ) def recording_load(macro_files: tuple[DiscoveredMacroFile, ...]) -> dict[str, LoadedMacro]: diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py index 1927e81a71..459d0a388b 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py @@ -1,6 +1,6 @@ """Test-case models for the same-runner compile performance ratio guard.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from scripts.compile_performance_ratio.models import CompileComparison @@ -12,6 +12,7 @@ class RatioFailuresTestCase: modes: tuple[str, ...] max_ratio: float expected_failures: tuple[str, ...] + mode_max_ratios: dict[str, float] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py index 6da3ed0a93..74ebe3f291 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py @@ -66,6 +66,25 @@ max_ratio=1.10, expected_failures=("dense 3000 warm: wall ratio 1.518 exceeds 1.10",), ), + RatioFailuresTestCase( + description="an edit within its own looser limit passes while warm keeps the default", + comparisons=( + comparison(mode="warm", wall=(7.6, 7.6), cpu=(9.0, 9.0)), + comparison(mode="edit", wall=(8.0, 10.0), cpu=(10.0, 12.0)), + ), + modes=("warm", "edit"), + max_ratio=1.10, + mode_max_ratios={"edit": 1.30}, + expected_failures=(), + ), + RatioFailuresTestCase( + description="an edit beyond its own limit fails against that limit", + comparisons=(comparison(mode="edit", wall=(8.0, 11.0), cpu=(10.0, 10.0)),), + modes=("edit",), + max_ratio=1.10, + mode_max_ratios={"edit": 1.30}, + expected_failures=("dense 3000 edit: wall ratio 1.375 exceeds 1.30",), + ), RatioFailuresTestCase( description="a requested mode without a measurement fails", comparisons=(comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)),), @@ -102,6 +121,7 @@ def test_given_mode_comparisons_when_judging_then_reports_each_failed_limit( modes=test_case.modes, max_ratio=test_case.max_ratio, noise_floor_seconds=NOISE_FLOOR_SECONDS, + mode_max_ratios=test_case.mode_max_ratios, ) assert failures == test_case.expected_failures diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index de8e40f8c8..f1be1b4fac 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -120,12 +120,20 @@ prints `Recording compile for reuse...` and a completion line to stderr before t Any change runs the whole compile again: every model, test, and audit is rendered again, and contracts and semantic validation run for the whole project. No render is carried over from the -previous compile. A few caches under `target/cache/` still apply: an unchanged model takes its -column analysis from the analysis cache when its query and the analyses of its upstream models are -unchanged, rule results are reused for unchanged inputs, and macro call results may be reused as -described in [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). -Until the native compiler ships, editing one model therefore costs close to a full compile. The -output is byte-identical to `sqb compile --no-cache`. +previous compile, so until the native compiler ships, an edited model is recompiled without any +cache. A few caches under `target/cache/` shorten the rest: + +- An unchanged model takes its column analysis from the analysis cache when its query and the + analyses of its upstream models are unchanged. +- An unchanged SQL test file reuses its stored test scan, the CTEs and expected models read from + it, from `sql-test-scans.bin`. Entries are keyed by the file's expanded text, the adapter's SQL + lexical rules, and the SQLBuild build; a damaged or unreadable entry is scanned again. +- Rule results are reused for unchanged inputs. +- Macro call results may be reused as described in + [Macros must be deterministic](/docs/concepts/macros#macros-must-be-deterministic). + +An edit therefore costs a full render of the project plus the analysis, test scans, and rules of +what changed. The output is byte-identical to `sqb compile --no-cache`. Known limits: @@ -146,7 +154,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings and cache counters such as `sql_test_scan_cache_hits` and +`sql_test_scan_cache_misses`. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. From 0e376682cd5b679c24fee8000f5f608784313cef Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:21:42 +0100 Subject: [PATCH 4/5] ci(perf): tighten the one-model edit ratio limits to the observed ratios --- .github/workflows/verify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 973adcb5f6..a5a48b0136 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -566,10 +566,10 @@ jobs: include: - kind: dense max_ratio: "1.10" - edit_max_ratio: "1.45" + edit_max_ratio: "1.25" - kind: fresh max_ratio: "1.10" - edit_max_ratio: "1.25" + edit_max_ratio: "1.20" steps: - uses: actions/checkout@v4 - name: Check out the comparison base From 5c7abf23910f0b9806c215da35e8c2806d7a4c3b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:31:53 +0100 Subject: [PATCH 5/5] fix(compile): key SQL-test scans by installed code and delete retired cache files Stored expected-model names can come from the Python extractor, so the scan store environment now includes the installed code identity, which moves from the rule engine to the compiler frontier for both to share. Opening the compiler cache root deletes every engine's retired fact cache, and storing a reuse entry deletes render files older releases left in the store directory. --- .../cli/compile_reuse/_helpers/entry_file.py | 5 ++ src/sqlbuild/cli/compile_reuse/constants.py | 1 + .../compile/_helpers/attachment/target.py | 23 +++++++++- .../compile/classes/sql_test_scan_cache.py | 5 +- src/sqlbuild/compiler/compile/constants.py | 1 + .../frontier/_helpers}/code_identity.py | 14 +----- .../frontier/main/compiled_code_identity.py | 14 ++++++ .../_helpers/engine/custom_rules.py | 2 +- .../rule_engine/_helpers/engine/native.py | 2 +- .../rule_engine/_helpers/run/native_memo.py | 2 +- .../cli/commands/main/compile/_test_types.py | 9 ++++ .../cli/commands/main/compile/helpers.py | 46 ++++++++++++++++++- .../compile/test_compile_incremental_edits.py | 6 +++ .../main/compile/test_compile_reuse.py | 29 ++++++++++++ 14 files changed, 139 insertions(+), 20 deletions(-) rename src/sqlbuild/{rule_engine/_helpers/run => compiler/frontier/_helpers}/code_identity.py (87%) create mode 100644 src/sqlbuild/compiler/frontier/main/compiled_code_identity.py diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py index 37f8c3f597..9ffe463834 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py @@ -21,6 +21,7 @@ REUSE_FORMAT_VERSION, REUSE_MAX_ENTRY_BYTES, REUSE_MAX_STORED_ENTRIES, + REUSE_RETIRED_RENDER_SUFFIX, REUSE_STDOUT_SEPARATOR, REUSE_STDOUT_SUFFIX, ) @@ -179,6 +180,10 @@ def _read_section(*, handle: BinaryIO) -> bytes: def _prune_entries(*, directory: Path, keep: Path) -> None: entries: list[tuple[int, Path]] = [] for candidate in directory.iterdir(): + if candidate.suffix == REUSE_RETIRED_RENDER_SUFFIX: + with contextlib.suppress(OSError): + candidate.unlink() + continue if candidate.suffix != REUSE_ENTRY_SUFFIX or candidate == keep: continue with contextlib.suppress(OSError): diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 1fb957fa41..d4da1c210d 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -19,6 +19,7 @@ REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" +REUSE_RETIRED_RENDER_SUFFIX: str = ".render" REUSE_STDOUT_SEPARATOR: str = "-" REUSE_ENTRY_LENGTH_BYTES: int = 8 REUSE_ENTRY_CHECKSUM_BYTES: int = 4 diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py index 335e83bec4..384a535141 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/target.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/target.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +import shutil from pathlib import Path from sqlbuild.compiler.compile._helpers.attachment.core import build_effective_vars @@ -13,8 +14,13 @@ from sqlbuild.compiler.compile.constants import ( COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, + RETIRED_FACT_CACHE_DIRECTORY_NAME, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.constants import ( + COMPILER_CACHE_DIRECTORY_NAME, + ENGINE_CACHE_NAMESPACE_SUFFIXES, +) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.spec.contracts.main.resolve_target_config import resolve_target_config from sqlbuild.spec.contracts.main.resolve_target_name import resolve_target_name @@ -63,7 +69,22 @@ def compile_cache_root( or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE ): return None - return compiler_cache_directory(project_dir) + root: Path = compiler_cache_directory(project_dir) + _remove_retired_fact_caches(root=root) + return root + + +def _remove_retired_fact_caches(*, root: Path) -> None: + """Delete the per-file fact caches older releases kept under every engine's compiler root.""" + + for suffix in ENGINE_CACHE_NAMESPACE_SUFFIXES.values(): + retired: Path = ( + root.parent + / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" + / (RETIRED_FACT_CACHE_DIRECTORY_NAME) + ) + if retired.is_dir(): + shutil.rmtree(retired, ignore_errors=True) def build_effective_target_namespace( diff --git a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py index 306157d1c7..2e0cd9bd95 100644 --- a/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py +++ b/src/sqlbuild/compiler/compile/classes/sql_test_scan_cache.py @@ -11,12 +11,13 @@ SQL_TEST_SCAN_STORE_FILE_NAME, SQL_TEST_SCAN_STORE_VERSION, ) +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.profiling.main._metric import record_compile_metric from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax class SqlTestScanCache: - """Whole per-file scan results keyed by scan algorithm, lexical rules and scanned text.""" + """Whole per-file scan results keyed by installed code, scan algorithm, rules and text.""" def __init__(self, *, cache_dir: Path | None) -> None: self._path: Path | None = ( @@ -73,7 +74,7 @@ def _opened(self) -> _native.SqlTestScanStore | None: return None if self._store is None: environment: str = _native.content_digest( - [SQL_TEST_SCAN_STORE_VERSION, _native.BUILD_IDENTITY] + [SQL_TEST_SCAN_STORE_VERSION, compiled_code_identity()] ) try: self._store = _native.SqlTestScanStore(str(self._path), environment) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 5b51523ff2..d4777ac0ee 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -188,6 +188,7 @@ COMPACT_ANALYSIS_SOURCE_LENGTH: int = 3 MODEL_SCHEMA_CONFIG_KEY: str = "schema" SQL_TEST_SCAN_STORE_FILE_NAME: str = "sql-test-scans.bin" +RETIRED_FACT_CACHE_DIRECTORY_NAME: str = "facts-v1" SQL_TEST_SCAN_STORE_VERSION: str = "sql-test-scan-store-v1" SQL_TEST_CTE_SCAN_ALGORITHM: str = "expanded-sql-test-ctes-v1" SQL_TEST_EXPECTED_MODELS_SCAN_ALGORITHM: str = "sql-test-expected-models-v1" diff --git a/src/sqlbuild/rule_engine/_helpers/run/code_identity.py b/src/sqlbuild/compiler/frontier/_helpers/code_identity.py similarity index 87% rename from src/sqlbuild/rule_engine/_helpers/run/code_identity.py rename to src/sqlbuild/compiler/frontier/_helpers/code_identity.py index 1514af90b1..b173a9509d 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/code_identity.py +++ b/src/sqlbuild/compiler/frontier/_helpers/code_identity.py @@ -1,4 +1,4 @@ -"""Identity of the installed code that produces cached rule results.""" +"""Identity of the installed SQLBuild code that produces cached compiler and rule results.""" from __future__ import annotations @@ -11,8 +11,6 @@ from pathlib import Path from typing import Any -import sqlbuild._native as _native - _PACKAGE_NAME: str = "sqlbuild" _POLYGLOT_PACKAGE_NAME: str = "polyglot-sql-chio" _YAML_PACKAGE_NAME: str = "PyYAML" @@ -22,16 +20,8 @@ _PACKAGE_ROOT_PARENT_DEPTH: int = 3 -def compiled_code_identity() -> str: - """Return the digest that changes with every released, locally edited, or rebuilt version.""" - - return hashlib.sha256( - f"{_installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() - ).hexdigest() - - @cache -def _installed_code_identity() -> str: +def installed_code_identity() -> str: """Return a digest that changes with every released or locally edited code version.""" digest: Any = hashlib.sha256() diff --git a/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py b/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py new file mode 100644 index 0000000000..103cd40e4f --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/compiled_code_identity.py @@ -0,0 +1,14 @@ +"""Identity of the installed and native code behind cached compiler and rule results.""" + +import hashlib + +import sqlbuild._native as _native +from sqlbuild.compiler.frontier._helpers.code_identity import installed_code_identity + + +def compiled_code_identity() -> str: + """Return the digest that changes with every released, locally edited, or rebuilt version.""" + + return hashlib.sha256( + f"{installed_code_identity()}\0{_native.BUILD_IDENTITY}".encode() + ).hexdigest() diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 0b88c31df3..b806fb398c 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -14,6 +14,7 @@ import orjson from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.models import ScopeIndex from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, @@ -26,7 +27,6 @@ run_custom_hosts, ) from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine._helpers.run.code_identity import compiled_code_identity from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 75f1599fdb..d9cd1a569b 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -27,6 +27,7 @@ ) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.discovery.models import ConstantDeclaration, EnumDeclaration +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.compiler.scopes.main.scope_metadata import scope_metadata_projection from sqlbuild.rule_engine._helpers.engine.custom_rule_evidence import ( custom_rule_implementation_fingerprint, @@ -35,7 +36,6 @@ ) from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine._helpers.run.code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.run.native_memo import ( native_payload_digests, native_request_identity, diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py index 93964edf80..d1ceddcb9f 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -10,8 +10,8 @@ import orjson +from sqlbuild.compiler.frontier.main.compiled_code_identity import compiled_code_identity from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path -from sqlbuild.rule_engine._helpers.run.code_identity import compiled_code_identity from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ca7d026a0c..bc3989c839 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -753,6 +753,15 @@ class SqlTestScanStoreTestCase: expected_matches_uncached: bool = True +@dataclass(frozen=True) +class RetiredCompilerCacheTestCase: + """A compile over a project holding cache files older releases wrote.""" + + description: str + compile_args: tuple[str, ...] + expected_removed: bool + + @dataclass(frozen=True) class DenseBatchedPreparationTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 9d45e57f88..0acd2a0030 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -41,6 +41,7 @@ import sqlbuild.compiler.compile._helpers.native_stages.assembly as native_stages import sqlbuild.compiler.compile._helpers.native_stages.sql_tests as native_sql_test_stage import sqlbuild.compiler.contracts.main.validate as contract_validation +import sqlbuild.compiler.frontier.main.compiled_code_identity as compiled_code_identity_module import sqlbuild.compiler.lineage.main.columns as column_lineage import sqlbuild.compiler.macro_bridge.classes.macro_bridge as macro_bridge_class from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement @@ -64,8 +65,15 @@ StoredCompileInputs, ) from sqlbuild.compiler.compile.classes.sql_test_scan_cache import SqlTestScanCache -from sqlbuild.compiler.compile.constants import SQL_TEST_SCAN_STORE_FILE_NAME -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.compile.constants import ( + RETIRED_FACT_CACHE_DIRECTORY_NAME, + SQL_TEST_SCAN_STORE_FILE_NAME, +) +from sqlbuild.compiler.frontier.constants import ( + COMPILER_CACHE_DIRECTORY_NAME, + COMPILER_ENGINE_ENV_VAR, + ENGINE_CACHE_NAMESPACE_SUFFIXES, +) from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.macro_bridge.constants import MACRO_CALL_STORE_FILE_NAME from sqlbuild.compiler.sql_analysis.models import SqlLexicalSyntax @@ -4127,6 +4135,40 @@ def corrupt_sql_test_scan_store(root: Path, _monkeypatch: pytest.MonkeyPatch) -> _ = path.write_bytes(b"not a native store") +def edit_installed_code(_root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Pretend the installed Python code changed, as a local edit of an editable install does.""" + + monkeypatch.setattr( + compiled_code_identity_module, "installed_code_identity", lambda: "edited-python-code" + ) + + +RETIRED_RENDER_FILES: tuple[str, ...] = ( + "0123456789abcdef-0000.render", + "fedcba9876543210-1111.render", +) + + +def write_retired_compiler_cache_files(project_dir: Path) -> tuple[Path, ...]: + """Leave every engine's fact cache and render files of other target slots from old releases.""" + + written: list[Path] = [ + compiler_cache_directory(project_dir).parent + / f"{COMPILER_CACHE_DIRECTORY_NAME}{suffix}" + / RETIRED_FACT_CACHE_DIRECTORY_NAME + / "sql-tests.sqlite3" + for suffix in ENGINE_CACHE_NAMESPACE_SUFFIXES.values() + ] + written.extend( + compiler_cache_directory(project_dir) / REUSE_ENTRY_DIRECTORY_NAME / name + for name in RETIRED_RENDER_FILES + ) + for path in written: + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_bytes(b"retired") + return tuple(written) + + _ORIGINAL_SCAN_WRITE: Callable[..., None] = SqlTestScanCache.write diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index 864e326272..cea28c600b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -33,6 +33,7 @@ completed_order_udf_signature_change, corrupt_sql_test_scan_store, disable_project_reuse, + edit_installed_code, edit_sql_test_scan_input, edit_step, edit_unrelated_model, @@ -473,6 +474,11 @@ def test_given_models_sharing_an_analysis_cache_key_when_upstream_changes_then_i change=upgrade_native_build, expected_rescans=12, ), + SqlTestScanStoreTestCase( + description="installed_python_code_changed", + change=edit_installed_code, + expected_rescans=12, + ), SqlTestScanStoreTestCase( description="store_file_corrupted", change=corrupt_sql_test_scan_store, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py index daf0d72598..f107cbb22b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py @@ -23,6 +23,7 @@ CompileReuseReplayTestCase, CompileReuseStoreFailureTestCase, CompileReuseTimingsTestCase, + RetiredCompilerCacheTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( COMPILE_REUSE_HIT_LINE, @@ -72,6 +73,7 @@ write_orders_api_timeout_file, write_project_file, write_recording_sink, + write_retired_compiler_cache_files, ) _CONTRACT_ERROR_MODEL: str = ( @@ -807,5 +809,32 @@ def test_given_store_is_interrupted_when_compiling_then_the_interrupt_propagates assert raised.type is test_case.expected_raised +@pytest.mark.parametrize( + "test_case", + [ + RetiredCompilerCacheTestCase( + description="cached_compile_removes_them", compile_args=(), expected_removed=True + ), + RetiredCompilerCacheTestCase( + description="uncached_compile_leaves_the_cache_alone", + compile_args=("--no-cache",), + expected_removed=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_cache_files_of_older_releases_when_compiling_then_cached_compile_deletes_them( + compile_reuse_project: Path, test_case: RetiredCompilerCacheTestCase +) -> None: + retired: tuple[Path, ...] = write_retired_compiler_cache_files(compile_reuse_project) + + run: CompileReuseRun = run_reuse_compile( + project_dir=compile_reuse_project, args=test_case.compile_args + ) + + assert run.returncode == 0, run.stderr + assert [path.exists() for path in retired] == [not test_case.expected_removed] * len(retired) + + if __name__ == "__main__": pytest.main([__file__, "-vv"])