diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index f4ca9887b3..bc37009e7e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -117,8 +117,31 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data or the project has very many files, compile -prints `Recording compile for reuse...` and a completion line to stderr before the report. +report is ready. When it has to hash a lot of data, the project has very many files, or it records +thousands of new model renders, compile prints `Recording compile for reuse...` and a completion +line to stderr before the report. + +### Compiling after model-only edits + +When the only changes since the stored compile are edits to existing model files, compile runs in +full but reuses the stored render of every unchanged model, together with the diagnostics and +environment reads that rendering reported. Only the edited models are rendered again. Analysis, +contracts, rules, and artifact writes still run for the whole project, so downstream effects of an +edit, such as a changed column type, are reported exactly as in an uncached compile. The output is +byte-identical to `sqb compile --no-cache`. + +Every other change renders the whole project: adding, removing, or renaming a model, and editing +macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope +folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings +is always rendered again. + +Renders are stored only when a compile result for the same project and target is already stored, +so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. +The next full compile stores them, and edits after that reuse them. Stored renders are compressed +and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints +`Loading stored renders...` and a completion line to stderr before it starts. While renders are +reused, the Python modules recorded by the previous compile stay tracked even if this compile did +not import them, so editing such a module runs the next compile in full. Known limits: @@ -138,7 +161,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or +rendered again after model-only edits. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index aa35dbc92c..466362096a 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -28,6 +28,9 @@ CompileCommandResult, CompileWriteResult, ) +from sqlbuild.cli.compile_render_reuse.main._activated_render_reuse import activated_render_reuse +from sqlbuild.cli.compile_render_reuse.main._render_reuse_session import render_reuse_session +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings @@ -64,34 +67,35 @@ def run_compile( else reuse_attempt ) output: RecordedCompileOutput = RecordedCompileOutput() + render_reuse: CompileRenderReuse | None = render_reuse_session(attempt=attempt) status: TransientStatusReporter | None = start_compile_status( json_output=request.json_output, no_color=request.no_color, ) - try: - with ( - paused_cyclic_collection(), - collect_compile_timings() as detailed_timings, - COMPILE_INPUT_READS.recording() as input_reads, - COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, - PreparedCompileArtifacts( - enabled=not request.profile_flags.skip_write - ) as prepared_artifacts, - ): - result: CompileCommandResult = _run_compile_with_status( - request=effective_request, - total_start=total_start, - status=status, - detailed_timings=detailed_timings, - prepared_artifacts=prepared_artifacts, - output=output, - reuse_timings_ms=compile_reuse_timings(attempt=attempt), - ) - finally: - if status is not None: - status.close() - try: - with paused_cyclic_collection(): + with paused_cyclic_collection(): + try: + with ( + collect_compile_timings() as detailed_timings, + COMPILE_INPUT_READS.recording() as input_reads, + COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, + activated_render_reuse(render_reuse=render_reuse), + PreparedCompileArtifacts( + enabled=not request.profile_flags.skip_write + ) as prepared_artifacts, + ): + result: CompileCommandResult = _run_compile_with_status( + request=effective_request, + total_start=total_start, + status=status, + detailed_timings=detailed_timings, + prepared_artifacts=prepared_artifacts, + output=output, + reuse_timings_ms=compile_reuse_timings(attempt=attempt), + ) + finally: + if status is not None: + status.close() + try: write_reusable_compile( attempt=attempt, output=output, @@ -102,9 +106,10 @@ def run_compile( artifacts_written=result.artifacts_written, dag_artifact_path=result.dag_artifact_path, json_output=request.json_output, + render_reuse=render_reuse, ) - finally: - output.print_stdout() + finally: + output.print_stdout() return result.exit_code diff --git a/src/sqlbuild/cli/compile_render_reuse/__init__.py b/src/sqlbuild/cli/compile_render_reuse/__init__.py new file mode 100644 index 0000000000..1b6139a171 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/__init__.py @@ -0,0 +1 @@ +"""Reuse unaffected renders of the stored compile in the next full compile.""" diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py new file mode 100644 index 0000000000..b89b7a1c87 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py @@ -0,0 +1,49 @@ +"""Lifecycle lines for loading a large render store before the compile status starts.""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path + +from sqlbuild.cli.compile_render_reuse.constants import ( + RENDER_LOAD_DONE_MESSAGE, + RENDER_LOAD_FAILED_MESSAGE, + RENDER_LOAD_NOTICE_BYTES, + RENDER_LOAD_START_MESSAGE, +) +from sqlbuild.cli.compile_reuse.constants import BYTES_PER_MEBIBYTE, REUSE_RENDER_STATE_SUFFIX + + +def start_load_notice(*, entry_path: Path) -> float | None: + """Announce loading on stderr when the slot's stored render files are large.""" + + stored_bytes: int = sum( + _file_size(path=path) + for path in entry_path.parent.glob(f"{entry_path.stem}-*{REUSE_RENDER_STATE_SUFFIX}") + ) + if stored_bytes < RENDER_LOAD_NOTICE_BYTES: + return None + print( + RENDER_LOAD_START_MESSAGE.format(mebibytes=stored_bytes / BYTES_PER_MEBIBYTE), + file=sys.stderr, + ) + return time.monotonic() + + +def _file_size(*, path: Path) -> int: + """Size a stored render file, treating one removed by a concurrent compile as empty.""" + + try: + return path.stat().st_size + except OSError: + return 0 + + +def finish_load_notice(*, started: float | None, loaded: bool) -> None: + """Report whether an announced load produced usable renders.""" + + if started is None: + return + message: str = RENDER_LOAD_DONE_MESSAGE if loaded else RENDER_LOAD_FAILED_MESSAGE + print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py new file mode 100644 index 0000000000..f68ef0b514 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py @@ -0,0 +1,230 @@ +"""Checksummed, layered storage for the renders of the compile one stored entry describes.""" + +from __future__ import annotations + +import json +import zlib +from collections.abc import Iterable +from pathlib import Path +from typing import cast + +from sqlbuild.cli.compile_render_reuse.constants import ( + RENDER_COMPRESSION_LEVEL, + RENDER_INDEX_FIELD_COUNT, + RENDER_OVERLAY_MAX_SHARE, + RENDER_STATE_MAGIC, +) +from sqlbuild.cli.compile_render_reuse.models import ( + StoredRenderLayer, + StoredRenderLayers, + StoredRenderState, +) +from sqlbuild.cli.compile_reuse.constants import ( + REUSE_ENTRY_BYTE_ORDER, + REUSE_ENTRY_CHECKSUM_BYTES, + REUSE_ENTRY_LENGTH_BYTES, + REUSE_MAX_ENTRY_BYTES, + REUSE_RENDER_STATE_SUFFIX, +) +from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError +from sqlbuild.cli.compile_reuse.main._framed_section import framed_stored_section +from sqlbuild.cli.compile_reuse.main._read_framed_section import read_stored_section +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState + + +def read_render_state(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderState | None: + """Read verified renders when every changed path is a stored model; any fault means none.""" + + try: + newest: StoredRenderLayer | None = _read_layer(path=path, changed_paths=changed_paths) + if newest is None: + return None + if newest.base_file is None: + return _stored_state(base_file=path.name, base=newest, overlay=None) + base: StoredRenderLayer | None = _read_layer( + path=path.parent / _layer_name(newest.base_file), changed_paths=frozenset() + ) + if base is None or base.base_file is not None: + return None + return _stored_state(base_file=newest.base_file, base=base, overlay=newest) + except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError, zlib.error): + return None + + +def render_state_layer( + *, state: RenderReuseState, stored: StoredRenderLayers | None +) -> RenderStateLayer | None: + """Return an overlay on the stored base, a new base, or None when released bytes are needed.""" + + if ( + stored is not None + and state.model_paths == stored.model_paths + and stored.model_keys <= state.model_payloads.keys() + and stored.group_keys <= state.group_payloads.keys() + ): + models: list[tuple[str, memoryview]] = _held( + (path, state.model_payloads.get(path)) for path in state.model_paths + ) + groups: list[tuple[str, memoryview]] = _held(sorted(state.group_payloads.items())) + if ( + sum(len(item) for _, item in (*models, *groups)) + <= stored.base_bytes * RENDER_OVERLAY_MAX_SHARE + ): + return _layer( + model_paths=state.model_paths, + base_file=stored.base_file, + models=models, + groups=groups, + ) + if None in state.model_payloads.values() or None in state.group_payloads.values(): + return None + return _layer( + model_paths=state.model_paths, + base_file=None, + models=_held((path, state.model_payloads.get(path)) for path in state.model_paths), + groups=_held(sorted(state.group_payloads.items())), + ) + + +def _held(items: Iterable[tuple[str, memoryview | None]]) -> list[tuple[str, memoryview]]: + return [(key, payload) for key, payload in items if payload is not None] + + +def _layer( + *, + model_paths: tuple[str, ...], + base_file: str | None, + models: list[tuple[str, memoryview]], + groups: list[tuple[str, memoryview]], +) -> RenderStateLayer: + index: bytes = json.dumps( + { + "model_paths": list(model_paths), + "base": base_file, + "models": [[path, len(payload)] for path, payload in models], + "groups": [[name, len(payload)] for name, payload in groups], + }, + separators=(",", ":"), + ).encode("utf-8", "surrogateescape") + compressor: zlib._Compress = zlib.compressobj(RENDER_COMPRESSION_LEVEL) + compressed: list[bytes] = [compressor.compress(item) for _, item in (*models, *groups)] + compressed.append(compressor.flush()) + return RenderStateLayer( + chunks=( + RENDER_STATE_MAGIC, + framed_stored_section(data=index), + _section_header(payloads=compressed), + *compressed, + ), + base_file=base_file, + ) + + +def _section_header(*, payloads: Iterable[bytes]) -> bytes: + length: int = 0 + checksum: int = 0 + for payload in payloads: + length += len(payload) + checksum = zlib.crc32(payload, checksum) + return length.to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER) + checksum.to_bytes( + REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER + ) + + +def _read_layer(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderLayer | None: + with open(path, "rb") as handle: + if handle.read(len(RENDER_STATE_MAGIC)) != RENDER_STATE_MAGIC: + return None + index: object = json.loads(read_stored_section(handle=handle)) + if not isinstance(index, dict): + return None + fields: dict[str, object] = cast(dict[str, object], index) + model_paths: tuple[str, ...] = tuple(_string(item) for item in _list(fields["model_paths"])) + if not changed_paths.issubset(model_paths): + return None + compressed: bytes = read_stored_section(handle=handle) + if handle.read(1) or len(compressed) > REUSE_MAX_ENTRY_BYTES: + return None + base: object = fields["base"] + model_rows: list[tuple[str, int]] = _rows(fields["models"]) + group_rows: list[tuple[str, int]] = _rows(fields["groups"]) + size: int = sum(length for _, length in (*model_rows, *group_rows)) + if size > REUSE_MAX_ENTRY_BYTES: + return None + decompressor: zlib._Decompress = zlib.decompressobj() + payload: bytes = decompressor.decompress(compressed, size + 1) + del compressed + if len(payload) != size or not decompressor.eof or decompressor.unconsumed_tail: + return None + view: memoryview = memoryview(payload) + offset: int = 0 + models: dict[str, memoryview] = {} + for name, length in model_rows: + models[name] = view[offset : offset + length] + offset += length + groups: dict[str, memoryview] = {} + for name, length in group_rows: + groups[name] = view[offset : offset + length] + offset += length + return StoredRenderLayer( + model_paths=model_paths, + base_file=None if base is None else _string(base), + models=models, + groups=groups, + size=len(payload), + ) + + +def _stored_state( + *, base_file: str, base: StoredRenderLayer, overlay: StoredRenderLayer | None +) -> StoredRenderState: + newest: StoredRenderLayer = base if overlay is None else overlay + state: RenderReuseState = RenderReuseState( + model_paths=newest.model_paths, + model_payloads=dict(base.models) if overlay is None else {**base.models, **overlay.models}, + group_payloads=dict(base.groups) if overlay is None else {**base.groups, **overlay.groups}, + ) + return StoredRenderState( + state=state, + layers=StoredRenderLayers( + model_paths=newest.model_paths, + model_keys=frozenset(state.model_payloads), + group_keys=frozenset(state.group_payloads), + base_file=base_file, + base_bytes=base.size, + overlay_models=frozenset() if overlay is None else frozenset(overlay.models), + overlay_groups=frozenset() if overlay is None else frozenset(overlay.groups), + ), + ) + + +def _layer_name(name: str) -> str: + if Path(name).name != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): + raise CompileReuseEntryError("invalid stored render base name") + return name + + +def _rows(value: object) -> list[tuple[str, int]]: + rows: list[tuple[str, int]] = [] + for row in _list(value): + items: list[object] = _list(row) + if len(items) != RENDER_INDEX_FIELD_COUNT: + raise CompileReuseEntryError("invalid stored render index") + length: object = items[1] + if not isinstance(length, int) or isinstance(length, bool) or length < 0: + raise CompileReuseEntryError("invalid stored render length") + rows.append((_string(items[0]), length)) + return rows + + +def _list(value: object) -> list[object]: + if not isinstance(value, list): + raise CompileReuseEntryError("invalid stored render list") + return cast(list[object], value) + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise CompileReuseEntryError("invalid stored render string") + return value diff --git a/src/sqlbuild/cli/compile_render_reuse/constants.py b/src/sqlbuild/cli/compile_render_reuse/constants.py new file mode 100644 index 0000000000..be5e190f0a --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/constants.py @@ -0,0 +1,14 @@ +"""Render storage format constants.""" + +from __future__ import annotations + +RENDER_STATE_MAGIC: bytes = b"SQBRENDER2\n" +RENDER_COMPRESSION_LEVEL: int = 1 +RENDER_OVERLAY_MAX_SHARE: float = 0.25 +RENDER_INDEX_FIELD_COUNT: int = 2 +RENDER_LOAD_NOTICE_BYTES: int = 8 * 1024 * 1024 +RENDER_LOAD_START_MESSAGE: str = "Loading stored renders ({mebibytes:.0f} MiB)..." +RENDER_LOAD_DONE_MESSAGE: str = "Loaded stored renders ({seconds:.1f} s)" +RENDER_LOAD_FAILED_MESSAGE: str = ( + "Stored renders not reused; rendering every model again ({seconds:.1f} s)" +) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/__init__.py b/src/sqlbuild/cli/compile_render_reuse/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py b/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py new file mode 100644 index 0000000000..a40d737c78 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py @@ -0,0 +1,20 @@ +"""Public entry for exposing one compile's render reuse to the rendering it runs.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager, nullcontext + +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse +from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE + + +def activated_render_reuse( + *, render_reuse: CompileRenderReuse | None +) -> AbstractContextManager[object]: + """Activate the render reuse session for the compile block, or do nothing without one.""" + + return ( + nullcontext() + if render_reuse is None + else COMPILE_RENDER_REUSE.activated(render_reuse.session) + ) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py new file mode 100644 index 0000000000..399a33b9e5 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py @@ -0,0 +1,53 @@ +"""Public entry for the render reuse one full compile runs with.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.cli.compile_render_reuse._helpers.load_notice import ( + finish_load_notice, + start_load_notice, +) +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import read_render_state +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse, StoredRenderState +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession + + +def render_reuse_session(*, attempt: CompileReuseAttempt) -> CompileRenderReuse | None: + """Record renders once a compile is stored, reusing those the changed paths cannot affect.""" + + if ( + attempt.outcome is not CompileReuseOutcome.MISS + or attempt.entry_path is None + or not attempt.prior_entry + ): + return None + stored: StoredRenderState | None = ( + None + if attempt.changed_paths is None or attempt.render_state_path is None + else _loaded_render_state( + entry_path=attempt.entry_path, + path=attempt.render_state_path, + changed_paths=attempt.changed_paths, + ) + ) + return CompileRenderReuse( + session=CompileRenderReuseSession( + prior=None if stored is None else stored.state, + changed_paths=attempt.changed_paths, + retained_models=frozenset() if stored is None else stored.layers.overlay_models, + retained_groups=frozenset() if stored is None else stored.layers.overlay_groups, + ), + stored=None if stored is None else stored.layers, + ) + + +def _loaded_render_state( + *, entry_path: Path, path: Path, changed_paths: frozenset[str] +) -> StoredRenderState | None: + started: float | None = start_load_notice(entry_path=entry_path) + stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) + _ = finish_load_notice(started=started, loaded=stored is not None) + return stored diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py new file mode 100644 index 0000000000..83467f8b70 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py @@ -0,0 +1,23 @@ +"""Public entry for the render layer a finished compile stores with its entry.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import render_state_layer +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState + + +def stored_render_layer(*, render_reuse: CompileRenderReuse | None) -> RenderStateLayer | None: + """Return this compile's renders as an overlay on the stored ones, or as a new base.""" + + state: RenderReuseState | None = ( + None if render_reuse is None else render_reuse.session.stored_state() + ) + if render_reuse is None or state is None: + return None + layer: RenderStateLayer | None = render_state_layer(state=state, stored=render_reuse.stored) + if layer is not None: + return layer + complete: RenderReuseState | None = render_reuse.session.stored_state(complete=True) + return None if complete is None else render_state_layer(state=complete, stored=None) diff --git a/src/sqlbuild/cli/compile_render_reuse/models.py b/src/sqlbuild/cli/compile_render_reuse/models.py new file mode 100644 index 0000000000..a7595e63c1 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/models.py @@ -0,0 +1,48 @@ +"""Stored renders of a previous compile and the render reuse of one full compile.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.models import RenderReuseState + + +@dataclass(frozen=True) +class StoredRenderLayer: + """One decoded render file: its index and its payloads by key.""" + + model_paths: tuple[str, ...] + base_file: str | None + models: dict[str, memoryview] + groups: dict[str, memoryview] + size: int + + +@dataclass(frozen=True) +class StoredRenderLayers: + """How a stored compile's renders are split between a base file and one overlay.""" + + model_paths: tuple[str, ...] + model_keys: frozenset[str] + group_keys: frozenset[str] + base_file: str + base_bytes: int + overlay_models: frozenset[str] + overlay_groups: frozenset[str] + + +@dataclass(frozen=True) +class StoredRenderState: + """Renders a stored compile recorded, merged across layers, with their layout.""" + + state: RenderReuseState + layers: StoredRenderLayers + + +@dataclass(frozen=True) +class CompileRenderReuse: + """One full compile's render reuse session with the stored renders it started from.""" + + session: CompileRenderReuseSession + stored: StoredRenderLayers | None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index ccb90f2e46..e963bbc831 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -9,12 +9,14 @@ from pathlib import Path from sqlbuild.cli.compile_reuse._helpers.entry_file import ( + entry_render_state_path, read_entry_header, read_entry_stdout, rewrite_entry_inputs, ) from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, + changed_project_paths, compare_project_files, needs_refresh, restamped_paths, @@ -102,7 +104,14 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ) attempt = replace( attempt, + prior_entry=True, + prior_modules=header.inputs.modules, restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), + render_state_path=( + None + if attempt.changed_paths is None or attempt.entry_path is None + else entry_render_state_path(path=attempt.entry_path, header=header) + ), ) return replace( attempt, @@ -155,12 +164,22 @@ def _checked_attempt( unchanged: bool = comparison.unchanged and target_files_unchanged( stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree ) + changed_paths: frozenset[str] | None = None + verified: dict[str, str] = comparison.verified + if identity_unchanged and not unchanged: + changed_paths, verified = changed_project_paths( + project_dir=project_dir, + stored=inputs.project_files, + current=attempt.snapshot, + verified=comparison.verified, + ) return replace( attempt, outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, digests=carried_forward_digests( - stored=inputs.project_files, current=attempt.snapshot, verified=comparison.verified + stored=inputs.project_files, current=attempt.snapshot, verified=verified ), + changed_paths=changed_paths, ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py index f7da96aeb7..c7453b448c 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py @@ -20,12 +20,15 @@ REUSE_ENTRY_SUFFIX, REUSE_FORMAT_VERSION, REUSE_MAX_ENTRY_BYTES, + REUSE_MAX_RENDER_STATES, REUSE_MAX_STORED_ENTRIES, + REUSE_RENDER_STATE_SUFFIX, REUSE_STDOUT_SEPARATOR, REUSE_STDOUT_SUFFIX, ) from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError from sqlbuild.cli.compile_reuse.models import ( + RenderStateLayer, SettingsEnvironmentInputs, StoredCompileHeader, StoredCompileInputs, @@ -48,7 +51,7 @@ def read_entry_header(*, path: Path) -> StoredCompileHeader | None: with open(path, "rb") as handle: if handle.read(len(REUSE_ENTRY_MAGIC)) != REUSE_ENTRY_MAGIC: return None - metadata: bytes = _read_section(handle=handle) + metadata: bytes = read_framed_section(handle=handle) if handle.read(1): return None payload: object = json.loads(metadata) @@ -80,13 +83,24 @@ def write_entry( inputs: StoredCompileInputs, output: StoredCompileOutput, stdout: str, + render_state: RenderStateLayer | None = None, ) -> None: - """Publish one entry and its stdout atomically, filling in the stdout file and checksum.""" + """Publish one entry, its stdout, and its renders atomically, naming them in the entry.""" encoded_stdout: bytes = stdout.encode("utf-8", "surrogateescape") - stdout_file: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}{REUSE_STDOUT_SUFFIX}" + stamp: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}" + stdout_file: str = f"{stamp}{REUSE_STDOUT_SUFFIX}" + render_state_file: str | None = ( + None + if render_state is None + or sum(len(chunk) for chunk in render_state.chunks) > REUSE_MAX_ENTRY_BYTES + else f"{stamp}{REUSE_RENDER_STATE_SUFFIX}" + ) stored_output: StoredCompileOutput = replace( - output, stdout_file=stdout_file, stdout_checksum=zlib.crc32(encoded_stdout) + output, + stdout_file=stdout_file, + stdout_checksum=zlib.crc32(encoded_stdout), + render_state_file=render_state_file, ) metadata: bytes = _metadata(inputs=inputs, output=stored_output) if len(metadata) + len(encoded_stdout) > REUSE_MAX_ENTRY_BYTES: @@ -94,14 +108,31 @@ def write_entry( return try: path.parent.mkdir(parents=True, exist_ok=True) - _publish_file(path=path.parent / stdout_file, contents=encoded_stdout) - _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) - _remove_stdout_files(entry_path=path, keep=stdout_file) + _publish_file(path=path.parent / stdout_file, contents=(encoded_stdout,)) + if render_state is not None and render_state_file is not None: + _publish_file(path=path.parent / render_state_file, contents=render_state.chunks) + _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) + _remove_stdout_files(entry_path=path, keep=(stdout_file,)) + _remove_render_files( + entry_path=path, + keep=( + () + if render_state is None or render_state_file is None + else (render_state_file, render_state.base_file) + ), + ) _prune_entries(directory=path.parent, keep=path) except OSError: remove_entry(path=path) +def entry_render_state_path(*, path: Path, header: StoredCompileHeader) -> Path | None: + """Return where the entry's renders are stored, when it stored any.""" + + name: str | None = header.output.render_state_file + return None if name is None else path.parent / name + + def rewrite_entry_inputs( *, path: Path, header: StoredCompileHeader, inputs: StoredCompileInputs ) -> None: @@ -109,7 +140,7 @@ def rewrite_entry_inputs( metadata: bytes = _metadata(inputs=inputs, output=header.output) with contextlib.suppress(OSError): - _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) + _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) def remove_entry(*, path: Path) -> None: @@ -117,16 +148,18 @@ def remove_entry(*, path: Path) -> None: with contextlib.suppress(OSError): path.unlink(missing_ok=True) - _remove_stdout_files(entry_path=path, keep=None) + _remove_stdout_files(entry_path=path, keep=()) + _remove_render_files(entry_path=path, keep=()) -def _publish_file(*, path: Path, contents: bytes) -> None: +def _publish_file(*, path: Path, contents: tuple[bytes | memoryview, ...]) -> None: descriptor, temporary_path = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent ) try: with os.fdopen(descriptor, "wb") as handle: - _ = handle.write(contents) + for chunk in contents: + _ = handle.write(chunk) os.replace(temporary_path, path) except BaseException: with contextlib.suppress(OSError): @@ -141,17 +174,30 @@ def _metadata(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> by def _framed_metadata(*, metadata: bytes) -> bytes: + return REUSE_ENTRY_MAGIC + framed_section(data=metadata) + + +def framed_section(*, data: bytes) -> bytes: + """Prefix one section with its length and checksum so a reader can verify it.""" + return b"".join( ( - REUSE_ENTRY_MAGIC, - len(metadata).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), - zlib.crc32(metadata).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), - metadata, + len(data).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), + zlib.crc32(data).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), + data, ) ) -def _remove_stdout_files(*, entry_path: Path, keep: str | None) -> None: +def _remove_stdout_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: + _remove_entry_files(entry_path=entry_path, suffix=REUSE_STDOUT_SUFFIX, keep=keep) + + +def _remove_render_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: + _remove_entry_files(entry_path=entry_path, suffix=REUSE_RENDER_STATE_SUFFIX, keep=keep) + + +def _remove_entry_files(*, entry_path: Path, suffix: str, keep: tuple[str | None, ...]) -> None: prefix: str = f"{entry_path.stem}{REUSE_STDOUT_SEPARATOR}" try: candidates: list[Path] = list(entry_path.parent.iterdir()) @@ -160,14 +206,16 @@ def _remove_stdout_files(*, entry_path: Path, keep: str | None) -> None: for candidate in candidates: if ( candidate.name.startswith(prefix) - and candidate.suffix == REUSE_STDOUT_SUFFIX - and candidate.name != keep + and candidate.suffix == suffix + and candidate.name not in keep ): with contextlib.suppress(OSError): candidate.unlink() -def _read_section(*, handle: BinaryIO) -> bytes: +def read_framed_section(*, handle: BinaryIO) -> bytes: + """Read one length- and checksum-framed section, raising when it is damaged.""" + length: int = int.from_bytes(handle.read(REUSE_ENTRY_LENGTH_BYTES), REUSE_ENTRY_BYTE_ORDER) checksum: int = int.from_bytes(handle.read(REUSE_ENTRY_CHECKSUM_BYTES), REUSE_ENTRY_BYTE_ORDER) if length > REUSE_MAX_ENTRY_BYTES: @@ -188,6 +236,8 @@ def _prune_entries(*, directory: Path, keep: Path) -> None: entries.sort(reverse=True) for _, stale in entries[REUSE_MAX_STORED_ENTRIES - 1 :]: remove_entry(path=stale) + for _, older in entries[REUSE_MAX_RENDER_STATES - 1 : REUSE_MAX_STORED_ENTRIES - 1]: + _remove_render_files(entry_path=older, keep=()) def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dict[str, object]: @@ -210,6 +260,7 @@ def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dic "stdout_length": output.stdout_length, "stdout_checksum": output.stdout_checksum, "stdout_file": output.stdout_file, + "render_state_file": output.render_state_file, "target_tree": inputs.target_tree, "settings_inputs": [ { @@ -266,6 +317,7 @@ def _header(*, payload: object) -> StoredCompileHeader: stdout_length=_integer(fields["stdout_length"]), stdout_checksum=_integer(fields["stdout_checksum"]), stdout_file=_stdout_file(fields["stdout_file"]), + render_state_file=_render_state_file(fields.get("render_state_file")), ), ) @@ -288,6 +340,15 @@ def _stdout_file(value: object) -> str: return name +def _render_state_file(value: object) -> str | None: + if value is None: + return None + name: str = _string(value) + if os.path.basename(name) != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): + raise CompileReuseEntryError("invalid stored render file name") + return name + + def _timings_span(value: object) -> tuple[int, int] | None: if value is None: return None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py index 5c0472b33c..e68d78330c 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py @@ -101,6 +101,37 @@ def compare_project_files( return ProjectFilesComparison(unchanged=True, verified=verified) +def changed_project_paths( + *, + project_dir: str, + stored: dict[str, StoredProjectFile], + current: dict[str, FileStamp], + verified: dict[str, str], +) -> tuple[frozenset[str], dict[str, str]]: + """Return every path added, removed, or changed since the stored compile, and new digests.""" + + digests: dict[str, str] = dict(verified) + changed: set[str] = {path for path in stored if path not in current} + for relative_path, stamp in current.items(): + previous: StoredProjectFile | None = stored.get(relative_path) + if previous is None or not _same_identity(stamp=stamp, recorded=previous.stamp): + changed.add(relative_path) + continue + if stamp.kind in _PRESENCE_ONLY_KINDS or (stamp == previous.stamp and not previous.racy): + continue + if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: + changed.add(relative_path) + continue + digest: str | None = digests.get(relative_path) or file_digest( + path=os.path.join(project_dir, relative_path) + ) + if digest is None or digest != previous.digest: + changed.add(relative_path) + else: + digests[relative_path] = digest + return frozenset(changed), digests + + def carried_forward_digests( *, stored: dict[str, StoredProjectFile], diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index bad781b93b..b3eddc3024 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -69,6 +69,19 @@ def loaded_module_stamps(*, covered_paths: frozenset[str]) -> tuple[tuple[str, i return tuple(sorted(stamps.values())) +def with_carried_module_stamps( + *, + current: tuple[tuple[str, int, int], ...], + carried: tuple[tuple[str, int, int], ...], + covered_paths: frozenset[str], +) -> tuple[tuple[str, int, int], ...]: + """Add stored stamps of modules a reused render imported but this process did not load.""" + + stamps: dict[str, tuple[str, int, int]] = {stamp[0]: stamp for stamp in carried} + stamps.update((stamp[0], stamp) for stamp in current) + return tuple(sorted(stamp for path, stamp in stamps.items() if path not in covered_paths)) + + def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool: """Return whether every recorded module file still has the same stat identity.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index 7bed444a1d..7ead497bbc 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -8,6 +8,8 @@ import time from pathlib import Path +from sqlbuild.cli.compile_render_reuse.main._render_state_layer import stored_render_layer +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.entry_file import remove_entry, write_entry from sqlbuild.cli.compile_reuse._helpers.project_files import ( pending_digest_bytes, @@ -21,6 +23,7 @@ loaded_module_stamps, settings_inputs_digest, tracked_environment_names, + with_carried_module_stamps, ) from sqlbuild.cli.compile_reuse._helpers.target_files import verified_target_files from sqlbuild.cli.compile_reuse._helpers.timings import compile_timings_span @@ -32,6 +35,7 @@ REUSE_STORE_DONE_MESSAGE, REUSE_STORE_NOTICE_BYTES, REUSE_STORE_NOTICE_PATHS, + REUSE_STORE_NOTICE_RENDERS, REUSE_STORE_SKIPPED_MESSAGE, REUSE_STORE_START_MESSAGE, ) @@ -58,6 +62,7 @@ def write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, + render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" @@ -66,7 +71,6 @@ def write_compile_entry( try: _write_compile_entry( attempt=attempt, - entry_path=attempt.entry_path, output=output, exit_code=exit_code, input_reads=input_reads, @@ -75,6 +79,7 @@ def write_compile_entry( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, + render_reuse=render_reuse, ) except Exception: _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) @@ -84,7 +89,6 @@ def write_compile_entry( def _write_compile_entry( *, attempt: CompileReuseAttempt, - entry_path: Path, output: RecordedCompileOutput, exit_code: int, input_reads: CompileInputReads, @@ -93,7 +97,11 @@ def _write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, + render_reuse: CompileRenderReuse | None, ) -> None: + if attempt.entry_path is None: + return + entry_path: Path = attempt.entry_path stdout: str | None = output.stdout settings: SettingsInputsResult = provider_settings_inputs( settings_classes=input_reads.settings_classes @@ -109,7 +117,10 @@ def _write_compile_entry( remove_entry(path=entry_path) return project_dir: str = str(attempt.project_dir) - notice_started: float | None = _start_notice(attempt=attempt) + notice_started: float | None = _start_notice( + attempt=attempt, + pending_renders=0 if render_reuse is None else render_reuse.session.pending_renders(), + ) digests: dict[str, str] = with_missing_digests( project_dir=project_dir, snapshot=attempt.snapshot, @@ -143,10 +154,8 @@ def _write_compile_entry( environment_names=environment_names, environment_digest=environment_digest(names=environment_names), search_path=attempt.search_path, - modules=loaded_module_stamps( - covered_paths=frozenset( - os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot - ) + modules=_module_stamps( + attempt=attempt, project_dir=project_dir, render_reuse=render_reuse ), project_files=stored_project_files( snapshot=attempt.snapshot, digests=digests, snapshot_ns=attempt.snapshot_ns @@ -165,21 +174,40 @@ def _write_compile_entry( stdout_file="", ), stdout=stdout, + render_state=stored_render_layer(render_reuse=render_reuse), ) _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) -def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: +def _module_stamps( + *, attempt: CompileReuseAttempt, project_dir: str, render_reuse: CompileRenderReuse | None +) -> tuple[tuple[str, int, int], ...]: + covered_paths: frozenset[str] = frozenset( + os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot + ) + current: tuple[tuple[str, int, int], ...] = loaded_module_stamps(covered_paths=covered_paths) + if render_reuse is None or not render_reuse.session.reused_any(): + return current + return with_carried_module_stamps( + current=current, carried=attempt.prior_modules, covered_paths=covered_paths + ) + + +def _start_notice(*, attempt: CompileReuseAttempt, pending_renders: int) -> float | None: pending_bytes: int = pending_digest_bytes( snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped ) - if pending_bytes < REUSE_STORE_NOTICE_BYTES and len(attempt.snapshot) < ( - REUSE_STORE_NOTICE_PATHS + if ( + pending_bytes < REUSE_STORE_NOTICE_BYTES + and len(attempt.snapshot) < REUSE_STORE_NOTICE_PATHS + and pending_renders < REUSE_STORE_NOTICE_RENDERS ): return None print( REUSE_STORE_START_MESSAGE.format( - paths=len(attempt.snapshot), mebibytes=pending_bytes / BYTES_PER_MEBIBYTE + paths=len(attempt.snapshot), + mebibytes=pending_bytes / BYTES_PER_MEBIBYTE, + renders=pending_renders, ), file=sys.stderr, ) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index b5af8a4d5b..95f701e062 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -14,6 +14,8 @@ REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" REUSE_STDOUT_SEPARATOR: str = "-" +REUSE_RENDER_STATE_SUFFIX: str = ".render" +REUSE_MAX_RENDER_STATES: int = 2 REUSE_ENTRY_LENGTH_BYTES: int = 8 REUSE_ENTRY_CHECKSUM_BYTES: int = 4 REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" @@ -79,9 +81,10 @@ MILLISECONDS_PER_SECOND: int = 1000 REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 REUSE_STORE_NOTICE_PATHS: int = 100_000 +REUSE_STORE_NOTICE_RENDERS: int = 4_000 BYTES_PER_MEBIBYTE: int = 1024 * 1024 REUSE_STORE_START_MESSAGE: str = ( - "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." + "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash, {renders} renders)..." ) REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" REUSE_STORE_SKIPPED_MESSAGE: str = ( diff --git a/src/sqlbuild/cli/compile_reuse/main/_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_framed_section.py new file mode 100644 index 0000000000..9259128048 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_framed_section.py @@ -0,0 +1,11 @@ +"""Public entry for framing one stored section with its length and checksum.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_reuse._helpers.entry_file import framed_section + + +def framed_stored_section(*, data: bytes) -> bytes: + """Prefix one section with its length and checksum so a reader can verify it.""" + + return framed_section(data=data) diff --git a/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py new file mode 100644 index 0000000000..08583cd0f7 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py @@ -0,0 +1,13 @@ +"""Public entry for reading one framed stored section.""" + +from __future__ import annotations + +from typing import BinaryIO + +from sqlbuild.cli.compile_reuse._helpers.entry_file import read_framed_section + + +def read_stored_section(*, handle: BinaryIO) -> bytes: + """Read one length- and checksum-framed section, raising when it is damaged.""" + + return read_framed_section(handle=handle) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py index 0736318972..0fbf395025 100644 --- a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py +++ b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py @@ -4,6 +4,7 @@ from pathlib import Path +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput @@ -22,6 +23,7 @@ def write_reusable_compile( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, + render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable full compile, or drop the stored one when it cannot be reused.""" @@ -35,4 +37,5 @@ def write_reusable_compile( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, + render_reuse=render_reuse, ) diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index cc2bd55c4b..12d6acfc49 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -93,6 +93,7 @@ class StoredCompileOutput: stdout_length: int stdout_checksum: int stdout_file: str + render_state_file: str | None = None @dataclass(frozen=True) @@ -128,3 +129,15 @@ class CompileReuseAttempt: snapshot_ns: int = 0 digests: dict[str, str] = field(default_factory=dict) restamped: frozenset[str] = frozenset() + changed_paths: frozenset[str] | None = None + render_state_path: Path | None = None + prior_entry: bool = False + prior_modules: tuple[tuple[str, int, int], ...] = () + + +@dataclass(frozen=True) +class RenderStateLayer: + """One render layer to publish: its framed chunks, and the base layer it extends.""" + + chunks: tuple[bytes | memoryview, ...] + base_file: str | None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index b2737a43b7..4234550ba8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -2,7 +2,9 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass, field +from functools import partial from pathlib import Path from sqlbuild.compiler.auditing.constants import ( @@ -35,10 +37,16 @@ render_parameterized_sql, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics +from sqlbuild.compiler.compile._helpers.render.reuse import reused_or_rendered from sqlbuild.compiler.compile._helpers.render.sql_vars import ( expand_authored_sql_result, ) -from sqlbuild.compiler.compile.constants import SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import ( + RENDER_REUSE_ATTACHED_AUDITS_GROUP, + RENDER_REUSE_SINGULAR_AUDITS_GROUP, + SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -127,6 +135,7 @@ def build_project_audit_inputs( model_inputs: tuple[CompileModelInput, ...], source_inputs: tuple[CompileSourceInput, ...], seed_inputs: tuple[CompileSeedInput, ...], + render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: """Resolve built-in-aware generic audits and build every project audit input.""" @@ -142,6 +151,7 @@ def build_project_audit_inputs( source_inputs=source_inputs, generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, + render_reuse=render_reuse, ) return audit_inputs, diagnostics @@ -155,6 +165,7 @@ def build_audit_inputs( generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] | None = None, seed_inputs: tuple[CompileSeedInput, ...] = (), + render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileAuditInput, ...]: """Build compile-time audit inputs from discovered SQL audit blocks.""" @@ -184,6 +195,57 @@ def build_audit_inputs( declaration_expansion=declaration_expansion, sql_lexical_syntax=sql_lexical_syntax, ) + audit_inputs: list[CompileAuditInput] = list( + reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SINGULAR_AUDITS_GROUP, + render=partial( + _build_singular_audit_inputs, + discovered_inputs=discovered_inputs, + context=attachment_context, + ), + ) + ) + model_input: CompileModelInput + for model_input in model_inputs: + if model_input.schema_entry is None: + continue + render: Callable[[], tuple[CompileAuditInput, ...]] = partial( + build_model_attached_audit_inputs, model_input=model_input, context=attachment_context + ) + audit_inputs.extend( + render() + if render_reuse is None + else render_reuse.model_audits(model_input=model_input, render=render) + ) + audit_inputs.extend( + reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_ATTACHED_AUDITS_GROUP, + render=partial( + _build_source_and_seed_audit_inputs, + source_inputs=source_inputs, + seed_inputs=seed_inputs, + context=attachment_context, + ), + ) + ) + return tuple(audit_inputs) + + +def _build_singular_audit_inputs( + *, discovered_inputs: DiscoveredProjectInputs, context: _AuditAttachmentContext +) -> tuple[CompileAuditInput, ...]: + attachment_context: _AuditAttachmentContext = context + effective_vars: dict[str, object] = context.effective_vars + macro_context: MacroContext = context.macro_context + loaded_macros: dict[str, LoadedMacro] = context.loaded_macros + sql_lexical_syntax: SqlLexicalSyntax = context.sql_lexical_syntax + known_model_names: set[str] = context.known_model_names + known_seed_names: set[str] = context.known_seed_names + known_source_names: set[str] = context.known_source_names + default_audit_severity: AuditSeverity | None = context.default_audit_severity + default_audit_run_scope: str | None = context.default_audit_run_scope audit_inputs: list[CompileAuditInput] = [] audit_file: DiscoveredAuditFile for audit_file in discovered_inputs.audit_files: @@ -326,28 +388,28 @@ def build_audit_inputs( ), ) ) - model_input: CompileModelInput - for model_input in model_inputs: - if model_input.schema_entry is None: - continue - audit_inputs.extend( - build_model_attached_audit_inputs( - model_input=model_input, - context=attachment_context, - ) - ) + return tuple(audit_inputs) + + +def _build_source_and_seed_audit_inputs( + *, + source_inputs: tuple[CompileSourceInput, ...], + seed_inputs: tuple[CompileSeedInput, ...], + context: _AuditAttachmentContext, +) -> tuple[CompileAuditInput, ...]: + audit_inputs: list[CompileAuditInput] = [] source_input: CompileSourceInput for source_input in source_inputs: audit_inputs.extend( build_source_attached_audit_inputs( source_input=source_input, - context=attachment_context, + context=context, ) ) seed_input: CompileSeedInput for seed_input in seed_inputs: audit_inputs.extend( - build_seed_attached_audit_inputs(seed_input=seed_input, context=attachment_context) + build_seed_attached_audit_inputs(seed_input=seed_input, context=context) ) return tuple(audit_inputs) diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c2260c1862..6e3fb87045 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -6,6 +6,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field, fields, replace +from functools import partial from inspect import Parameter, Signature from pathlib import Path from typing import Any, cast @@ -95,6 +96,7 @@ expand_effective_vars, expand_template_data, ) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_FULL_REFRESH_CONFIG_KEY, @@ -255,6 +257,19 @@ class _ModelValidationContext: custom_materialization_names: frozenset[str] +@dataclass(frozen=True) +class _ModelInputLoop: + discovered_inputs: DiscoveredProjectInputs + context: ModelInputBuildContext + validation_context: _ModelValidationContext + sql_hook_definitions: dict[str, DiscoveredSqlHookFile] + legacy_schema_files: tuple[DiscoveredSchemaFile, ...] + model_header_column_cache: ModelHeaderColumnCache + config_scan_cache: ModelConfigScanCache + reusable_config_cache: _ReusableModelConfigCache + declaration_cache: _VisibleModelDeclarationCache + + @dataclass(frozen=True) class _HookExpansionContext: file_path: Path @@ -414,6 +429,7 @@ def build_model_inputs( defer_model_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, reference_cache_dir: Path | None = None, + render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileModelInput, ...]: """Attach schema metadata to discovered model files.""" @@ -431,6 +447,7 @@ def build_model_inputs( external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, legacy_schema_files=legacy_schema_files, + render_reuse=render_reuse, ) @@ -443,15 +460,11 @@ def _build_model_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, extract_references: Callable[[str], tuple[CompileSqlReference, ...]], legacy_schema_files: tuple[DiscoveredSchemaFile, ...], + render_reuse: CompileRenderReuseSession | None, ) -> tuple[CompileModelInput, ...]: effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings - target_config: TargetConfig | None = context.target_config - effective_target_name: str | None = context.effective_target_name - run_id: str = context.run_id - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -479,305 +492,352 @@ def _build_model_inputs( sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( discovered_inputs.sql_hook_files ) - model_inputs: list[CompileModelInput] = [] - model_header_column_cache: ModelHeaderColumnCache = ModelHeaderColumnCache() - config_scan_cache: ModelConfigScanCache = ModelConfigScanCache() - reusable_config_cache: _ReusableModelConfigCache = _ReusableModelConfigCache( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - target_config=target_config, + render_files: tuple[DiscoveredSqlModelFile, ...] = tuple( + model_file + for model_file in discovered_inputs.model_files + if render_reuse is None or not render_reuse.has_reusable_model(model_file=model_file) ) - declaration_cache: _VisibleModelDeclarationCache = _VisibleModelDeclarationCache.build(context) - model_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files - prepared_var_substituted_sqls: tuple[str | None, ...] = prepare_static_project_vars_batch( - sqls=tuple(model_file.query_sql for model_file in model_files), - effective_vars=effective_vars, + prepared_var_substituted_sqls: dict[Path, str | None] = dict( + zip( + (model_file.file_path for model_file in render_files), + prepare_static_project_vars_batch( + sqls=tuple(model_file.query_sql for model_file in render_files), + effective_vars=effective_vars, + ), + strict=True, + ) ) + loop: _ModelInputLoop = _ModelInputLoop( + discovered_inputs=discovered_inputs, + context=context, + validation_context=validation_context, + sql_hook_definitions=sql_hook_definitions, + legacy_schema_files=legacy_schema_files, + model_header_column_cache=ModelHeaderColumnCache(), + config_scan_cache=ModelConfigScanCache(), + reusable_config_cache=_ReusableModelConfigCache( + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, + target_config=context.target_config, + ), + declaration_cache=_VisibleModelDeclarationCache.build(context), + ) + model_inputs: list[CompileModelInput] = [] model_file: DiscoveredSqlModelFile - prepared_var_substituted_sql: str | None - for model_file, prepared_var_substituted_sql in zip( - model_files, prepared_var_substituted_sqls, strict=True - ): - model_identity: ResourceIdentity = ResourceIdentity( - ResourceKind.MODEL, model_file.file_path.stem - ) - declarations: _VisibleModelDeclarations = declaration_cache.for_model( - model_file=model_file, consumer=model_identity + for model_file in discovered_inputs.model_files: + reused: CompileModelInput | None = ( + None if render_reuse is None else render_reuse.reused_model(model_file=model_file) ) - matched_path_default: str | None = find_matching_path_default( + if reused is not None: + model_inputs.append(reused) + continue + render: Callable[[], CompileModelInput] = partial( + _build_model_input, + loop=loop, model_file=model_file, - path_defaults=discovered_inputs.project_config.path_defaults, + prepared_var_substituted_sql=prepared_var_substituted_sqls.get(model_file.file_path), ) - effective_config: CompileModelConfig | None = reusable_config_cache.get( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, + model_inputs.append( + render() + if render_reuse is None + else render_reuse.rendered_model(model_file=model_file, render=render) ) - if effective_config is None: - effective_config = build_model_config( - request=ModelConfigBuildRequest( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - effective_vars=effective_vars, - target_config=target_config, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - materialization_defaults=( - discovered_inputs.project_config.materialization_defaults - ), - scan_cache=config_scan_cache, - ) - ) - reusable_config_cache.remember( + + validate_declared_schema_models_are_attached( + model_inputs=tuple(model_inputs), + schema_files=discovered_inputs.schema_files, + ) + return tuple(model_inputs) + + +def _build_model_input( + *, + loop: _ModelInputLoop, + model_file: DiscoveredSqlModelFile, + prepared_var_substituted_sql: str | None, +) -> CompileModelInput: + discovered_inputs: DiscoveredProjectInputs = loop.discovered_inputs + context: ModelInputBuildContext = loop.context + validation_context: _ModelValidationContext = loop.validation_context + sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = loop.sql_hook_definitions + legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = loop.legacy_schema_files + model_header_column_cache: ModelHeaderColumnCache = loop.model_header_column_cache + config_scan_cache: ModelConfigScanCache = loop.config_scan_cache + reusable_config_cache: _ReusableModelConfigCache = loop.reusable_config_cache + declaration_cache: _VisibleModelDeclarationCache = loop.declaration_cache + effective_vars: dict[str, object] = context.effective_vars + target_config: TargetConfig | None = context.target_config + effective_target_name: str | None = context.effective_target_name + run_id: str = context.run_id + macro_context: MacroContext = context.macro_context + loaded_macros: dict[str, LoadedMacro] = context.loaded_macros + model_identity: ResourceIdentity = ResourceIdentity( + ResourceKind.MODEL, model_file.file_path.stem + ) + declarations: _VisibleModelDeclarations = declaration_cache.for_model( + model_file=model_file, consumer=model_identity + ) + matched_path_default: str | None = find_matching_path_default( + model_file=model_file, + path_defaults=discovered_inputs.project_config.path_defaults, + ) + effective_config: CompileModelConfig | None = reusable_config_cache.get( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + ) + if effective_config is None: + effective_config = build_model_config( + request=ModelConfigBuildRequest( + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, matched_path_default=matched_path_default, model_header_values=model_file.header_values, - config=effective_config, + effective_vars=effective_vars, + target_config=target_config, + model_name=model_file.file_path.stem, + effective_target_name=effective_target_name, + run_id=run_id, + materialization_defaults=( + discovered_inputs.project_config.materialization_defaults + ), + scan_cache=config_scan_cache, ) - model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( - values=effective_config.values, - model_name=model_file.file_path.stem, - public_model_schemas=context.public_model_schemas, - ) - model_schema_columns: tuple[SchemaColumn, ...] | None = ( - model_schema.columns if model_schema is not None else None ) - validate_python_hook_config( - values=effective_config.values, - model_name=model_file.file_path.stem, - hook_functions=discovered_inputs.hook_functions, - provider_names=frozenset(provider.name for provider in discovered_inputs.providers), + reusable_config_cache.remember( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + config=effective_config, ) - named_usages: tuple[UsageRecord, ...] = ( - *( - named_declaration_usages( - resolver=context.declaration_resolver, - kind=DeclarationKind.SCHEMA, - name=model_schema.name, - consumer=model_identity, - consumer_path=model_file.relative_path, - ) - if model_schema is not None - else () - ), - *_python_hook_usages( - values=effective_config.values, + model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( + values=effective_config.values, + model_name=model_file.file_path.stem, + public_model_schemas=context.public_model_schemas, + ) + model_schema_columns: tuple[SchemaColumn, ...] | None = ( + model_schema.columns if model_schema is not None else None + ) + validate_python_hook_config( + values=effective_config.values, + model_name=model_file.file_path.stem, + hook_functions=discovered_inputs.hook_functions, + provider_names=frozenset(provider.name for provider in discovered_inputs.providers), + ) + named_usages: tuple[UsageRecord, ...] = ( + *( + named_declaration_usages( resolver=context.declaration_resolver, + kind=DeclarationKind.SCHEMA, + name=model_schema.name, consumer=model_identity, consumer_path=model_file.relative_path, - ), - ) - var_substituted_sql: str = ( - prepared_var_substituted_sql - if prepared_var_substituted_sql is not None - else substitute_sql_vars( - sql=model_file.query_sql, - file_path=model_file.file_path, - effective_vars=effective_vars, ) - ) - declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - enum_visibility=declarations.enum_visibility, - constant_visibility=declarations.constant_visibility, - macros=declarations.macros, - macro_records=declarations.macro_records, - inaccessible_macros=declarations.inaccessible_macros, + if model_schema is not None + else () + ), + *_python_hook_usages( + values=effective_config.values, + resolver=context.declaration_resolver, consumer=model_identity, - ) - declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( - sql=var_substituted_sql, - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) - declaration_expanded_sql: str = declaration_expansion.sql - macro_expansion: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_expanded_sql, + consumer_path=model_file.relative_path, + ), + ) + var_substituted_sql: str = ( + prepared_var_substituted_sql + if prepared_var_substituted_sql is not None + else substitute_sql_vars( + sql=model_file.query_sql, file_path=model_file.file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=context.declaration_resolver, - declarations=( - declaration_context if context.declaration_resolver is not None else None - ), - consumer=model_identity, - ) - expanded_query_sql: str = macro_expansion.sql - expanded_query_sql = get_validated_model_cursor_intrinsics( - sql=expanded_query_sql, - config_values=effective_config.values, - model_name=model_file.file_path.stem, - ) - raw_placeholders: object | None = effective_config.values.get("placeholders") - sql_validation_placeholders: dict[str, str] | None = ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - sql_validation_enabled, references, rejected_opt_out = _validate_model_input( - context=validation_context, - model_file=model_file, - config=effective_config, - expanded_query_sql=expanded_query_sql, - sql_validation_placeholders=sql_validation_placeholders, - model_schema_columns=model_schema_columns, - argument_references=macro_expansion.argument_references, + effective_vars=effective_vars, ) - hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + ) + declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, + enum_visibility=declarations.enum_visibility, + constant_visibility=declarations.constant_visibility, + macros=declarations.macros, + macro_records=declarations.macro_records, + inaccessible_macros=declarations.inaccessible_macros, + consumer=model_identity, + ) + declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( + sql=var_substituted_sql, + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + ) + declaration_expanded_sql: str = declaration_expansion.sql + macro_expansion: MacroExpansionResult = expand_sql_macros_result( + sql=declaration_expanded_sql, + file_path=model_file.file_path, + loaded_macros=loaded_macros, + macro_context=macro_context, + declaration_resolver=context.declaration_resolver, + declarations=(declaration_context if context.declaration_resolver is not None else None), + consumer=model_identity, + ) + expanded_query_sql: str = macro_expansion.sql + expanded_query_sql = get_validated_model_cursor_intrinsics( + sql=expanded_query_sql, + config_values=effective_config.values, + model_name=model_file.file_path.stem, + ) + raw_placeholders: object | None = effective_config.values.get("placeholders") + sql_validation_placeholders: dict[str, str] | None = ( + {str(k): str(v) for k, v in raw_placeholders.items()} + if isinstance(raw_placeholders, dict) + else None + ) + sql_validation_enabled, references, rejected_opt_out = _validate_model_input( + context=validation_context, + model_file=model_file, + config=effective_config, + expanded_query_sql=expanded_query_sql, + sql_validation_placeholders=sql_validation_placeholders, + model_schema_columns=model_schema_columns, + argument_references=macro_expansion.argument_references, + ) + hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + values=effective_config.values, + file_path=model_file.file_path, + effective_vars=effective_vars, + context_values=build_model_context_values( values=effective_config.values, - file_path=model_file.file_path, - effective_vars=effective_vars, - context_values=build_model_context_values( - values=effective_config.values, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=True, + model_name=model_file.file_path.stem, + effective_target_name=effective_target_name, + run_id=run_id, + include_target_values=True, + ), + loaded_macros=loaded_macros, + macro_context=macro_context, + declaration_expansion=DeclarationExpansionContext( + declarations=DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, ), - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + resolver=context.declaration_resolver, + ), + sql_lexical_syntax=context.sql_lexical_syntax, + sql_hook_definitions=sql_hook_definitions, + consumer=model_identity, + ) + expanded_config: CompileModelConfig = _hook_expanded_config( + config=effective_config, + hook_expansion=hook_expansion, + model_file=model_file, + validation_context=validation_context, + ) + hook_name: str + for hook_name in ("pre_hooks", "post_hooks"): + hook_value: object | None = expanded_config.values.get(hook_name) + if isinstance(hook_value, list | tuple): + hook_entry: object + for hook_entry in hook_value: + if isinstance(hook_entry, SqlHookEntry): + reject_cursor_intrinsics( + sql=hook_entry.statement, + context=f"Model '{model_file.file_path.stem}' {hook_name}", + ) + header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( + model_name=model_file.file_path.stem, + model_header_values=expanded_config.values, + file_path=model_file.relative_path, + column_locations=model_file.header_column_locations, + model_schema_columns=model_schema_columns, + model_schema_name=model_schema.name if model_schema is not None else None, + model_schema_description=model_schema.description if model_schema is not None else None, + audit_factories=discovered_inputs.audit_factories, + column_cache=model_header_column_cache, + ) + model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) + header_schema_entry, enum_columns = resolve_enum_contract_columns( + schema_entry=header_schema_entry, + config_values=model_config.values, + enums=( + declarations.enums + if model_schema is None + else { + **declarations.enums, + **model_schema_enum_declarations( + schema=model_schema, resolver=context.declaration_resolver ), - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - resolver=context.declaration_resolver, - ), - sql_lexical_syntax=context.sql_lexical_syntax, - sql_hook_definitions=sql_hook_definitions, - consumer=model_identity, - ) - expanded_config: CompileModelConfig = _hook_expanded_config( - config=effective_config, - hook_expansion=hook_expansion, - model_file=model_file, - validation_context=validation_context, - ) - hook_name: str - for hook_name in ("pre_hooks", "post_hooks"): - hook_value: object | None = expanded_config.values.get(hook_name) - if isinstance(hook_value, list | tuple): - hook_entry: object - for hook_entry in hook_value: - if isinstance(hook_entry, SqlHookEntry): - reject_cursor_intrinsics( - sql=hook_entry.statement, - context=f"Model '{model_file.file_path.stem}' {hook_name}", - ) - header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( - model_name=model_file.file_path.stem, - model_header_values=expanded_config.values, - file_path=model_file.relative_path, - column_locations=model_file.header_column_locations, - model_schema_columns=model_schema_columns, - model_schema_name=model_schema.name if model_schema is not None else None, - model_schema_description=model_schema.description if model_schema is not None else None, - audit_factories=discovered_inputs.audit_factories, - column_cache=model_header_column_cache, + } + ), + ) + generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( + enum_columns=enum_columns, + declarations=declarations, + consumer=model_identity, + ) + model_declaration_usages: tuple[UsageRecord, ...] = tuple( + dict.fromkeys( + ( + *declaration_expansion.usages, + *generated_usages, + *hook_expansion.usages, + *named_usages, + ) ) - model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) - header_schema_entry, enum_columns = resolve_enum_contract_columns( - schema_entry=header_schema_entry, - config_values=model_config.values, - enums=( - declarations.enums - if model_schema is None - else { - **declarations.enums, - **model_schema_enum_declarations( - schema=model_schema, resolver=context.declaration_resolver - ), - } + ) + _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) + if header_schema_entry is None: + return CompileModelInput( + sql_expansion=( + CompiledSqlExpansion( + authored_sql=model_file.query_sql, + expanded_sql=macro_expansion.sql, + passes=(declaration_expansion.spans, macro_expansion.spans), + ) + if var_substituted_sql == model_file.query_sql + else None ), - ) - generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( + model_file=model_file, + config=model_config, + query_sql=expanded_query_sql, + macro_source_sql=declaration_expanded_sql, + references=references, + sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, + enum_declarations=tuple(declarations.local_enums.values()), + constant_declarations=tuple(declarations.local_constants.values()), enum_columns=enum_columns, - declarations=declarations, - consumer=model_identity, - ) - model_declaration_usages: tuple[UsageRecord, ...] = tuple( - dict.fromkeys( - ( - *declaration_expansion.usages, - *generated_usages, - *hook_expansion.usages, - *named_usages, - ) - ) + macro_deps=tuple(item.name for item in macro_expansion.dependencies), + macro_usages=macro_expansion.usages, + declaration_usages=model_declaration_usages, ) - _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) - if header_schema_entry is None: - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None - ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, - ) - ) - continue - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None - ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - schema_entry=header_schema_entry, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, + return CompileModelInput( + sql_expansion=( + CompiledSqlExpansion( + authored_sql=model_file.query_sql, + expanded_sql=macro_expansion.sql, + passes=(declaration_expansion.spans, macro_expansion.spans), ) - ) - - validate_declared_schema_models_are_attached( - model_inputs=tuple(model_inputs), - schema_files=discovered_inputs.schema_files, + if var_substituted_sql == model_file.query_sql + else None + ), + model_file=model_file, + config=model_config, + query_sql=expanded_query_sql, + macro_source_sql=declaration_expanded_sql, + references=references, + schema_entry=header_schema_entry, + sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, + enum_declarations=tuple(declarations.local_enums.values()), + constant_declarations=tuple(declarations.local_constants.values()), + enum_columns=enum_columns, + macro_deps=tuple(item.name for item in macro_expansion.dependencies), + macro_usages=macro_expansion.usages, + declaration_usages=model_declaration_usages, ) - return tuple(model_inputs) def _hook_expanded_config( diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index ae6080acb7..9a7ee0f036 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -21,6 +21,23 @@ _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None ) +_ACTIVE_TAPS: ContextVar[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = ContextVar( + "sqlbuild_tapped_compile_diagnostics", default=() +) + + +@contextmanager +def tapped_compile_diagnostics() -> Iterator[list[tuple[tuple[str, ...], CompilerDiagnostic]]]: + """Also note every diagnostic reported inside the block, in report order, for replay.""" + + tap: list[tuple[tuple[str, ...], CompilerDiagnostic]] = [] + token: Token[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = _ACTIVE_TAPS.set( + (*_ACTIVE_TAPS.get(), tap) + ) + try: + yield tap + finally: + _ACTIVE_TAPS.reset(token) @contextmanager @@ -54,6 +71,8 @@ def collecting(*args: P.args, **kwargs: P.kwargs) -> R: def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: """Record a diagnostic in the active collector, or raise it when none is active.""" + for tap in _ACTIVE_TAPS.get(): + tap.append((key, diagnostic)) collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() if collector is not None: collector.add(key=key, diagnostic=diagnostic) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/reuse.py b/src/sqlbuild/compiler/compile/_helpers/render/reuse.py new file mode 100644 index 0000000000..70132c44da --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/render/reuse.py @@ -0,0 +1,111 @@ +"""Route project-wide renders through the active render reuse session, if any.""" + +from __future__ import annotations + +from collections.abc import Callable +from functools import partial +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs +from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( + build_scenario_inputs, + build_test_inputs_with_cache, +) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import ( + COMPILE_RENDER_REUSE, + RENDER_REUSE_SCENARIOS_GROUP, + RENDER_REUSE_SOURCES_GROUP, + RENDER_REUSE_TESTS_GROUP, +) +from sqlbuild.compiler.compile.models import ( + CompileSourceInput, + CompileSqlFunctionInput, + CompileSqlScenarioInput, + CompileSqlTestInput, + ModelInputBuildContext, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver + + +def reused_or_rendered[T]( + *, render_reuse: CompileRenderReuseSession | None, name: str, render: Callable[[], T] +) -> T: + """Return the stored render of one project-wide group, or render and record it.""" + + return render() if render_reuse is None else render_reuse.group(name=name, render=render) + + +def claimed_render_reuse( + *, discovered_inputs: DiscoveredProjectInputs +) -> CompileRenderReuseSession | None: + """Claim the active render reuse session, if any, and plan it for the discovered models.""" + + render_reuse: CompileRenderReuseSession | None = COMPILE_RENDER_REUSE.claim() + if render_reuse is not None: + render_reuse.plan_models(model_files=discovered_inputs.model_files) + return render_reuse + + +def build_sql_resource_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + sql_function_inputs: tuple[CompileSqlFunctionInput, ...], + no_sql_validation: bool, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, + compile_cache_dir: Path | None, + render_reuse: CompileRenderReuseSession | None, +) -> tuple[ + tuple[CompileSourceInput, ...], + tuple[CompileSqlTestInput, ...], + tuple[CompileSqlScenarioInput, ...], +]: + """Render sources, tests, and scenarios, each reused whole after model-only edits.""" + + source_inputs: tuple[CompileSourceInput, ...] = reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SOURCES_GROUP, + render=partial( + build_source_inputs, + discovered_inputs=discovered_inputs, + effective_vars=context.effective_vars, + effective_settings=context.effective_settings, + macro_context=context.macro_context, + loaded_macros=context.loaded_macros, + declaration_expansion=context.declaration_expansion, + no_sql_validation=no_sql_validation, + ), + ) + test_inputs: tuple[CompileSqlTestInput, ...] = reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_TESTS_GROUP, + render=partial( + build_test_inputs_with_cache, + discovered_inputs=discovered_inputs, + effective_vars=context.effective_vars, + macro_context=context.macro_context, + loaded_macros=context.loaded_macros, + declaration_expansion=context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_function_inputs=sql_function_inputs, + compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=context.sql_lexical_syntax, + ), + ) + scenario_inputs: tuple[CompileSqlScenarioInput, ...] = reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SCENARIOS_GROUP, + render=partial( + build_scenario_inputs, + discovered_inputs=discovered_inputs, + effective_vars=context.effective_vars, + macro_context=context.macro_context, + loaded_macros=context.loaded_macros, + declaration_expansion=context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=context.sql_lexical_syntax, + ), + ) + return source_inputs, test_inputs, scenario_inputs diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py b/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py new file mode 100644 index 0000000000..c6a35d2e99 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py @@ -0,0 +1,41 @@ +"""Process-wide access to the render reuse session of the running compile command.""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession + + +class CompileRenderReuseRegistry: + """Expose one compile command's render reuse session to the rendering code it runs.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._session: CompileRenderReuseSession | None = None + + @contextmanager + def activated(self, session: CompileRenderReuseSession) -> Iterator[CompileRenderReuseSession]: + """Make the session available to rendering until the block exits.""" + + with self._lock: + previous: CompileRenderReuseSession | None = self._session + self._session = session + try: + yield session + finally: + with self._lock: + self._session = previous + + def claim(self) -> CompileRenderReuseSession | None: + """Hand the active session to the first project render only, so it is used once.""" + + with self._lock: + session: CompileRenderReuseSession | None = self._session + if session is None or not session.claim(): + return None + return session diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py new file mode 100644 index 0000000000..f011a76c43 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py @@ -0,0 +1,290 @@ +"""Reuse unaffected rendered inputs from the previous compile and record this compile's.""" + +from __future__ import annotations + +import pickle +import threading +from collections.abc import Callable +from dataclasses import replace +from typing import cast + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( + report_compile_diagnostic, + tapped_compile_diagnostics, +) +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads +from sqlbuild.compiler.compile.constants import ( + COMPILE_INPUT_READS, + RENDER_REUSE_MODEL_AUDITS_PREFIX, +) +from sqlbuild.compiler.compile.models import ( + CompileAuditInput, + CompileModelInput, + CompilerDiagnostic, + RenderReuseState, + StoredRender, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError +from sqlbuild.compiler.fact_cache.main._dump_fact_payload import dumped_fact_payload +from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload +from sqlbuild.compiler.profiling.main._metric import record_compile_metric + +_LOAD_ERRORS: tuple[type[Exception], ...] = ( + FactCachePayloadError, + pickle.UnpicklingError, + AttributeError, + EOFError, + ImportError, + IndexError, + KeyError, + TypeError, + ValueError, + RecursionError, + MemoryError, + OverflowError, +) + + +class CompileRenderReuseSession: + """Serve stored renders of inputs a change set cannot affect, and record every render.""" + + def __init__( + self, + *, + prior: RenderReuseState | None, + changed_paths: frozenset[str] | None, + retained_models: frozenset[str] = frozenset(), + retained_groups: frozenset[str] = frozenset(), + ) -> None: + self._lock: threading.Lock = threading.Lock() + self._claimed: bool = False + self._prior: RenderReuseState | None = prior if changed_paths is not None else None + self._changed_paths: frozenset[str] = changed_paths or frozenset() + self._reusable: bool = False + self._model_paths: tuple[str, ...] | None = None + self._retained_models: frozenset[str] = retained_models + self._retained_groups: frozenset[str] = retained_groups + self._model_records: dict[str, memoryview | None] = {} + self._group_records: dict[str, memoryview | None] = {} + self._decoded_models: dict[str, StoredRender] = {} + self._decoded_groups: dict[str, StoredRender] = {} + self._pending_models: list[tuple[str, StoredRender]] = [] + self._pending_groups: list[tuple[str, StoredRender]] = [] + self._serialized_models: dict[str, memoryview] = {} + self._serialized_groups: dict[str, memoryview] = {} + self._reused_paths: set[str] = set() + + def claim(self) -> bool: + """Return True for the first claim only; later project renders neither reuse nor record.""" + + with self._lock: + claimed: bool = self._claimed + self._claimed = True + return not claimed + + def plan_models(self, *, model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: + """Reuse stored renders only when the change set touches nothing but existing models.""" + + model_paths: tuple[str, ...] = tuple(str(item.relative_path) for item in model_files) + self._model_paths = model_paths + prior: RenderReuseState | None = self._prior + self._reusable = ( + prior is not None + and prior.model_paths == model_paths + and self._changed_paths.issubset(model_paths) + ) + + def has_reusable_model(self, *, model_file: DiscoveredSqlModelFile) -> bool: + """Return whether a stored render of this model may stand in for rendering it.""" + + path: str = str(model_file.relative_path) + return ( + self._reusable + and self._prior is not None + and path not in self._changed_paths + and path in self._prior.model_payloads + ) + + def reused_model(self, *, model_file: DiscoveredSqlModelFile) -> CompileModelInput | None: + """Return the stored render of an unchanged model, replaying what rendering reported.""" + + if not self.has_reusable_model(model_file=model_file) or self._prior is None: + return None + path: str = str(model_file.relative_path) + payload: memoryview | None = self._prior.model_payloads[path] + stored: StoredRender | None = None if payload is None else _loaded_render(payload) + if stored is None or not isinstance(stored.value, CompileModelInput): + return None + _replay(stored=stored) + self._model_records[path] = payload + self._decoded_models[path] = stored + self._reused_paths.add(path) + record_compile_metric(metric="render_reuse_hits", value=1) + return replace(stored.value, model_file=model_file) + + def rendered_model( + self, + *, + model_file: DiscoveredSqlModelFile, + render: Callable[[], CompileModelInput], + ) -> CompileModelInput: + """Render one model and record its render with what rendering reported and read.""" + + with ( + COMPILE_INPUT_READS.recording() as reads, + tapped_compile_diagnostics() as reported, + ): + model_input: CompileModelInput = render() + record_compile_metric(metric="render_reuse_misses", value=1) + stored: StoredRender | None = _stored_render( + reads=reads, value=replace(model_input, model_file=None), reported=reported + ) + if stored is not None: + self._pending_models.append((str(model_file.relative_path), stored)) + return model_input + + def model_audits( + self, + *, + model_input: CompileModelInput, + render: Callable[[], tuple[CompileAuditInput, ...]], + ) -> tuple[CompileAuditInput, ...]: + """Return a reused model's stored attached audits, or render and record them.""" + + path: str = str(model_input.model_file.relative_path) + return self.group( + name=f"{RENDER_REUSE_MODEL_AUDITS_PREFIX}{path}", + render=render, + reusable=path in self._reused_paths, + ) + + def group[T](self, *, name: str, render: Callable[[], T], reusable: bool = True) -> T: + """Return a stored project-wide render no model edit can affect, or render and record it.""" + + payload: memoryview | None = ( + self._prior.group_payloads.get(name) + if reusable and self._reusable and self._prior is not None + else None + ) + stored: StoredRender | None = None if payload is None else _loaded_render(payload) + if stored is not None and payload is not None: + _replay(stored=stored) + self._group_records[name] = payload + self._decoded_groups[name] = stored + return cast(T, stored.value) + with ( + COMPILE_INPUT_READS.recording() as reads, + tapped_compile_diagnostics() as reported, + ): + value: T = render() + rendered: StoredRender | None = _stored_render(reads=reads, value=value, reported=reported) + if rendered is not None: + self._pending_groups.append((name, rendered)) + return value + + def pending_renders(self) -> int: + """Return how many new renders storing this compile still has to serialize.""" + + return len(self._pending_models) + len(self._pending_groups) + + def reused_any(self) -> bool: + """Return whether any render was reused, so imports it made were skipped this run.""" + + return bool(self._decoded_models or self._decoded_groups) + + def release_stored(self) -> None: + """Drop stored base bytes once every render is decoded, recording reused renders as None.""" + + self._prior = None + self._reusable = False + for records, retained in ( + (self._model_records, self._retained_models), + (self._group_records, self._retained_groups), + ): + for key in records.keys() - retained: + records[key] = None + + def stored_state(self, *, complete: bool = False) -> RenderReuseState | None: + """Serialize this compile's renders; with complete, re-serialize released reused ones.""" + + if self._model_paths is None: + return None + self.release_stored() + self._serialize_pending() + if complete: + self._model_records = _reserialized( + records=self._model_records, decoded=self._decoded_models + ) + self._group_records = _reserialized( + records=self._group_records, decoded=self._decoded_groups + ) + return RenderReuseState( + model_paths=self._model_paths, + model_payloads={**self._model_records, **self._serialized_models}, + group_payloads={**self._group_records, **self._serialized_groups}, + ) + + def _serialize_pending(self) -> None: + for pending, serialized in ( + (self._pending_models, self._serialized_models), + (self._pending_groups, self._serialized_groups), + ): + for key, stored in pending: + payload: bytes | None = _dumped(stored) + if payload is not None: + serialized[key] = memoryview(payload) + pending.clear() + + +def _reserialized( + *, records: dict[str, memoryview | None], decoded: dict[str, StoredRender] +) -> dict[str, memoryview | None]: + complete: dict[str, memoryview | None] = {} + key: str + payload: memoryview | None + for key, payload in records.items(): + if payload is None: + dumped: bytes | None = _dumped(decoded[key]) + if dumped is None: + continue + payload = memoryview(dumped) + complete[key] = payload + return complete + + +def _stored_render( + *, + reads: CompileInputReads, + value: object, + reported: list[tuple[tuple[str, ...], CompilerDiagnostic]], +) -> StoredRender | None: + """Keep one render for reuse unless it read the run identity or provider settings.""" + + if reads.read_run_id or reads.settings_classes: + return None + return StoredRender( + value=value, diagnostics=tuple(reported), environment_names=reads.environment_names + ) + + +def _dumped(stored: StoredRender) -> bytes | None: + try: + return dumped_fact_payload(stored) + except (pickle.PicklingError, TypeError, AttributeError, RecursionError): + return None + + +def _replay(*, stored: StoredRender) -> None: + for name in stored.environment_names: + COMPILE_INPUT_READS.environment_read(name) + for key, diagnostic in stored.diagnostics: + report_compile_diagnostic(key=key, diagnostic=diagnostic) + + +def _loaded_render(payload: memoryview) -> StoredRender | None: + try: + loaded: object = loaded_fact_payload(payload) + except _LOAD_ERRORS: + return None + return loaded if isinstance(loaded, StoredRender) else None diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index cabdcb1316..44905feecc 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( CompileInputReadRegistry, ) +from sqlbuild.compiler.compile.classes.render_reuse_registry import CompileRenderReuseRegistry from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind @@ -171,6 +172,13 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() +COMPILE_RENDER_REUSE: CompileRenderReuseRegistry = CompileRenderReuseRegistry() +RENDER_REUSE_SOURCES_GROUP: str = "sources" +RENDER_REUSE_TESTS_GROUP: str = "tests" +RENDER_REUSE_SCENARIOS_GROUP: str = "scenarios" +RENDER_REUSE_SINGULAR_AUDITS_GROUP: str = "singular_audits" +RENDER_REUSE_ATTACHED_AUDITS_GROUP: str = "source_and_seed_audits" +RENDER_REUSE_MODEL_AUDITS_PREFIX: str = "model_audits:" SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v2" SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 0ffd4eb089..af15698a4f 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -18,11 +18,6 @@ from sqlbuild.compiler.compile._helpers.attachment.references import ( validate_table_function_call_arities, ) -from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs -from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( - build_scenario_inputs, - build_test_inputs_with_cache, -) from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, @@ -39,6 +34,11 @@ build_public_model_schema_index, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile._helpers.render.reuse import ( + build_sql_resource_inputs, + claimed_render_reuse, +) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAuditInput, @@ -119,6 +119,9 @@ def build_compile_inputs( _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), ) resolved_run_id: str = resolve_run_id(selected_run_id=run_id) + render_reuse: CompileRenderReuseSession | None = claimed_render_reuse( + discovered_inputs=discovered_inputs + ) loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) declaration_scope: DeclarationScopeBuild = build_declaration_scope( discovered_inputs=discovered_inputs, @@ -146,6 +149,7 @@ def build_compile_inputs( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=compile_cache_dir, + render_reuse=render_reuse, ) model_context = model_build.context seed_inputs: tuple[CompileSeedInput, ...] = build_seed_inputs(discovered_inputs) @@ -159,34 +163,17 @@ def build_compile_inputs( adapter_context.python_functions_inherit_default_namespace ), ) - source_inputs: tuple[CompileSourceInput, ...] = build_source_inputs( + source_inputs: tuple[CompileSourceInput, ...] + test_inputs: tuple[CompileSqlTestInput, ...] + scenario_inputs: tuple[CompileSqlScenarioInput, ...] + source_inputs, test_inputs, scenario_inputs = build_sql_resource_inputs( discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - effective_settings=effective_settings, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, + context=model_context, + sql_function_inputs=sql_function_inputs, no_sql_validation=no_sql_validation, - ) - test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs_with_cache( - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, compile_cache_dir=compile_cache_dir, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - ) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, + render_reuse=render_reuse, ) audit_inputs: tuple[CompileAuditInput, ...] audit_diagnostics: tuple[CompilerDiagnostic, ...] @@ -196,7 +183,10 @@ def build_compile_inputs( model_inputs=model_build.inputs, source_inputs=source_inputs, seed_inputs=seed_inputs, + render_reuse=render_reuse, ) + if render_reuse is not None: + render_reuse.release_stored() diagnostics: tuple[CompilerDiagnostic, ...] = ( *audit_diagnostics, *model_build.diagnostics, @@ -284,6 +274,7 @@ def _build_models_with_declarations( defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, reference_cache_dir: Path | None, + render_reuse: CompileRenderReuseSession | None, ) -> ModelInputScopeBuild: public_enums: dict[str, EnumDeclaration] public_constants: dict[str, ConstantDeclaration] @@ -309,6 +300,7 @@ def _build_models_with_declarations( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=reference_cache_dir, + render_reuse=render_reuse, ) return ModelInputScopeBuild( inputs=model_inputs, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 93bfc4495e..e07afcffe8 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1579,3 +1579,21 @@ class SqlAnalysisOptOutRequest: query_sql: str placeholders: dict[str, str] | None project_config_path: Path + + +@dataclass(frozen=True) +class StoredRender: + """One stored render with the diagnostics it reported and environment variables it read.""" + + value: object + diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] + environment_names: tuple[str, ...] + + +@dataclass(frozen=True) +class RenderReuseState: + """Serialized renders of one compile by model path and group; None marks released bytes.""" + + model_paths: tuple[str, ...] + model_payloads: dict[str, memoryview | None] + group_payloads: dict[str, memoryview | None] diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py b/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py index 05b7a1bf8d..757e20e635 100644 --- a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py +++ b/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py @@ -44,7 +44,7 @@ def dump_fact_payload(value: object) -> bytes: return pickle.dumps(value, protocol=FACT_CACHE_PICKLE_PROTOCOL) -def load_fact_payload(payload: bytes) -> object: +def load_fact_payload(payload: bytes | memoryview) -> object: """Deserialize one verified cache payload through the restricted class allowlist.""" return _RestrictedUnpickler(io.BytesIO(payload)).load() diff --git a/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py new file mode 100644 index 0000000000..35f3cdd661 --- /dev/null +++ b/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py @@ -0,0 +1,11 @@ +"""Public entry for serializing one local cache value with the fact-cache codec.""" + +from __future__ import annotations + +from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import dump_fact_payload + + +def dumped_fact_payload(value: object) -> bytes: + """Serialize one cache value with the fixed fact-cache protocol.""" + + return dump_fact_payload(value) diff --git a/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py new file mode 100644 index 0000000000..1687bed1ed --- /dev/null +++ b/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py @@ -0,0 +1,11 @@ +"""Public entry for reading one local cache value through the restricted fact-cache codec.""" + +from __future__ import annotations + +from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import load_fact_payload + + +def loaded_fact_payload(payload: bytes | memoryview) -> object: + """Deserialize one verified cache payload through the restricted class allowlist.""" + + return load_fact_payload(payload) diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 6ca434213c..b969bb04cf 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -24,4 +24,6 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "render_reuse_hits", + "render_reuse_misses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index abb9028ae4..f3e9a1c2b4 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -24,4 +24,6 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "render_reuse_hits", + "render_reuse_misses", ] diff --git a/src/sqlbuild/lint/_helpers/project_files.py b/src/sqlbuild/lint/_helpers/project_files.py index 01e0f21c60..316ff051ae 100644 --- a/src/sqlbuild/lint/_helpers/project_files.py +++ b/src/sqlbuild/lint/_helpers/project_files.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Callable +from functools import cache from pathlib import Path from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -17,11 +19,14 @@ def collect_project_files( ) -> dict[Path, str]: """Return authored SQL files keyed by path, sharing identical text discovery already holds.""" + resolved_directory: Callable[[Path], Path] = cache(_resolved_path) if source_files is not None: return { file_path: contents for file_path, contents in source_files.items() - if selected_paths is None or file_path.resolve() in selected_paths + if selected_paths is None + or _resolved_file(path=file_path, resolved_directory=resolved_directory) + in selected_paths } discovered: dict[Path, str] = ( {} if discovered_inputs is None else discovered_sql_contents(discovered_inputs) @@ -34,7 +39,11 @@ def collect_project_files( continue file_path: Path for file_path in sorted(root.rglob("*.sql")): - if selected_paths is not None and file_path.resolve() not in selected_paths: + if ( + selected_paths is not None + and _resolved_file(path=file_path, resolved_directory=resolved_directory) + not in selected_paths + ): continue with file_path.open("r", encoding="utf-8", newline="") as handle: contents: str = handle.read() @@ -43,6 +52,18 @@ def collect_project_files( return files +def _resolved_file(*, path: Path, resolved_directory: Callable[[Path], Path]) -> Path: + """Resolve a file as Path.resolve() does, resolving each parent directory only once.""" + + if path.is_symlink(): + return path.resolve() + return resolved_directory(path.parent) / path.name + + +def _resolved_path(path: Path) -> Path: + return path.resolve() + + def discovered_sql_contents(discovered_inputs: DiscoveredProjectInputs) -> dict[Path, str]: """Return the text discovery read for every SQL file it retains, keyed by path.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 47e87b2de1..7701ffa63f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -90,6 +90,8 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "render_reuse_hits", + "render_reuse_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 3ca4ed3d78..a4f6dea82a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -681,3 +681,81 @@ class CompileReuseStoreFailureTestCase: expected_returncode: int = 0 expected_stored_entries: int = 0 expected_raised: type[BaseException] | None = None + + +@dataclass(frozen=True) +class IncrementalEditStep: + """One authored edit, and whether its compile may reuse unaffected renders.""" + + description: str + edit: Callable[[Path], None] + expected_render_reuse: bool + + +@dataclass(frozen=True) +class IncrementalEditSequenceTestCase: + """Edits applied in order, each compiled incrementally and compared with --no-cache.""" + + description: str + steps: tuple[IncrementalEditStep, ...] + expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class RandomEditChainTestCase: + """A seeded random chain of edits over a generated project.""" + + description: str + seed: int + model_count: int + step_count: int + expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class BrokenChangeDetectionTestCase: + """A model edit compiled while change detection reports nothing changed.""" + + description: str + edit: Callable[[Path], None] + expected_matches_uncached: bool + + +@dataclass(frozen=True) +class RenderStoreNoticeTestCase: + """A render-count notice threshold and whether storing renders announces itself.""" + + description: str + notice_renders: int + expected_notice: bool + + +@dataclass(frozen=True) +class RenderSavePolicyTestCase: + """Compiles after a cold compile, and the render files and reuse each one must leave.""" + + description: str + edits: tuple[Callable[[Path], None], ...] + expected_cold_render_files: int + expected_render_files: tuple[int, ...] + expected_reused: tuple[bool, ...] + + +@dataclass(frozen=True) +class ExternalModuleEditTestCase: + """An outside module imported only while rendering, rewritten after its render is reused.""" + + description: str + initial_value: str + edited_value: str + expected_matches_uncached: bool + expected_compiled_value: str + + +@dataclass(frozen=True) +class RenderLoadNoticeTestCase: + """A stored-render size threshold and whether loading the store announces itself.""" + + description: str + notice_bytes: int + expected_notice: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 29858c0293..a5709dcc3f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2,8 +2,10 @@ from __future__ import annotations +import itertools import json import os +import random import re import shutil import signal @@ -24,6 +26,8 @@ import pytest import sqlbuild.cli.commands.main.project._compile as compile_command_module +import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice +import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files import sqlbuild.cli.compile_reuse._helpers.store as reuse_store from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement @@ -36,10 +40,15 @@ read_entry_stdout, write_entry, ) -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.cli.compile_reuse.constants import ( + REUSE_DISABLE_ENV_VAR, + REUSE_ENTRY_DIRECTORY_PARTS, + REUSE_RENDER_STATE_SUFFIX, +) from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, + IncrementalEditStep, SemanticCorpusCase, SetOperationModel, ) @@ -3433,3 +3442,391 @@ def compile_in_process_output( code: int = main(["--project-dir", str(project_dir), "--no-color", "compile", "--json"]) out, err = capsys.readouterr() return code, out, err + + +class IncrementalEditComparison(NamedTuple): + """An incremental compile and the --no-cache compile of the same edited project.""" + + incremental: CompileReuseRun + reference: CompileReuseRun + + @property + def reused_renders(self) -> int: + """Return how many model renders the incremental compile reused.""" + + return self.incremental.timings.get("render_reuse_hits", 0) + + @property + def matches(self) -> bool: + """Return whether exit code, report, and every compiled artifact are byte-identical.""" + + return _edit_outcome(self.incremental) == _edit_outcome(self.reference) + + @property + def mismatched_artifacts(self) -> list[str]: + """Return the first compiled artifacts that differ from the --no-cache compile.""" + + incremental: dict[str, bytes] = self.incremental.compiled + reference: dict[str, bytes] = self.reference.compiled + return sorted( + filter( + lambda path: incremental.get(path) != reference.get(path), + incremental.keys() | reference.keys(), + ) + )[:5] + + +def _edit_outcome(run: CompileReuseRun) -> tuple[int, str, dict[str, bytes]]: + return run.returncode, run.report, run.compiled + + +def compare_incremental_compile(*, project_dir: Path) -> IncrementalEditComparison: + """Compile incrementally, then compile the same inputs without any cache.""" + + incremental: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + return IncrementalEditComparison(incremental=incremental, reference=reference) + + +def in_process_reuse_run( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> CompileReuseRun: + """Compile with --json in this process and capture output comparable with fresh processes.""" + + code, out, err = compile_in_process_output(project_dir=project_dir, capsys=capsys) + payload: dict[str, object] = cast(dict[str, object], json.loads(out)) + return CompileReuseRun( + returncode=code, + report=_COMPILE_TIMINGS_PATTERN.sub("", out), + stderr=err, + timings=cast(dict[str, int], payload.get("compile_timings", {})), + compiled=compiled_artifacts(project_dir=project_dir), + ) + + +GENERATED_EDIT_MODEL_PREFIX: str = "orders_step_" +_GENERATED_EDIT_COLUMNS: tuple[str, ...] = ("order_id", "customer_id", "quantity", "status") +_PLAIN_QUANTITY: str = " quantity,\n" +_CAST_QUANTITY: str = " CAST(quantity AS BIGINT) AS quantity,\n" +_SWAP_PLACEHOLDER: str = "\x00swap\x00" +_RANDOM_EDIT_KINDS: tuple[str, ...] = ( + "comment", + "comment", + "add_column", + "type_change", + "header_change", + "introduce_error", + "add_model", + "macro_edit", + "test_edit", +) +_RANDOM_EDIT_FOLLOW_UPS: dict[str, tuple[str, ...]] = { + "introduce_error": ("introduce_error", "fix_error"), +} +_MODEL_ONLY_EDIT_KINDS: frozenset[str] = frozenset( + {"comment", "add_column", "type_change", "header_change", "introduce_error", "fix_error"} +) + + +def write_generated_edit_models(*, project_dir: Path, model_count: int, seed: int) -> None: + """Add a seeded chain of models over the staging orders, each reading an earlier one.""" + + chooser: random.Random = random.Random(seed) + upstreams: tuple[str, ...] = ( + "stg_orders", + *( + f"{GENERATED_EDIT_MODEL_PREFIX}{chooser.randrange(index):03d}" + for index in range(1, model_count) + ), + ) + for index, upstream in enumerate(upstreams): + write_project_file( + project_dir, + f"models/generated/{GENERATED_EDIT_MODEL_PREFIX}{index:03d}.sql", + _generated_edit_model_sql(upstream=upstream), + ) + + +def _generated_edit_model_sql(*, upstream: str) -> str: + columns: str = ",\n".join(f" {column}" for column in _GENERATED_EDIT_COLUMNS) + return ( + "MODEL (description 'Generated order step.', materialized view);\n\n" + f'SELECT\n{columns}\nFROM __ref("{upstream}")\n' + ) + + +def random_edit_plan(*, seed: int, step_count: int) -> tuple[str, ...]: + """Draw seeded edit kinds; every introduced error is fixed by the following step.""" + + chooser: random.Random = random.Random(seed) + return tuple( + itertools.chain.from_iterable( + map( + lambda drawn: _RANDOM_EDIT_FOLLOW_UPS.get(drawn, (drawn,)), + chooser.choices(_RANDOM_EDIT_KINDS, k=step_count), + ) + ) + ) + + +def is_model_only_edit(kind: str) -> bool: + """Return whether an edit kind touches model files only, so renders may be reused.""" + + return kind in _MODEL_ONLY_EDIT_KINDS + + +def _swapped(contents: str, first: str, second: str) -> str: + return ( + contents.replace(first, _SWAP_PLACEHOLDER) + .replace(second, first) + .replace(_SWAP_PLACEHOLDER, second) + ) + + +class RandomEditChain: + """Apply seeded random edits of every supported kind to a generated project.""" + + def __init__(self, *, project_dir: Path, seed: int) -> None: + self._project_dir: Path = project_dir + self._random: random.Random = random.Random(seed) + self._broken: Path = project_dir / "models" / "generated" / "unbroken.sql" + self._added: int = 0 + + def apply(self, kind: str) -> None: + """Apply one edit of the given kind to a randomly chosen generated model.""" + + models: list[Path] = sorted((self._project_dir / "models" / "generated").glob("*.sql")) + edit: Callable[[Path], None] = getattr(self, f"_{kind}") + edit(self._random.choice(models)) + + def _rewrite(self, model: Path, transform: Callable[[str], str]) -> None: + model.write_text(transform(model.read_text(encoding="utf-8")), encoding="utf-8") + + def _comment(self, model: Path) -> None: + note: str = f"-- note {self._random.random()}\nFROM __ref" + self._rewrite(model, lambda contents: contents.replace("FROM __ref", note)) + + def _add_column(self, model: Path) -> None: + column: str = f",\n status AS status_{self._random.randrange(1000)}\nFROM __ref" + self._rewrite(model, lambda contents: contents.replace("\nFROM __ref", column)) + + def _type_change(self, model: Path) -> None: + self._rewrite(model, lambda contents: _swapped(contents, _PLAIN_QUANTITY, _CAST_QUANTITY)) + + def _header_change(self, model: Path) -> None: + self._rewrite( + model, + lambda contents: _swapped(contents, "materialized view", "materialized table"), + ) + + def _introduce_error(self, model: Path) -> None: + self._rewrite( + model, lambda contents: contents.replace("SELECT\n", "SELECT\n unknown_column,\n", 1) + ) + self._broken = model + + def _fix_error(self, _model: Path) -> None: + self._rewrite(self._broken, lambda contents: contents.replace(" unknown_column,\n", "")) + + def _add_model(self, model: Path) -> None: + self._added += 1 + write_project_file( + self._project_dir, + f"models/generated/added_step_{self._added:03d}.sql", + _generated_edit_model_sql(upstream=model.stem), + ) + + def _macro_edit(self, _model: Path) -> None: + scale: str = f"_SCALE: int = {2 + self._random.randrange(3)}" + self._rewrite( + self._project_dir / "macros" / "_rounding.py", + lambda contents: re.sub(r"_SCALE: int = \d+", scale, contents), + ) + + def _test_edit(self, _model: Path) -> None: + revision: str = f"\n-- revision {self._random.random()}\n" + self._rewrite( + self._project_dir / "tests" / "unit" / "test_stg_orders.sql", + lambda contents: contents + revision, + ) + + +STG_ORDERS_MODEL: str = "models/staging/stg_orders.sql" +FACT_ORDERS_MODEL: str = "models/marts/fact_orders.sql" + + +def model_edit_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: + """Return an edit to model files only, whose compile must reuse unaffected renders.""" + + return IncrementalEditStep(description=description, edit=edit, expected_render_reuse=True) + + +def full_edit_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: + """Return an edit beyond model contents, whose compile must render everything again.""" + + return IncrementalEditStep(description=description, edit=edit, expected_render_reuse=False) + + +def staging_comment(root: Path) -> None: + """Add a comment to the staging orders model without changing its shape.""" + + replace_project_text( + root, + STG_ORDERS_MODEL, + 'FROM __source("raw__orders")', + '-- staged\nFROM __source("raw__orders")', + ) + + +def staging_type_change(root: Path) -> None: + """Change one staging output column type, which propagates downstream.""" + + replace_project_text( + root, STG_ORDERS_MODEL, " quantity,\n", " CAST(quantity AS BIGINT) AS quantity,\n" + ) + + +def staging_new_column(root: Path) -> None: + """Add an output column to the staging orders model.""" + + replace_project_text( + root, STG_ORDERS_MODEL, " status\n", " status,\n status AS raw_status\n" + ) + + +def staging_header_change(root: Path) -> None: + """Change the staging orders materialization in its header.""" + + replace_project_text(root, STG_ORDERS_MODEL, "materialized view,", "materialized table,") + + +def staging_contract_change(root: Path) -> None: + """Add a declared type to one staging column contract.""" + + replace_project_text( + root, + STG_ORDERS_MODEL, + "customer_id (nullable false, audits [not_null]),", + "customer_id (type BIGINT, nullable false, audits [not_null]),", + ) + + +def fact_audit_added(root: Path) -> None: + """Attach one more column audit to the fact orders model.""" + + replace_project_text( + root, + FACT_ORDERS_MODEL, + " order_id (nullable false, audits [not_null]),", + " order_id (nullable false, audits [not_null, unique]),", + ) + + +def fact_error_introduced(root: Path) -> None: + """Make the fact orders model read a column that does not exist.""" + + replace_project_text(root, FACT_ORDERS_MODEL, " o.quantity,\n", " o.missing_quantity,\n") + + +def fact_error_fixed(root: Path) -> None: + """Restore the column the fact orders model reads.""" + + replace_project_text(root, FACT_ORDERS_MODEL, " o.missing_quantity,\n", " o.quantity,\n") + + +def fact_comment(root: Path) -> None: + """Add a comment to the leaf fact orders model.""" + + replace_project_text(root, FACT_ORDERS_MODEL, "FROM __ref", "-- leaf\nFROM __ref") + + +def disable_change_detection(monkeypatch: pytest.MonkeyPatch) -> None: + """Make compile reuse believe no project file changed since the stored compile.""" + + monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: (frozenset(), {})) + + +def set_store_notice_renders(monkeypatch: pytest.MonkeyPatch, renders: int) -> None: + """Announce storing a compile once it has at least this many new renders to record.""" + + monkeypatch.setattr(reuse_store, "REUSE_STORE_NOTICE_RENDERS", renders) + + +def prime_render_store(project_dir: Path) -> CompileReuseRun: + """After a cold compile, compile one leaf edit so the stored compile also holds renders.""" + + fact_comment(project_dir) + return run_reuse_compile(project_dir=project_dir) + + +def prime_render_store_in_process(project_dir: Path) -> int: + """In this process: compile cold, then compile one leaf edit so renders are stored.""" + + cold: int = compile_in_process(project_dir=project_dir) + fact_comment(project_dir) + return max(cold, compile_in_process(project_dir=project_dir)) + + +def render_store_files(project_dir: Path) -> int: + """Return how many stored render files the project's compile reuse folder holds.""" + + return len( + list( + project_dir.joinpath(*REUSE_ENTRY_DIRECTORY_PARTS).glob(f"*{REUSE_RENDER_STATE_SUFFIX}") + ) + ) + + +def edit_and_compile(*, project_dir: Path, edit: Callable[[Path], None]) -> CompileReuseRun: + """Apply one edit, compile with reuse enabled, and return the run.""" + + edit(project_dir) + return run_reuse_compile(project_dir=project_dir) + + +EXTERNAL_FLAVOR_MODULE: str = "extflavor" + + +def write_external_flavor(extlib: Path, value: str) -> None: + """Write, or rewrite in place, an outside module that a macro imports only while rendering.""" + + extlib.mkdir(parents=True, exist_ok=True) + (extlib / f"{EXTERNAL_FLAVOR_MODULE}.py").write_text(f"VALUE = {value!r}\n", encoding="utf-8") + + +def add_external_flavor_macro(*, project_dir: Path, extlib: Path, value: str) -> dict[str, str]: + """Make the fact orders render import an outside module; return the compile environment.""" + + write_external_flavor(extlib, value) + write_project_file( + project_dir, + "models/marts/_sqlbuild/_macros/flavor.py", + '"""Flavor macros backed by an outside module."""\n\n\n' + "def flavor() -> str:\n" + ' """Return the configured flavor literal."""\n' + f" import {EXTERNAL_FLAVOR_MODULE}\n\n" + f" return {EXTERNAL_FLAVOR_MODULE}.VALUE\n", + ) + replace_project_text( + project_dir, FACT_ORDERS_MODEL, " o.quantity,\n", " o.quantity,\n @flavor() AS flavor,\n" + ) + return {"PYTHONPATH": str(extlib), "PYTHONDONTWRITEBYTECODE": "1"} + + +def compiled_text(*, run: CompileReuseRun, suffix: str) -> str: + """Return the compiled artifact whose path ends with the suffix.""" + + path: str = next(filter(lambda path: path.endswith(suffix), sorted(run.compiled))) + return run.compiled[path].decode("utf-8") + + +def set_render_load_notice_bytes(monkeypatch: pytest.MonkeyPatch, stored_bytes: int) -> None: + """Announce loading stored renders once their files hold at least this many bytes.""" + + monkeypatch.setattr(render_load_notice, "RENDER_LOAD_NOTICE_BYTES", stored_bytes) + + +def json_report_keys(stdout: str) -> tuple[str, ...]: + """Parse a JSON compile report and return its top-level keys.""" + + return tuple(cast(dict[str, object], json.loads(stdout))) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py new file mode 100644 index 0000000000..fc0550161c --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -0,0 +1,438 @@ +"""Incremental edit compiles must match an uncached compile of the same edited project.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + BrokenChangeDetectionTestCase, + ExternalModuleEditTestCase, + IncrementalEditSequenceTestCase, + RandomEditChainTestCase, + RenderLoadNoticeTestCase, + RenderSavePolicyTestCase, + RenderStoreNoticeTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + IncrementalEditComparison, + RandomEditChain, + add_external_flavor_macro, + compare_incremental_compile, + compile_in_process, + compile_in_process_output, + compiled_text, + disable_change_detection, + edit_and_compile, + enable_compile_reuse, + fact_audit_added, + fact_comment, + fact_error_fixed, + fact_error_introduced, + full_edit_step, + in_process_reuse_run, + is_model_only_edit, + json_report_keys, + model_edit_step, + move_project_file, + prime_render_store, + prime_render_store_in_process, + random_edit_plan, + render_store_files, + replace_project_text, + run_reuse_compile, + set_render_load_notice_bytes, + set_store_notice_renders, + staging_comment, + staging_contract_change, + staging_header_change, + staging_new_column, + staging_type_change, + write_external_flavor, + write_generated_edit_models, + write_project_file, +) + + +@pytest.mark.parametrize( + "test_case", + [ + IncrementalEditSequenceTestCase( + description="model_edits", + steps=( + model_edit_step("comment_without_shape_change", staging_comment), + model_edit_step("type_change_propagates_downstream", staging_type_change), + model_edit_step("new_output_column", staging_new_column), + model_edit_step("header_config_change", staging_header_change), + model_edit_step("column_contract_change", staging_contract_change), + model_edit_step("attached_audit_added", fact_audit_added), + model_edit_step("error_introduced", fact_error_introduced), + model_edit_step("error_fixed", fact_error_fixed), + ), + ), + IncrementalEditSequenceTestCase( + description="model_set_changes", + steps=( + full_edit_step( + "model_added", + lambda root: write_project_file( + root, + "models/marts/order_quantities.sql", + "MODEL (description 'Order quantities.');\n\n" + 'SELECT order_id, quantity FROM __ref("stg_orders")\n', + ), + ), + model_edit_step("edit_after_add", staging_comment), + full_edit_step( + "model_renamed", + lambda root: move_project_file( + root, + "models/marts/order_quantities.sql", + "models/marts/order_quantity_totals.sql", + ), + ), + full_edit_step( + "model_removed", + lambda root: (root / "models/marts/order_quantity_totals.sql").unlink(), + ), + model_edit_step("edit_after_remove", fact_comment), + ), + ), + IncrementalEditSequenceTestCase( + description="declaration_edits", + steps=( + full_edit_step( + "macro_edit", + lambda root: replace_project_text( + root, + "macros/currency.py", + "{price_cents} * {quantity}", + "{quantity} * {price_cents}", + ), + ), + model_edit_step("edit_after_macro", fact_comment), + full_edit_step( + "module_imported_by_macro_edit", + lambda root: replace_project_text( + root, "macros/_rounding.py", "_SCALE: int = 2", "_SCALE: int = 3" + ), + ), + full_edit_step( + "scoped_macro_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_macros/currency.py", + "/ 100.0, 2)", + "/ 100.0, 3)", + ), + ), + full_edit_step( + "path_defaults_edit", + lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + '[path_defaults.staging]\nmaterialized = "view"', + '[path_defaults.staging]\nmaterialized = "table"', + ), + ), + full_edit_step( + "enum_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_enums/order_channel.sql", + 'WEB "web"', + 'WEB "online"', + ), + ), + full_edit_step( + "constant_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_constants/min_quantity.sql", + "value 1)", + "value 2)", + ), + ), + full_edit_step( + "new_scope_folder", + lambda root: write_project_file( + root, + "models/staging/_sqlbuild/_macros/staging_totals.py", + "def staging_total_cents(price_cents: str, quantity: str) -> str:\n" + ' """Calculate a staging line total."""\n' + ' return f"({price_cents}) * ({quantity})"\n', + ), + ), + model_edit_step("edit_after_declarations", staging_comment), + ), + ), + IncrementalEditSequenceTestCase( + description="resource_edits", + steps=( + full_edit_step( + "seed_edit", + lambda root: replace_project_text( + root, "seeds/waffle_types.csv", "Classic Belgian", "Classic Brussels" + ), + ), + full_edit_step( + "source_edit", + lambda root: replace_project_text( + root, + "sources/raw.yml", + " - name: quantity\n type: INTEGER", + " - name: quantity\n type: BIGINT", + ), + ), + model_edit_step("edit_after_source", staging_type_change), + full_edit_step( + "test_edit", + lambda root: replace_project_text( + root, + "tests/unit/test_stg_orders.sql", + "100 AS customer_id", + "101 AS customer_id", + ), + ), + full_edit_step( + "audit_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql", + "WHERE NOT (@expression)", + "WHERE NOT (@expression) AND 1 = 1", + ), + ), + model_edit_step("edit_after_resources", fact_comment), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches_uncached_compile( + compile_reuse_project: Path, test_case: IncrementalEditSequenceTestCase +) -> None: + project_dir: Path = compile_reuse_project + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + primed: CompileReuseRun = prime_render_store(project_dir) + assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr + + for step in test_case.steps: + step.edit(project_dir) + comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) + + assert comparison.incremental.reused is False, step.description + assert comparison.matches is test_case.expected_matches_uncached, ( + step.description, + comparison.mismatched_artifacts, + comparison.incremental.stderr, + ) + assert (comparison.reused_renders > 0) is step.expected_render_reuse, step.description + + +@pytest.mark.parametrize( + "test_case", + [ + RandomEditChainTestCase(description="seed_7", seed=7, model_count=24, step_count=12), + RandomEditChainTestCase(description="seed_19", seed=19, model_count=24, step_count=12), + ], + ids=lambda case: case.description, +) +def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_matches_uncached_compile( + compile_reuse_project: Path, test_case: RandomEditChainTestCase +) -> None: + project_dir: Path = compile_reuse_project + write_generated_edit_models( + project_dir=project_dir, model_count=test_case.model_count, seed=test_case.seed + ) + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + primed: CompileReuseRun = prime_render_store(project_dir) + assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr + chain: RandomEditChain = RandomEditChain(project_dir=project_dir, seed=test_case.seed) + + for step, kind in enumerate( + random_edit_plan(seed=test_case.seed, step_count=test_case.step_count) + ): + chain.apply(kind) + comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) + + assert comparison.matches is test_case.expected_matches_uncached, ( + step, + kind, + comparison.mismatched_artifacts, + comparison.incremental.stderr, + ) + assert (comparison.reused_renders > 0) is is_model_only_edit(kind), (step, kind) + + +@pytest.mark.parametrize( + "test_case", + [ + BrokenChangeDetectionTestCase( + description="new_output_column", + edit=staging_new_column, + expected_matches_uncached=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( + compile_reuse_project: Path, + test_case: BrokenChangeDetectionTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = compile_reuse_project + enable_compile_reuse(monkeypatch) + assert prime_render_store_in_process(project_dir) == 0 + test_case.edit(project_dir) + disable_change_detection(monkeypatch) + + broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=broken, reference=reference + ) + + assert comparison.reused_renders > 0 + assert comparison.matches is test_case.expected_matches_uncached + + +@pytest.mark.parametrize( + "test_case", + [ + RenderStoreNoticeTestCase( + description="few_renders_store_quietly", notice_renders=100_000, expected_notice=False + ), + RenderStoreNoticeTestCase( + description="many_renders_announce_recording", notice_renders=1, expected_notice=True + ), + ], + ids=lambda case: case.description, +) +def test_given_new_renders_when_storing_after_an_edit_then_slow_recording_is_announced( + compile_reuse_project: Path, + test_case: RenderStoreNoticeTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + enable_compile_reuse(monkeypatch) + set_store_notice_renders(monkeypatch, test_case.notice_renders) + assert compile_in_process(project_dir=compile_reuse_project) == 0 + fact_comment(compile_reuse_project) + + code, _, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) + + assert code == 0 + assert ("renders)..." in err) is test_case.expected_notice + assert ("Recorded compile for reuse" in err) is test_case.expected_notice + + +@pytest.mark.parametrize( + "test_case", + [ + RenderSavePolicyTestCase( + description="cold_then_two_model_edits", + edits=(fact_comment, staging_comment), + expected_cold_render_files=0, + expected_render_files=(1, 2), + expected_reused=(False, True), + ), + ], + ids=lambda case: case.description, +) +def test_given_cold_compile_when_editing_then_renders_are_stored_only_once_a_compile_exists( + compile_reuse_project: Path, test_case: RenderSavePolicyTestCase +) -> None: + cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project) + cold_render_files: int = render_store_files(compile_reuse_project) + render_files: list[int] = [] + reused: list[bool] = [] + + for edit in test_case.edits: + run: CompileReuseRun = edit_and_compile(project_dir=compile_reuse_project, edit=edit) + assert run.returncode == 0, run.stderr + render_files.append(render_store_files(compile_reuse_project)) + reused.append(run.timings.get("render_reuse_hits", 0) > 0) + + assert cold.returncode == 0, cold.stderr + assert cold_render_files == test_case.expected_cold_render_files + assert tuple(render_files) == test_case.expected_render_files + assert tuple(reused) == test_case.expected_reused + + +@pytest.mark.parametrize( + "test_case", + [ + ExternalModuleEditTestCase( + description="module_rewritten_in_place_after_reuse", + initial_value="'vanilla'", + edited_value="'choco'", + expected_matches_uncached=True, + expected_compiled_value="'choco' AS flavor", + ), + ], + ids=lambda case: case.description, +) +def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_then_output_is_fresh( + compile_reuse_project: Path, tmp_path: Path, test_case: ExternalModuleEditTestCase +) -> None: + extlib: Path = tmp_path / "extlib" + env: dict[str, str] = add_external_flavor_macro( + project_dir=compile_reuse_project, extlib=extlib, value=test_case.initial_value + ) + cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + staging_comment(compile_reuse_project) + recorded: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + staging_comment(compile_reuse_project) + reused: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + + write_external_flavor(extlib, test_case.edited_value) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=run_reuse_compile(project_dir=compile_reuse_project, env=env), + reference=run_reuse_compile( + project_dir=compile_reuse_project, env=env, args=("--no-cache",) + ), + ) + + assert (cold.returncode, recorded.returncode, reused.returncode) == (0, 0, 0), reused.stderr + assert reused.timings.get("render_reuse_hits", 0) > 0 + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) + assert test_case.expected_compiled_value in compiled_text( + run=comparison.incremental, suffix="fact_orders.sql" + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderLoadNoticeTestCase( + description="small_store_loads_quietly", notice_bytes=1 << 40, expected_notice=False + ), + RenderLoadNoticeTestCase( + description="large_store_announces_loading", notice_bytes=1, expected_notice=True + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_renders_when_loading_them_then_large_loads_are_announced_on_stderr( + compile_reuse_project: Path, + test_case: RenderLoadNoticeTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + enable_compile_reuse(monkeypatch) + set_render_load_notice_bytes(monkeypatch, test_case.notice_bytes) + assert prime_render_store_in_process(compile_reuse_project) == 0 + staging_comment(compile_reuse_project) + + code, out, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) + + assert code == 0 + assert "compile_timings" in json_report_keys(out) + assert ("Loading stored renders" in err) is test_case.expected_notice + assert ("Loaded stored renders" in err) is test_case.expected_notice diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py new file mode 100644 index 0000000000..913e647485 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py @@ -0,0 +1,46 @@ +"""Test case types for layered render storage.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RenderLayerTestCase: + """Renders changed after a stored compile, and whether an overlay can hold them.""" + + description: str + changed_models: tuple[str, ...] + removed_models: tuple[str, ...] + expected_overlay: bool + + +@dataclass(frozen=True) +class RenderEditChainTestCase: + """Successive one-model edits, each stored on top of the previous compile.""" + + description: str + edits: tuple[str, ...] + expected_overlay_models: frozenset[str] + + +@dataclass(frozen=True) +class StoredRenderReadTestCase: + """A stored render file, possibly damaged, and whether reading it yields renders.""" + + description: str + prepare: Callable[[Path], tuple[Path, frozenset[str]]] + expected_readable: bool + + +@dataclass(frozen=True) +class RenderLoadNoticeTestCase: + """Stored render files, some removed while being sized, and the expected notice.""" + + description: str + file_sizes: tuple[int, ...] + removed_while_sizing: int + notice_bytes: int + expected_notice: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py new file mode 100644 index 0000000000..c215b98d84 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py @@ -0,0 +1,167 @@ +"""Shared builders for layered render storage unit tests.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( + read_render_state, + render_state_layer, +) +from sqlbuild.cli.compile_render_reuse.models import StoredRenderState +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState + +MODEL_PATHS: tuple[str, ...] = tuple(f"models/orders_{index:02d}.sql" for index in range(20)) + + +def publish_render_state(*, directory: Path, name: str, layer: RenderStateLayer | None) -> Path: + """Write one render layer as the store would publish it.""" + + assert layer is not None + path: Path = directory / name + path.write_bytes(b"".join(bytes(chunk) for chunk in layer.chunks)) + return path + + +def read_stored(path: Path) -> StoredRenderState: + """Read a render file that must decode.""" + + stored: StoredRenderState | None = read_render_state(path=path, changed_paths=frozenset()) + assert stored is not None + return stored + + +def stored_base(directory: Path) -> tuple[Path, StoredRenderState]: + """Publish a base render file holding every model and one group, and read it back.""" + + state: RenderReuseState = RenderReuseState( + model_paths=MODEL_PATHS, + model_payloads={path: memoryview(f"render {path}".encode() * 20) for path in MODEL_PATHS}, + group_payloads={"sources": memoryview(b"source renders" * 20)}, + ) + path: Path = publish_render_state( + directory=directory, + name="orders-base.render", + layer=render_state_layer(state=state, stored=None), + ) + return path, read_stored(path) + + +def next_render_state( + *, stored: StoredRenderState, edited: tuple[str, ...], removed: tuple[str, ...] = () +) -> RenderReuseState: + """Return the next compile's renders as a session stores them after releasing reused bytes. + + Renders reused from the stored base are None; reused overlay renders keep their bytes. + """ + + kept: dict[str, memoryview | None] = { + path: retained_payload(stored=stored, path=path, payload=payload) + for path, payload in filter( + lambda item: item[0] not in removed, stored.state.model_payloads.items() + ) + } + return RenderReuseState( + model_paths=MODEL_PATHS, + model_payloads={ + **kept, + **{path: memoryview(f"edited {path}".encode() * 20) for path in edited}, + }, + group_payloads=dict.fromkeys(stored.state.group_payloads), + ) + + +def retained_payload( + *, stored: StoredRenderState, path: str, payload: memoryview | None +) -> memoryview | None: + """Keep a reused render's bytes only when it lives in the stored overlay.""" + + return {True: payload, False: None}[path in stored.layers.overlay_models] + + +def payload_bytes(state: RenderReuseState) -> dict[str, bytes]: + """Return every model payload as bytes for comparison.""" + + return {path: bytes(payload or b"") for path, payload in state.model_payloads.items()} + + +def store_edit_chain(*, directory: Path, edits: tuple[str, ...]) -> StoredRenderState: + """Store one compile per edited model, each layered on the previous one, and read the last.""" + + stored: StoredRenderState + _, stored = stored_base(directory) + for index, edited in enumerate(edits): + path: Path = publish_render_state( + directory=directory, + name=f"orders-edit-{index}.render", + layer=render_state_layer( + state=next_render_state(stored=stored, edited=(edited,)), stored=stored.layers + ), + ) + stored = read_stored(path) + return stored + + +def chain_payload_bytes(edits: tuple[str, ...]) -> dict[str, bytes]: + """Return the model payloads an edit chain must leave readable.""" + + return { + **{path: f"render {path}".encode() * 20 for path in MODEL_PATHS}, + **{path: f"edited {path}".encode() * 20 for path in edits}, + } + + +def flipped_last_byte(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and corrupt its last payload byte.""" + + path, _ = stored_base(directory) + data: bytearray = bytearray(path.read_bytes()) + data[-1] ^= 0xFF + path.write_bytes(bytes(data)) + return path, frozenset() + + +def truncated(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and cut off its tail.""" + + path, _ = stored_base(directory) + path.write_bytes(path.read_bytes()[:-7]) + return path, frozenset() + + +def emptied(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and empty it.""" + + path, _ = stored_base(directory) + path.write_bytes(b"") + return path, frozenset() + + +def overlay_without_base(directory: Path) -> tuple[Path, frozenset[str]]: + """Store an overlay and delete the base file it layers on.""" + + base_path, stored = stored_base(directory) + overlay_path: Path = publish_render_state( + directory=directory, + name="orders-overlay.render", + layer=render_state_layer( + state=next_render_state(stored=stored, edited=(MODEL_PATHS[0],)), stored=stored.layers + ), + ) + base_path.unlink() + return overlay_path, frozenset() + + +def changed_macro(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and report a changed path that is not a stored model.""" + + path, _ = stored_base(directory) + return path, frozenset({"macros/currency.py"}) + + +def intact(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and report one stored model as changed.""" + + path, _ = stored_base(directory) + return path, frozenset({MODEL_PATHS[4]}) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py new file mode 100644 index 0000000000..3feb8dc648 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py @@ -0,0 +1,68 @@ +"""Sizing the render store for the load notice must tolerate files removed by a concurrent compile.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_render_reuse._helpers import load_notice +from sqlbuild.cli.compile_reuse.constants import REUSE_RENDER_STATE_SUFFIX +from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( + RenderLoadNoticeTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderLoadNoticeTestCase( + description="large_store_announced", + file_sizes=(64, 64), + removed_while_sizing=0, + notice_bytes=100, + expected_notice=True, + ), + RenderLoadNoticeTestCase( + description="small_store_silent", + file_sizes=(16, 16), + removed_while_sizing=0, + notice_bytes=100, + expected_notice=False, + ), + RenderLoadNoticeTestCase( + description="file_removed_while_sizing_counts_as_empty", + file_sizes=(64, 64), + removed_while_sizing=1, + notice_bytes=100, + expected_notice=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_render_files_when_starting_load_notice_then_sizes_without_failing( + test_case: RenderLoadNoticeTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + # Arrange + entry_path: Path = tmp_path / "dev.entry" + stored_paths: list[Path] = [ + tmp_path / f"dev-{index}{REUSE_RENDER_STATE_SUFFIX}" + for index in range(len(test_case.file_sizes)) + ] + for path, size in zip(stored_paths, test_case.file_sizes, strict=True): + path.write_bytes(b"x" * size) + listed: list[Path] = sorted(stored_paths) + monkeypatch.setattr(load_notice, "RENDER_LOAD_NOTICE_BYTES", test_case.notice_bytes) + monkeypatch.setattr(Path, "glob", lambda self, pattern: iter(listed)) + for path in listed[: test_case.removed_while_sizing]: + path.unlink() + + # Act + started: float | None = load_notice.start_load_notice(entry_path=entry_path) + + # Assert + assert (started is not None) is test_case.expected_notice + assert capsys.readouterr().out == "" diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py new file mode 100644 index 0000000000..bca44fbe3e --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py @@ -0,0 +1,136 @@ +"""Layered render storage must round-trip exactly and treat every fault as no stored renders.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( + read_render_state, + render_state_layer, +) +from sqlbuild.cli.compile_render_reuse.models import StoredRenderState +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState +from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( + RenderEditChainTestCase, + RenderLayerTestCase, + StoredRenderReadTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers.helpers import ( + MODEL_PATHS, + chain_payload_bytes, + changed_macro, + emptied, + flipped_last_byte, + intact, + next_render_state, + overlay_without_base, + payload_bytes, + store_edit_chain, + stored_base, + truncated, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderLayerTestCase( + description="one_model_changed", + changed_models=(MODEL_PATHS[3],), + removed_models=(), + expected_overlay=True, + ), + RenderLayerTestCase( + description="most_models_changed", + changed_models=MODEL_PATHS[:15], + removed_models=(), + expected_overlay=False, + ), + RenderLayerTestCase( + description="stored_render_dropped", + changed_models=(MODEL_PATHS[3],), + removed_models=(MODEL_PATHS[5],), + expected_overlay=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_released_reused_renders_when_storing_then_only_a_small_overlay_needs_no_rewrite( + test_case: RenderLayerTestCase, tmp_path: Path +) -> None: + _, stored = stored_base(tmp_path) + current: RenderReuseState = next_render_state( + stored=stored, edited=test_case.changed_models, removed=test_case.removed_models + ) + + layer: RenderStateLayer | None = render_state_layer(state=current, stored=stored.layers) + + assert (layer is not None) is test_case.expected_overlay + + +@pytest.mark.parametrize( + "test_case", + [ + RenderEditChainTestCase( + description="different_models_edited", + edits=(MODEL_PATHS[1], MODEL_PATHS[2]), + expected_overlay_models=frozenset({MODEL_PATHS[1], MODEL_PATHS[2]}), + ), + RenderEditChainTestCase( + description="same_model_edited_twice", + edits=(MODEL_PATHS[1], MODEL_PATHS[1]), + expected_overlay_models=frozenset({MODEL_PATHS[1]}), + ), + ], + ids=lambda case: case.description, +) +def test_given_overlay_when_storing_again_then_earlier_overlay_renders_are_kept( + test_case: RenderEditChainTestCase, tmp_path: Path +) -> None: + stored: StoredRenderState = store_edit_chain(directory=tmp_path, edits=test_case.edits) + + assert stored.layers.overlay_models == test_case.expected_overlay_models + assert payload_bytes(stored.state) == chain_payload_bytes(test_case.edits) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredRenderReadTestCase( + description="intact_with_changed_model", prepare=intact, expected_readable=True + ), + StoredRenderReadTestCase( + description="flipped_payload_byte", prepare=flipped_last_byte, expected_readable=False + ), + StoredRenderReadTestCase( + description="truncated", prepare=truncated, expected_readable=False + ), + StoredRenderReadTestCase(description="empty", prepare=emptied, expected_readable=False), + StoredRenderReadTestCase( + description="overlay_whose_base_is_missing", + prepare=overlay_without_base, + expected_readable=False, + ), + StoredRenderReadTestCase( + description="changed_path_is_not_a_stored_model", + prepare=changed_macro, + expected_readable=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_render_file_when_reading_then_only_intact_relevant_renders_are_returned( + test_case: StoredRenderReadTestCase, tmp_path: Path +) -> None: + path, changed_paths = test_case.prepare(tmp_path) + + stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) + + assert (stored is not None) is test_case.expected_readable + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index 77b9011266..ea3686abfc 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -8,6 +8,9 @@ import pytest +from sqlbuild.cli.compile_reuse.models import StoredProjectFile +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import digested_project_files + @dataclass(frozen=True) class ProjectFingerprintTestCase: @@ -78,3 +81,13 @@ class SettingsDigestTestCase: description: str change: Callable[[pytest.MonkeyPatch, Path], None] expected_changed: bool + + +@dataclass(frozen=True) +class ChangedPathsTestCase: + """One project change and every project path a change set must name for it.""" + + description: str + change: Callable[[Path], object] + expected_paths: tuple[str, ...] + store: Callable[[Path], dict[str, StoredProjectFile]] = digested_project_files diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index 3dcf51f238..efacd4c1c0 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -124,6 +124,12 @@ def break_link(project_dir: Path) -> None: (project_dir.parent / "shared/customers.sql").unlink() +def digested_project_files(project_dir: Path) -> dict[str, StoredProjectFile]: + """Record the project with digests and a snapshot time that leaves no file racy.""" + + return stored_project_files(project_dir=project_dir, snapshot_ns=later_snapshot_ns()) + + def stat_only_project_files(*, project_dir: Path) -> dict[str, StoredProjectFile]: """Record the project as a stored compile does by default: stat stamps without digests.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py index 9f0eeb019f..f4867265a1 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py @@ -10,6 +10,7 @@ from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, + changed_project_paths, compare_project_files, restamped_paths, snapshot_project_files, @@ -17,6 +18,7 @@ from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile from sqlbuild.cli.compile_reuse.types import FileStamp from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + ChangedPathsTestCase, DigestCarryForwardTestCase, LinkCycleTestCase, ProjectFingerprintTestCase, @@ -147,13 +149,68 @@ def test_given_stored_fingerprint_when_project_changes_then_change_is_detected_e ) test_case.change(project_dir) + current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) comparison: ProjectFilesComparison = compare_project_files( + project_dir=str(project_dir), stored=stored, current=current + ) + changed, _ = changed_project_paths( + project_dir=str(project_dir), stored=stored, current=current, verified={} + ) + + assert comparison.unchanged is test_case.expected_unchanged + assert (not changed) is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedPathsTestCase( + description="touched_without_change", + change=touch_without_change, + expected_paths=(), + ), + ChangedPathsTestCase( + description="model_edited", + change=lambda root: write_file(root / "models/orders.sql", "SELECT 10 AS order_id\n"), + expected_paths=("models/orders.sql",), + ), + ChangedPathsTestCase( + description="same_size_and_mtime_new_content", + change=rewrite_same_size_keeping_mtime, + expected_paths=("models/orders.sql",), + ), + ChangedPathsTestCase( + description="file_added_and_removed", + change=lambda root: ( + write_file(root / "models/returns.sql", "SELECT 1\n"), + (root / "macros/currency.py").unlink(), + ), + expected_paths=("macros/currency.py", "models/returns.sql"), + ), + ChangedPathsTestCase( + description="touched_without_stored_digest", + change=touch_without_change, + expected_paths=("models/orders.sql",), + store=lambda root: stat_only_project_files(project_dir=root), + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_fingerprint_when_listing_changes_then_returns_every_changed_path( + test_case: ChangedPathsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + stored: dict[str, StoredProjectFile] = test_case.store(project_dir) + + test_case.change(project_dir) + changed, _ = changed_project_paths( project_dir=str(project_dir), stored=stored, current=snapshot_project_files(project_dir=str(project_dir)), + verified={}, ) - assert comparison.unchanged is test_case.expected_unchanged + assert changed == frozenset(test_case.expected_paths) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py new file mode 100644 index 0000000000..23fea2a02f --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py @@ -0,0 +1,48 @@ +"""Test case types for the compile render reuse session.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import CompilerDiagnostic + + +@dataclass(frozen=True) +class RenderReplayTestCase: + """A reused render and what reusing it must report again.""" + + description: str + model_name: str + expected_diagnostics: tuple[CompilerDiagnostic, ...] + expected_environment_names: tuple[str, ...] + + +@dataclass(frozen=True) +class ReusableModelsTestCase: + """Stored and current models, and which current models may reuse their stored render.""" + + description: str + stored_models: tuple[str, ...] + current_models: tuple[str, ...] + changed_paths: frozenset[str] + run_id_readers: frozenset[str] + expected_reusable: dict[str, bool] + + +@dataclass(frozen=True) +class CorruptRenderTestCase: + """A stored render payload that cannot be decoded.""" + + description: str + model_name: str + expected_rendered_again: bool + + +@dataclass(frozen=True) +class ReleasedRenderTestCase: + """A reused and an edited render, and which stored bytes storing the compile releases.""" + + description: str + retained_models: frozenset[str] + expected_released: dict[str, bool] + expected_query_sqls: dict[str, str] diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py new file mode 100644 index 0000000000..265ff3b7fc --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py @@ -0,0 +1,158 @@ +"""Shared builders for render reuse session unit tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path +from typing import cast + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( + collect_compile_diagnostics, + report_compile_diagnostic, +) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompilerDiagnostic, + RenderReuseState, + StoredRender, +) +from sqlbuild.compiler.compile.types import CompileContextKey, DiagnosticPhase, DiagnosticSeverity +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload + +REGION_ENV_VAR: str = "ORDERS_REGION" + + +def model_file(name: str, sql: str = "SELECT 1 AS order_id") -> DiscoveredSqlModelFile: + """Return one discovered SQL model.""" + + return DiscoveredSqlModelFile( + file_path=Path("/orders") / "models" / f"{name}.sql", + relative_path=Path("models") / f"{name}.sql", + contents=sql, + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql=sql, + ) + + +def missing_description(name: str) -> CompilerDiagnostic: + """Return the warning a render of the named model reports.""" + + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.WARNING, + code="P010", + message=f"Model '{name}' has no description", + ) + + +def _render(item: DiscoveredSqlModelFile) -> CompileModelInput: + report_compile_diagnostic( + key=(item.file_path.stem,), diagnostic=missing_description(item.file_path.stem) + ) + COMPILE_INPUT_READS.environment_read(REGION_ENV_VAR) + return CompileModelInput(model_file=item, query_sql=item.query_sql) + + +def _render_reading_run_id(item: DiscoveredSqlModelFile) -> CompileModelInput: + COMPILE_INPUT_READS.context_read(CompileContextKey.RUN_ID) + return _render(item) + + +def recorded_state( + model_files: tuple[DiscoveredSqlModelFile, ...], + *, + run_id_readers: frozenset[str] = frozenset(), +) -> RenderReuseState: + """Render every model through a fresh session and return what it stores.""" + + session: CompileRenderReuseSession = CompileRenderReuseSession(prior=None, changed_paths=None) + session.plan_models(model_files=model_files) + renderers: dict[bool, Callable[[DiscoveredSqlModelFile], CompileModelInput]] = { + False: _render, + True: _render_reading_run_id, + } + with collect_compile_diagnostics(): + for item in model_files: + render: Callable[[DiscoveredSqlModelFile], CompileModelInput] = renderers[ + item.file_path.stem in run_id_readers + ] + _ = session.rendered_model(model_file=item, render=lambda item=item, r=render: r(item)) + state: RenderReuseState | None = session.stored_state() + assert state is not None + return state + + +def planned_session( + *, + stored: tuple[DiscoveredSqlModelFile, ...], + current: tuple[DiscoveredSqlModelFile, ...], + changed_paths: frozenset[str] = frozenset(), + run_id_readers: frozenset[str] = frozenset(), +) -> CompileRenderReuseSession: + """Record renders for the stored models, then plan a session for the current models.""" + + session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=recorded_state(stored, run_id_readers=run_id_readers), changed_paths=changed_paths + ) + session.plan_models(model_files=current) + return session + + +def corrupted_session(item: DiscoveredSqlModelFile) -> CompileRenderReuseSession: + """Plan a session whose only stored render payload cannot be decoded.""" + + state: RenderReuseState = recorded_state((item,)) + session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=replace( + state, + model_payloads={str(item.relative_path): memoryview(b"not a render")}, + ), + changed_paths=frozenset(), + ) + session.plan_models(model_files=(item,)) + return session + + +def edited_session(*, retained_models: frozenset[str]) -> CompileRenderReuseSession: + """Reuse the stored orders render and render an edited customers model.""" + + orders: DiscoveredSqlModelFile = model_file("orders") + customers: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") + session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=recorded_state((orders, model_file("customers"))), + changed_paths=frozenset({str(customers.relative_path)}), + retained_models=retained_models, + ) + session.plan_models(model_files=(orders, customers)) + with collect_compile_diagnostics(): + _ = session.reused_model(model_file=orders) + _ = session.rendered_model(model_file=customers, render=lambda: _render(customers)) + return session + + +def released_paths(state: RenderReuseState | None) -> dict[str, bool]: + """Return, per stored model, whether its bytes were released.""" + + assert state is not None + return {path: payload is None for path, payload in state.model_payloads.items()} + + +def stored_query_sqls(state: RenderReuseState | None) -> dict[str, str]: + """Decode every stored model render and return its query SQL.""" + + assert state is not None + return { + path: _stored_query_sql(cast(memoryview, payload)) + for path, payload in filter(lambda item: item[1] is not None, state.model_payloads.items()) + } + + +def _stored_query_sql(payload: memoryview) -> str: + stored: StoredRender = cast(StoredRender, loaded_fact_payload(payload)) + return cast(CompileModelInput, stored.value).query_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py new file mode 100644 index 0000000000..a906fdd2db --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py @@ -0,0 +1,188 @@ +"""Render reuse must replay what an unchanged render reported and never serve a changed one.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import CompileModelInput +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( + CorruptRenderTestCase, + ReleasedRenderTestCase, + RenderReplayTestCase, + ReusableModelsTestCase, +) +from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( + REGION_ENV_VAR, + corrupted_session, + edited_session, + missing_description, + model_file, + planned_session, + released_paths, + stored_query_sqls, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderReplayTestCase( + description="unchanged_model", + model_name="orders", + expected_diagnostics=(missing_description("orders"),), + expected_environment_names=(REGION_ENV_VAR,), + ), + ], + ids=lambda case: case.description, +) +def test_given_unchanged_model_when_reusing_then_render_and_reports_are_replayed( + test_case: RenderReplayTestCase, +) -> None: + unchanged: DiscoveredSqlModelFile = model_file(test_case.model_name) + edited: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") + session: CompileRenderReuseSession = planned_session( + stored=(unchanged, model_file("customers")), + current=(unchanged, edited), + changed_paths=frozenset({str(edited.relative_path)}), + ) + + with ( + COMPILE_INPUT_READS.recording() as reads, + collect_compile_diagnostics() as collected, + ): + reused: CompileModelInput | None = session.reused_model(model_file=unchanged) + + assert reused == CompileModelInput(model_file=unchanged, query_sql=unchanged.query_sql) + assert reused is not None + assert reused.model_file is unchanged + assert collected.diagnostics == test_case.expected_diagnostics + assert reads.environment_names == test_case.expected_environment_names + + +@pytest.mark.parametrize( + "test_case", + [ + ReusableModelsTestCase( + description="changed_model", + stored_models=("orders", "customers"), + current_models=("orders", "customers"), + changed_paths=frozenset({"models/customers.sql"}), + run_id_readers=frozenset(), + expected_reusable={"orders": True, "customers": False}, + ), + ReusableModelsTestCase( + description="render_read_run_identity", + stored_models=("orders", "customers"), + current_models=("orders", "customers"), + changed_paths=frozenset(), + run_id_readers=frozenset({"orders"}), + expected_reusable={"orders": False, "customers": True}, + ), + ReusableModelsTestCase( + description="model_added", + stored_models=("orders",), + current_models=("orders", "customers"), + changed_paths=frozenset({"models/customers.sql"}), + run_id_readers=frozenset(), + expected_reusable={"orders": False, "customers": False}, + ), + ReusableModelsTestCase( + description="model_removed", + stored_models=("orders", "customers", "returns"), + current_models=("orders", "customers"), + changed_paths=frozenset(), + run_id_readers=frozenset(), + expected_reusable={"orders": False, "customers": False}, + ), + ReusableModelsTestCase( + description="non_model_input_changed", + stored_models=("orders", "customers"), + current_models=("orders", "customers"), + changed_paths=frozenset({"macros/currency.py"}), + run_id_readers=frozenset(), + expected_reusable={"orders": False, "customers": False}, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_renders_when_planning_then_only_unchanged_identical_model_sets_reuse( + test_case: ReusableModelsTestCase, +) -> None: + current: tuple[DiscoveredSqlModelFile, ...] = tuple(map(model_file, test_case.current_models)) + session: CompileRenderReuseSession = planned_session( + stored=tuple(map(model_file, test_case.stored_models)), + current=current, + changed_paths=test_case.changed_paths, + run_id_readers=test_case.run_id_readers, + ) + + reusable: dict[str, bool] = { + item.file_path.stem: session.has_reusable_model(model_file=item) for item in current + } + + assert reusable == test_case.expected_reusable + + +@pytest.mark.parametrize( + "test_case", + [ + CorruptRenderTestCase( + description="undecodable_payload", model_name="orders", expected_rendered_again=True + ), + ], + ids=lambda case: case.description, +) +def test_given_corrupt_stored_render_when_reusing_then_the_model_is_rendered_again( + test_case: CorruptRenderTestCase, +) -> None: + item: DiscoveredSqlModelFile = model_file(test_case.model_name) + session: CompileRenderReuseSession = corrupted_session(item) + + with collect_compile_diagnostics(): + reused: CompileModelInput | None = session.reused_model(model_file=item) + + assert (reused is None) is test_case.expected_rendered_again + + +@pytest.mark.parametrize( + "test_case", + [ + ReleasedRenderTestCase( + description="reused_from_base", + retained_models=frozenset(), + expected_released={"models/orders.sql": True, "models/customers.sql": False}, + expected_query_sqls={ + "models/orders.sql": "SELECT 1 AS order_id", + "models/customers.sql": "SELECT 2 AS customer_id", + }, + ), + ReleasedRenderTestCase( + description="reused_from_overlay", + retained_models=frozenset({"models/orders.sql"}), + expected_released={"models/orders.sql": False, "models/customers.sql": False}, + expected_query_sqls={ + "models/orders.sql": "SELECT 1 AS order_id", + "models/customers.sql": "SELECT 2 AS customer_id", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_reused_render_when_storing_then_base_bytes_are_released_and_recoverable( + test_case: ReleasedRenderTestCase, +) -> None: + session: CompileRenderReuseSession = edited_session(retained_models=test_case.retained_models) + + released: dict[str, bool] = released_paths(session.stored_state()) + complete: dict[str, str] = stored_query_sqls(session.stored_state(complete=True)) + + assert released == test_case.expected_released + assert complete == test_case.expected_query_sqls + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index ac1722759e..d4385d6164 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -243,3 +243,12 @@ class CollectProjectFilesTestCase: description: str model_bytes: bytes expected_shared: bool + + +@dataclass(frozen=True) +class SelectedProjectFilesTestCase: + """Selected lint inputs named by resolved path, possibly through symbolic links.""" + + description: str + selected: str + expected_present: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index e46fed4c20..8a44dae9ea 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -112,3 +112,22 @@ def write_orders_lint_project(*, project_dir: Path, model_bytes: bytes) -> Path: encoding="utf-8", ) return model_path + + +def write_linked_orders_project(*, project_dir: Path) -> None: + """Write models reached both directly and through file and directory links.""" + + (project_dir / "models" / "orders").mkdir(parents=True) + (project_dir / "shared").mkdir() + (project_dir / "models" / "orders" / "orders.sql").write_text("SELECT 1 AS order_id\n") + (project_dir / "shared" / "customers.sql").write_text("SELECT 1 AS customer_id\n") + (project_dir / "models" / "orders" / "customers_link.sql").symlink_to( + project_dir / "shared" / "customers.sql" + ) + (project_dir / "models" / "shared_link").symlink_to(project_dir / "shared") + + +def files_resolving_to(*, files: dict[Path, str], selected: frozenset[Path]) -> dict[Path, str]: + """Return the files whose fully resolved path is one of the selected paths.""" + + return dict(filter(lambda item: item[0].resolve() in selected, files.items())) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index 693e961d64..c9f239e882 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -11,8 +11,15 @@ from sqlbuild.lint._helpers.project_files import collect_project_files from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult -from tests.unit.src.sqlbuild.lint._helpers._test_types import CollectProjectFilesTestCase -from tests.unit.src.sqlbuild.lint._helpers.helpers import write_orders_lint_project +from tests.unit.src.sqlbuild.lint._helpers._test_types import ( + CollectProjectFilesTestCase, + SelectedProjectFilesTestCase, +) +from tests.unit.src.sqlbuild.lint._helpers.helpers import ( + files_resolving_to, + write_linked_orders_project, + write_orders_lint_project, +) @pytest.mark.parametrize( @@ -54,5 +61,41 @@ def test_given_discovered_project_when_collecting_files_then_returns_exact_autho assert shared.files_checked == fresh.files_checked == 2 +@pytest.mark.parametrize( + "test_case", + [ + SelectedProjectFilesTestCase( + description="regular_file", + selected="models/orders/orders.sql", + expected_present=("models/orders/orders.sql",), + ), + SelectedProjectFilesTestCase( + description="file_reached_through_links", + selected="shared/customers.sql", + expected_present=("models/orders/customers_link.sql",), + ), + ], + ids=lambda case: case.description, +) +def test_given_selected_resolved_paths_when_collecting_files_then_matches_resolving_each_file( + test_case: SelectedProjectFilesTestCase, tmp_path: Path +) -> None: + write_linked_orders_project(project_dir=tmp_path) + selected: frozenset[Path] = frozenset({(tmp_path / test_case.selected).resolve()}) + every_file: dict[Path, str] = collect_project_files(project_dir=tmp_path) + + collected: dict[Path, str] = collect_project_files( + project_dir=tmp_path, selected_paths=selected + ) + from_sources: dict[Path, str] = collect_project_files( + project_dir=tmp_path, selected_paths=selected, source_files=every_file + ) + + relative: set[str] = {path.relative_to(tmp_path).as_posix() for path in collected} + assert relative >= set(test_case.expected_present) + assert from_sources == collected + assert collected == files_resolving_to(files=every_file, selected=selected) + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index b039cbc86d..affbda9659 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -108,8 +108,31 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data or the project has very many files, compile -prints `Recording compile for reuse...` and a completion line to stderr before the report. +report is ready. When it has to hash a lot of data, the project has very many files, or it records +thousands of new model renders, compile prints `Recording compile for reuse...` and a completion +line to stderr before the report. + +### Compiling after model-only edits + +When the only changes since the stored compile are edits to existing model files, compile runs in +full but reuses the stored render of every unchanged model, together with the diagnostics and +environment reads that rendering reported. Only the edited models are rendered again. Analysis, +contracts, rules, and artifact writes still run for the whole project, so downstream effects of an +edit, such as a changed column type, are reported exactly as in an uncached compile. The output is +byte-identical to `sqb compile --no-cache`. + +Every other change renders the whole project: adding, removing, or renaming a model, and editing +macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope +folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings +is always rendered again. + +Renders are stored only when a compile result for the same project and target is already stored, +so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. +The next full compile stores them, and edits after that reuse them. Stored renders are compressed +and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints +`Loading stored renders...` and a completion line to stderr before it starts. While renders are +reused, the Python modules recorded by the previous compile stay tracked even if this compile did +not import them, so editing such a module runs the next compile in full. Known limits: @@ -130,7 +153,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or +rendered again after model-only edits. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs.