From 96cc018dd0ce7c2d2e4b68fed5c6817e99bf79b8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:27:30 +0100 Subject: [PATCH 1/4] perf(compile): reuse unaffected model renders after model-only edits --- .../cli/commands/main/project/_compile.py | 70 +- .../cli/compile_reuse/_helpers/attempt.py | 21 +- .../cli/compile_reuse/_helpers/entry_file.py | 99 ++- .../compile_reuse/_helpers/project_files.py | 30 + .../_helpers/render_state_file.py | 234 +++++++ .../cli/compile_reuse/_helpers/store.py | 22 +- src/sqlbuild/cli/compile_reuse/constants.py | 4 + .../main/_render_reuse_session.py | 30 + .../main/_write_reusable_compile.py | 4 +- src/sqlbuild/cli/compile_reuse/models.py | 53 ++ .../compile/_helpers/attachment/core.py | 610 ++++++++++-------- .../compile/_helpers/diagnostics/collector.py | 19 + .../compile/classes/render_reuse_registry.py | 41 ++ .../compile/classes/render_reuse_session.py | 224 +++++++ src/sqlbuild/compiler/compile/constants.py | 5 + .../compile/main/_build_compile_inputs.py | 90 ++- src/sqlbuild/compiler/compile/models.py | 18 + .../fact_cache/_helpers/restricted_pickle.py | 2 +- src/sqlbuild/compiler/profiling/constants.py | 2 + src/sqlbuild/compiler/profiling/types.py | 2 + .../main/build/test_compile_json_behavior.py | 2 + .../cli/commands/main/compile/_test_types.py | 27 + .../cli/commands/main/compile/helpers.py | 178 +++++ .../compile/test_compile_incremental_edits.py | 296 +++++++++ 24 files changed, 1730 insertions(+), 353 deletions(-) create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py create mode 100644 src/sqlbuild/compiler/compile/classes/render_reuse_registry.py create mode 100644 src/sqlbuild/compiler/compile/classes/render_reuse_session.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index aa35dbc92c..4e6422e05a 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -3,6 +3,7 @@ from __future__ import annotations import time +from contextlib import AbstractContextManager, nullcontext from dataclasses import replace from pathlib import Path @@ -31,11 +32,12 @@ from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings +from sqlbuild.cli.compile_reuse.main._render_reuse_session import render_reuse_session from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.models import CompileRenderReuse, CompileReuseAttempt from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.cli.entry.models import CompileCommandRequest -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, COMPILE_RENDER_REUSE from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic from sqlbuild.compiler.compile.types import DiagnosticPhase from sqlbuild.compiler.profiling.main.collect import collect_compile_timings @@ -64,34 +66,35 @@ def run_compile( else reuse_attempt ) output: RecordedCompileOutput = RecordedCompileOutput() + render_reuse: CompileRenderReuse | None = render_reuse_session(attempt=attempt) status: TransientStatusReporter | None = start_compile_status( json_output=request.json_output, no_color=request.no_color, ) - try: - with ( - paused_cyclic_collection(), - collect_compile_timings() as detailed_timings, - COMPILE_INPUT_READS.recording() as input_reads, - COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, - PreparedCompileArtifacts( - enabled=not request.profile_flags.skip_write - ) as prepared_artifacts, - ): - result: CompileCommandResult = _run_compile_with_status( - request=effective_request, - total_start=total_start, - status=status, - detailed_timings=detailed_timings, - prepared_artifacts=prepared_artifacts, - output=output, - reuse_timings_ms=compile_reuse_timings(attempt=attempt), - ) - finally: - if status is not None: - status.close() - try: - with paused_cyclic_collection(): + with paused_cyclic_collection(): + try: + with ( + collect_compile_timings() as detailed_timings, + COMPILE_INPUT_READS.recording() as input_reads, + COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, + _activated_render_reuse(render_reuse=render_reuse), + PreparedCompileArtifacts( + enabled=not request.profile_flags.skip_write + ) as prepared_artifacts, + ): + result: CompileCommandResult = _run_compile_with_status( + request=effective_request, + total_start=total_start, + status=status, + detailed_timings=detailed_timings, + prepared_artifacts=prepared_artifacts, + output=output, + reuse_timings_ms=compile_reuse_timings(attempt=attempt), + ) + finally: + if status is not None: + status.close() + try: write_reusable_compile( attempt=attempt, output=output, @@ -102,12 +105,23 @@ def run_compile( artifacts_written=result.artifacts_written, dag_artifact_path=result.dag_artifact_path, json_output=request.json_output, + render_reuse=render_reuse, ) - finally: - output.print_stdout() + finally: + output.print_stdout() return result.exit_code +def _activated_render_reuse( + *, render_reuse: CompileRenderReuse | None +) -> AbstractContextManager[object]: + return ( + nullcontext() + if render_reuse is None + else COMPILE_RENDER_REUSE.activated(render_reuse.session) + ) + + def _run_compile_with_status( *, request: CompileCommandRequest, diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index ccb90f2e46..db1caaadf7 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -9,12 +9,14 @@ from pathlib import Path from sqlbuild.cli.compile_reuse._helpers.entry_file import ( + entry_render_state_path, read_entry_header, read_entry_stdout, rewrite_entry_inputs, ) from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, + changed_project_paths, compare_project_files, needs_refresh, restamped_paths, @@ -103,6 +105,11 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: attempt = replace( attempt, restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), + render_state_path=( + None + if attempt.changed_paths is None or attempt.entry_path is None + else entry_render_state_path(path=attempt.entry_path, header=header) + ), ) return replace( attempt, @@ -155,12 +162,24 @@ def _checked_attempt( unchanged: bool = comparison.unchanged and target_files_unchanged( stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree ) + verified: dict[str, str] = dict(comparison.verified) + changed_paths: frozenset[str] | None = ( + changed_project_paths( + project_dir=project_dir, + stored=inputs.project_files, + current=attempt.snapshot, + verified=verified, + ) + if identity_unchanged and not unchanged + else None + ) return replace( attempt, outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, digests=carried_forward_digests( - stored=inputs.project_files, current=attempt.snapshot, verified=comparison.verified + stored=inputs.project_files, current=attempt.snapshot, verified=verified ), + changed_paths=changed_paths, ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py index f7da96aeb7..c7453b448c 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py @@ -20,12 +20,15 @@ REUSE_ENTRY_SUFFIX, REUSE_FORMAT_VERSION, REUSE_MAX_ENTRY_BYTES, + REUSE_MAX_RENDER_STATES, REUSE_MAX_STORED_ENTRIES, + REUSE_RENDER_STATE_SUFFIX, REUSE_STDOUT_SEPARATOR, REUSE_STDOUT_SUFFIX, ) from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError from sqlbuild.cli.compile_reuse.models import ( + RenderStateLayer, SettingsEnvironmentInputs, StoredCompileHeader, StoredCompileInputs, @@ -48,7 +51,7 @@ def read_entry_header(*, path: Path) -> StoredCompileHeader | None: with open(path, "rb") as handle: if handle.read(len(REUSE_ENTRY_MAGIC)) != REUSE_ENTRY_MAGIC: return None - metadata: bytes = _read_section(handle=handle) + metadata: bytes = read_framed_section(handle=handle) if handle.read(1): return None payload: object = json.loads(metadata) @@ -80,13 +83,24 @@ def write_entry( inputs: StoredCompileInputs, output: StoredCompileOutput, stdout: str, + render_state: RenderStateLayer | None = None, ) -> None: - """Publish one entry and its stdout atomically, filling in the stdout file and checksum.""" + """Publish one entry, its stdout, and its renders atomically, naming them in the entry.""" encoded_stdout: bytes = stdout.encode("utf-8", "surrogateescape") - stdout_file: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}{REUSE_STDOUT_SUFFIX}" + stamp: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}" + stdout_file: str = f"{stamp}{REUSE_STDOUT_SUFFIX}" + render_state_file: str | None = ( + None + if render_state is None + or sum(len(chunk) for chunk in render_state.chunks) > REUSE_MAX_ENTRY_BYTES + else f"{stamp}{REUSE_RENDER_STATE_SUFFIX}" + ) stored_output: StoredCompileOutput = replace( - output, stdout_file=stdout_file, stdout_checksum=zlib.crc32(encoded_stdout) + output, + stdout_file=stdout_file, + stdout_checksum=zlib.crc32(encoded_stdout), + render_state_file=render_state_file, ) metadata: bytes = _metadata(inputs=inputs, output=stored_output) if len(metadata) + len(encoded_stdout) > REUSE_MAX_ENTRY_BYTES: @@ -94,14 +108,31 @@ def write_entry( return try: path.parent.mkdir(parents=True, exist_ok=True) - _publish_file(path=path.parent / stdout_file, contents=encoded_stdout) - _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) - _remove_stdout_files(entry_path=path, keep=stdout_file) + _publish_file(path=path.parent / stdout_file, contents=(encoded_stdout,)) + if render_state is not None and render_state_file is not None: + _publish_file(path=path.parent / render_state_file, contents=render_state.chunks) + _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) + _remove_stdout_files(entry_path=path, keep=(stdout_file,)) + _remove_render_files( + entry_path=path, + keep=( + () + if render_state is None or render_state_file is None + else (render_state_file, render_state.base_file) + ), + ) _prune_entries(directory=path.parent, keep=path) except OSError: remove_entry(path=path) +def entry_render_state_path(*, path: Path, header: StoredCompileHeader) -> Path | None: + """Return where the entry's renders are stored, when it stored any.""" + + name: str | None = header.output.render_state_file + return None if name is None else path.parent / name + + def rewrite_entry_inputs( *, path: Path, header: StoredCompileHeader, inputs: StoredCompileInputs ) -> None: @@ -109,7 +140,7 @@ def rewrite_entry_inputs( metadata: bytes = _metadata(inputs=inputs, output=header.output) with contextlib.suppress(OSError): - _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) + _publish_file(path=path, contents=(_framed_metadata(metadata=metadata),)) def remove_entry(*, path: Path) -> None: @@ -117,16 +148,18 @@ def remove_entry(*, path: Path) -> None: with contextlib.suppress(OSError): path.unlink(missing_ok=True) - _remove_stdout_files(entry_path=path, keep=None) + _remove_stdout_files(entry_path=path, keep=()) + _remove_render_files(entry_path=path, keep=()) -def _publish_file(*, path: Path, contents: bytes) -> None: +def _publish_file(*, path: Path, contents: tuple[bytes | memoryview, ...]) -> None: descriptor, temporary_path = tempfile.mkstemp( prefix=f".{path.name}.", suffix=".tmp", dir=path.parent ) try: with os.fdopen(descriptor, "wb") as handle: - _ = handle.write(contents) + for chunk in contents: + _ = handle.write(chunk) os.replace(temporary_path, path) except BaseException: with contextlib.suppress(OSError): @@ -141,17 +174,30 @@ def _metadata(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> by def _framed_metadata(*, metadata: bytes) -> bytes: + return REUSE_ENTRY_MAGIC + framed_section(data=metadata) + + +def framed_section(*, data: bytes) -> bytes: + """Prefix one section with its length and checksum so a reader can verify it.""" + return b"".join( ( - REUSE_ENTRY_MAGIC, - len(metadata).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), - zlib.crc32(metadata).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), - metadata, + len(data).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), + zlib.crc32(data).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), + data, ) ) -def _remove_stdout_files(*, entry_path: Path, keep: str | None) -> None: +def _remove_stdout_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: + _remove_entry_files(entry_path=entry_path, suffix=REUSE_STDOUT_SUFFIX, keep=keep) + + +def _remove_render_files(*, entry_path: Path, keep: tuple[str | None, ...]) -> None: + _remove_entry_files(entry_path=entry_path, suffix=REUSE_RENDER_STATE_SUFFIX, keep=keep) + + +def _remove_entry_files(*, entry_path: Path, suffix: str, keep: tuple[str | None, ...]) -> None: prefix: str = f"{entry_path.stem}{REUSE_STDOUT_SEPARATOR}" try: candidates: list[Path] = list(entry_path.parent.iterdir()) @@ -160,14 +206,16 @@ def _remove_stdout_files(*, entry_path: Path, keep: str | None) -> None: for candidate in candidates: if ( candidate.name.startswith(prefix) - and candidate.suffix == REUSE_STDOUT_SUFFIX - and candidate.name != keep + and candidate.suffix == suffix + and candidate.name not in keep ): with contextlib.suppress(OSError): candidate.unlink() -def _read_section(*, handle: BinaryIO) -> bytes: +def read_framed_section(*, handle: BinaryIO) -> bytes: + """Read one length- and checksum-framed section, raising when it is damaged.""" + length: int = int.from_bytes(handle.read(REUSE_ENTRY_LENGTH_BYTES), REUSE_ENTRY_BYTE_ORDER) checksum: int = int.from_bytes(handle.read(REUSE_ENTRY_CHECKSUM_BYTES), REUSE_ENTRY_BYTE_ORDER) if length > REUSE_MAX_ENTRY_BYTES: @@ -188,6 +236,8 @@ def _prune_entries(*, directory: Path, keep: Path) -> None: entries.sort(reverse=True) for _, stale in entries[REUSE_MAX_STORED_ENTRIES - 1 :]: remove_entry(path=stale) + for _, older in entries[REUSE_MAX_RENDER_STATES - 1 : REUSE_MAX_STORED_ENTRIES - 1]: + _remove_render_files(entry_path=older, keep=()) def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dict[str, object]: @@ -210,6 +260,7 @@ def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dic "stdout_length": output.stdout_length, "stdout_checksum": output.stdout_checksum, "stdout_file": output.stdout_file, + "render_state_file": output.render_state_file, "target_tree": inputs.target_tree, "settings_inputs": [ { @@ -266,6 +317,7 @@ def _header(*, payload: object) -> StoredCompileHeader: stdout_length=_integer(fields["stdout_length"]), stdout_checksum=_integer(fields["stdout_checksum"]), stdout_file=_stdout_file(fields["stdout_file"]), + render_state_file=_render_state_file(fields.get("render_state_file")), ), ) @@ -288,6 +340,15 @@ def _stdout_file(value: object) -> str: return name +def _render_state_file(value: object) -> str | None: + if value is None: + return None + name: str = _string(value) + if os.path.basename(name) != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): + raise CompileReuseEntryError("invalid stored render file name") + return name + + def _timings_span(value: object) -> tuple[int, int] | None: if value is None: return None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py index 5c0472b33c..0c2ed267ae 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py @@ -101,6 +101,36 @@ def compare_project_files( return ProjectFilesComparison(unchanged=True, verified=verified) +def changed_project_paths( + *, + project_dir: str, + stored: dict[str, StoredProjectFile], + current: dict[str, FileStamp], + verified: dict[str, str], +) -> frozenset[str]: + """Return every path added, removed, or changed in content since the stored compile.""" + + changed: set[str] = {path for path in stored if path not in current} + for relative_path, stamp in current.items(): + previous: StoredProjectFile | None = stored.get(relative_path) + if previous is None or not _same_identity(stamp=stamp, recorded=previous.stamp): + changed.add(relative_path) + continue + if stamp.kind in _PRESENCE_ONLY_KINDS or (stamp == previous.stamp and not previous.racy): + continue + if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: + changed.add(relative_path) + continue + digest: str | None = verified.get(relative_path) or file_digest( + path=os.path.join(project_dir, relative_path) + ) + if digest is None or digest != previous.digest: + changed.add(relative_path) + else: + verified[relative_path] = digest + return frozenset(changed) + + def carried_forward_digests( *, stored: dict[str, StoredProjectFile], diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py new file mode 100644 index 0000000000..a90eb3af18 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py @@ -0,0 +1,234 @@ +"""Checksummed, layered storage for the renders of the compile one stored entry describes.""" + +from __future__ import annotations + +import json +import zlib +from collections.abc import Iterable +from pathlib import Path +from typing import cast + +from sqlbuild.cli.compile_reuse._helpers.entry_file import framed_section, read_framed_section +from sqlbuild.cli.compile_reuse.constants import ( + REUSE_ENTRY_BYTE_ORDER, + REUSE_ENTRY_CHECKSUM_BYTES, + REUSE_ENTRY_LENGTH_BYTES, + REUSE_MAX_ENTRY_BYTES, + REUSE_RENDER_OVERLAY_MAX_SHARE, + REUSE_RENDER_STATE_MAGIC, + REUSE_RENDER_STATE_SUFFIX, +) +from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError +from sqlbuild.cli.compile_reuse.models import ( + RenderStateLayer, + StoredRenderLayer, + StoredRenderLayers, + StoredRenderState, +) +from sqlbuild.compiler.compile.models import RenderReuseState + +_INDEX_FIELD_COUNT: int = 2 + + +def read_render_state(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderState | None: + """Read verified renders when every changed path is a stored model; any fault means none.""" + + try: + newest: StoredRenderLayer | None = _read_layer(path=path, changed_paths=changed_paths) + if newest is None: + return None + if newest.base_file is None: + return _stored_state(base_file=path.name, base=newest, overlay=None) + base: StoredRenderLayer | None = _read_layer( + path=path.parent / _layer_name(newest.base_file), changed_paths=frozenset() + ) + if base is None or base.base_file is not None: + return None + return _stored_state(base_file=newest.base_file, base=base, overlay=newest) + except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError): + return None + + +def render_state_layer( + *, state: RenderReuseState, stored: StoredRenderState | None +) -> RenderStateLayer: + """Extend the stored base with a small overlay of changed renders, or write a new base.""" + + if stored is not None and state.model_paths == stored.state.model_paths: + overlay_models: frozenset[str] | None = _overlay_keys( + current=state.model_payloads, + previous=stored.state.model_payloads, + overlay=stored.layers.overlay_models, + ) + overlay_groups: frozenset[str] | None = _overlay_keys( + current=state.group_payloads, + previous=stored.state.group_payloads, + overlay=stored.layers.overlay_groups, + ) + if overlay_models is not None and overlay_groups is not None: + models: list[tuple[str, memoryview]] = [ + (path, state.model_payloads[path]) + for path in state.model_paths + if path in overlay_models + ] + groups: list[tuple[str, memoryview]] = sorted( + (name, state.group_payloads[name]) for name in overlay_groups + ) + if ( + sum(len(item) for _, item in (*models, *groups)) + <= stored.layers.base_bytes * REUSE_RENDER_OVERLAY_MAX_SHARE + ): + return _layer( + model_paths=state.model_paths, + base_file=stored.layers.base_file, + models=models, + groups=groups, + ) + return _layer( + model_paths=state.model_paths, + base_file=None, + models=[ + (path, state.model_payloads[path]) + for path in state.model_paths + if path in state.model_payloads + ], + groups=sorted(state.group_payloads.items()), + ) + + +def _overlay_keys( + *, + current: dict[str, memoryview], + previous: dict[str, memoryview], + overlay: frozenset[str], +) -> frozenset[str] | None: + """Return keys the overlay must hold, or None when a stored render is no longer valid.""" + + if not previous.keys() <= current.keys(): + return None + return overlay | {key for key, payload in current.items() if previous.get(key) is not payload} + + +def _layer( + *, + model_paths: tuple[str, ...], + base_file: str | None, + models: list[tuple[str, memoryview]], + groups: list[tuple[str, memoryview]], +) -> RenderStateLayer: + index: bytes = json.dumps( + { + "model_paths": list(model_paths), + "base": base_file, + "models": [[path, len(payload)] for path, payload in models], + "groups": [[name, len(payload)] for name, payload in groups], + }, + separators=(",", ":"), + ).encode("utf-8", "surrogateescape") + payloads: tuple[memoryview, ...] = tuple(item for _, item in (*models, *groups)) + return RenderStateLayer( + chunks=( + REUSE_RENDER_STATE_MAGIC, + framed_section(data=index), + _section_header(payloads=payloads), + *payloads, + ), + base_file=base_file, + ) + + +def _section_header(*, payloads: Iterable[memoryview]) -> bytes: + length: int = 0 + checksum: int = 0 + for payload in payloads: + length += len(payload) + checksum = zlib.crc32(payload, checksum) + return length.to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER) + checksum.to_bytes( + REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER + ) + + +def _read_layer(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderLayer | None: + with open(path, "rb") as handle: + if handle.read(len(REUSE_RENDER_STATE_MAGIC)) != REUSE_RENDER_STATE_MAGIC: + return None + index: object = json.loads(read_framed_section(handle=handle)) + if not isinstance(index, dict): + return None + fields: dict[str, object] = cast(dict[str, object], index) + model_paths: tuple[str, ...] = tuple(_string(item) for item in _list(fields["model_paths"])) + if not changed_paths.issubset(model_paths): + return None + payload: bytes = read_framed_section(handle=handle) + if handle.read(1) or len(payload) > REUSE_MAX_ENTRY_BYTES: + return None + base: object = fields["base"] + view: memoryview = memoryview(payload) + offset: int = 0 + models: dict[str, memoryview] = {} + for name, length in _rows(fields["models"]): + models[name] = view[offset : offset + length] + offset += length + groups: dict[str, memoryview] = {} + for name, length in _rows(fields["groups"]): + groups[name] = view[offset : offset + length] + offset += length + if offset != len(view): + return None + return StoredRenderLayer( + model_paths=model_paths, + base_file=None if base is None else _string(base), + models=models, + groups=groups, + size=len(payload), + ) + + +def _stored_state( + *, base_file: str, base: StoredRenderLayer, overlay: StoredRenderLayer | None +) -> StoredRenderState: + newest: StoredRenderLayer = base if overlay is None else overlay + return StoredRenderState( + state=RenderReuseState( + model_paths=newest.model_paths, + model_payloads=base.models if overlay is None else {**base.models, **overlay.models}, + group_payloads=base.groups if overlay is None else {**base.groups, **overlay.groups}, + ), + layers=StoredRenderLayers( + base_file=base_file, + base_bytes=base.size, + overlay_models=frozenset() if overlay is None else frozenset(overlay.models), + overlay_groups=frozenset() if overlay is None else frozenset(overlay.groups), + ), + ) + + +def _layer_name(name: str) -> str: + if Path(name).name != name or not name.endswith(REUSE_RENDER_STATE_SUFFIX): + raise CompileReuseEntryError("invalid stored render base name") + return name + + +def _rows(value: object) -> list[tuple[str, int]]: + rows: list[tuple[str, int]] = [] + for row in _list(value): + items: list[object] = _list(row) + if len(items) != _INDEX_FIELD_COUNT: + raise CompileReuseEntryError("invalid stored render index") + length: object = items[1] + if not isinstance(length, int) or isinstance(length, bool) or length < 0: + raise CompileReuseEntryError("invalid stored render length") + rows.append((_string(items[0]), length)) + return rows + + +def _list(value: object) -> list[object]: + if not isinstance(value, list): + raise CompileReuseEntryError("invalid stored render list") + return cast(list[object], value) + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise CompileReuseEntryError("invalid stored render string") + return value diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index 7bed444a1d..cfcde2ffd9 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -16,6 +16,7 @@ with_missing_digests, ) from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs +from sqlbuild.cli.compile_reuse._helpers.render_state_file import render_state_layer from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( environment_digest, loaded_module_stamps, @@ -36,13 +37,16 @@ REUSE_STORE_START_MESSAGE, ) from sqlbuild.cli.compile_reuse.models import ( + CompileRenderReuse, CompileReuseAttempt, + RenderStateLayer, SettingsInputsResult, StoredCompileInputs, StoredCompileOutput, ) from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads +from sqlbuild.compiler.compile.models import RenderReuseState _LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) @@ -58,6 +62,7 @@ def write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, + render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" @@ -66,7 +71,6 @@ def write_compile_entry( try: _write_compile_entry( attempt=attempt, - entry_path=attempt.entry_path, output=output, exit_code=exit_code, input_reads=input_reads, @@ -75,6 +79,7 @@ def write_compile_entry( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, + render_reuse=render_reuse, ) except Exception: _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) @@ -84,7 +89,6 @@ def write_compile_entry( def _write_compile_entry( *, attempt: CompileReuseAttempt, - entry_path: Path, output: RecordedCompileOutput, exit_code: int, input_reads: CompileInputReads, @@ -93,7 +97,11 @@ def _write_compile_entry( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, + render_reuse: CompileRenderReuse | None, ) -> None: + if attempt.entry_path is None: + return + entry_path: Path = attempt.entry_path stdout: str | None = output.stdout settings: SettingsInputsResult = provider_settings_inputs( settings_classes=input_reads.settings_classes @@ -165,10 +173,20 @@ def _write_compile_entry( stdout_file="", ), stdout=stdout, + render_state=_render_state_layer(render_reuse=render_reuse), ) _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) +def _render_state_layer(*, render_reuse: CompileRenderReuse | None) -> RenderStateLayer | None: + state: RenderReuseState | None = ( + None if render_reuse is None else render_reuse.session.stored_state() + ) + if render_reuse is None or state is None: + return None + return render_state_layer(state=state, stored=render_reuse.stored) + + def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: pending_bytes: int = pending_digest_bytes( snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index b5af8a4d5b..ea40480ddf 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -14,6 +14,10 @@ REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" REUSE_STDOUT_SEPARATOR: str = "-" +REUSE_RENDER_STATE_SUFFIX: str = ".render" +REUSE_RENDER_STATE_MAGIC: bytes = b"SQBRENDER1\n" +REUSE_MAX_RENDER_STATES: int = 2 +REUSE_RENDER_OVERLAY_MAX_SHARE: float = 0.25 REUSE_ENTRY_LENGTH_BYTES: int = 8 REUSE_ENTRY_CHECKSUM_BYTES: int = 4 REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" diff --git a/src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py b/src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py new file mode 100644 index 0000000000..919a6bfd95 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py @@ -0,0 +1,30 @@ +"""Public entry for the render reuse one full compile runs with.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_reuse._helpers.render_state_file import read_render_state +from sqlbuild.cli.compile_reuse.models import ( + CompileRenderReuse, + CompileReuseAttempt, + StoredRenderState, +) +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession + + +def render_reuse_session(*, attempt: CompileReuseAttempt) -> CompileRenderReuse | None: + """Start recording renders, reusing the stored ones the changed paths cannot affect.""" + + if attempt.outcome is not CompileReuseOutcome.MISS or attempt.entry_path is None: + return None + stored: StoredRenderState | None = ( + None + if attempt.changed_paths is None or attempt.render_state_path is None + else read_render_state(path=attempt.render_state_path, changed_paths=attempt.changed_paths) + ) + return CompileRenderReuse( + session=CompileRenderReuseSession( + prior=None if stored is None else stored.state, changed_paths=attempt.changed_paths + ), + stored=stored, + ) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py index 0736318972..ff1dc933a7 100644 --- a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py +++ b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py @@ -7,7 +7,7 @@ from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.models import CompileRenderReuse, CompileReuseAttempt from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads @@ -22,6 +22,7 @@ def write_reusable_compile( artifacts_written: bool, dag_artifact_path: Path | None, json_output: bool, + render_reuse: CompileRenderReuse | None = None, ) -> None: """Store a reusable full compile, or drop the stored one when it cannot be reused.""" @@ -35,4 +36,5 @@ def write_reusable_compile( artifacts_written=artifacts_written, dag_artifact_path=dag_artifact_path, json_output=json_output, + render_reuse=render_reuse, ) diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index cc2bd55c4b..0fcd66b9e3 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -4,9 +4,14 @@ from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp +if TYPE_CHECKING: + from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession + from sqlbuild.compiler.compile.models import RenderReuseState + @dataclass(frozen=True) class CompileReuseRequest: @@ -93,6 +98,7 @@ class StoredCompileOutput: stdout_length: int stdout_checksum: int stdout_file: str + render_state_file: str | None = None @dataclass(frozen=True) @@ -128,3 +134,50 @@ class CompileReuseAttempt: snapshot_ns: int = 0 digests: dict[str, str] = field(default_factory=dict) restamped: frozenset[str] = frozenset() + changed_paths: frozenset[str] | None = None + render_state_path: Path | None = None + + +@dataclass(frozen=True) +class StoredRenderLayer: + """One decoded render file: its index and its payloads by key.""" + + model_paths: tuple[str, ...] + base_file: str | None + models: dict[str, memoryview] + groups: dict[str, memoryview] + size: int + + +@dataclass(frozen=True) +class StoredRenderLayers: + """How a stored compile's renders are split between a base file and one overlay.""" + + base_file: str + base_bytes: int + overlay_models: frozenset[str] + overlay_groups: frozenset[str] + + +@dataclass(frozen=True) +class StoredRenderState: + """Renders a stored compile recorded, merged across layers, with their layout.""" + + state: RenderReuseState + layers: StoredRenderLayers + + +@dataclass(frozen=True) +class CompileRenderReuse: + """One full compile's render reuse session with the stored renders it started from.""" + + session: CompileRenderReuseSession + stored: StoredRenderState | None + + +@dataclass(frozen=True) +class RenderStateLayer: + """One render layer to publish: its framed chunks, and the base layer it extends.""" + + chunks: tuple[bytes | memoryview, ...] + base_file: str | None diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c2260c1862..6e3fb87045 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -6,6 +6,7 @@ import re from collections.abc import Callable from dataclasses import dataclass, field, fields, replace +from functools import partial from inspect import Parameter, Signature from pathlib import Path from typing import Any, cast @@ -95,6 +96,7 @@ expand_effective_vars, expand_template_data, ) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_FULL_REFRESH_CONFIG_KEY, @@ -255,6 +257,19 @@ class _ModelValidationContext: custom_materialization_names: frozenset[str] +@dataclass(frozen=True) +class _ModelInputLoop: + discovered_inputs: DiscoveredProjectInputs + context: ModelInputBuildContext + validation_context: _ModelValidationContext + sql_hook_definitions: dict[str, DiscoveredSqlHookFile] + legacy_schema_files: tuple[DiscoveredSchemaFile, ...] + model_header_column_cache: ModelHeaderColumnCache + config_scan_cache: ModelConfigScanCache + reusable_config_cache: _ReusableModelConfigCache + declaration_cache: _VisibleModelDeclarationCache + + @dataclass(frozen=True) class _HookExpansionContext: file_path: Path @@ -414,6 +429,7 @@ def build_model_inputs( defer_model_sql_validation: bool = False, external_sql_reference_resolver: ExternalSqlReferenceResolver | None = None, reference_cache_dir: Path | None = None, + render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileModelInput, ...]: """Attach schema metadata to discovered model files.""" @@ -431,6 +447,7 @@ def build_model_inputs( external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, legacy_schema_files=legacy_schema_files, + render_reuse=render_reuse, ) @@ -443,15 +460,11 @@ def _build_model_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, extract_references: Callable[[str], tuple[CompileSqlReference, ...]], legacy_schema_files: tuple[DiscoveredSchemaFile, ...], + render_reuse: CompileRenderReuseSession | None, ) -> tuple[CompileModelInput, ...]: effective_vars: dict[str, object] = context.effective_vars effective_settings: SettingsConfig = context.effective_settings - target_config: TargetConfig | None = context.target_config - effective_target_name: str | None = context.effective_target_name - run_id: str = context.run_id - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros known_model_names: set[str] = build_known_ref_names(discovered_inputs) known_seed_names: set[str] = build_known_seed_names(discovered_inputs) known_source_names: set[str] = build_known_source_names(discovered_inputs) @@ -479,305 +492,352 @@ def _build_model_inputs( sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( discovered_inputs.sql_hook_files ) - model_inputs: list[CompileModelInput] = [] - model_header_column_cache: ModelHeaderColumnCache = ModelHeaderColumnCache() - config_scan_cache: ModelConfigScanCache = ModelConfigScanCache() - reusable_config_cache: _ReusableModelConfigCache = _ReusableModelConfigCache( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - target_config=target_config, + render_files: tuple[DiscoveredSqlModelFile, ...] = tuple( + model_file + for model_file in discovered_inputs.model_files + if render_reuse is None or not render_reuse.has_reusable_model(model_file=model_file) ) - declaration_cache: _VisibleModelDeclarationCache = _VisibleModelDeclarationCache.build(context) - model_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files - prepared_var_substituted_sqls: tuple[str | None, ...] = prepare_static_project_vars_batch( - sqls=tuple(model_file.query_sql for model_file in model_files), - effective_vars=effective_vars, + prepared_var_substituted_sqls: dict[Path, str | None] = dict( + zip( + (model_file.file_path for model_file in render_files), + prepare_static_project_vars_batch( + sqls=tuple(model_file.query_sql for model_file in render_files), + effective_vars=effective_vars, + ), + strict=True, + ) ) + loop: _ModelInputLoop = _ModelInputLoop( + discovered_inputs=discovered_inputs, + context=context, + validation_context=validation_context, + sql_hook_definitions=sql_hook_definitions, + legacy_schema_files=legacy_schema_files, + model_header_column_cache=ModelHeaderColumnCache(), + config_scan_cache=ModelConfigScanCache(), + reusable_config_cache=_ReusableModelConfigCache( + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, + target_config=context.target_config, + ), + declaration_cache=_VisibleModelDeclarationCache.build(context), + ) + model_inputs: list[CompileModelInput] = [] model_file: DiscoveredSqlModelFile - prepared_var_substituted_sql: str | None - for model_file, prepared_var_substituted_sql in zip( - model_files, prepared_var_substituted_sqls, strict=True - ): - model_identity: ResourceIdentity = ResourceIdentity( - ResourceKind.MODEL, model_file.file_path.stem - ) - declarations: _VisibleModelDeclarations = declaration_cache.for_model( - model_file=model_file, consumer=model_identity + for model_file in discovered_inputs.model_files: + reused: CompileModelInput | None = ( + None if render_reuse is None else render_reuse.reused_model(model_file=model_file) ) - matched_path_default: str | None = find_matching_path_default( + if reused is not None: + model_inputs.append(reused) + continue + render: Callable[[], CompileModelInput] = partial( + _build_model_input, + loop=loop, model_file=model_file, - path_defaults=discovered_inputs.project_config.path_defaults, + prepared_var_substituted_sql=prepared_var_substituted_sqls.get(model_file.file_path), ) - effective_config: CompileModelConfig | None = reusable_config_cache.get( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, + model_inputs.append( + render() + if render_reuse is None + else render_reuse.rendered_model(model_file=model_file, render=render) ) - if effective_config is None: - effective_config = build_model_config( - request=ModelConfigBuildRequest( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - effective_vars=effective_vars, - target_config=target_config, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - materialization_defaults=( - discovered_inputs.project_config.materialization_defaults - ), - scan_cache=config_scan_cache, - ) - ) - reusable_config_cache.remember( + + validate_declared_schema_models_are_attached( + model_inputs=tuple(model_inputs), + schema_files=discovered_inputs.schema_files, + ) + return tuple(model_inputs) + + +def _build_model_input( + *, + loop: _ModelInputLoop, + model_file: DiscoveredSqlModelFile, + prepared_var_substituted_sql: str | None, +) -> CompileModelInput: + discovered_inputs: DiscoveredProjectInputs = loop.discovered_inputs + context: ModelInputBuildContext = loop.context + validation_context: _ModelValidationContext = loop.validation_context + sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = loop.sql_hook_definitions + legacy_schema_files: tuple[DiscoveredSchemaFile, ...] = loop.legacy_schema_files + model_header_column_cache: ModelHeaderColumnCache = loop.model_header_column_cache + config_scan_cache: ModelConfigScanCache = loop.config_scan_cache + reusable_config_cache: _ReusableModelConfigCache = loop.reusable_config_cache + declaration_cache: _VisibleModelDeclarationCache = loop.declaration_cache + effective_vars: dict[str, object] = context.effective_vars + target_config: TargetConfig | None = context.target_config + effective_target_name: str | None = context.effective_target_name + run_id: str = context.run_id + macro_context: MacroContext = context.macro_context + loaded_macros: dict[str, LoadedMacro] = context.loaded_macros + model_identity: ResourceIdentity = ResourceIdentity( + ResourceKind.MODEL, model_file.file_path.stem + ) + declarations: _VisibleModelDeclarations = declaration_cache.for_model( + model_file=model_file, consumer=model_identity + ) + matched_path_default: str | None = find_matching_path_default( + model_file=model_file, + path_defaults=discovered_inputs.project_config.path_defaults, + ) + effective_config: CompileModelConfig | None = reusable_config_cache.get( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + ) + if effective_config is None: + effective_config = build_model_config( + request=ModelConfigBuildRequest( + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, matched_path_default=matched_path_default, model_header_values=model_file.header_values, - config=effective_config, + effective_vars=effective_vars, + target_config=target_config, + model_name=model_file.file_path.stem, + effective_target_name=effective_target_name, + run_id=run_id, + materialization_defaults=( + discovered_inputs.project_config.materialization_defaults + ), + scan_cache=config_scan_cache, ) - model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( - values=effective_config.values, - model_name=model_file.file_path.stem, - public_model_schemas=context.public_model_schemas, - ) - model_schema_columns: tuple[SchemaColumn, ...] | None = ( - model_schema.columns if model_schema is not None else None ) - validate_python_hook_config( - values=effective_config.values, - model_name=model_file.file_path.stem, - hook_functions=discovered_inputs.hook_functions, - provider_names=frozenset(provider.name for provider in discovered_inputs.providers), + reusable_config_cache.remember( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + config=effective_config, ) - named_usages: tuple[UsageRecord, ...] = ( - *( - named_declaration_usages( - resolver=context.declaration_resolver, - kind=DeclarationKind.SCHEMA, - name=model_schema.name, - consumer=model_identity, - consumer_path=model_file.relative_path, - ) - if model_schema is not None - else () - ), - *_python_hook_usages( - values=effective_config.values, + model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( + values=effective_config.values, + model_name=model_file.file_path.stem, + public_model_schemas=context.public_model_schemas, + ) + model_schema_columns: tuple[SchemaColumn, ...] | None = ( + model_schema.columns if model_schema is not None else None + ) + validate_python_hook_config( + values=effective_config.values, + model_name=model_file.file_path.stem, + hook_functions=discovered_inputs.hook_functions, + provider_names=frozenset(provider.name for provider in discovered_inputs.providers), + ) + named_usages: tuple[UsageRecord, ...] = ( + *( + named_declaration_usages( resolver=context.declaration_resolver, + kind=DeclarationKind.SCHEMA, + name=model_schema.name, consumer=model_identity, consumer_path=model_file.relative_path, - ), - ) - var_substituted_sql: str = ( - prepared_var_substituted_sql - if prepared_var_substituted_sql is not None - else substitute_sql_vars( - sql=model_file.query_sql, - file_path=model_file.file_path, - effective_vars=effective_vars, ) - ) - declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - enum_visibility=declarations.enum_visibility, - constant_visibility=declarations.constant_visibility, - macros=declarations.macros, - macro_records=declarations.macro_records, - inaccessible_macros=declarations.inaccessible_macros, + if model_schema is not None + else () + ), + *_python_hook_usages( + values=effective_config.values, + resolver=context.declaration_resolver, consumer=model_identity, - ) - declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( - sql=var_substituted_sql, - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) - declaration_expanded_sql: str = declaration_expansion.sql - macro_expansion: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_expanded_sql, + consumer_path=model_file.relative_path, + ), + ) + var_substituted_sql: str = ( + prepared_var_substituted_sql + if prepared_var_substituted_sql is not None + else substitute_sql_vars( + sql=model_file.query_sql, file_path=model_file.file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=context.declaration_resolver, - declarations=( - declaration_context if context.declaration_resolver is not None else None - ), - consumer=model_identity, - ) - expanded_query_sql: str = macro_expansion.sql - expanded_query_sql = get_validated_model_cursor_intrinsics( - sql=expanded_query_sql, - config_values=effective_config.values, - model_name=model_file.file_path.stem, - ) - raw_placeholders: object | None = effective_config.values.get("placeholders") - sql_validation_placeholders: dict[str, str] | None = ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - sql_validation_enabled, references, rejected_opt_out = _validate_model_input( - context=validation_context, - model_file=model_file, - config=effective_config, - expanded_query_sql=expanded_query_sql, - sql_validation_placeholders=sql_validation_placeholders, - model_schema_columns=model_schema_columns, - argument_references=macro_expansion.argument_references, + effective_vars=effective_vars, ) - hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + ) + declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, + enum_visibility=declarations.enum_visibility, + constant_visibility=declarations.constant_visibility, + macros=declarations.macros, + macro_records=declarations.macro_records, + inaccessible_macros=declarations.inaccessible_macros, + consumer=model_identity, + ) + declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( + sql=var_substituted_sql, + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + ) + declaration_expanded_sql: str = declaration_expansion.sql + macro_expansion: MacroExpansionResult = expand_sql_macros_result( + sql=declaration_expanded_sql, + file_path=model_file.file_path, + loaded_macros=loaded_macros, + macro_context=macro_context, + declaration_resolver=context.declaration_resolver, + declarations=(declaration_context if context.declaration_resolver is not None else None), + consumer=model_identity, + ) + expanded_query_sql: str = macro_expansion.sql + expanded_query_sql = get_validated_model_cursor_intrinsics( + sql=expanded_query_sql, + config_values=effective_config.values, + model_name=model_file.file_path.stem, + ) + raw_placeholders: object | None = effective_config.values.get("placeholders") + sql_validation_placeholders: dict[str, str] | None = ( + {str(k): str(v) for k, v in raw_placeholders.items()} + if isinstance(raw_placeholders, dict) + else None + ) + sql_validation_enabled, references, rejected_opt_out = _validate_model_input( + context=validation_context, + model_file=model_file, + config=effective_config, + expanded_query_sql=expanded_query_sql, + sql_validation_placeholders=sql_validation_placeholders, + model_schema_columns=model_schema_columns, + argument_references=macro_expansion.argument_references, + ) + hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + values=effective_config.values, + file_path=model_file.file_path, + effective_vars=effective_vars, + context_values=build_model_context_values( values=effective_config.values, - file_path=model_file.file_path, - effective_vars=effective_vars, - context_values=build_model_context_values( - values=effective_config.values, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=True, + model_name=model_file.file_path.stem, + effective_target_name=effective_target_name, + run_id=run_id, + include_target_values=True, + ), + loaded_macros=loaded_macros, + macro_context=macro_context, + declaration_expansion=DeclarationExpansionContext( + declarations=DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, ), - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + resolver=context.declaration_resolver, + ), + sql_lexical_syntax=context.sql_lexical_syntax, + sql_hook_definitions=sql_hook_definitions, + consumer=model_identity, + ) + expanded_config: CompileModelConfig = _hook_expanded_config( + config=effective_config, + hook_expansion=hook_expansion, + model_file=model_file, + validation_context=validation_context, + ) + hook_name: str + for hook_name in ("pre_hooks", "post_hooks"): + hook_value: object | None = expanded_config.values.get(hook_name) + if isinstance(hook_value, list | tuple): + hook_entry: object + for hook_entry in hook_value: + if isinstance(hook_entry, SqlHookEntry): + reject_cursor_intrinsics( + sql=hook_entry.statement, + context=f"Model '{model_file.file_path.stem}' {hook_name}", + ) + header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( + model_name=model_file.file_path.stem, + model_header_values=expanded_config.values, + file_path=model_file.relative_path, + column_locations=model_file.header_column_locations, + model_schema_columns=model_schema_columns, + model_schema_name=model_schema.name if model_schema is not None else None, + model_schema_description=model_schema.description if model_schema is not None else None, + audit_factories=discovered_inputs.audit_factories, + column_cache=model_header_column_cache, + ) + model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) + header_schema_entry, enum_columns = resolve_enum_contract_columns( + schema_entry=header_schema_entry, + config_values=model_config.values, + enums=( + declarations.enums + if model_schema is None + else { + **declarations.enums, + **model_schema_enum_declarations( + schema=model_schema, resolver=context.declaration_resolver ), - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - resolver=context.declaration_resolver, - ), - sql_lexical_syntax=context.sql_lexical_syntax, - sql_hook_definitions=sql_hook_definitions, - consumer=model_identity, - ) - expanded_config: CompileModelConfig = _hook_expanded_config( - config=effective_config, - hook_expansion=hook_expansion, - model_file=model_file, - validation_context=validation_context, - ) - hook_name: str - for hook_name in ("pre_hooks", "post_hooks"): - hook_value: object | None = expanded_config.values.get(hook_name) - if isinstance(hook_value, list | tuple): - hook_entry: object - for hook_entry in hook_value: - if isinstance(hook_entry, SqlHookEntry): - reject_cursor_intrinsics( - sql=hook_entry.statement, - context=f"Model '{model_file.file_path.stem}' {hook_name}", - ) - header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( - model_name=model_file.file_path.stem, - model_header_values=expanded_config.values, - file_path=model_file.relative_path, - column_locations=model_file.header_column_locations, - model_schema_columns=model_schema_columns, - model_schema_name=model_schema.name if model_schema is not None else None, - model_schema_description=model_schema.description if model_schema is not None else None, - audit_factories=discovered_inputs.audit_factories, - column_cache=model_header_column_cache, + } + ), + ) + generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( + enum_columns=enum_columns, + declarations=declarations, + consumer=model_identity, + ) + model_declaration_usages: tuple[UsageRecord, ...] = tuple( + dict.fromkeys( + ( + *declaration_expansion.usages, + *generated_usages, + *hook_expansion.usages, + *named_usages, + ) ) - model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) - header_schema_entry, enum_columns = resolve_enum_contract_columns( - schema_entry=header_schema_entry, - config_values=model_config.values, - enums=( - declarations.enums - if model_schema is None - else { - **declarations.enums, - **model_schema_enum_declarations( - schema=model_schema, resolver=context.declaration_resolver - ), - } + ) + _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) + if header_schema_entry is None: + return CompileModelInput( + sql_expansion=( + CompiledSqlExpansion( + authored_sql=model_file.query_sql, + expanded_sql=macro_expansion.sql, + passes=(declaration_expansion.spans, macro_expansion.spans), + ) + if var_substituted_sql == model_file.query_sql + else None ), - ) - generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( + model_file=model_file, + config=model_config, + query_sql=expanded_query_sql, + macro_source_sql=declaration_expanded_sql, + references=references, + sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, + enum_declarations=tuple(declarations.local_enums.values()), + constant_declarations=tuple(declarations.local_constants.values()), enum_columns=enum_columns, - declarations=declarations, - consumer=model_identity, - ) - model_declaration_usages: tuple[UsageRecord, ...] = tuple( - dict.fromkeys( - ( - *declaration_expansion.usages, - *generated_usages, - *hook_expansion.usages, - *named_usages, - ) - ) + macro_deps=tuple(item.name for item in macro_expansion.dependencies), + macro_usages=macro_expansion.usages, + declaration_usages=model_declaration_usages, ) - _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) - if header_schema_entry is None: - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None - ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, - ) - ) - continue - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None - ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - schema_entry=header_schema_entry, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, + return CompileModelInput( + sql_expansion=( + CompiledSqlExpansion( + authored_sql=model_file.query_sql, + expanded_sql=macro_expansion.sql, + passes=(declaration_expansion.spans, macro_expansion.spans), ) - ) - - validate_declared_schema_models_are_attached( - model_inputs=tuple(model_inputs), - schema_files=discovered_inputs.schema_files, + if var_substituted_sql == model_file.query_sql + else None + ), + model_file=model_file, + config=model_config, + query_sql=expanded_query_sql, + macro_source_sql=declaration_expanded_sql, + references=references, + schema_entry=header_schema_entry, + sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, + enum_declarations=tuple(declarations.local_enums.values()), + constant_declarations=tuple(declarations.local_constants.values()), + enum_columns=enum_columns, + macro_deps=tuple(item.name for item in macro_expansion.dependencies), + macro_usages=macro_expansion.usages, + declaration_usages=model_declaration_usages, ) - return tuple(model_inputs) def _hook_expanded_config( diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index ae6080acb7..9a7ee0f036 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -21,6 +21,23 @@ _ACTIVE_COLLECTOR: ContextVar[CollectedCompileDiagnostics | None] = ContextVar( "sqlbuild_collected_compile_diagnostics", default=None ) +_ACTIVE_TAPS: ContextVar[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = ContextVar( + "sqlbuild_tapped_compile_diagnostics", default=() +) + + +@contextmanager +def tapped_compile_diagnostics() -> Iterator[list[tuple[tuple[str, ...], CompilerDiagnostic]]]: + """Also note every diagnostic reported inside the block, in report order, for replay.""" + + tap: list[tuple[tuple[str, ...], CompilerDiagnostic]] = [] + token: Token[tuple[list[tuple[tuple[str, ...], CompilerDiagnostic]], ...]] = _ACTIVE_TAPS.set( + (*_ACTIVE_TAPS.get(), tap) + ) + try: + yield tap + finally: + _ACTIVE_TAPS.reset(token) @contextmanager @@ -54,6 +71,8 @@ def collecting(*args: P.args, **kwargs: P.kwargs) -> R: def report_compile_diagnostic(*, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: """Record a diagnostic in the active collector, or raise it when none is active.""" + for tap in _ACTIVE_TAPS.get(): + tap.append((key, diagnostic)) collector: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() if collector is not None: collector.add(key=key, diagnostic=diagnostic) diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py b/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py new file mode 100644 index 0000000000..c6a35d2e99 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_registry.py @@ -0,0 +1,41 @@ +"""Process-wide access to the render reuse session of the running compile command.""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession + + +class CompileRenderReuseRegistry: + """Expose one compile command's render reuse session to the rendering code it runs.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._session: CompileRenderReuseSession | None = None + + @contextmanager + def activated(self, session: CompileRenderReuseSession) -> Iterator[CompileRenderReuseSession]: + """Make the session available to rendering until the block exits.""" + + with self._lock: + previous: CompileRenderReuseSession | None = self._session + self._session = session + try: + yield session + finally: + with self._lock: + self._session = previous + + def claim(self) -> CompileRenderReuseSession | None: + """Hand the active session to the first project render only, so it is used once.""" + + with self._lock: + session: CompileRenderReuseSession | None = self._session + if session is None or not session.claim(): + return None + return session diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py new file mode 100644 index 0000000000..a68c70eeeb --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py @@ -0,0 +1,224 @@ +"""Reuse unaffected rendered inputs from the previous compile and record this compile's.""" + +from __future__ import annotations + +import pickle +import threading +from collections.abc import Callable +from dataclasses import replace +from typing import cast + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( + report_compile_diagnostic, + tapped_compile_diagnostics, +) +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompilerDiagnostic, + RenderReuseState, + StoredRender, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import ( + dump_fact_payload, + load_fact_payload, +) +from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError +from sqlbuild.compiler.profiling.main._metric import record_compile_metric + +_SERIALIZER_THREAD_NAME: str = "sqlbuild-render-reuse-serializer" +_LOAD_ERRORS: tuple[type[Exception], ...] = ( + FactCachePayloadError, + pickle.UnpicklingError, + AttributeError, + EOFError, + ImportError, + IndexError, + KeyError, + TypeError, + ValueError, +) + + +class CompileRenderReuseSession: + """Serve stored renders of inputs a change set cannot affect, and record every render.""" + + def __init__( + self, *, prior: RenderReuseState | None, changed_paths: frozenset[str] | None + ) -> None: + self._lock: threading.Lock = threading.Lock() + self._claimed: bool = False + self._prior: RenderReuseState | None = prior if changed_paths is not None else None + self._changed_paths: frozenset[str] = changed_paths or frozenset() + self._reusable: bool = False + self._model_paths: tuple[str, ...] | None = None + self._model_records: dict[str, memoryview] = {} + self._group_records: dict[str, memoryview] = {} + self._pending_models: list[tuple[str, StoredRender]] = [] + self._pending_groups: list[tuple[str, StoredRender]] = [] + self._serialized_models: dict[str, memoryview] = {} + self._serialized_groups: dict[str, memoryview] = {} + self._serializer: threading.Thread | None = None + + def claim(self) -> bool: + """Return True for the first claim only; later project renders neither reuse nor record.""" + + with self._lock: + claimed: bool = self._claimed + self._claimed = True + return not claimed + + def plan_models(self, *, model_files: tuple[DiscoveredSqlModelFile, ...]) -> None: + """Reuse stored renders only when the change set touches nothing but existing models.""" + + model_paths: tuple[str, ...] = tuple(str(item.relative_path) for item in model_files) + self._model_paths = model_paths + prior: RenderReuseState | None = self._prior + self._reusable = ( + prior is not None + and prior.model_paths == model_paths + and self._changed_paths.issubset(model_paths) + ) + + def has_reusable_model(self, *, model_file: DiscoveredSqlModelFile) -> bool: + """Return whether a stored render of this model may stand in for rendering it.""" + + path: str = str(model_file.relative_path) + return ( + self._reusable + and self._prior is not None + and path not in self._changed_paths + and path in self._prior.model_payloads + ) + + def reused_model(self, *, model_file: DiscoveredSqlModelFile) -> CompileModelInput | None: + """Return the stored render of an unchanged model, replaying what rendering reported.""" + + if not self.has_reusable_model(model_file=model_file) or self._prior is None: + return None + path: str = str(model_file.relative_path) + payload: memoryview = self._prior.model_payloads[path] + stored: StoredRender | None = _loaded_render(payload) + if stored is None or not isinstance(stored.value, CompileModelInput): + return None + _replay(stored=stored) + self._model_records[path] = payload + record_compile_metric(metric="render_reuse_hits", value=1) + return replace(stored.value, model_file=model_file) + + def rendered_model( + self, + *, + model_file: DiscoveredSqlModelFile, + render: Callable[[], CompileModelInput], + ) -> CompileModelInput: + """Render one model and record its render with what rendering reported and read.""" + + with ( + COMPILE_INPUT_READS.recording() as reads, + tapped_compile_diagnostics() as reported, + ): + model_input: CompileModelInput = render() + record_compile_metric(metric="render_reuse_misses", value=1) + stored: StoredRender | None = _stored_render( + reads=reads, value=replace(model_input, model_file=None), reported=reported + ) + if stored is not None: + self._pending_models.append((str(model_file.relative_path), stored)) + return model_input + + def group[T](self, *, name: str, render: Callable[[], T]) -> T: + """Return a stored project-wide render no model edit can affect, or render and record it.""" + + payload: memoryview | None = ( + self._prior.group_payloads.get(name) + if self._reusable and self._prior is not None + else None + ) + stored: StoredRender | None = None if payload is None else _loaded_render(payload) + if stored is not None and payload is not None: + _replay(stored=stored) + self._group_records[name] = payload + return cast(T, stored.value) + with ( + COMPILE_INPUT_READS.recording() as reads, + tapped_compile_diagnostics() as reported, + ): + value: T = render() + rendered: StoredRender | None = _stored_render(reads=reads, value=value, reported=reported) + if rendered is not None: + self._pending_groups.append((name, rendered)) + return value + + def renders_complete(self) -> None: + """Serialize this compile's new renders in the background while later phases run.""" + + if self._serializer is None and (self._pending_models or self._pending_groups): + self._serializer = threading.Thread( + target=self._serialize_pending, name=_SERIALIZER_THREAD_NAME, daemon=True + ) + self._serializer.start() + + def stored_state(self) -> RenderReuseState | None: + """Serialize this compile's renders, keeping stored bytes of reused ones as they were.""" + + if self._model_paths is None: + return None + if self._serializer is None: + self._serialize_pending() + else: + self._serializer.join() + return RenderReuseState( + model_paths=self._model_paths, + model_payloads={**self._model_records, **self._serialized_models}, + group_payloads={**self._group_records, **self._serialized_groups}, + ) + + def _serialize_pending(self) -> None: + for pending, serialized in ( + (self._pending_models, self._serialized_models), + (self._pending_groups, self._serialized_groups), + ): + for key, stored in pending: + payload: bytes | None = _dumped(stored) + if payload is not None: + serialized[key] = memoryview(payload) + + +def _stored_render( + *, + reads: CompileInputReads, + value: object, + reported: list[tuple[tuple[str, ...], CompilerDiagnostic]], +) -> StoredRender | None: + """Keep one render for reuse unless it read the run identity or provider settings.""" + + if reads.read_run_id or reads.settings_classes: + return None + return StoredRender( + value=value, diagnostics=tuple(reported), environment_names=reads.environment_names + ) + + +def _dumped(stored: StoredRender) -> bytes | None: + try: + return dump_fact_payload(stored) + except (pickle.PicklingError, TypeError, AttributeError, RecursionError): + return None + + +def _replay(*, stored: StoredRender) -> None: + for name in stored.environment_names: + COMPILE_INPUT_READS.environment_read(name) + for key, diagnostic in stored.diagnostics: + report_compile_diagnostic(key=key, diagnostic=diagnostic) + + +def _loaded_render(payload: memoryview) -> StoredRender | None: + try: + loaded: object = load_fact_payload(payload) + except _LOAD_ERRORS: + return None + return loaded if isinstance(loaded, StoredRender) else None diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index cabdcb1316..b1959ce97d 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( CompileInputReadRegistry, ) +from sqlbuild.compiler.compile.classes.render_reuse_registry import CompileRenderReuseRegistry from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind @@ -171,6 +172,10 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() +COMPILE_RENDER_REUSE: CompileRenderReuseRegistry = CompileRenderReuseRegistry() +RENDER_REUSE_SOURCES_GROUP: str = "sources" +RENDER_REUSE_TESTS_GROUP: str = "tests" +RENDER_REUSE_SCENARIOS_GROUP: str = "scenarios" SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v2" SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 0ffd4eb089..ec0fe775cb 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -2,7 +2,9 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import replace +from functools import partial from pathlib import Path from sqlbuild.compiler.compile._helpers.attachment.audits import build_project_audit_inputs @@ -39,6 +41,13 @@ build_public_model_schema_index, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import ( + COMPILE_RENDER_REUSE, + RENDER_REUSE_SCENARIOS_GROUP, + RENDER_REUSE_SOURCES_GROUP, + RENDER_REUSE_TESTS_GROUP, +) from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAuditInput, @@ -119,6 +128,9 @@ def build_compile_inputs( _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), ) resolved_run_id: str = resolve_run_id(selected_run_id=run_id) + render_reuse: CompileRenderReuseSession | None = COMPILE_RENDER_REUSE.claim() + if render_reuse is not None: + render_reuse.plan_models(model_files=discovered_inputs.model_files) loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) declaration_scope: DeclarationScopeBuild = build_declaration_scope( discovered_inputs=discovered_inputs, @@ -146,6 +158,7 @@ def build_compile_inputs( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=compile_cache_dir, + render_reuse=render_reuse, ) model_context = model_build.context seed_inputs: tuple[CompileSeedInput, ...] = build_seed_inputs(discovered_inputs) @@ -159,35 +172,52 @@ def build_compile_inputs( adapter_context.python_functions_inherit_default_namespace ), ) - source_inputs: tuple[CompileSourceInput, ...] = build_source_inputs( - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - effective_settings=effective_settings, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - no_sql_validation=no_sql_validation, + source_inputs: tuple[CompileSourceInput, ...] = _reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SOURCES_GROUP, + render=partial( + build_source_inputs, + discovered_inputs=discovered_inputs, + effective_vars=effective_vars, + effective_settings=effective_settings, + macro_context=macro_context, + loaded_macros=declaration_scope.loaded_macros, + declaration_expansion=model_context.declaration_expansion, + no_sql_validation=no_sql_validation, + ), ) - test_inputs: tuple[CompileSqlTestInput, ...] = build_test_inputs_with_cache( - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, - compile_cache_dir=compile_cache_dir, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, + test_inputs: tuple[CompileSqlTestInput, ...] = _reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_TESTS_GROUP, + render=partial( + build_test_inputs_with_cache, + discovered_inputs=discovered_inputs, + effective_vars=effective_vars, + macro_context=macro_context, + loaded_macros=declaration_scope.loaded_macros, + declaration_expansion=model_context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_function_inputs=sql_function_inputs, + compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, + ), ) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = build_scenario_inputs( - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, + scenario_inputs: tuple[CompileSqlScenarioInput, ...] = _reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SCENARIOS_GROUP, + render=partial( + build_scenario_inputs, + discovered_inputs=discovered_inputs, + effective_vars=effective_vars, + macro_context=macro_context, + loaded_macros=declaration_scope.loaded_macros, + declaration_expansion=model_context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=adapter_context.sql_lexical_syntax, + ), ) + if render_reuse is not None: + render_reuse.renders_complete() audit_inputs: tuple[CompileAuditInput, ...] audit_diagnostics: tuple[CompilerDiagnostic, ...] audit_inputs, audit_diagnostics = build_project_audit_inputs( @@ -248,6 +278,12 @@ def build_compile_inputs( ) +def _reused_or_rendered[T]( + *, render_reuse: CompileRenderReuseSession | None, name: str, render: Callable[[], T] +) -> T: + return render() if render_reuse is None else render_reuse.group(name=name, render=render) + + def _build_sql_functions( *, discovered_inputs: DiscoveredProjectInputs, @@ -284,6 +320,7 @@ def _build_models_with_declarations( defer_model_sql_validation: bool, external_sql_reference_resolver: ExternalSqlReferenceResolver | None, reference_cache_dir: Path | None, + render_reuse: CompileRenderReuseSession | None, ) -> ModelInputScopeBuild: public_enums: dict[str, EnumDeclaration] public_constants: dict[str, ConstantDeclaration] @@ -309,6 +346,7 @@ def _build_models_with_declarations( defer_model_sql_validation=defer_model_sql_validation, external_sql_reference_resolver=external_sql_reference_resolver, reference_cache_dir=reference_cache_dir, + render_reuse=render_reuse, ) return ModelInputScopeBuild( inputs=model_inputs, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 93bfc4495e..30ba05a4ca 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1579,3 +1579,21 @@ class SqlAnalysisOptOutRequest: query_sql: str placeholders: dict[str, str] | None project_config_path: Path + + +@dataclass(frozen=True) +class StoredRender: + """One stored render with the diagnostics it reported and environment variables it read.""" + + value: object + diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] + environment_names: tuple[str, ...] + + +@dataclass(frozen=True) +class RenderReuseState: + """Serialized renders of one compile: each model by project path and project-wide groups.""" + + model_paths: tuple[str, ...] + model_payloads: dict[str, memoryview] + group_payloads: dict[str, memoryview] diff --git a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py b/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py index 05b7a1bf8d..757e20e635 100644 --- a/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py +++ b/src/sqlbuild/compiler/fact_cache/_helpers/restricted_pickle.py @@ -44,7 +44,7 @@ def dump_fact_payload(value: object) -> bytes: return pickle.dumps(value, protocol=FACT_CACHE_PICKLE_PROTOCOL) -def load_fact_payload(payload: bytes) -> object: +def load_fact_payload(payload: bytes | memoryview) -> object: """Deserialize one verified cache payload through the restricted class allowlist.""" return _RestrictedUnpickler(io.BytesIO(payload)).load() diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 6ca434213c..b969bb04cf 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -24,4 +24,6 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "render_reuse_hits", + "render_reuse_misses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index abb9028ae4..f3e9a1c2b4 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -24,4 +24,6 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "render_reuse_hits", + "render_reuse_misses", ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index 47e87b2de1..7701ffa63f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -90,6 +90,8 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "render_reuse_hits", + "render_reuse_misses", "project_reuse_hits", "project_reuse_misses", "project_reuse_bypasses", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 3ca4ed3d78..2ab6912d66 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -681,3 +681,30 @@ class CompileReuseStoreFailureTestCase: expected_returncode: int = 0 expected_stored_entries: int = 0 expected_raised: type[BaseException] | None = None + + +@dataclass(frozen=True) +class IncrementalEditStep: + """One authored edit, and whether its compile may reuse unaffected renders.""" + + description: str + edit: Callable[[Path], None] + reuses_renders: bool + + +@dataclass(frozen=True) +class IncrementalEditSequenceTestCase: + """Edits applied in order, each compiled incrementally and compared with --no-cache.""" + + description: str + steps: tuple[IncrementalEditStep, ...] + + +@dataclass(frozen=True) +class RandomEditChainTestCase: + """A seeded random chain of edits over a generated project.""" + + description: str + seed: int + model_count: int + step_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 29858c0293..5515760c42 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -4,6 +4,7 @@ import json import os +import random import re import shutil import signal @@ -3433,3 +3434,180 @@ def compile_in_process_output( code: int = main(["--project-dir", str(project_dir), "--no-color", "compile", "--json"]) out, err = capsys.readouterr() return code, out, err + + +class IncrementalEditComparison(NamedTuple): + """An incremental compile and the --no-cache compile of the same edited project.""" + + incremental: CompileReuseRun + reference: CompileReuseRun + + @property + def reused_renders(self) -> int: + """Return how many model renders the incremental compile reused.""" + + return self.incremental.timings.get("render_reuse_hits", 0) + + def mismatch(self) -> str | None: + """Describe the first difference from the --no-cache compile, if any.""" + + if self.incremental.returncode != self.reference.returncode: + return f"exit code {self.incremental.returncode} != {self.reference.returncode}" + if self.incremental.report != self.reference.report: + return "compile report differs" + if self.incremental.compiled != self.reference.compiled: + changed: list[str] = sorted( + path + for path in self.incremental.compiled.keys() | self.reference.compiled.keys() + if self.incremental.compiled.get(path) != self.reference.compiled.get(path) + ) + return f"compiled artifacts differ: {changed[:5]}" + return None + + +def compare_incremental_compile(*, project_dir: Path) -> IncrementalEditComparison: + """Compile incrementally, then compile the same inputs without any cache.""" + + incremental: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + return IncrementalEditComparison(incremental=incremental, reference=reference) + + +def in_process_reuse_run( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> CompileReuseRun: + """Compile with --json in this process and capture output comparable with fresh processes.""" + + code, out, err = compile_in_process_output(project_dir=project_dir, capsys=capsys) + payload: dict[str, object] = cast(dict[str, object], json.loads(out)) + return CompileReuseRun( + returncode=code, + report=_COMPILE_TIMINGS_PATTERN.sub("", out), + stderr=err, + timings=cast(dict[str, int], payload.get("compile_timings", {})), + compiled=compiled_artifacts(project_dir=project_dir), + ) + + +GENERATED_EDIT_MODEL_PREFIX: str = "orders_step_" +_GENERATED_EDIT_COLUMNS: tuple[str, ...] = ("order_id", "customer_id", "quantity", "status") + + +def write_generated_edit_models(*, project_dir: Path, model_count: int, seed: int) -> None: + """Add a seeded chain of models over the staging orders, each reading an earlier one.""" + + chooser: random.Random = random.Random(seed) + for index in range(model_count): + upstream: str = ( + "stg_orders" + if index == 0 + else f"{GENERATED_EDIT_MODEL_PREFIX}{chooser.randrange(index):03d}" + ) + write_project_file( + project_dir, + f"models/generated/{GENERATED_EDIT_MODEL_PREFIX}{index:03d}.sql", + _generated_edit_model_sql(upstream=upstream), + ) + + +def _generated_edit_model_sql(*, upstream: str) -> str: + columns: str = ",\n".join(f" {column}" for column in _GENERATED_EDIT_COLUMNS) + return ( + "MODEL (description 'Generated order step.', materialized view);\n\n" + f'SELECT\n{columns}\nFROM __ref("{upstream}")\n' + ) + + +class RandomEditChain: + """Apply seeded random edits of every supported kind to a generated project.""" + + def __init__(self, *, project_dir: Path, seed: int) -> None: + self._project_dir: Path = project_dir + self._random: random.Random = random.Random(seed) + self._broken: Path | None = None + self._added: int = 0 + + def apply(self) -> tuple[str, bool]: + """Apply one edit; return its kind and whether it touches model files only.""" + + if self._broken is not None: + path: Path = self._broken + path.write_text( + path.read_text(encoding="utf-8").replace(" unknown_column,\n", ""), + encoding="utf-8", + ) + self._broken = None + return "fix_error", True + kind: str = self._random.choice(_RANDOM_EDIT_KINDS) + models: list[Path] = sorted((self._project_dir / "models" / "generated").glob("*.sql")) + model: Path = self._random.choice(models) + contents: str = model.read_text(encoding="utf-8") + if kind == "comment": + model.write_text( + contents.replace("FROM __ref", f"-- note {self._random.random()}\nFROM __ref"), + encoding="utf-8", + ) + elif kind == "add_column": + model.write_text( + contents.replace( + "\nFROM __ref", + f",\n status AS status_{self._random.randrange(1000)}\nFROM __ref", + ), + encoding="utf-8", + ) + elif kind == "type_change": + model.write_text( + contents.replace(" quantity", " CAST(quantity AS BIGINT) AS quantity", 1) + if "CAST(quantity" not in contents + else contents.replace("CAST(quantity AS BIGINT) AS quantity", "quantity", 1), + encoding="utf-8", + ) + elif kind == "header_change": + model.write_text( + contents.replace("materialized view", "materialized table") + if "materialized view" in contents + else contents.replace("materialized table", "materialized view"), + encoding="utf-8", + ) + elif kind == "introduce_error": + model.write_text( + contents.replace("SELECT\n", "SELECT\n unknown_column,\n", 1), encoding="utf-8" + ) + self._broken = model + elif kind == "add_model": + self._added += 1 + write_project_file( + self._project_dir, + f"models/generated/added_step_{self._added:03d}.sql", + _generated_edit_model_sql(upstream=model.stem), + ) + return kind, False + elif kind == "macro_edit": + path = self._project_dir / "macros" / "_rounding.py" + scale: int = 2 + self._random.randrange(3) + path.write_text( + re.sub(r"_SCALE: int = \d+", f"_SCALE: int = {scale}", path.read_text("utf-8")), + encoding="utf-8", + ) + return kind, False + elif kind == "test_edit": + path = self._project_dir / "tests" / "unit" / "test_stg_orders.sql" + path.write_text( + path.read_text(encoding="utf-8") + f"\n-- revision {self._random.random()}\n", + encoding="utf-8", + ) + return kind, False + return kind, True + + +_RANDOM_EDIT_KINDS: tuple[str, ...] = ( + "comment", + "comment", + "add_column", + "type_change", + "header_change", + "introduce_error", + "add_model", + "macro_edit", + "test_edit", +) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py new file mode 100644 index 0000000000..bff7c9e418 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -0,0 +1,296 @@ +"""Incremental edit compiles must match an uncached compile of the same edited project.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path + +import pytest + +import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + IncrementalEditSequenceTestCase, + IncrementalEditStep, + RandomEditChainTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + IncrementalEditComparison, + RandomEditChain, + compare_incremental_compile, + compile_in_process, + enable_compile_reuse, + in_process_reuse_run, + move_project_file, + replace_project_text, + run_reuse_compile, + write_generated_edit_models, + write_project_file, +) + +_STG_ORDERS: str = "models/staging/stg_orders.sql" +_FACT_ORDERS: str = "models/marts/fact_orders.sql" + + +def _model_comment(root: Path) -> None: + replace_project_text( + root, _STG_ORDERS, 'FROM __source("raw__orders")', '-- staged\nFROM __source("raw__orders")' + ) + + +def _model_type_change(root: Path) -> None: + replace_project_text( + root, _STG_ORDERS, " quantity,\n", " CAST(quantity AS BIGINT) AS quantity,\n" + ) + + +def _model_new_column(root: Path) -> None: + replace_project_text(root, _STG_ORDERS, " status\n", " status,\n status AS raw_status\n") + + +def _model_header_change(root: Path) -> None: + replace_project_text(root, _STG_ORDERS, "materialized view,", "materialized table,") + + +def _model_contract_change(root: Path) -> None: + replace_project_text( + root, + _STG_ORDERS, + "customer_id (nullable false, audits [not_null]),", + "customer_id (type BIGINT, nullable false, audits [not_null]),", + ) + + +def _introduce_error(root: Path) -> None: + replace_project_text(root, _FACT_ORDERS, " o.quantity,\n", " o.missing_quantity,\n") + + +def _fix_error(root: Path) -> None: + replace_project_text(root, _FACT_ORDERS, " o.missing_quantity,\n", " o.quantity,\n") + + +def _leaf_comment(root: Path) -> None: + replace_project_text(root, _FACT_ORDERS, "FROM __ref", "-- leaf\nFROM __ref") + + +def _model_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: + return IncrementalEditStep(description=description, edit=edit, reuses_renders=True) + + +def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: + return IncrementalEditStep(description=description, edit=edit, reuses_renders=False) + + +@pytest.mark.parametrize( + "test_case", + [ + IncrementalEditSequenceTestCase( + description="model_edits", + steps=( + _model_step("comment_without_shape_change", _model_comment), + _model_step("type_change_propagates_downstream", _model_type_change), + _model_step("new_output_column", _model_new_column), + _model_step("header_config_change", _model_header_change), + _model_step("column_contract_change", _model_contract_change), + _model_step("error_introduced", _introduce_error), + _model_step("error_fixed", _fix_error), + ), + ), + IncrementalEditSequenceTestCase( + description="model_set_changes", + steps=( + _full_step( + "model_added", + lambda root: write_project_file( + root, + "models/marts/order_quantities.sql", + "MODEL (description 'Order quantities.');\n\n" + 'SELECT order_id, quantity FROM __ref("stg_orders")\n', + ), + ), + _model_step("edit_after_add", _model_comment), + _full_step( + "model_renamed", + lambda root: move_project_file( + root, + "models/marts/order_quantities.sql", + "models/marts/order_quantity_totals.sql", + ), + ), + _full_step( + "model_removed", + lambda root: (root / "models/marts/order_quantity_totals.sql").unlink(), + ), + _model_step("edit_after_remove", _leaf_comment), + ), + ), + IncrementalEditSequenceTestCase( + description="declaration_edits", + steps=( + _full_step( + "macro_edit", + lambda root: replace_project_text( + root, + "macros/currency.py", + "{price_cents} * {quantity}", + "{quantity} * {price_cents}", + ), + ), + _model_step("edit_after_macro", _leaf_comment), + _full_step( + "module_imported_by_macro_edit", + lambda root: replace_project_text( + root, "macros/_rounding.py", "_SCALE: int = 2", "_SCALE: int = 3" + ), + ), + _full_step( + "scoped_macro_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_macros/currency.py", + "/ 100.0, 2)", + "/ 100.0, 3)", + ), + ), + _full_step( + "path_defaults_edit", + lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + '[path_defaults.staging]\nmaterialized = "view"', + '[path_defaults.staging]\nmaterialized = "table"', + ), + ), + _full_step( + "enum_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_enums/order_channel.sql", + 'WEB "web"', + 'WEB "online"', + ), + ), + _full_step( + "constant_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_constants/min_quantity.sql", + "value 1)", + "value 2)", + ), + ), + _full_step( + "new_scope_folder", + lambda root: write_project_file( + root, + "models/staging/_sqlbuild/_macros/staging_totals.py", + "def staging_total_cents(price_cents: str, quantity: str) -> str:\n" + ' """Calculate a staging line total."""\n' + ' return f"({price_cents}) * ({quantity})"\n', + ), + ), + _model_step("edit_after_declarations", _model_comment), + ), + ), + IncrementalEditSequenceTestCase( + description="resource_edits", + steps=( + _full_step( + "seed_edit", + lambda root: replace_project_text( + root, "seeds/waffle_types.csv", "Classic Belgian", "Classic Brussels" + ), + ), + _full_step( + "source_edit", + lambda root: replace_project_text( + root, + "sources/raw.yml", + " - name: quantity\n type: INTEGER", + " - name: quantity\n type: BIGINT", + ), + ), + _model_step("edit_after_source", _model_type_change), + _full_step( + "test_edit", + lambda root: replace_project_text( + root, + "tests/unit/test_stg_orders.sql", + "100 AS customer_id", + "101 AS customer_id", + ), + ), + _full_step( + "audit_edit", + lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql", + "WHERE NOT (@expression)", + "WHERE NOT (@expression) AND 1 = 1", + ), + ), + _model_step("edit_after_resources", _leaf_comment), + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches_uncached_compile( + compile_reuse_project: Path, test_case: IncrementalEditSequenceTestCase +) -> None: + project_dir: Path = compile_reuse_project + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + assert cold.returncode == 0, cold.stderr + + for step in test_case.steps: + step.edit(project_dir) + comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) + + assert comparison.incremental.reused is False, step.description + assert comparison.mismatch() is None, (step.description, comparison.incremental.stderr) + assert (comparison.reused_renders > 0) is step.reuses_renders, step.description + + +@pytest.mark.parametrize( + "test_case", + [ + RandomEditChainTestCase(description="seed_7", seed=7, model_count=24, step_count=14), + RandomEditChainTestCase(description="seed_19", seed=19, model_count=24, step_count=14), + ], + ids=lambda case: case.description, +) +def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_matches_uncached_compile( + compile_reuse_project: Path, test_case: RandomEditChainTestCase +) -> None: + project_dir: Path = compile_reuse_project + write_generated_edit_models( + project_dir=project_dir, model_count=test_case.model_count, seed=test_case.seed + ) + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + assert cold.returncode == 0, cold.stderr + chain: RandomEditChain = RandomEditChain(project_dir=project_dir, seed=test_case.seed) + + for step in range(test_case.step_count): + kind, model_only = chain.apply() + comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) + + assert comparison.mismatch() is None, (step, kind, comparison.incremental.stderr) + assert (comparison.reused_renders > 0) is model_only, (step, kind) + + +def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( + compile_reuse_project: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + project_dir: Path = compile_reuse_project + enable_compile_reuse(monkeypatch) + assert compile_in_process(project_dir=project_dir) == 0 + _model_new_column(project_dir) + monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: frozenset()) + + broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + + assert broken.timings["render_reuse_hits"] > 0 + assert IncrementalEditComparison(incremental=broken, reference=reference).mismatch() is not None From ccd9994393131cbd0da196296727dea58af075b7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:02:36 +0100 Subject: [PATCH 2/4] perf(compile): layer stored renders, reuse audits, and serialize renders off the critical path Reuse per-model attached audits and whole audit groups after model-only edits, store renders as a base file plus a bounded overlay, pickle new renders on a background thread, keep GC paused across compile and store, and resolve lint project files once per parent directory. --- .../sqlbuild/references/docs/cli/compile.md | 17 +- .../cli/commands/main/project/_compile.py | 21 +- .../cli/compile_render_reuse/__init__.py | 1 + .../compile_render_reuse/_helpers/__init__.py | 0 .../_helpers/render_state_file.py | 38 +- .../cli/compile_render_reuse/constants.py | 7 + .../cli/compile_render_reuse/main/__init__.py | 0 .../main/_activated_render_reuse.py | 20 ++ .../main/_render_reuse_session.py | 9 +- .../main/_render_state_layer.py | 19 + .../cli/compile_render_reuse/models.py | 45 +++ .../cli/compile_reuse/_helpers/attempt.py | 12 +- .../compile_reuse/_helpers/project_files.py | 11 +- .../cli/compile_reuse/_helpers/store.py | 17 +- src/sqlbuild/cli/compile_reuse/constants.py | 2 - .../cli/compile_reuse/main/_framed_section.py | 11 + .../main/_read_framed_section.py | 13 + .../main/_write_reusable_compile.py | 3 +- src/sqlbuild/cli/compile_reuse/models.py | 42 --- .../compile/_helpers/attachment/audits.py | 88 ++++- .../compiler/compile/_helpers/render/reuse.py | 111 ++++++ .../compile/classes/render_reuse_session.py | 37 +- src/sqlbuild/compiler/compile/constants.py | 3 + .../compile/main/_build_compile_inputs.py | 86 ++--- .../fact_cache/main/_dump_fact_payload.py | 11 + .../fact_cache/main/_load_fact_payload.py | 11 + src/sqlbuild/lint/_helpers/project_files.py | 25 +- .../cli/commands/main/compile/_test_types.py | 13 +- .../cli/commands/main/compile/helpers.py | 330 ++++++++++++------ .../compile/test_compile_incremental_edits.py | 175 +++++----- .../cli/compile_render_reuse/__init__.py | 0 .../compile_render_reuse/_helpers/__init__.py | 0 .../_helpers/_test_types.py | 36 ++ .../compile_render_reuse/_helpers/helpers.py | 152 ++++++++ .../_helpers/test_render_state_file.py | 147 ++++++++ .../cli/compile_reuse/_helpers/_test_types.py | 13 + .../cli/compile_reuse/_helpers/helpers.py | 6 + .../_helpers/test_project_files.py | 59 +++- .../compiler/compile/classes/__init__.py | 0 .../compiler/compile/classes/_test_types.py | 38 ++ .../compiler/compile/classes/helpers.py | 117 +++++++ .../classes/test_render_reuse_session.py | 148 ++++++++ .../src/sqlbuild/lint/_helpers/_test_types.py | 9 + .../src/sqlbuild/lint/_helpers/helpers.py | 19 + .../lint/_helpers/test_project_files.py | 47 ++- website/src/content/docs/docs/cli/compile.mdx | 17 +- 46 files changed, 1585 insertions(+), 401 deletions(-) create mode 100644 src/sqlbuild/cli/compile_render_reuse/__init__.py create mode 100644 src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py rename src/sqlbuild/cli/{compile_reuse => compile_render_reuse}/_helpers/render_state_file.py (90%) create mode 100644 src/sqlbuild/cli/compile_render_reuse/constants.py create mode 100644 src/sqlbuild/cli/compile_render_reuse/main/__init__.py create mode 100644 src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py rename src/sqlbuild/cli/{compile_reuse => compile_render_reuse}/main/_render_reuse_session.py (80%) create mode 100644 src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py create mode 100644 src/sqlbuild/cli/compile_render_reuse/models.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/_framed_section.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py create mode 100644 src/sqlbuild/compiler/compile/_helpers/render/reuse.py create mode 100644 src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py create mode 100644 src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index f4ca9887b3..5e61478d7e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -120,6 +120,20 @@ ignored and compile runs in full. Storing a result normally takes a fraction of report is ready. When it has to hash a lot of data or the project has very many files, compile prints `Recording compile for reuse...` and a completion line to stderr before the report. +### Compiling after model-only edits + +When the only changes since the stored compile are edits to existing model files, compile runs in +full but reuses the stored render of every unchanged model, together with the diagnostics and +environment reads that rendering reported. Only the edited models are rendered again. Analysis, +contracts, rules, and artifact writes still run for the whole project, so downstream effects of an +edit, such as a changed column type, are reported exactly as in an uncached compile. The output is +byte-identical to `sqb compile --no-cache`. + +Every other change renders the whole project: adding, removing, or renaming a model, and editing +macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope +folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings +is always rendered again. Stored renders are kept for the two newest compiles in the same folder. + Known limits: - Python packages outside the project are checked by the stamps of the module files the compile @@ -138,7 +152,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or +rendered again after model-only edits. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 4e6422e05a..466362096a 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -3,7 +3,6 @@ from __future__ import annotations import time -from contextlib import AbstractContextManager, nullcontext from dataclasses import replace from pathlib import Path @@ -29,15 +28,17 @@ CompileCommandResult, CompileWriteResult, ) +from sqlbuild.cli.compile_render_reuse.main._activated_render_reuse import activated_render_reuse +from sqlbuild.cli.compile_render_reuse.main._render_reuse_session import render_reuse_session +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings -from sqlbuild.cli.compile_reuse.main._render_reuse_session import render_reuse_session from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile -from sqlbuild.cli.compile_reuse.models import CompileRenderReuse, CompileReuseAttempt +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.cli.entry.models import CompileCommandRequest -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS, COMPILE_RENDER_REUSE +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic from sqlbuild.compiler.compile.types import DiagnosticPhase from sqlbuild.compiler.profiling.main.collect import collect_compile_timings @@ -77,7 +78,7 @@ def run_compile( collect_compile_timings() as detailed_timings, COMPILE_INPUT_READS.recording() as input_reads, COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, - _activated_render_reuse(render_reuse=render_reuse), + activated_render_reuse(render_reuse=render_reuse), PreparedCompileArtifacts( enabled=not request.profile_flags.skip_write ) as prepared_artifacts, @@ -112,16 +113,6 @@ def run_compile( return result.exit_code -def _activated_render_reuse( - *, render_reuse: CompileRenderReuse | None -) -> AbstractContextManager[object]: - return ( - nullcontext() - if render_reuse is None - else COMPILE_RENDER_REUSE.activated(render_reuse.session) - ) - - def _run_compile_with_status( *, request: CompileCommandRequest, diff --git a/src/sqlbuild/cli/compile_render_reuse/__init__.py b/src/sqlbuild/cli/compile_render_reuse/__init__.py new file mode 100644 index 0000000000..1b6139a171 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/__init__.py @@ -0,0 +1 @@ +"""Reuse unaffected renders of the stored compile in the next full compile.""" diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py similarity index 90% rename from src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py rename to src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py index a90eb3af18..44b8001b00 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/render_state_file.py +++ b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py @@ -8,27 +8,29 @@ from pathlib import Path from typing import cast -from sqlbuild.cli.compile_reuse._helpers.entry_file import framed_section, read_framed_section +from sqlbuild.cli.compile_render_reuse.constants import ( + RENDER_INDEX_FIELD_COUNT, + RENDER_OVERLAY_MAX_SHARE, + RENDER_STATE_MAGIC, +) +from sqlbuild.cli.compile_render_reuse.models import ( + StoredRenderLayer, + StoredRenderLayers, + StoredRenderState, +) from sqlbuild.cli.compile_reuse.constants import ( REUSE_ENTRY_BYTE_ORDER, REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_LENGTH_BYTES, REUSE_MAX_ENTRY_BYTES, - REUSE_RENDER_OVERLAY_MAX_SHARE, - REUSE_RENDER_STATE_MAGIC, REUSE_RENDER_STATE_SUFFIX, ) from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError -from sqlbuild.cli.compile_reuse.models import ( - RenderStateLayer, - StoredRenderLayer, - StoredRenderLayers, - StoredRenderState, -) +from sqlbuild.cli.compile_reuse.main._framed_section import framed_stored_section +from sqlbuild.cli.compile_reuse.main._read_framed_section import read_stored_section +from sqlbuild.cli.compile_reuse.models import RenderStateLayer from sqlbuild.compiler.compile.models import RenderReuseState -_INDEX_FIELD_COUNT: int = 2 - def read_render_state(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderState | None: """Read verified renders when every changed path is a stored model; any fault means none.""" @@ -76,7 +78,7 @@ def render_state_layer( ) if ( sum(len(item) for _, item in (*models, *groups)) - <= stored.layers.base_bytes * REUSE_RENDER_OVERLAY_MAX_SHARE + <= stored.layers.base_bytes * RENDER_OVERLAY_MAX_SHARE ): return _layer( model_paths=state.model_paths, @@ -128,8 +130,8 @@ def _layer( payloads: tuple[memoryview, ...] = tuple(item for _, item in (*models, *groups)) return RenderStateLayer( chunks=( - REUSE_RENDER_STATE_MAGIC, - framed_section(data=index), + RENDER_STATE_MAGIC, + framed_stored_section(data=index), _section_header(payloads=payloads), *payloads, ), @@ -150,16 +152,16 @@ def _section_header(*, payloads: Iterable[memoryview]) -> bytes: def _read_layer(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderLayer | None: with open(path, "rb") as handle: - if handle.read(len(REUSE_RENDER_STATE_MAGIC)) != REUSE_RENDER_STATE_MAGIC: + if handle.read(len(RENDER_STATE_MAGIC)) != RENDER_STATE_MAGIC: return None - index: object = json.loads(read_framed_section(handle=handle)) + index: object = json.loads(read_stored_section(handle=handle)) if not isinstance(index, dict): return None fields: dict[str, object] = cast(dict[str, object], index) model_paths: tuple[str, ...] = tuple(_string(item) for item in _list(fields["model_paths"])) if not changed_paths.issubset(model_paths): return None - payload: bytes = read_framed_section(handle=handle) + payload: bytes = read_stored_section(handle=handle) if handle.read(1) or len(payload) > REUSE_MAX_ENTRY_BYTES: return None base: object = fields["base"] @@ -213,7 +215,7 @@ def _rows(value: object) -> list[tuple[str, int]]: rows: list[tuple[str, int]] = [] for row in _list(value): items: list[object] = _list(row) - if len(items) != _INDEX_FIELD_COUNT: + if len(items) != RENDER_INDEX_FIELD_COUNT: raise CompileReuseEntryError("invalid stored render index") length: object = items[1] if not isinstance(length, int) or isinstance(length, bool) or length < 0: diff --git a/src/sqlbuild/cli/compile_render_reuse/constants.py b/src/sqlbuild/cli/compile_render_reuse/constants.py new file mode 100644 index 0000000000..c6625ead49 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/constants.py @@ -0,0 +1,7 @@ +"""Render storage format constants.""" + +from __future__ import annotations + +RENDER_STATE_MAGIC: bytes = b"SQBRENDER1\n" +RENDER_OVERLAY_MAX_SHARE: float = 0.25 +RENDER_INDEX_FIELD_COUNT: int = 2 diff --git a/src/sqlbuild/cli/compile_render_reuse/main/__init__.py b/src/sqlbuild/cli/compile_render_reuse/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py b/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py new file mode 100644 index 0000000000..a40d737c78 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/main/_activated_render_reuse.py @@ -0,0 +1,20 @@ +"""Public entry for exposing one compile's render reuse to the rendering it runs.""" + +from __future__ import annotations + +from contextlib import AbstractContextManager, nullcontext + +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse +from sqlbuild.compiler.compile.constants import COMPILE_RENDER_REUSE + + +def activated_render_reuse( + *, render_reuse: CompileRenderReuse | None +) -> AbstractContextManager[object]: + """Activate the render reuse session for the compile block, or do nothing without one.""" + + return ( + nullcontext() + if render_reuse is None + else COMPILE_RENDER_REUSE.activated(render_reuse.session) + ) diff --git a/src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py similarity index 80% rename from src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py rename to src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py index 919a6bfd95..7a97880456 100644 --- a/src/sqlbuild/cli/compile_reuse/main/_render_reuse_session.py +++ b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py @@ -2,12 +2,9 @@ from __future__ import annotations -from sqlbuild.cli.compile_reuse._helpers.render_state_file import read_render_state -from sqlbuild.cli.compile_reuse.models import ( - CompileRenderReuse, - CompileReuseAttempt, - StoredRenderState, -) +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import read_render_state +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse, StoredRenderState +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py new file mode 100644 index 0000000000..6976765a3b --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py @@ -0,0 +1,19 @@ +"""Public entry for the render layer a finished compile stores with its entry.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import render_state_layer +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState + + +def stored_render_layer(*, render_reuse: CompileRenderReuse | None) -> RenderStateLayer | None: + """Return this compile's renders as an overlay on the stored ones, or as a new base.""" + + state: RenderReuseState | None = ( + None if render_reuse is None else render_reuse.session.stored_state() + ) + if render_reuse is None or state is None: + return None + return render_state_layer(state=state, stored=render_reuse.stored) diff --git a/src/sqlbuild/cli/compile_render_reuse/models.py b/src/sqlbuild/cli/compile_render_reuse/models.py new file mode 100644 index 0000000000..8080c25992 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/models.py @@ -0,0 +1,45 @@ +"""Stored renders of a previous compile and the render reuse of one full compile.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.models import RenderReuseState + + +@dataclass(frozen=True) +class StoredRenderLayer: + """One decoded render file: its index and its payloads by key.""" + + model_paths: tuple[str, ...] + base_file: str | None + models: dict[str, memoryview] + groups: dict[str, memoryview] + size: int + + +@dataclass(frozen=True) +class StoredRenderLayers: + """How a stored compile's renders are split between a base file and one overlay.""" + + base_file: str + base_bytes: int + overlay_models: frozenset[str] + overlay_groups: frozenset[str] + + +@dataclass(frozen=True) +class StoredRenderState: + """Renders a stored compile recorded, merged across layers, with their layout.""" + + state: RenderReuseState + layers: StoredRenderLayers + + +@dataclass(frozen=True) +class CompileRenderReuse: + """One full compile's render reuse session with the stored renders it started from.""" + + session: CompileRenderReuseSession + stored: StoredRenderState | None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index db1caaadf7..2e98b4a6f5 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -162,17 +162,15 @@ def _checked_attempt( unchanged: bool = comparison.unchanged and target_files_unchanged( stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree ) - verified: dict[str, str] = dict(comparison.verified) - changed_paths: frozenset[str] | None = ( - changed_project_paths( + changed_paths: frozenset[str] | None = None + verified: dict[str, str] = comparison.verified + if identity_unchanged and not unchanged: + changed_paths, verified = changed_project_paths( project_dir=project_dir, stored=inputs.project_files, current=attempt.snapshot, - verified=verified, + verified=comparison.verified, ) - if identity_unchanged and not unchanged - else None - ) return replace( attempt, outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py index 0c2ed267ae..e68d78330c 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py @@ -107,9 +107,10 @@ def changed_project_paths( stored: dict[str, StoredProjectFile], current: dict[str, FileStamp], verified: dict[str, str], -) -> frozenset[str]: - """Return every path added, removed, or changed in content since the stored compile.""" +) -> tuple[frozenset[str], dict[str, str]]: + """Return every path added, removed, or changed since the stored compile, and new digests.""" + digests: dict[str, str] = dict(verified) changed: set[str] = {path for path in stored if path not in current} for relative_path, stamp in current.items(): previous: StoredProjectFile | None = stored.get(relative_path) @@ -121,14 +122,14 @@ def changed_project_paths( if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: changed.add(relative_path) continue - digest: str | None = verified.get(relative_path) or file_digest( + digest: str | None = digests.get(relative_path) or file_digest( path=os.path.join(project_dir, relative_path) ) if digest is None or digest != previous.digest: changed.add(relative_path) else: - verified[relative_path] = digest - return frozenset(changed) + digests[relative_path] = digest + return frozenset(changed), digests def carried_forward_digests( diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index cfcde2ffd9..6b0ce71a49 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -8,6 +8,8 @@ import time from pathlib import Path +from sqlbuild.cli.compile_render_reuse.main._render_state_layer import stored_render_layer +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.entry_file import remove_entry, write_entry from sqlbuild.cli.compile_reuse._helpers.project_files import ( pending_digest_bytes, @@ -16,7 +18,6 @@ with_missing_digests, ) from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs -from sqlbuild.cli.compile_reuse._helpers.render_state_file import render_state_layer from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( environment_digest, loaded_module_stamps, @@ -37,16 +38,13 @@ REUSE_STORE_START_MESSAGE, ) from sqlbuild.cli.compile_reuse.models import ( - CompileRenderReuse, CompileReuseAttempt, - RenderStateLayer, SettingsInputsResult, StoredCompileInputs, StoredCompileOutput, ) from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads -from sqlbuild.compiler.compile.models import RenderReuseState _LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) @@ -173,20 +171,11 @@ def _write_compile_entry( stdout_file="", ), stdout=stdout, - render_state=_render_state_layer(render_reuse=render_reuse), + render_state=stored_render_layer(render_reuse=render_reuse), ) _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) -def _render_state_layer(*, render_reuse: CompileRenderReuse | None) -> RenderStateLayer | None: - state: RenderReuseState | None = ( - None if render_reuse is None else render_reuse.session.stored_state() - ) - if render_reuse is None or state is None: - return None - return render_state_layer(state=state, stored=render_reuse.stored) - - def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: pending_bytes: int = pending_digest_bytes( snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index ea40480ddf..c6ebe8a0a3 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -15,9 +15,7 @@ REUSE_STDOUT_SUFFIX: str = ".stdout" REUSE_STDOUT_SEPARATOR: str = "-" REUSE_RENDER_STATE_SUFFIX: str = ".render" -REUSE_RENDER_STATE_MAGIC: bytes = b"SQBRENDER1\n" REUSE_MAX_RENDER_STATES: int = 2 -REUSE_RENDER_OVERLAY_MAX_SHARE: float = 0.25 REUSE_ENTRY_LENGTH_BYTES: int = 8 REUSE_ENTRY_CHECKSUM_BYTES: int = 4 REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" diff --git a/src/sqlbuild/cli/compile_reuse/main/_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_framed_section.py new file mode 100644 index 0000000000..9259128048 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_framed_section.py @@ -0,0 +1,11 @@ +"""Public entry for framing one stored section with its length and checksum.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_reuse._helpers.entry_file import framed_section + + +def framed_stored_section(*, data: bytes) -> bytes: + """Prefix one section with its length and checksum so a reader can verify it.""" + + return framed_section(data=data) diff --git a/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py b/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py new file mode 100644 index 0000000000..08583cd0f7 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_read_framed_section.py @@ -0,0 +1,13 @@ +"""Public entry for reading one framed stored section.""" + +from __future__ import annotations + +from typing import BinaryIO + +from sqlbuild.cli.compile_reuse._helpers.entry_file import read_framed_section + + +def read_stored_section(*, handle: BinaryIO) -> bytes: + """Read one length- and checksum-framed section, raising when it is damaged.""" + + return read_framed_section(handle=handle) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py index ff1dc933a7..0fbf395025 100644 --- a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py +++ b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py @@ -4,10 +4,11 @@ from pathlib import Path +from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput -from sqlbuild.cli.compile_reuse.models import CompileRenderReuse, CompileReuseAttempt +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index 0fcd66b9e3..b4763717d1 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -4,14 +4,9 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import TYPE_CHECKING from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp -if TYPE_CHECKING: - from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession - from sqlbuild.compiler.compile.models import RenderReuseState - @dataclass(frozen=True) class CompileReuseRequest: @@ -138,43 +133,6 @@ class CompileReuseAttempt: render_state_path: Path | None = None -@dataclass(frozen=True) -class StoredRenderLayer: - """One decoded render file: its index and its payloads by key.""" - - model_paths: tuple[str, ...] - base_file: str | None - models: dict[str, memoryview] - groups: dict[str, memoryview] - size: int - - -@dataclass(frozen=True) -class StoredRenderLayers: - """How a stored compile's renders are split between a base file and one overlay.""" - - base_file: str - base_bytes: int - overlay_models: frozenset[str] - overlay_groups: frozenset[str] - - -@dataclass(frozen=True) -class StoredRenderState: - """Renders a stored compile recorded, merged across layers, with their layout.""" - - state: RenderReuseState - layers: StoredRenderLayers - - -@dataclass(frozen=True) -class CompileRenderReuse: - """One full compile's render reuse session with the stored renders it started from.""" - - session: CompileRenderReuseSession - stored: StoredRenderState | None - - @dataclass(frozen=True) class RenderStateLayer: """One render layer to publish: its framed chunks, and the base layer it extends.""" diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py index b2737a43b7..4234550ba8 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/audits.py @@ -2,7 +2,9 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass, field +from functools import partial from pathlib import Path from sqlbuild.compiler.auditing.constants import ( @@ -35,10 +37,16 @@ render_parameterized_sql, ) from sqlbuild.compiler.compile._helpers.render.cursor_intrinsics import reject_cursor_intrinsics +from sqlbuild.compiler.compile._helpers.render.reuse import reused_or_rendered from sqlbuild.compiler.compile._helpers.render.sql_vars import ( expand_authored_sql_result, ) -from sqlbuild.compiler.compile.constants import SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import ( + RENDER_REUSE_ATTACHED_AUDITS_GROUP, + RENDER_REUSE_SINGULAR_AUDITS_GROUP, + SINGULAR_AUDIT_NOT_CROSS_RESOURCE_CODE, +) from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AuthoredSqlExpansionResult, @@ -127,6 +135,7 @@ def build_project_audit_inputs( model_inputs: tuple[CompileModelInput, ...], source_inputs: tuple[CompileSourceInput, ...], seed_inputs: tuple[CompileSeedInput, ...], + render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[tuple[CompileAuditInput, ...], tuple[CompilerDiagnostic, ...]]: """Resolve built-in-aware generic audits and build every project audit input.""" @@ -142,6 +151,7 @@ def build_project_audit_inputs( source_inputs=source_inputs, generic_audit_definitions=generic_audit_definitions, seed_inputs=seed_inputs, + render_reuse=render_reuse, ) return audit_inputs, diagnostics @@ -155,6 +165,7 @@ def build_audit_inputs( generic_audit_definitions: dict[str, tuple[DiscoveredAuditFile, DiscoveredAuditBlock]] | None = None, seed_inputs: tuple[CompileSeedInput, ...] = (), + render_reuse: CompileRenderReuseSession | None = None, ) -> tuple[CompileAuditInput, ...]: """Build compile-time audit inputs from discovered SQL audit blocks.""" @@ -184,6 +195,57 @@ def build_audit_inputs( declaration_expansion=declaration_expansion, sql_lexical_syntax=sql_lexical_syntax, ) + audit_inputs: list[CompileAuditInput] = list( + reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SINGULAR_AUDITS_GROUP, + render=partial( + _build_singular_audit_inputs, + discovered_inputs=discovered_inputs, + context=attachment_context, + ), + ) + ) + model_input: CompileModelInput + for model_input in model_inputs: + if model_input.schema_entry is None: + continue + render: Callable[[], tuple[CompileAuditInput, ...]] = partial( + build_model_attached_audit_inputs, model_input=model_input, context=attachment_context + ) + audit_inputs.extend( + render() + if render_reuse is None + else render_reuse.model_audits(model_input=model_input, render=render) + ) + audit_inputs.extend( + reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_ATTACHED_AUDITS_GROUP, + render=partial( + _build_source_and_seed_audit_inputs, + source_inputs=source_inputs, + seed_inputs=seed_inputs, + context=attachment_context, + ), + ) + ) + return tuple(audit_inputs) + + +def _build_singular_audit_inputs( + *, discovered_inputs: DiscoveredProjectInputs, context: _AuditAttachmentContext +) -> tuple[CompileAuditInput, ...]: + attachment_context: _AuditAttachmentContext = context + effective_vars: dict[str, object] = context.effective_vars + macro_context: MacroContext = context.macro_context + loaded_macros: dict[str, LoadedMacro] = context.loaded_macros + sql_lexical_syntax: SqlLexicalSyntax = context.sql_lexical_syntax + known_model_names: set[str] = context.known_model_names + known_seed_names: set[str] = context.known_seed_names + known_source_names: set[str] = context.known_source_names + default_audit_severity: AuditSeverity | None = context.default_audit_severity + default_audit_run_scope: str | None = context.default_audit_run_scope audit_inputs: list[CompileAuditInput] = [] audit_file: DiscoveredAuditFile for audit_file in discovered_inputs.audit_files: @@ -326,28 +388,28 @@ def build_audit_inputs( ), ) ) - model_input: CompileModelInput - for model_input in model_inputs: - if model_input.schema_entry is None: - continue - audit_inputs.extend( - build_model_attached_audit_inputs( - model_input=model_input, - context=attachment_context, - ) - ) + return tuple(audit_inputs) + + +def _build_source_and_seed_audit_inputs( + *, + source_inputs: tuple[CompileSourceInput, ...], + seed_inputs: tuple[CompileSeedInput, ...], + context: _AuditAttachmentContext, +) -> tuple[CompileAuditInput, ...]: + audit_inputs: list[CompileAuditInput] = [] source_input: CompileSourceInput for source_input in source_inputs: audit_inputs.extend( build_source_attached_audit_inputs( source_input=source_input, - context=attachment_context, + context=context, ) ) seed_input: CompileSeedInput for seed_input in seed_inputs: audit_inputs.extend( - build_seed_attached_audit_inputs(seed_input=seed_input, context=attachment_context) + build_seed_attached_audit_inputs(seed_input=seed_input, context=context) ) return tuple(audit_inputs) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/reuse.py b/src/sqlbuild/compiler/compile/_helpers/render/reuse.py new file mode 100644 index 0000000000..70132c44da --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/render/reuse.py @@ -0,0 +1,111 @@ +"""Route project-wide renders through the active render reuse session, if any.""" + +from __future__ import annotations + +from collections.abc import Callable +from functools import partial +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs +from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( + build_scenario_inputs, + build_test_inputs_with_cache, +) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import ( + COMPILE_RENDER_REUSE, + RENDER_REUSE_SCENARIOS_GROUP, + RENDER_REUSE_SOURCES_GROUP, + RENDER_REUSE_TESTS_GROUP, +) +from sqlbuild.compiler.compile.models import ( + CompileSourceInput, + CompileSqlFunctionInput, + CompileSqlScenarioInput, + CompileSqlTestInput, + ModelInputBuildContext, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver + + +def reused_or_rendered[T]( + *, render_reuse: CompileRenderReuseSession | None, name: str, render: Callable[[], T] +) -> T: + """Return the stored render of one project-wide group, or render and record it.""" + + return render() if render_reuse is None else render_reuse.group(name=name, render=render) + + +def claimed_render_reuse( + *, discovered_inputs: DiscoveredProjectInputs +) -> CompileRenderReuseSession | None: + """Claim the active render reuse session, if any, and plan it for the discovered models.""" + + render_reuse: CompileRenderReuseSession | None = COMPILE_RENDER_REUSE.claim() + if render_reuse is not None: + render_reuse.plan_models(model_files=discovered_inputs.model_files) + return render_reuse + + +def build_sql_resource_inputs( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + sql_function_inputs: tuple[CompileSqlFunctionInput, ...], + no_sql_validation: bool, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, + compile_cache_dir: Path | None, + render_reuse: CompileRenderReuseSession | None, +) -> tuple[ + tuple[CompileSourceInput, ...], + tuple[CompileSqlTestInput, ...], + tuple[CompileSqlScenarioInput, ...], +]: + """Render sources, tests, and scenarios, each reused whole after model-only edits.""" + + source_inputs: tuple[CompileSourceInput, ...] = reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SOURCES_GROUP, + render=partial( + build_source_inputs, + discovered_inputs=discovered_inputs, + effective_vars=context.effective_vars, + effective_settings=context.effective_settings, + macro_context=context.macro_context, + loaded_macros=context.loaded_macros, + declaration_expansion=context.declaration_expansion, + no_sql_validation=no_sql_validation, + ), + ) + test_inputs: tuple[CompileSqlTestInput, ...] = reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_TESTS_GROUP, + render=partial( + build_test_inputs_with_cache, + discovered_inputs=discovered_inputs, + effective_vars=context.effective_vars, + macro_context=context.macro_context, + loaded_macros=context.loaded_macros, + declaration_expansion=context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_function_inputs=sql_function_inputs, + compile_cache_dir=compile_cache_dir, + sql_lexical_syntax=context.sql_lexical_syntax, + ), + ) + scenario_inputs: tuple[CompileSqlScenarioInput, ...] = reused_or_rendered( + render_reuse=render_reuse, + name=RENDER_REUSE_SCENARIOS_GROUP, + render=partial( + build_scenario_inputs, + discovered_inputs=discovered_inputs, + effective_vars=context.effective_vars, + macro_context=context.macro_context, + loaded_macros=context.loaded_macros, + declaration_expansion=context.declaration_expansion, + external_sql_reference_resolver=external_sql_reference_resolver, + sql_lexical_syntax=context.sql_lexical_syntax, + ), + ) + return source_inputs, test_inputs, scenario_inputs diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py index a68c70eeeb..701464d137 100644 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py @@ -13,19 +13,21 @@ tapped_compile_diagnostics, ) from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads -from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.constants import ( + COMPILE_INPUT_READS, + RENDER_REUSE_MODEL_AUDITS_PREFIX, +) from sqlbuild.compiler.compile.models import ( + CompileAuditInput, CompileModelInput, CompilerDiagnostic, RenderReuseState, StoredRender, ) from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile -from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import ( - dump_fact_payload, - load_fact_payload, -) from sqlbuild.compiler.fact_cache.exceptions import FactCachePayloadError +from sqlbuild.compiler.fact_cache.main._dump_fact_payload import dumped_fact_payload +from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload from sqlbuild.compiler.profiling.main._metric import record_compile_metric _SERIALIZER_THREAD_NAME: str = "sqlbuild-render-reuse-serializer" @@ -61,6 +63,7 @@ def __init__( self._serialized_models: dict[str, memoryview] = {} self._serialized_groups: dict[str, memoryview] = {} self._serializer: threading.Thread | None = None + self._reused_paths: set[str] = set() def claim(self) -> bool: """Return True for the first claim only; later project renders neither reuse nor record.""" @@ -105,6 +108,7 @@ def reused_model(self, *, model_file: DiscoveredSqlModelFile) -> CompileModelInp return None _replay(stored=stored) self._model_records[path] = payload + self._reused_paths.add(path) record_compile_metric(metric="render_reuse_hits", value=1) return replace(stored.value, model_file=model_file) @@ -129,12 +133,27 @@ def rendered_model( self._pending_models.append((str(model_file.relative_path), stored)) return model_input - def group[T](self, *, name: str, render: Callable[[], T]) -> T: + def model_audits( + self, + *, + model_input: CompileModelInput, + render: Callable[[], tuple[CompileAuditInput, ...]], + ) -> tuple[CompileAuditInput, ...]: + """Return a reused model's stored attached audits, or render and record them.""" + + path: str = str(model_input.model_file.relative_path) + return self.group( + name=f"{RENDER_REUSE_MODEL_AUDITS_PREFIX}{path}", + render=render, + reusable=path in self._reused_paths, + ) + + def group[T](self, *, name: str, render: Callable[[], T], reusable: bool = True) -> T: """Return a stored project-wide render no model edit can affect, or render and record it.""" payload: memoryview | None = ( self._prior.group_payloads.get(name) - if self._reusable and self._prior is not None + if reusable and self._reusable and self._prior is not None else None ) stored: StoredRender | None = None if payload is None else _loaded_render(payload) @@ -204,7 +223,7 @@ def _stored_render( def _dumped(stored: StoredRender) -> bytes | None: try: - return dump_fact_payload(stored) + return dumped_fact_payload(stored) except (pickle.PicklingError, TypeError, AttributeError, RecursionError): return None @@ -218,7 +237,7 @@ def _replay(*, stored: StoredRender) -> None: def _loaded_render(payload: memoryview) -> StoredRender | None: try: - loaded: object = load_fact_payload(payload) + loaded: object = loaded_fact_payload(payload) except _LOAD_ERRORS: return None return loaded if isinstance(loaded, StoredRender) else None diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index b1959ce97d..44905feecc 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -176,6 +176,9 @@ RENDER_REUSE_SOURCES_GROUP: str = "sources" RENDER_REUSE_TESTS_GROUP: str = "tests" RENDER_REUSE_SCENARIOS_GROUP: str = "scenarios" +RENDER_REUSE_SINGULAR_AUDITS_GROUP: str = "singular_audits" +RENDER_REUSE_ATTACHED_AUDITS_GROUP: str = "source_and_seed_audits" +RENDER_REUSE_MODEL_AUDITS_PREFIX: str = "model_audits:" SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v2" SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index ec0fe775cb..55d1fd4393 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -2,9 +2,7 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import replace -from functools import partial from pathlib import Path from sqlbuild.compiler.compile._helpers.attachment.audits import build_project_audit_inputs @@ -20,11 +18,6 @@ from sqlbuild.compiler.compile._helpers.attachment.references import ( validate_table_function_call_arities, ) -from sqlbuild.compiler.compile._helpers.attachment.sources import build_source_inputs -from sqlbuild.compiler.compile._helpers.attachment.sql_tests import ( - build_scenario_inputs, - build_test_inputs_with_cache, -) from sqlbuild.compiler.compile._helpers.attachment.target import build_compile_target_context from sqlbuild.compiler.compile._helpers.audit_factories.core import ( build_audit_factory_orphan_diagnostics, @@ -41,13 +34,11 @@ build_public_model_schema_index, ) from sqlbuild.compiler.compile._helpers.render.macros import load_project_macros -from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession -from sqlbuild.compiler.compile.constants import ( - COMPILE_RENDER_REUSE, - RENDER_REUSE_SCENARIOS_GROUP, - RENDER_REUSE_SOURCES_GROUP, - RENDER_REUSE_TESTS_GROUP, +from sqlbuild.compiler.compile._helpers.render.reuse import ( + build_sql_resource_inputs, + claimed_render_reuse, ) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession from sqlbuild.compiler.compile.models import ( CompileAdapterContext, CompileAuditInput, @@ -128,9 +119,9 @@ def build_compile_inputs( _enforce_explicit_references=(discovered_inputs.project_config.references.enforce_explicit), ) resolved_run_id: str = resolve_run_id(selected_run_id=run_id) - render_reuse: CompileRenderReuseSession | None = COMPILE_RENDER_REUSE.claim() - if render_reuse is not None: - render_reuse.plan_models(model_files=discovered_inputs.model_files) + render_reuse: CompileRenderReuseSession | None = claimed_render_reuse( + discovered_inputs=discovered_inputs + ) loaded_macros: dict[str, LoadedMacro] = load_project_macros(discovered_inputs.macro_files) declaration_scope: DeclarationScopeBuild = build_declaration_scope( discovered_inputs=discovered_inputs, @@ -172,52 +163,18 @@ def build_compile_inputs( adapter_context.python_functions_inherit_default_namespace ), ) - source_inputs: tuple[CompileSourceInput, ...] = _reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_SOURCES_GROUP, - render=partial( - build_source_inputs, - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - effective_settings=effective_settings, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - no_sql_validation=no_sql_validation, - ), - ) - test_inputs: tuple[CompileSqlTestInput, ...] = _reused_or_rendered( - render_reuse=render_reuse, - name=RENDER_REUSE_TESTS_GROUP, - render=partial( - build_test_inputs_with_cache, - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_function_inputs=sql_function_inputs, - compile_cache_dir=compile_cache_dir, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - ), - ) - scenario_inputs: tuple[CompileSqlScenarioInput, ...] = _reused_or_rendered( + source_inputs: tuple[CompileSourceInput, ...] + test_inputs: tuple[CompileSqlTestInput, ...] + scenario_inputs: tuple[CompileSqlScenarioInput, ...] + source_inputs, test_inputs, scenario_inputs = build_sql_resource_inputs( + discovered_inputs=discovered_inputs, + context=model_context, + sql_function_inputs=sql_function_inputs, + no_sql_validation=no_sql_validation, + external_sql_reference_resolver=external_sql_reference_resolver, + compile_cache_dir=compile_cache_dir, render_reuse=render_reuse, - name=RENDER_REUSE_SCENARIOS_GROUP, - render=partial( - build_scenario_inputs, - discovered_inputs=discovered_inputs, - effective_vars=effective_vars, - macro_context=macro_context, - loaded_macros=declaration_scope.loaded_macros, - declaration_expansion=model_context.declaration_expansion, - external_sql_reference_resolver=external_sql_reference_resolver, - sql_lexical_syntax=adapter_context.sql_lexical_syntax, - ), ) - if render_reuse is not None: - render_reuse.renders_complete() audit_inputs: tuple[CompileAuditInput, ...] audit_diagnostics: tuple[CompilerDiagnostic, ...] audit_inputs, audit_diagnostics = build_project_audit_inputs( @@ -226,7 +183,10 @@ def build_compile_inputs( model_inputs=model_build.inputs, source_inputs=source_inputs, seed_inputs=seed_inputs, + render_reuse=render_reuse, ) + if render_reuse is not None: + render_reuse.renders_complete() diagnostics: tuple[CompilerDiagnostic, ...] = ( *audit_diagnostics, *model_build.diagnostics, @@ -278,12 +238,6 @@ def build_compile_inputs( ) -def _reused_or_rendered[T]( - *, render_reuse: CompileRenderReuseSession | None, name: str, render: Callable[[], T] -) -> T: - return render() if render_reuse is None else render_reuse.group(name=name, render=render) - - def _build_sql_functions( *, discovered_inputs: DiscoveredProjectInputs, diff --git a/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py new file mode 100644 index 0000000000..35f3cdd661 --- /dev/null +++ b/src/sqlbuild/compiler/fact_cache/main/_dump_fact_payload.py @@ -0,0 +1,11 @@ +"""Public entry for serializing one local cache value with the fact-cache codec.""" + +from __future__ import annotations + +from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import dump_fact_payload + + +def dumped_fact_payload(value: object) -> bytes: + """Serialize one cache value with the fixed fact-cache protocol.""" + + return dump_fact_payload(value) diff --git a/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py b/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py new file mode 100644 index 0000000000..1687bed1ed --- /dev/null +++ b/src/sqlbuild/compiler/fact_cache/main/_load_fact_payload.py @@ -0,0 +1,11 @@ +"""Public entry for reading one local cache value through the restricted fact-cache codec.""" + +from __future__ import annotations + +from sqlbuild.compiler.fact_cache._helpers.restricted_pickle import load_fact_payload + + +def loaded_fact_payload(payload: bytes | memoryview) -> object: + """Deserialize one verified cache payload through the restricted class allowlist.""" + + return load_fact_payload(payload) diff --git a/src/sqlbuild/lint/_helpers/project_files.py b/src/sqlbuild/lint/_helpers/project_files.py index 01e0f21c60..316ff051ae 100644 --- a/src/sqlbuild/lint/_helpers/project_files.py +++ b/src/sqlbuild/lint/_helpers/project_files.py @@ -2,6 +2,8 @@ from __future__ import annotations +from collections.abc import Callable +from functools import cache from pathlib import Path from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs @@ -17,11 +19,14 @@ def collect_project_files( ) -> dict[Path, str]: """Return authored SQL files keyed by path, sharing identical text discovery already holds.""" + resolved_directory: Callable[[Path], Path] = cache(_resolved_path) if source_files is not None: return { file_path: contents for file_path, contents in source_files.items() - if selected_paths is None or file_path.resolve() in selected_paths + if selected_paths is None + or _resolved_file(path=file_path, resolved_directory=resolved_directory) + in selected_paths } discovered: dict[Path, str] = ( {} if discovered_inputs is None else discovered_sql_contents(discovered_inputs) @@ -34,7 +39,11 @@ def collect_project_files( continue file_path: Path for file_path in sorted(root.rglob("*.sql")): - if selected_paths is not None and file_path.resolve() not in selected_paths: + if ( + selected_paths is not None + and _resolved_file(path=file_path, resolved_directory=resolved_directory) + not in selected_paths + ): continue with file_path.open("r", encoding="utf-8", newline="") as handle: contents: str = handle.read() @@ -43,6 +52,18 @@ def collect_project_files( return files +def _resolved_file(*, path: Path, resolved_directory: Callable[[Path], Path]) -> Path: + """Resolve a file as Path.resolve() does, resolving each parent directory only once.""" + + if path.is_symlink(): + return path.resolve() + return resolved_directory(path.parent) / path.name + + +def _resolved_path(path: Path) -> Path: + return path.resolve() + + def discovered_sql_contents(discovered_inputs: DiscoveredProjectInputs) -> dict[Path, str]: """Return the text discovery read for every SQL file it retains, keyed by path.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 2ab6912d66..7083ea496a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -689,7 +689,7 @@ class IncrementalEditStep: description: str edit: Callable[[Path], None] - reuses_renders: bool + expected_render_reuse: bool @dataclass(frozen=True) @@ -698,6 +698,7 @@ class IncrementalEditSequenceTestCase: description: str steps: tuple[IncrementalEditStep, ...] + expected_matches_uncached: bool = True @dataclass(frozen=True) @@ -708,3 +709,13 @@ class RandomEditChainTestCase: seed: int model_count: int step_count: int + expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class BrokenChangeDetectionTestCase: + """A model edit compiled while change detection reports nothing changed.""" + + description: str + edit: Callable[[Path], None] + expected_matches_uncached: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 5515760c42..744742c9a0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools import json import os import random @@ -25,6 +26,7 @@ import pytest import sqlbuild.cli.commands.main.project._compile as compile_command_module +import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files import sqlbuild.cli.compile_reuse._helpers.store as reuse_store from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement @@ -41,6 +43,7 @@ from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, + IncrementalEditStep, SemanticCorpusCase, SetOperationModel, ) @@ -3448,21 +3451,28 @@ def reused_renders(self) -> int: return self.incremental.timings.get("render_reuse_hits", 0) - def mismatch(self) -> str | None: - """Describe the first difference from the --no-cache compile, if any.""" - - if self.incremental.returncode != self.reference.returncode: - return f"exit code {self.incremental.returncode} != {self.reference.returncode}" - if self.incremental.report != self.reference.report: - return "compile report differs" - if self.incremental.compiled != self.reference.compiled: - changed: list[str] = sorted( - path - for path in self.incremental.compiled.keys() | self.reference.compiled.keys() - if self.incremental.compiled.get(path) != self.reference.compiled.get(path) + @property + def matches(self) -> bool: + """Return whether exit code, report, and every compiled artifact are byte-identical.""" + + return _edit_outcome(self.incremental) == _edit_outcome(self.reference) + + @property + def mismatched_artifacts(self) -> list[str]: + """Return the first compiled artifacts that differ from the --no-cache compile.""" + + incremental: dict[str, bytes] = self.incremental.compiled + reference: dict[str, bytes] = self.reference.compiled + return sorted( + filter( + lambda path: incremental.get(path) != reference.get(path), + incremental.keys() | reference.keys(), ) - return f"compiled artifacts differ: {changed[:5]}" - return None + )[:5] + + +def _edit_outcome(run: CompileReuseRun) -> tuple[int, str, dict[str, bytes]]: + return run.returncode, run.report, run.compiled def compare_incremental_compile(*, project_dir: Path) -> IncrementalEditComparison: @@ -3491,18 +3501,40 @@ def in_process_reuse_run( GENERATED_EDIT_MODEL_PREFIX: str = "orders_step_" _GENERATED_EDIT_COLUMNS: tuple[str, ...] = ("order_id", "customer_id", "quantity", "status") +_PLAIN_QUANTITY: str = " quantity,\n" +_CAST_QUANTITY: str = " CAST(quantity AS BIGINT) AS quantity,\n" +_SWAP_PLACEHOLDER: str = "\x00swap\x00" +_RANDOM_EDIT_KINDS: tuple[str, ...] = ( + "comment", + "comment", + "add_column", + "type_change", + "header_change", + "introduce_error", + "add_model", + "macro_edit", + "test_edit", +) +_RANDOM_EDIT_FOLLOW_UPS: dict[str, tuple[str, ...]] = { + "introduce_error": ("introduce_error", "fix_error"), +} +_MODEL_ONLY_EDIT_KINDS: frozenset[str] = frozenset( + {"comment", "add_column", "type_change", "header_change", "introduce_error", "fix_error"} +) def write_generated_edit_models(*, project_dir: Path, model_count: int, seed: int) -> None: """Add a seeded chain of models over the staging orders, each reading an earlier one.""" chooser: random.Random = random.Random(seed) - for index in range(model_count): - upstream: str = ( - "stg_orders" - if index == 0 - else f"{GENERATED_EDIT_MODEL_PREFIX}{chooser.randrange(index):03d}" - ) + upstreams: tuple[str, ...] = ( + "stg_orders", + *( + f"{GENERATED_EDIT_MODEL_PREFIX}{chooser.randrange(index):03d}" + for index in range(1, model_count) + ), + ) + for index, upstream in enumerate(upstreams): write_project_file( project_dir, f"models/generated/{GENERATED_EDIT_MODEL_PREFIX}{index:03d}.sql", @@ -3518,96 +3550,192 @@ def _generated_edit_model_sql(*, upstream: str) -> str: ) +def random_edit_plan(*, seed: int, step_count: int) -> tuple[str, ...]: + """Draw seeded edit kinds; every introduced error is fixed by the following step.""" + + chooser: random.Random = random.Random(seed) + return tuple( + itertools.chain.from_iterable( + map( + lambda drawn: _RANDOM_EDIT_FOLLOW_UPS.get(drawn, (drawn,)), + chooser.choices(_RANDOM_EDIT_KINDS, k=step_count), + ) + ) + ) + + +def is_model_only_edit(kind: str) -> bool: + """Return whether an edit kind touches model files only, so renders may be reused.""" + + return kind in _MODEL_ONLY_EDIT_KINDS + + +def _swapped(contents: str, first: str, second: str) -> str: + return ( + contents.replace(first, _SWAP_PLACEHOLDER) + .replace(second, first) + .replace(_SWAP_PLACEHOLDER, second) + ) + + class RandomEditChain: """Apply seeded random edits of every supported kind to a generated project.""" def __init__(self, *, project_dir: Path, seed: int) -> None: self._project_dir: Path = project_dir self._random: random.Random = random.Random(seed) - self._broken: Path | None = None + self._broken: Path = project_dir / "models" / "generated" / "unbroken.sql" self._added: int = 0 - def apply(self) -> tuple[str, bool]: - """Apply one edit; return its kind and whether it touches model files only.""" + def apply(self, kind: str) -> None: + """Apply one edit of the given kind to a randomly chosen generated model.""" - if self._broken is not None: - path: Path = self._broken - path.write_text( - path.read_text(encoding="utf-8").replace(" unknown_column,\n", ""), - encoding="utf-8", - ) - self._broken = None - return "fix_error", True - kind: str = self._random.choice(_RANDOM_EDIT_KINDS) models: list[Path] = sorted((self._project_dir / "models" / "generated").glob("*.sql")) - model: Path = self._random.choice(models) - contents: str = model.read_text(encoding="utf-8") - if kind == "comment": - model.write_text( - contents.replace("FROM __ref", f"-- note {self._random.random()}\nFROM __ref"), - encoding="utf-8", - ) - elif kind == "add_column": - model.write_text( - contents.replace( - "\nFROM __ref", - f",\n status AS status_{self._random.randrange(1000)}\nFROM __ref", - ), - encoding="utf-8", - ) - elif kind == "type_change": - model.write_text( - contents.replace(" quantity", " CAST(quantity AS BIGINT) AS quantity", 1) - if "CAST(quantity" not in contents - else contents.replace("CAST(quantity AS BIGINT) AS quantity", "quantity", 1), - encoding="utf-8", - ) - elif kind == "header_change": - model.write_text( - contents.replace("materialized view", "materialized table") - if "materialized view" in contents - else contents.replace("materialized table", "materialized view"), - encoding="utf-8", - ) - elif kind == "introduce_error": - model.write_text( - contents.replace("SELECT\n", "SELECT\n unknown_column,\n", 1), encoding="utf-8" - ) - self._broken = model - elif kind == "add_model": - self._added += 1 - write_project_file( - self._project_dir, - f"models/generated/added_step_{self._added:03d}.sql", - _generated_edit_model_sql(upstream=model.stem), - ) - return kind, False - elif kind == "macro_edit": - path = self._project_dir / "macros" / "_rounding.py" - scale: int = 2 + self._random.randrange(3) - path.write_text( - re.sub(r"_SCALE: int = \d+", f"_SCALE: int = {scale}", path.read_text("utf-8")), - encoding="utf-8", - ) - return kind, False - elif kind == "test_edit": - path = self._project_dir / "tests" / "unit" / "test_stg_orders.sql" - path.write_text( - path.read_text(encoding="utf-8") + f"\n-- revision {self._random.random()}\n", - encoding="utf-8", - ) - return kind, False - return kind, True + edit: Callable[[Path], None] = getattr(self, f"_{kind}") + edit(self._random.choice(models)) + def _rewrite(self, model: Path, transform: Callable[[str], str]) -> None: + model.write_text(transform(model.read_text(encoding="utf-8")), encoding="utf-8") -_RANDOM_EDIT_KINDS: tuple[str, ...] = ( - "comment", - "comment", - "add_column", - "type_change", - "header_change", - "introduce_error", - "add_model", - "macro_edit", - "test_edit", -) + def _comment(self, model: Path) -> None: + note: str = f"-- note {self._random.random()}\nFROM __ref" + self._rewrite(model, lambda contents: contents.replace("FROM __ref", note)) + + def _add_column(self, model: Path) -> None: + column: str = f",\n status AS status_{self._random.randrange(1000)}\nFROM __ref" + self._rewrite(model, lambda contents: contents.replace("\nFROM __ref", column)) + + def _type_change(self, model: Path) -> None: + self._rewrite(model, lambda contents: _swapped(contents, _PLAIN_QUANTITY, _CAST_QUANTITY)) + + def _header_change(self, model: Path) -> None: + self._rewrite( + model, + lambda contents: _swapped(contents, "materialized view", "materialized table"), + ) + + def _introduce_error(self, model: Path) -> None: + self._rewrite( + model, lambda contents: contents.replace("SELECT\n", "SELECT\n unknown_column,\n", 1) + ) + self._broken = model + + def _fix_error(self, _model: Path) -> None: + self._rewrite(self._broken, lambda contents: contents.replace(" unknown_column,\n", "")) + + def _add_model(self, model: Path) -> None: + self._added += 1 + write_project_file( + self._project_dir, + f"models/generated/added_step_{self._added:03d}.sql", + _generated_edit_model_sql(upstream=model.stem), + ) + + def _macro_edit(self, _model: Path) -> None: + scale: str = f"_SCALE: int = {2 + self._random.randrange(3)}" + self._rewrite( + self._project_dir / "macros" / "_rounding.py", + lambda contents: re.sub(r"_SCALE: int = \d+", scale, contents), + ) + + def _test_edit(self, _model: Path) -> None: + revision: str = f"\n-- revision {self._random.random()}\n" + self._rewrite( + self._project_dir / "tests" / "unit" / "test_stg_orders.sql", + lambda contents: contents + revision, + ) + + +STG_ORDERS_MODEL: str = "models/staging/stg_orders.sql" +FACT_ORDERS_MODEL: str = "models/marts/fact_orders.sql" + + +def model_edit_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: + """Return an edit to model files only, whose compile must reuse unaffected renders.""" + + return IncrementalEditStep(description=description, edit=edit, expected_render_reuse=True) + + +def full_edit_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: + """Return an edit beyond model contents, whose compile must render everything again.""" + + return IncrementalEditStep(description=description, edit=edit, expected_render_reuse=False) + + +def staging_comment(root: Path) -> None: + """Add a comment to the staging orders model without changing its shape.""" + + replace_project_text( + root, + STG_ORDERS_MODEL, + 'FROM __source("raw__orders")', + '-- staged\nFROM __source("raw__orders")', + ) + + +def staging_type_change(root: Path) -> None: + """Change one staging output column type, which propagates downstream.""" + + replace_project_text( + root, STG_ORDERS_MODEL, " quantity,\n", " CAST(quantity AS BIGINT) AS quantity,\n" + ) + + +def staging_new_column(root: Path) -> None: + """Add an output column to the staging orders model.""" + + replace_project_text( + root, STG_ORDERS_MODEL, " status\n", " status,\n status AS raw_status\n" + ) + + +def staging_header_change(root: Path) -> None: + """Change the staging orders materialization in its header.""" + + replace_project_text(root, STG_ORDERS_MODEL, "materialized view,", "materialized table,") + + +def staging_contract_change(root: Path) -> None: + """Add a declared type to one staging column contract.""" + + replace_project_text( + root, + STG_ORDERS_MODEL, + "customer_id (nullable false, audits [not_null]),", + "customer_id (type BIGINT, nullable false, audits [not_null]),", + ) + + +def fact_audit_added(root: Path) -> None: + """Attach one more column audit to the fact orders model.""" + + replace_project_text( + root, + FACT_ORDERS_MODEL, + " order_id (nullable false, audits [not_null]),", + " order_id (nullable false, audits [not_null, unique]),", + ) + + +def fact_error_introduced(root: Path) -> None: + """Make the fact orders model read a column that does not exist.""" + + replace_project_text(root, FACT_ORDERS_MODEL, " o.quantity,\n", " o.missing_quantity,\n") + + +def fact_error_fixed(root: Path) -> None: + """Restore the column the fact orders model reads.""" + + replace_project_text(root, FACT_ORDERS_MODEL, " o.missing_quantity,\n", " o.quantity,\n") + + +def fact_comment(root: Path) -> None: + """Add a comment to the leaf fact orders model.""" + + replace_project_text(root, FACT_ORDERS_MODEL, "FROM __ref", "-- leaf\nFROM __ref") + + +def disable_change_detection(monkeypatch: pytest.MonkeyPatch) -> None: + """Make compile reuse believe no project file changed since the stored compile.""" + + monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: (frozenset(), {})) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index bff7c9e418..5bd3cab376 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -2,15 +2,13 @@ from __future__ import annotations -from collections.abc import Callable from pathlib import Path import pytest -import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + BrokenChangeDetectionTestCase, IncrementalEditSequenceTestCase, - IncrementalEditStep, RandomEditChainTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( @@ -19,67 +17,29 @@ RandomEditChain, compare_incremental_compile, compile_in_process, + disable_change_detection, enable_compile_reuse, + fact_audit_added, + fact_comment, + fact_error_fixed, + fact_error_introduced, + full_edit_step, in_process_reuse_run, + is_model_only_edit, + model_edit_step, move_project_file, + random_edit_plan, replace_project_text, run_reuse_compile, + staging_comment, + staging_contract_change, + staging_header_change, + staging_new_column, + staging_type_change, write_generated_edit_models, write_project_file, ) -_STG_ORDERS: str = "models/staging/stg_orders.sql" -_FACT_ORDERS: str = "models/marts/fact_orders.sql" - - -def _model_comment(root: Path) -> None: - replace_project_text( - root, _STG_ORDERS, 'FROM __source("raw__orders")', '-- staged\nFROM __source("raw__orders")' - ) - - -def _model_type_change(root: Path) -> None: - replace_project_text( - root, _STG_ORDERS, " quantity,\n", " CAST(quantity AS BIGINT) AS quantity,\n" - ) - - -def _model_new_column(root: Path) -> None: - replace_project_text(root, _STG_ORDERS, " status\n", " status,\n status AS raw_status\n") - - -def _model_header_change(root: Path) -> None: - replace_project_text(root, _STG_ORDERS, "materialized view,", "materialized table,") - - -def _model_contract_change(root: Path) -> None: - replace_project_text( - root, - _STG_ORDERS, - "customer_id (nullable false, audits [not_null]),", - "customer_id (type BIGINT, nullable false, audits [not_null]),", - ) - - -def _introduce_error(root: Path) -> None: - replace_project_text(root, _FACT_ORDERS, " o.quantity,\n", " o.missing_quantity,\n") - - -def _fix_error(root: Path) -> None: - replace_project_text(root, _FACT_ORDERS, " o.missing_quantity,\n", " o.quantity,\n") - - -def _leaf_comment(root: Path) -> None: - replace_project_text(root, _FACT_ORDERS, "FROM __ref", "-- leaf\nFROM __ref") - - -def _model_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: - return IncrementalEditStep(description=description, edit=edit, reuses_renders=True) - - -def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEditStep: - return IncrementalEditStep(description=description, edit=edit, reuses_renders=False) - @pytest.mark.parametrize( "test_case", @@ -87,19 +47,20 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi IncrementalEditSequenceTestCase( description="model_edits", steps=( - _model_step("comment_without_shape_change", _model_comment), - _model_step("type_change_propagates_downstream", _model_type_change), - _model_step("new_output_column", _model_new_column), - _model_step("header_config_change", _model_header_change), - _model_step("column_contract_change", _model_contract_change), - _model_step("error_introduced", _introduce_error), - _model_step("error_fixed", _fix_error), + model_edit_step("comment_without_shape_change", staging_comment), + model_edit_step("type_change_propagates_downstream", staging_type_change), + model_edit_step("new_output_column", staging_new_column), + model_edit_step("header_config_change", staging_header_change), + model_edit_step("column_contract_change", staging_contract_change), + model_edit_step("attached_audit_added", fact_audit_added), + model_edit_step("error_introduced", fact_error_introduced), + model_edit_step("error_fixed", fact_error_fixed), ), ), IncrementalEditSequenceTestCase( description="model_set_changes", steps=( - _full_step( + full_edit_step( "model_added", lambda root: write_project_file( root, @@ -108,8 +69,8 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi 'SELECT order_id, quantity FROM __ref("stg_orders")\n', ), ), - _model_step("edit_after_add", _model_comment), - _full_step( + model_edit_step("edit_after_add", staging_comment), + full_edit_step( "model_renamed", lambda root: move_project_file( root, @@ -117,17 +78,17 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi "models/marts/order_quantity_totals.sql", ), ), - _full_step( + full_edit_step( "model_removed", lambda root: (root / "models/marts/order_quantity_totals.sql").unlink(), ), - _model_step("edit_after_remove", _leaf_comment), + model_edit_step("edit_after_remove", fact_comment), ), ), IncrementalEditSequenceTestCase( description="declaration_edits", steps=( - _full_step( + full_edit_step( "macro_edit", lambda root: replace_project_text( root, @@ -136,14 +97,14 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi "{quantity} * {price_cents}", ), ), - _model_step("edit_after_macro", _leaf_comment), - _full_step( + model_edit_step("edit_after_macro", fact_comment), + full_edit_step( "module_imported_by_macro_edit", lambda root: replace_project_text( root, "macros/_rounding.py", "_SCALE: int = 2", "_SCALE: int = 3" ), ), - _full_step( + full_edit_step( "scoped_macro_edit", lambda root: replace_project_text( root, @@ -152,7 +113,7 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi "/ 100.0, 3)", ), ), - _full_step( + full_edit_step( "path_defaults_edit", lambda root: replace_project_text( root, @@ -161,7 +122,7 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi '[path_defaults.staging]\nmaterialized = "table"', ), ), - _full_step( + full_edit_step( "enum_edit", lambda root: replace_project_text( root, @@ -170,7 +131,7 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi 'WEB "online"', ), ), - _full_step( + full_edit_step( "constant_edit", lambda root: replace_project_text( root, @@ -179,7 +140,7 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi "value 2)", ), ), - _full_step( + full_edit_step( "new_scope_folder", lambda root: write_project_file( root, @@ -189,19 +150,19 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi ' return f"({price_cents}) * ({quantity})"\n', ), ), - _model_step("edit_after_declarations", _model_comment), + model_edit_step("edit_after_declarations", staging_comment), ), ), IncrementalEditSequenceTestCase( description="resource_edits", steps=( - _full_step( + full_edit_step( "seed_edit", lambda root: replace_project_text( root, "seeds/waffle_types.csv", "Classic Belgian", "Classic Brussels" ), ), - _full_step( + full_edit_step( "source_edit", lambda root: replace_project_text( root, @@ -210,8 +171,8 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi " - name: quantity\n type: BIGINT", ), ), - _model_step("edit_after_source", _model_type_change), - _full_step( + model_edit_step("edit_after_source", staging_type_change), + full_edit_step( "test_edit", lambda root: replace_project_text( root, @@ -220,7 +181,7 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi "101 AS customer_id", ), ), - _full_step( + full_edit_step( "audit_edit", lambda root: replace_project_text( root, @@ -229,7 +190,7 @@ def _full_step(description: str, edit: Callable[[Path], None]) -> IncrementalEdi "WHERE NOT (@expression) AND 1 = 1", ), ), - _model_step("edit_after_resources", _leaf_comment), + model_edit_step("edit_after_resources", fact_comment), ), ), ], @@ -247,15 +208,19 @@ def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) assert comparison.incremental.reused is False, step.description - assert comparison.mismatch() is None, (step.description, comparison.incremental.stderr) - assert (comparison.reused_renders > 0) is step.reuses_renders, step.description + assert comparison.matches is test_case.expected_matches_uncached, ( + step.description, + comparison.mismatched_artifacts, + comparison.incremental.stderr, + ) + assert (comparison.reused_renders > 0) is step.expected_render_reuse, step.description @pytest.mark.parametrize( "test_case", [ - RandomEditChainTestCase(description="seed_7", seed=7, model_count=24, step_count=14), - RandomEditChainTestCase(description="seed_19", seed=19, model_count=24, step_count=14), + RandomEditChainTestCase(description="seed_7", seed=7, model_count=24, step_count=12), + RandomEditChainTestCase(description="seed_19", seed=19, model_count=24, step_count=12), ], ids=lambda case: case.description, ) @@ -270,27 +235,49 @@ def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_mat assert cold.returncode == 0, cold.stderr chain: RandomEditChain = RandomEditChain(project_dir=project_dir, seed=test_case.seed) - for step in range(test_case.step_count): - kind, model_only = chain.apply() + for step, kind in enumerate( + random_edit_plan(seed=test_case.seed, step_count=test_case.step_count) + ): + chain.apply(kind) comparison: IncrementalEditComparison = compare_incremental_compile(project_dir=project_dir) - assert comparison.mismatch() is None, (step, kind, comparison.incremental.stderr) - assert (comparison.reused_renders > 0) is model_only, (step, kind) + assert comparison.matches is test_case.expected_matches_uncached, ( + step, + kind, + comparison.mismatched_artifacts, + comparison.incremental.stderr, + ) + assert (comparison.reused_renders > 0) is is_model_only_edit(kind), (step, kind) +@pytest.mark.parametrize( + "test_case", + [ + BrokenChangeDetectionTestCase( + description="new_output_column", + edit=staging_new_column, + expected_matches_uncached=False, + ), + ], + ids=lambda case: case.description, +) def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_reports_a_mismatch( compile_reuse_project: Path, + test_case: BrokenChangeDetectionTestCase, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], ) -> None: project_dir: Path = compile_reuse_project enable_compile_reuse(monkeypatch) assert compile_in_process(project_dir=project_dir) == 0 - _model_new_column(project_dir) - monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: frozenset()) + test_case.edit(project_dir) + disable_change_detection(monkeypatch) broken: CompileReuseRun = in_process_reuse_run(project_dir=project_dir, capsys=capsys) reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=broken, reference=reference + ) - assert broken.timings["render_reuse_hits"] > 0 - assert IncrementalEditComparison(incremental=broken, reference=reference).mismatch() is not None + assert comparison.reused_renders > 0 + assert comparison.matches is test_case.expected_matches_uncached diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py new file mode 100644 index 0000000000..e59823a920 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py @@ -0,0 +1,36 @@ +"""Test case types for layered render storage.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RenderLayerTestCase: + """Renders changed after a stored compile, and whether they fit in an overlay.""" + + description: str + changed_models: tuple[str, ...] + removed_models: tuple[str, ...] + expected_overlay: bool + expected_overlay_models: frozenset[str] + + +@dataclass(frozen=True) +class RenderEditChainTestCase: + """Successive one-model edits, each stored on top of the previous compile.""" + + description: str + edits: tuple[str, ...] + expected_overlay_models: frozenset[str] + + +@dataclass(frozen=True) +class StoredRenderReadTestCase: + """A stored render file, possibly damaged, and whether reading it yields renders.""" + + description: str + prepare: Callable[[Path], tuple[Path, frozenset[str]]] + expected_readable: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py new file mode 100644 index 0000000000..590026c840 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py @@ -0,0 +1,152 @@ +"""Shared builders for layered render storage unit tests.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( + read_render_state, + render_state_layer, +) +from sqlbuild.cli.compile_render_reuse.models import StoredRenderState +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState + +MODEL_PATHS: tuple[str, ...] = tuple(f"models/orders_{index:02d}.sql" for index in range(20)) + + +def publish_render_state(*, directory: Path, name: str, layer: RenderStateLayer) -> Path: + """Write one render layer as the store would publish it.""" + + path: Path = directory / name + path.write_bytes(b"".join(bytes(chunk) for chunk in layer.chunks)) + return path + + +def read_stored(path: Path) -> StoredRenderState: + """Read a render file that must decode.""" + + stored: StoredRenderState | None = read_render_state(path=path, changed_paths=frozenset()) + assert stored is not None + return stored + + +def stored_base(directory: Path) -> tuple[Path, StoredRenderState]: + """Publish a base render file holding every model and one group, and read it back.""" + + state: RenderReuseState = RenderReuseState( + model_paths=MODEL_PATHS, + model_payloads={path: memoryview(f"render {path}".encode() * 20) for path in MODEL_PATHS}, + group_payloads={"sources": memoryview(b"source renders" * 20)}, + ) + path: Path = publish_render_state( + directory=directory, + name="orders-base.render", + layer=render_state_layer(state=state, stored=None), + ) + return path, read_stored(path) + + +def next_render_state( + *, stored: StoredRenderState, edited: tuple[str, ...], removed: tuple[str, ...] = () +) -> RenderReuseState: + """Return the renders of the next compile: some models edited, some stored renders dropped.""" + + kept: dict[str, memoryview] = dict( + filter(lambda item: item[0] not in removed, stored.state.model_payloads.items()) + ) + return RenderReuseState( + model_paths=MODEL_PATHS, + model_payloads={ + **kept, + **{path: memoryview(f"edited {path}".encode() * 20) for path in edited}, + }, + group_payloads=dict(stored.state.group_payloads), + ) + + +def payload_bytes(state: RenderReuseState) -> dict[str, bytes]: + """Return every model payload as bytes for comparison.""" + + return {path: bytes(payload) for path, payload in state.model_payloads.items()} + + +def store_edit_chain(*, directory: Path, edits: tuple[str, ...]) -> StoredRenderState: + """Store one compile per edited model, each layered on the previous one, and read the last.""" + + stored: StoredRenderState + _, stored = stored_base(directory) + for index, edited in enumerate(edits): + path: Path = publish_render_state( + directory=directory, + name=f"orders-edit-{index}.render", + layer=render_state_layer( + state=next_render_state(stored=stored, edited=(edited,)), stored=stored + ), + ) + stored = read_stored(path) + return stored + + +def chain_payload_bytes(edits: tuple[str, ...]) -> dict[str, bytes]: + """Return the model payloads an edit chain must leave readable.""" + + return { + **{path: f"render {path}".encode() * 20 for path in MODEL_PATHS}, + **{path: f"edited {path}".encode() * 20 for path in edits}, + } + + +def flipped_last_byte(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and corrupt its last payload byte.""" + + path, _ = stored_base(directory) + data: bytearray = bytearray(path.read_bytes()) + data[-1] ^= 0xFF + path.write_bytes(bytes(data)) + return path, frozenset() + + +def truncated(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and cut off its tail.""" + + path, _ = stored_base(directory) + path.write_bytes(path.read_bytes()[:-7]) + return path, frozenset() + + +def emptied(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and empty it.""" + + path, _ = stored_base(directory) + path.write_bytes(b"") + return path, frozenset() + + +def overlay_without_base(directory: Path) -> tuple[Path, frozenset[str]]: + """Store an overlay and delete the base file it layers on.""" + + base_path, stored = stored_base(directory) + overlay_path: Path = publish_render_state( + directory=directory, + name="orders-overlay.render", + layer=render_state_layer( + state=next_render_state(stored=stored, edited=(MODEL_PATHS[0],)), stored=stored + ), + ) + base_path.unlink() + return overlay_path, frozenset() + + +def changed_macro(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and report a changed path that is not a stored model.""" + + path, _ = stored_base(directory) + return path, frozenset({"macros/currency.py"}) + + +def intact(directory: Path) -> tuple[Path, frozenset[str]]: + """Store a base render file and report one stored model as changed.""" + + path, _ = stored_base(directory) + return path, frozenset({MODEL_PATHS[4]}) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py new file mode 100644 index 0000000000..71bccd8da9 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py @@ -0,0 +1,147 @@ +"""Layered render storage must round-trip exactly and treat every fault as no stored renders.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import ( + read_render_state, + render_state_layer, +) +from sqlbuild.cli.compile_render_reuse.models import StoredRenderState +from sqlbuild.cli.compile_reuse.models import RenderStateLayer +from sqlbuild.compiler.compile.models import RenderReuseState +from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( + RenderEditChainTestCase, + RenderLayerTestCase, + StoredRenderReadTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers.helpers import ( + MODEL_PATHS, + chain_payload_bytes, + changed_macro, + emptied, + flipped_last_byte, + intact, + next_render_state, + overlay_without_base, + payload_bytes, + publish_render_state, + read_stored, + store_edit_chain, + stored_base, + truncated, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderLayerTestCase( + description="one_model_changed", + changed_models=(MODEL_PATHS[3],), + removed_models=(), + expected_overlay=True, + expected_overlay_models=frozenset({MODEL_PATHS[3]}), + ), + RenderLayerTestCase( + description="most_models_changed", + changed_models=MODEL_PATHS[:15], + removed_models=(), + expected_overlay=False, + expected_overlay_models=frozenset(), + ), + RenderLayerTestCase( + description="stored_render_dropped", + changed_models=(MODEL_PATHS[3],), + removed_models=(MODEL_PATHS[5],), + expected_overlay=False, + expected_overlay_models=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_renders_when_storing_the_next_compile_then_reading_returns_its_renders( + test_case: RenderLayerTestCase, tmp_path: Path +) -> None: + _, stored = stored_base(tmp_path) + current: RenderReuseState = next_render_state( + stored=stored, edited=test_case.changed_models, removed=test_case.removed_models + ) + + layer: RenderStateLayer = render_state_layer(state=current, stored=stored) + reread: StoredRenderState = read_stored( + publish_render_state(directory=tmp_path, name="orders-next.render", layer=layer) + ) + + assert (layer.base_file is not None) is test_case.expected_overlay + assert payload_bytes(reread.state) == payload_bytes(current) + assert reread.state.group_payloads.keys() == current.group_payloads.keys() + assert reread.layers.overlay_models == test_case.expected_overlay_models + + +@pytest.mark.parametrize( + "test_case", + [ + RenderEditChainTestCase( + description="different_models_edited", + edits=(MODEL_PATHS[1], MODEL_PATHS[2]), + expected_overlay_models=frozenset({MODEL_PATHS[1], MODEL_PATHS[2]}), + ), + RenderEditChainTestCase( + description="same_model_edited_twice", + edits=(MODEL_PATHS[1], MODEL_PATHS[1]), + expected_overlay_models=frozenset({MODEL_PATHS[1]}), + ), + ], + ids=lambda case: case.description, +) +def test_given_overlay_when_storing_again_then_earlier_overlay_renders_are_kept( + test_case: RenderEditChainTestCase, tmp_path: Path +) -> None: + stored: StoredRenderState = store_edit_chain(directory=tmp_path, edits=test_case.edits) + + assert stored.layers.overlay_models == test_case.expected_overlay_models + assert payload_bytes(stored.state) == chain_payload_bytes(test_case.edits) + + +@pytest.mark.parametrize( + "test_case", + [ + StoredRenderReadTestCase( + description="intact_with_changed_model", prepare=intact, expected_readable=True + ), + StoredRenderReadTestCase( + description="flipped_payload_byte", prepare=flipped_last_byte, expected_readable=False + ), + StoredRenderReadTestCase( + description="truncated", prepare=truncated, expected_readable=False + ), + StoredRenderReadTestCase(description="empty", prepare=emptied, expected_readable=False), + StoredRenderReadTestCase( + description="overlay_whose_base_is_missing", + prepare=overlay_without_base, + expected_readable=False, + ), + StoredRenderReadTestCase( + description="changed_path_is_not_a_stored_model", + prepare=changed_macro, + expected_readable=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_render_file_when_reading_then_only_intact_relevant_renders_are_returned( + test_case: StoredRenderReadTestCase, tmp_path: Path +) -> None: + path, changed_paths = test_case.prepare(tmp_path) + + stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) + + assert (stored is not None) is test_case.expected_readable + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index 77b9011266..ea3686abfc 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -8,6 +8,9 @@ import pytest +from sqlbuild.cli.compile_reuse.models import StoredProjectFile +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import digested_project_files + @dataclass(frozen=True) class ProjectFingerprintTestCase: @@ -78,3 +81,13 @@ class SettingsDigestTestCase: description: str change: Callable[[pytest.MonkeyPatch, Path], None] expected_changed: bool + + +@dataclass(frozen=True) +class ChangedPathsTestCase: + """One project change and every project path a change set must name for it.""" + + description: str + change: Callable[[Path], object] + expected_paths: tuple[str, ...] + store: Callable[[Path], dict[str, StoredProjectFile]] = digested_project_files diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index 3dcf51f238..efacd4c1c0 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -124,6 +124,12 @@ def break_link(project_dir: Path) -> None: (project_dir.parent / "shared/customers.sql").unlink() +def digested_project_files(project_dir: Path) -> dict[str, StoredProjectFile]: + """Record the project with digests and a snapshot time that leaves no file racy.""" + + return stored_project_files(project_dir=project_dir, snapshot_ns=later_snapshot_ns()) + + def stat_only_project_files(*, project_dir: Path) -> dict[str, StoredProjectFile]: """Record the project as a stored compile does by default: stat stamps without digests.""" diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py index 9f0eeb019f..f4867265a1 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py @@ -10,6 +10,7 @@ from sqlbuild.cli.compile_reuse._helpers.project_files import ( carried_forward_digests, + changed_project_paths, compare_project_files, restamped_paths, snapshot_project_files, @@ -17,6 +18,7 @@ from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile from sqlbuild.cli.compile_reuse.types import FileStamp from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + ChangedPathsTestCase, DigestCarryForwardTestCase, LinkCycleTestCase, ProjectFingerprintTestCase, @@ -147,13 +149,68 @@ def test_given_stored_fingerprint_when_project_changes_then_change_is_detected_e ) test_case.change(project_dir) + current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) comparison: ProjectFilesComparison = compare_project_files( + project_dir=str(project_dir), stored=stored, current=current + ) + changed, _ = changed_project_paths( + project_dir=str(project_dir), stored=stored, current=current, verified={} + ) + + assert comparison.unchanged is test_case.expected_unchanged + assert (not changed) is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + ChangedPathsTestCase( + description="touched_without_change", + change=touch_without_change, + expected_paths=(), + ), + ChangedPathsTestCase( + description="model_edited", + change=lambda root: write_file(root / "models/orders.sql", "SELECT 10 AS order_id\n"), + expected_paths=("models/orders.sql",), + ), + ChangedPathsTestCase( + description="same_size_and_mtime_new_content", + change=rewrite_same_size_keeping_mtime, + expected_paths=("models/orders.sql",), + ), + ChangedPathsTestCase( + description="file_added_and_removed", + change=lambda root: ( + write_file(root / "models/returns.sql", "SELECT 1\n"), + (root / "macros/currency.py").unlink(), + ), + expected_paths=("macros/currency.py", "models/returns.sql"), + ), + ChangedPathsTestCase( + description="touched_without_stored_digest", + change=touch_without_change, + expected_paths=("models/orders.sql",), + store=lambda root: stat_only_project_files(project_dir=root), + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_fingerprint_when_listing_changes_then_returns_every_changed_path( + test_case: ChangedPathsTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + stored: dict[str, StoredProjectFile] = test_case.store(project_dir) + + test_case.change(project_dir) + changed, _ = changed_project_paths( project_dir=str(project_dir), stored=stored, current=snapshot_project_files(project_dir=str(project_dir)), + verified={}, ) - assert comparison.unchanged is test_case.expected_unchanged + assert changed == frozenset(test_case.expected_paths) @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py b/tests/unit/src/sqlbuild/compiler/compile/classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py new file mode 100644 index 0000000000..324c935c91 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py @@ -0,0 +1,38 @@ +"""Test case types for the compile render reuse session.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from sqlbuild.compiler.compile.models import CompilerDiagnostic + + +@dataclass(frozen=True) +class RenderReplayTestCase: + """A reused render and what reusing it must report again.""" + + description: str + model_name: str + expected_diagnostics: tuple[CompilerDiagnostic, ...] + expected_environment_names: tuple[str, ...] + + +@dataclass(frozen=True) +class ReusableModelsTestCase: + """Stored and current models, and which current models may reuse their stored render.""" + + description: str + stored_models: tuple[str, ...] + current_models: tuple[str, ...] + changed_paths: frozenset[str] + run_id_readers: frozenset[str] + expected_reusable: dict[str, bool] + + +@dataclass(frozen=True) +class CorruptRenderTestCase: + """A stored render payload that cannot be decoded.""" + + description: str + model_name: str + expected_rendered_again: bool diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py new file mode 100644 index 0000000000..c6c0b114b1 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py @@ -0,0 +1,117 @@ +"""Shared builders for render reuse session unit tests.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( + collect_compile_diagnostics, + report_compile_diagnostic, +) +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompilerDiagnostic, + RenderReuseState, +) +from sqlbuild.compiler.compile.types import CompileContextKey, DiagnosticPhase, DiagnosticSeverity +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile + +REGION_ENV_VAR: str = "ORDERS_REGION" + + +def model_file(name: str, sql: str = "SELECT 1 AS order_id") -> DiscoveredSqlModelFile: + """Return one discovered SQL model.""" + + return DiscoveredSqlModelFile( + file_path=Path("/orders") / "models" / f"{name}.sql", + relative_path=Path("models") / f"{name}.sql", + contents=sql, + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql=sql, + ) + + +def missing_description(name: str) -> CompilerDiagnostic: + """Return the warning a render of the named model reports.""" + + return CompilerDiagnostic( + phase=DiagnosticPhase.COMPILE, + severity=DiagnosticSeverity.WARNING, + code="P010", + message=f"Model '{name}' has no description", + ) + + +def _render(item: DiscoveredSqlModelFile) -> CompileModelInput: + report_compile_diagnostic( + key=(item.file_path.stem,), diagnostic=missing_description(item.file_path.stem) + ) + COMPILE_INPUT_READS.environment_read(REGION_ENV_VAR) + return CompileModelInput(model_file=item, query_sql=item.query_sql) + + +def _render_reading_run_id(item: DiscoveredSqlModelFile) -> CompileModelInput: + COMPILE_INPUT_READS.context_read(CompileContextKey.RUN_ID) + return _render(item) + + +def recorded_state( + model_files: tuple[DiscoveredSqlModelFile, ...], + *, + run_id_readers: frozenset[str] = frozenset(), +) -> RenderReuseState: + """Render every model through a fresh session and return what it stores.""" + + session: CompileRenderReuseSession = CompileRenderReuseSession(prior=None, changed_paths=None) + session.plan_models(model_files=model_files) + renderers: dict[bool, Callable[[DiscoveredSqlModelFile], CompileModelInput]] = { + False: _render, + True: _render_reading_run_id, + } + with collect_compile_diagnostics(): + for item in model_files: + render: Callable[[DiscoveredSqlModelFile], CompileModelInput] = renderers[ + item.file_path.stem in run_id_readers + ] + _ = session.rendered_model(model_file=item, render=lambda item=item, r=render: r(item)) + session.renders_complete() + state: RenderReuseState | None = session.stored_state() + assert state is not None + return state + + +def planned_session( + *, + stored: tuple[DiscoveredSqlModelFile, ...], + current: tuple[DiscoveredSqlModelFile, ...], + changed_paths: frozenset[str] = frozenset(), + run_id_readers: frozenset[str] = frozenset(), +) -> CompileRenderReuseSession: + """Record renders for the stored models, then plan a session for the current models.""" + + session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=recorded_state(stored, run_id_readers=run_id_readers), changed_paths=changed_paths + ) + session.plan_models(model_files=current) + return session + + +def corrupted_session(item: DiscoveredSqlModelFile) -> CompileRenderReuseSession: + """Plan a session whose only stored render payload cannot be decoded.""" + + state: RenderReuseState = recorded_state((item,)) + session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=replace( + state, + model_payloads={str(item.relative_path): memoryview(b"not a render")}, + ), + changed_paths=frozenset(), + ) + session.plan_models(model_files=(item,)) + return session diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py new file mode 100644 index 0000000000..45c765ab18 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py @@ -0,0 +1,148 @@ +"""Render reuse must replay what an unchanged render reported and never serve a changed one.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.compiler.compile._helpers.diagnostics.collector import collect_compile_diagnostics +from sqlbuild.compiler.compile.classes.render_reuse_session import CompileRenderReuseSession +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS +from sqlbuild.compiler.compile.models import CompileModelInput +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( + CorruptRenderTestCase, + RenderReplayTestCase, + ReusableModelsTestCase, +) +from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( + REGION_ENV_VAR, + corrupted_session, + missing_description, + model_file, + planned_session, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderReplayTestCase( + description="unchanged_model", + model_name="orders", + expected_diagnostics=(missing_description("orders"),), + expected_environment_names=(REGION_ENV_VAR,), + ), + ], + ids=lambda case: case.description, +) +def test_given_unchanged_model_when_reusing_then_render_and_reports_are_replayed( + test_case: RenderReplayTestCase, +) -> None: + unchanged: DiscoveredSqlModelFile = model_file(test_case.model_name) + edited: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") + session: CompileRenderReuseSession = planned_session( + stored=(unchanged, model_file("customers")), + current=(unchanged, edited), + changed_paths=frozenset({str(edited.relative_path)}), + ) + + with ( + COMPILE_INPUT_READS.recording() as reads, + collect_compile_diagnostics() as collected, + ): + reused: CompileModelInput | None = session.reused_model(model_file=unchanged) + + assert reused == CompileModelInput(model_file=unchanged, query_sql=unchanged.query_sql) + assert reused is not None + assert reused.model_file is unchanged + assert collected.diagnostics == test_case.expected_diagnostics + assert reads.environment_names == test_case.expected_environment_names + + +@pytest.mark.parametrize( + "test_case", + [ + ReusableModelsTestCase( + description="changed_model", + stored_models=("orders", "customers"), + current_models=("orders", "customers"), + changed_paths=frozenset({"models/customers.sql"}), + run_id_readers=frozenset(), + expected_reusable={"orders": True, "customers": False}, + ), + ReusableModelsTestCase( + description="render_read_run_identity", + stored_models=("orders", "customers"), + current_models=("orders", "customers"), + changed_paths=frozenset(), + run_id_readers=frozenset({"orders"}), + expected_reusable={"orders": False, "customers": True}, + ), + ReusableModelsTestCase( + description="model_added", + stored_models=("orders",), + current_models=("orders", "customers"), + changed_paths=frozenset({"models/customers.sql"}), + run_id_readers=frozenset(), + expected_reusable={"orders": False, "customers": False}, + ), + ReusableModelsTestCase( + description="model_removed", + stored_models=("orders", "customers", "returns"), + current_models=("orders", "customers"), + changed_paths=frozenset(), + run_id_readers=frozenset(), + expected_reusable={"orders": False, "customers": False}, + ), + ReusableModelsTestCase( + description="non_model_input_changed", + stored_models=("orders", "customers"), + current_models=("orders", "customers"), + changed_paths=frozenset({"macros/currency.py"}), + run_id_readers=frozenset(), + expected_reusable={"orders": False, "customers": False}, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_renders_when_planning_then_only_unchanged_identical_model_sets_reuse( + test_case: ReusableModelsTestCase, +) -> None: + current: tuple[DiscoveredSqlModelFile, ...] = tuple(map(model_file, test_case.current_models)) + session: CompileRenderReuseSession = planned_session( + stored=tuple(map(model_file, test_case.stored_models)), + current=current, + changed_paths=test_case.changed_paths, + run_id_readers=test_case.run_id_readers, + ) + + reusable: dict[str, bool] = { + item.file_path.stem: session.has_reusable_model(model_file=item) for item in current + } + + assert reusable == test_case.expected_reusable + + +@pytest.mark.parametrize( + "test_case", + [ + CorruptRenderTestCase( + description="undecodable_payload", model_name="orders", expected_rendered_again=True + ), + ], + ids=lambda case: case.description, +) +def test_given_corrupt_stored_render_when_reusing_then_the_model_is_rendered_again( + test_case: CorruptRenderTestCase, +) -> None: + item: DiscoveredSqlModelFile = model_file(test_case.model_name) + session: CompileRenderReuseSession = corrupted_session(item) + + with collect_compile_diagnostics(): + reused: CompileModelInput | None = session.reused_model(model_file=item) + + assert (reused is None) is test_case.expected_rendered_again + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py index ac1722759e..d4385d6164 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/_test_types.py @@ -243,3 +243,12 @@ class CollectProjectFilesTestCase: description: str model_bytes: bytes expected_shared: bool + + +@dataclass(frozen=True) +class SelectedProjectFilesTestCase: + """Selected lint inputs named by resolved path, possibly through symbolic links.""" + + description: str + selected: str + expected_present: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py index e46fed4c20..8a44dae9ea 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/helpers.py @@ -112,3 +112,22 @@ def write_orders_lint_project(*, project_dir: Path, model_bytes: bytes) -> Path: encoding="utf-8", ) return model_path + + +def write_linked_orders_project(*, project_dir: Path) -> None: + """Write models reached both directly and through file and directory links.""" + + (project_dir / "models" / "orders").mkdir(parents=True) + (project_dir / "shared").mkdir() + (project_dir / "models" / "orders" / "orders.sql").write_text("SELECT 1 AS order_id\n") + (project_dir / "shared" / "customers.sql").write_text("SELECT 1 AS customer_id\n") + (project_dir / "models" / "orders" / "customers_link.sql").symlink_to( + project_dir / "shared" / "customers.sql" + ) + (project_dir / "models" / "shared_link").symlink_to(project_dir / "shared") + + +def files_resolving_to(*, files: dict[Path, str], selected: frozenset[Path]) -> dict[Path, str]: + """Return the files whose fully resolved path is one of the selected paths.""" + + return dict(filter(lambda item: item[0].resolve() in selected, files.items())) diff --git a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py index 693e961d64..c9f239e882 100644 --- a/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py +++ b/tests/unit/src/sqlbuild/lint/_helpers/test_project_files.py @@ -11,8 +11,15 @@ from sqlbuild.lint._helpers.project_files import collect_project_files from sqlbuild.lint.main.run_lint import run_lint from sqlbuild.lint.models import LintConfig, LintRunResult -from tests.unit.src.sqlbuild.lint._helpers._test_types import CollectProjectFilesTestCase -from tests.unit.src.sqlbuild.lint._helpers.helpers import write_orders_lint_project +from tests.unit.src.sqlbuild.lint._helpers._test_types import ( + CollectProjectFilesTestCase, + SelectedProjectFilesTestCase, +) +from tests.unit.src.sqlbuild.lint._helpers.helpers import ( + files_resolving_to, + write_linked_orders_project, + write_orders_lint_project, +) @pytest.mark.parametrize( @@ -54,5 +61,41 @@ def test_given_discovered_project_when_collecting_files_then_returns_exact_autho assert shared.files_checked == fresh.files_checked == 2 +@pytest.mark.parametrize( + "test_case", + [ + SelectedProjectFilesTestCase( + description="regular_file", + selected="models/orders/orders.sql", + expected_present=("models/orders/orders.sql",), + ), + SelectedProjectFilesTestCase( + description="file_reached_through_links", + selected="shared/customers.sql", + expected_present=("models/orders/customers_link.sql",), + ), + ], + ids=lambda case: case.description, +) +def test_given_selected_resolved_paths_when_collecting_files_then_matches_resolving_each_file( + test_case: SelectedProjectFilesTestCase, tmp_path: Path +) -> None: + write_linked_orders_project(project_dir=tmp_path) + selected: frozenset[Path] = frozenset({(tmp_path / test_case.selected).resolve()}) + every_file: dict[Path, str] = collect_project_files(project_dir=tmp_path) + + collected: dict[Path, str] = collect_project_files( + project_dir=tmp_path, selected_paths=selected + ) + from_sources: dict[Path, str] = collect_project_files( + project_dir=tmp_path, selected_paths=selected, source_files=every_file + ) + + relative: set[str] = {path.relative_to(tmp_path).as_posix() for path in collected} + assert relative >= set(test_case.expected_present) + assert from_sources == collected + assert collected == files_resolving_to(files=every_file, selected=selected) + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index b039cbc86d..a71bdf5482 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -111,6 +111,20 @@ ignored and compile runs in full. Storing a result normally takes a fraction of report is ready. When it has to hash a lot of data or the project has very many files, compile prints `Recording compile for reuse...` and a completion line to stderr before the report. +### Compiling after model-only edits + +When the only changes since the stored compile are edits to existing model files, compile runs in +full but reuses the stored render of every unchanged model, together with the diagnostics and +environment reads that rendering reported. Only the edited models are rendered again. Analysis, +contracts, rules, and artifact writes still run for the whole project, so downstream effects of an +edit, such as a changed column type, are reported exactly as in an uncached compile. The output is +byte-identical to `sqb compile --no-cache`. + +Every other change renders the whole project: adding, removing, or renaming a model, and editing +macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope +folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings +is always rendered again. Stored renders are kept for the two newest compiles in the same folder. + Known limits: - Python packages outside the project are checked by the stamps of the module files the compile @@ -130,7 +144,8 @@ Known limits: A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, `project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside -the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +the phase timings, plus `render_reuse_hits` and `render_reuse_misses` for model renders reused or +rendered again after model-only edits. In text mode, a replayed compile does not print the per-phase progress lines. Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` events are not emitted because no compile phase runs. From 2bfb7ef928a0e025c026cf467c568f98cac7565e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 20:23:51 +0100 Subject: [PATCH 3/4] perf(compile): store compressed renders only after a stored compile and release them once decoded Serialize renders at store time instead of on a background thread, which contended for the GIL with analysis. Skip render storage on cold compiles, compress render files with zlib, release stored render bytes after rendering, and announce slow render loading and recording. Carry forward stored module stamps whenever a render is reused, so outside modules imported only while rendering stay tracked. --- .../sqlbuild/references/docs/cli/compile.md | 15 +- .../_helpers/load_notice.py | 40 +++++ .../_helpers/render_state_file.py | 124 +++++++------- .../cli/compile_render_reuse/constants.py | 9 +- .../main/_render_reuse_session.py | 36 +++- .../main/_render_state_layer.py | 6 +- .../cli/compile_render_reuse/models.py | 5 +- .../cli/compile_reuse/_helpers/attempt.py | 2 + .../_helpers/runtime_identity.py | 13 ++ .../cli/compile_reuse/_helpers/store.py | 39 ++++- src/sqlbuild/cli/compile_reuse/constants.py | 3 +- src/sqlbuild/cli/compile_reuse/models.py | 2 + .../compile/classes/render_reuse_session.py | 87 +++++++--- .../compile/main/_build_compile_inputs.py | 2 +- src/sqlbuild/compiler/compile/models.py | 6 +- .../cli/commands/main/compile/_test_types.py | 40 +++++ .../cli/commands/main/compile/helpers.py | 93 +++++++++- .../compile/test_compile_incremental_edits.py | 161 +++++++++++++++++- .../_helpers/_test_types.py | 3 +- .../compile_render_reuse/_helpers/helpers.py | 33 +++- .../_helpers/test_render_state_file.py | 17 +- .../compiler/compile/classes/_test_types.py | 10 ++ .../compiler/compile/classes/helpers.py | 43 ++++- .../classes/test_render_reuse_session.py | 40 +++++ website/src/content/docs/docs/cli/compile.mdx | 15 +- 25 files changed, 701 insertions(+), 143 deletions(-) create mode 100644 src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 5e61478d7e..bc37009e7e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -117,8 +117,9 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data or the project has very many files, compile -prints `Recording compile for reuse...` and a completion line to stderr before the report. +report is ready. When it has to hash a lot of data, the project has very many files, or it records +thousands of new model renders, compile prints `Recording compile for reuse...` and a completion +line to stderr before the report. ### Compiling after model-only edits @@ -132,7 +133,15 @@ byte-identical to `sqb compile --no-cache`. Every other change renders the whole project: adding, removing, or renaming a model, and editing macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings -is always rendered again. Stored renders are kept for the two newest compiles in the same folder. +is always rendered again. + +Renders are stored only when a compile result for the same project and target is already stored, +so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. +The next full compile stores them, and edits after that reuse them. Stored renders are compressed +and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints +`Loading stored renders...` and a completion line to stderr before it starts. While renders are +reused, the Python modules recorded by the previous compile stay tracked even if this compile did +not import them, so editing such a module runs the next compile in full. Known limits: diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py new file mode 100644 index 0000000000..5e330aaa68 --- /dev/null +++ b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py @@ -0,0 +1,40 @@ +"""Lifecycle lines for loading a large render store before the compile status starts.""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path + +from sqlbuild.cli.compile_render_reuse.constants import ( + RENDER_LOAD_DONE_MESSAGE, + RENDER_LOAD_FAILED_MESSAGE, + RENDER_LOAD_NOTICE_BYTES, + RENDER_LOAD_START_MESSAGE, +) +from sqlbuild.cli.compile_reuse.constants import BYTES_PER_MEBIBYTE, REUSE_RENDER_STATE_SUFFIX + + +def start_load_notice(*, entry_path: Path) -> float | None: + """Announce loading on stderr when the slot's stored render files are large.""" + + stored_bytes: int = sum( + path.stat().st_size + for path in entry_path.parent.glob(f"{entry_path.stem}-*{REUSE_RENDER_STATE_SUFFIX}") + ) + if stored_bytes < RENDER_LOAD_NOTICE_BYTES: + return None + print( + RENDER_LOAD_START_MESSAGE.format(mebibytes=stored_bytes / BYTES_PER_MEBIBYTE), + file=sys.stderr, + ) + return time.monotonic() + + +def finish_load_notice(*, started: float | None, loaded: bool) -> None: + """Report whether an announced load produced usable renders.""" + + if started is None: + return + message: str = RENDER_LOAD_DONE_MESSAGE if loaded else RENDER_LOAD_FAILED_MESSAGE + print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py index 44b8001b00..f68ef0b514 100644 --- a/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py +++ b/src/sqlbuild/cli/compile_render_reuse/_helpers/render_state_file.py @@ -9,6 +9,7 @@ from typing import cast from sqlbuild.cli.compile_render_reuse.constants import ( + RENDER_COMPRESSION_LEVEL, RENDER_INDEX_FIELD_COUNT, RENDER_OVERLAY_MAX_SHARE, RENDER_STATE_MAGIC, @@ -47,68 +48,47 @@ def read_render_state(*, path: Path, changed_paths: frozenset[str]) -> StoredRen if base is None or base.base_file is not None: return None return _stored_state(base_file=newest.base_file, base=base, overlay=newest) - except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError): + except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError, zlib.error): return None def render_state_layer( - *, state: RenderReuseState, stored: StoredRenderState | None -) -> RenderStateLayer: - """Extend the stored base with a small overlay of changed renders, or write a new base.""" - - if stored is not None and state.model_paths == stored.state.model_paths: - overlay_models: frozenset[str] | None = _overlay_keys( - current=state.model_payloads, - previous=stored.state.model_payloads, - overlay=stored.layers.overlay_models, + *, state: RenderReuseState, stored: StoredRenderLayers | None +) -> RenderStateLayer | None: + """Return an overlay on the stored base, a new base, or None when released bytes are needed.""" + + if ( + stored is not None + and state.model_paths == stored.model_paths + and stored.model_keys <= state.model_payloads.keys() + and stored.group_keys <= state.group_payloads.keys() + ): + models: list[tuple[str, memoryview]] = _held( + (path, state.model_payloads.get(path)) for path in state.model_paths ) - overlay_groups: frozenset[str] | None = _overlay_keys( - current=state.group_payloads, - previous=stored.state.group_payloads, - overlay=stored.layers.overlay_groups, - ) - if overlay_models is not None and overlay_groups is not None: - models: list[tuple[str, memoryview]] = [ - (path, state.model_payloads[path]) - for path in state.model_paths - if path in overlay_models - ] - groups: list[tuple[str, memoryview]] = sorted( - (name, state.group_payloads[name]) for name in overlay_groups + groups: list[tuple[str, memoryview]] = _held(sorted(state.group_payloads.items())) + if ( + sum(len(item) for _, item in (*models, *groups)) + <= stored.base_bytes * RENDER_OVERLAY_MAX_SHARE + ): + return _layer( + model_paths=state.model_paths, + base_file=stored.base_file, + models=models, + groups=groups, ) - if ( - sum(len(item) for _, item in (*models, *groups)) - <= stored.layers.base_bytes * RENDER_OVERLAY_MAX_SHARE - ): - return _layer( - model_paths=state.model_paths, - base_file=stored.layers.base_file, - models=models, - groups=groups, - ) + if None in state.model_payloads.values() or None in state.group_payloads.values(): + return None return _layer( model_paths=state.model_paths, base_file=None, - models=[ - (path, state.model_payloads[path]) - for path in state.model_paths - if path in state.model_payloads - ], - groups=sorted(state.group_payloads.items()), + models=_held((path, state.model_payloads.get(path)) for path in state.model_paths), + groups=_held(sorted(state.group_payloads.items())), ) -def _overlay_keys( - *, - current: dict[str, memoryview], - previous: dict[str, memoryview], - overlay: frozenset[str], -) -> frozenset[str] | None: - """Return keys the overlay must hold, or None when a stored render is no longer valid.""" - - if not previous.keys() <= current.keys(): - return None - return overlay | {key for key, payload in current.items() if previous.get(key) is not payload} +def _held(items: Iterable[tuple[str, memoryview | None]]) -> list[tuple[str, memoryview]]: + return [(key, payload) for key, payload in items if payload is not None] def _layer( @@ -127,19 +107,21 @@ def _layer( }, separators=(",", ":"), ).encode("utf-8", "surrogateescape") - payloads: tuple[memoryview, ...] = tuple(item for _, item in (*models, *groups)) + compressor: zlib._Compress = zlib.compressobj(RENDER_COMPRESSION_LEVEL) + compressed: list[bytes] = [compressor.compress(item) for _, item in (*models, *groups)] + compressed.append(compressor.flush()) return RenderStateLayer( chunks=( RENDER_STATE_MAGIC, framed_stored_section(data=index), - _section_header(payloads=payloads), - *payloads, + _section_header(payloads=compressed), + *compressed, ), base_file=base_file, ) -def _section_header(*, payloads: Iterable[memoryview]) -> bytes: +def _section_header(*, payloads: Iterable[bytes]) -> bytes: length: int = 0 checksum: int = 0 for payload in payloads: @@ -161,22 +143,30 @@ def _read_layer(*, path: Path, changed_paths: frozenset[str]) -> StoredRenderLay model_paths: tuple[str, ...] = tuple(_string(item) for item in _list(fields["model_paths"])) if not changed_paths.issubset(model_paths): return None - payload: bytes = read_stored_section(handle=handle) - if handle.read(1) or len(payload) > REUSE_MAX_ENTRY_BYTES: + compressed: bytes = read_stored_section(handle=handle) + if handle.read(1) or len(compressed) > REUSE_MAX_ENTRY_BYTES: return None base: object = fields["base"] + model_rows: list[tuple[str, int]] = _rows(fields["models"]) + group_rows: list[tuple[str, int]] = _rows(fields["groups"]) + size: int = sum(length for _, length in (*model_rows, *group_rows)) + if size > REUSE_MAX_ENTRY_BYTES: + return None + decompressor: zlib._Decompress = zlib.decompressobj() + payload: bytes = decompressor.decompress(compressed, size + 1) + del compressed + if len(payload) != size or not decompressor.eof or decompressor.unconsumed_tail: + return None view: memoryview = memoryview(payload) offset: int = 0 models: dict[str, memoryview] = {} - for name, length in _rows(fields["models"]): + for name, length in model_rows: models[name] = view[offset : offset + length] offset += length groups: dict[str, memoryview] = {} - for name, length in _rows(fields["groups"]): + for name, length in group_rows: groups[name] = view[offset : offset + length] offset += length - if offset != len(view): - return None return StoredRenderLayer( model_paths=model_paths, base_file=None if base is None else _string(base), @@ -190,13 +180,17 @@ def _stored_state( *, base_file: str, base: StoredRenderLayer, overlay: StoredRenderLayer | None ) -> StoredRenderState: newest: StoredRenderLayer = base if overlay is None else overlay + state: RenderReuseState = RenderReuseState( + model_paths=newest.model_paths, + model_payloads=dict(base.models) if overlay is None else {**base.models, **overlay.models}, + group_payloads=dict(base.groups) if overlay is None else {**base.groups, **overlay.groups}, + ) return StoredRenderState( - state=RenderReuseState( - model_paths=newest.model_paths, - model_payloads=base.models if overlay is None else {**base.models, **overlay.models}, - group_payloads=base.groups if overlay is None else {**base.groups, **overlay.groups}, - ), + state=state, layers=StoredRenderLayers( + model_paths=newest.model_paths, + model_keys=frozenset(state.model_payloads), + group_keys=frozenset(state.group_payloads), base_file=base_file, base_bytes=base.size, overlay_models=frozenset() if overlay is None else frozenset(overlay.models), diff --git a/src/sqlbuild/cli/compile_render_reuse/constants.py b/src/sqlbuild/cli/compile_render_reuse/constants.py index c6625ead49..f77740df1d 100644 --- a/src/sqlbuild/cli/compile_render_reuse/constants.py +++ b/src/sqlbuild/cli/compile_render_reuse/constants.py @@ -2,6 +2,13 @@ from __future__ import annotations -RENDER_STATE_MAGIC: bytes = b"SQBRENDER1\n" +RENDER_STATE_MAGIC: bytes = b"SQBRENDER2\n" +RENDER_COMPRESSION_LEVEL: int = 1 RENDER_OVERLAY_MAX_SHARE: float = 0.25 RENDER_INDEX_FIELD_COUNT: int = 2 +RENDER_LOAD_NOTICE_BYTES: int = 8 * 1024 * 1024 +RENDER_LOAD_START_MESSAGE: str = "Loading stored renders ({mebibytes:.0f} MiB)..." +RENDER_LOAD_DONE_MESSAGE: str = "Loaded stored renders ({seconds:.1f} s)" +RENDER_LOAD_FAILED_MESSAGE: str = ( + "Stored renders are unusable; rendering every model again ({seconds:.1f} s)" +) diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py index 7a97880456..399a33b9e5 100644 --- a/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py +++ b/src/sqlbuild/cli/compile_render_reuse/main/_render_reuse_session.py @@ -2,6 +2,12 @@ from __future__ import annotations +from pathlib import Path + +from sqlbuild.cli.compile_render_reuse._helpers.load_notice import ( + finish_load_notice, + start_load_notice, +) from sqlbuild.cli.compile_render_reuse._helpers.render_state_file import read_render_state from sqlbuild.cli.compile_render_reuse.models import CompileRenderReuse, StoredRenderState from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt @@ -10,18 +16,38 @@ def render_reuse_session(*, attempt: CompileReuseAttempt) -> CompileRenderReuse | None: - """Start recording renders, reusing the stored ones the changed paths cannot affect.""" + """Record renders once a compile is stored, reusing those the changed paths cannot affect.""" - if attempt.outcome is not CompileReuseOutcome.MISS or attempt.entry_path is None: + if ( + attempt.outcome is not CompileReuseOutcome.MISS + or attempt.entry_path is None + or not attempt.prior_entry + ): return None stored: StoredRenderState | None = ( None if attempt.changed_paths is None or attempt.render_state_path is None - else read_render_state(path=attempt.render_state_path, changed_paths=attempt.changed_paths) + else _loaded_render_state( + entry_path=attempt.entry_path, + path=attempt.render_state_path, + changed_paths=attempt.changed_paths, + ) ) return CompileRenderReuse( session=CompileRenderReuseSession( - prior=None if stored is None else stored.state, changed_paths=attempt.changed_paths + prior=None if stored is None else stored.state, + changed_paths=attempt.changed_paths, + retained_models=frozenset() if stored is None else stored.layers.overlay_models, + retained_groups=frozenset() if stored is None else stored.layers.overlay_groups, ), - stored=stored, + stored=None if stored is None else stored.layers, ) + + +def _loaded_render_state( + *, entry_path: Path, path: Path, changed_paths: frozenset[str] +) -> StoredRenderState | None: + started: float | None = start_load_notice(entry_path=entry_path) + stored: StoredRenderState | None = read_render_state(path=path, changed_paths=changed_paths) + _ = finish_load_notice(started=started, loaded=stored is not None) + return stored diff --git a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py index 6976765a3b..83467f8b70 100644 --- a/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py +++ b/src/sqlbuild/cli/compile_render_reuse/main/_render_state_layer.py @@ -16,4 +16,8 @@ def stored_render_layer(*, render_reuse: CompileRenderReuse | None) -> RenderSta ) if render_reuse is None or state is None: return None - return render_state_layer(state=state, stored=render_reuse.stored) + layer: RenderStateLayer | None = render_state_layer(state=state, stored=render_reuse.stored) + if layer is not None: + return layer + complete: RenderReuseState | None = render_reuse.session.stored_state(complete=True) + return None if complete is None else render_state_layer(state=complete, stored=None) diff --git a/src/sqlbuild/cli/compile_render_reuse/models.py b/src/sqlbuild/cli/compile_render_reuse/models.py index 8080c25992..a7595e63c1 100644 --- a/src/sqlbuild/cli/compile_render_reuse/models.py +++ b/src/sqlbuild/cli/compile_render_reuse/models.py @@ -23,6 +23,9 @@ class StoredRenderLayer: class StoredRenderLayers: """How a stored compile's renders are split between a base file and one overlay.""" + model_paths: tuple[str, ...] + model_keys: frozenset[str] + group_keys: frozenset[str] base_file: str base_bytes: int overlay_models: frozenset[str] @@ -42,4 +45,4 @@ class CompileRenderReuse: """One full compile's render reuse session with the stored renders it started from.""" session: CompileRenderReuseSession - stored: StoredRenderState | None + stored: StoredRenderLayers | None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index 2e98b4a6f5..e963bbc831 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -104,6 +104,8 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: ) attempt = replace( attempt, + prior_entry=True, + prior_modules=header.inputs.modules, restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), render_state_path=( None diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index bad781b93b..b3eddc3024 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -69,6 +69,19 @@ def loaded_module_stamps(*, covered_paths: frozenset[str]) -> tuple[tuple[str, i return tuple(sorted(stamps.values())) +def with_carried_module_stamps( + *, + current: tuple[tuple[str, int, int], ...], + carried: tuple[tuple[str, int, int], ...], + covered_paths: frozenset[str], +) -> tuple[tuple[str, int, int], ...]: + """Add stored stamps of modules a reused render imported but this process did not load.""" + + stamps: dict[str, tuple[str, int, int]] = {stamp[0]: stamp for stamp in carried} + stamps.update((stamp[0], stamp) for stamp in current) + return tuple(sorted(stamp for path, stamp in stamps.items() if path not in covered_paths)) + + def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool: """Return whether every recorded module file still has the same stat identity.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py index 6b0ce71a49..7ead497bbc 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/store.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -23,6 +23,7 @@ loaded_module_stamps, settings_inputs_digest, tracked_environment_names, + with_carried_module_stamps, ) from sqlbuild.cli.compile_reuse._helpers.target_files import verified_target_files from sqlbuild.cli.compile_reuse._helpers.timings import compile_timings_span @@ -34,6 +35,7 @@ REUSE_STORE_DONE_MESSAGE, REUSE_STORE_NOTICE_BYTES, REUSE_STORE_NOTICE_PATHS, + REUSE_STORE_NOTICE_RENDERS, REUSE_STORE_SKIPPED_MESSAGE, REUSE_STORE_START_MESSAGE, ) @@ -115,7 +117,10 @@ def _write_compile_entry( remove_entry(path=entry_path) return project_dir: str = str(attempt.project_dir) - notice_started: float | None = _start_notice(attempt=attempt) + notice_started: float | None = _start_notice( + attempt=attempt, + pending_renders=0 if render_reuse is None else render_reuse.session.pending_renders(), + ) digests: dict[str, str] = with_missing_digests( project_dir=project_dir, snapshot=attempt.snapshot, @@ -149,10 +154,8 @@ def _write_compile_entry( environment_names=environment_names, environment_digest=environment_digest(names=environment_names), search_path=attempt.search_path, - modules=loaded_module_stamps( - covered_paths=frozenset( - os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot - ) + modules=_module_stamps( + attempt=attempt, project_dir=project_dir, render_reuse=render_reuse ), project_files=stored_project_files( snapshot=attempt.snapshot, digests=digests, snapshot_ns=attempt.snapshot_ns @@ -176,17 +179,35 @@ def _write_compile_entry( _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) -def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: +def _module_stamps( + *, attempt: CompileReuseAttempt, project_dir: str, render_reuse: CompileRenderReuse | None +) -> tuple[tuple[str, int, int], ...]: + covered_paths: frozenset[str] = frozenset( + os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot + ) + current: tuple[tuple[str, int, int], ...] = loaded_module_stamps(covered_paths=covered_paths) + if render_reuse is None or not render_reuse.session.reused_any(): + return current + return with_carried_module_stamps( + current=current, carried=attempt.prior_modules, covered_paths=covered_paths + ) + + +def _start_notice(*, attempt: CompileReuseAttempt, pending_renders: int) -> float | None: pending_bytes: int = pending_digest_bytes( snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped ) - if pending_bytes < REUSE_STORE_NOTICE_BYTES and len(attempt.snapshot) < ( - REUSE_STORE_NOTICE_PATHS + if ( + pending_bytes < REUSE_STORE_NOTICE_BYTES + and len(attempt.snapshot) < REUSE_STORE_NOTICE_PATHS + and pending_renders < REUSE_STORE_NOTICE_RENDERS ): return None print( REUSE_STORE_START_MESSAGE.format( - paths=len(attempt.snapshot), mebibytes=pending_bytes / BYTES_PER_MEBIBYTE + paths=len(attempt.snapshot), + mebibytes=pending_bytes / BYTES_PER_MEBIBYTE, + renders=pending_renders, ), file=sys.stderr, ) diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index c6ebe8a0a3..95f701e062 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -81,9 +81,10 @@ MILLISECONDS_PER_SECOND: int = 1000 REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 REUSE_STORE_NOTICE_PATHS: int = 100_000 +REUSE_STORE_NOTICE_RENDERS: int = 4_000 BYTES_PER_MEBIBYTE: int = 1024 * 1024 REUSE_STORE_START_MESSAGE: str = ( - "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." + "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash, {renders} renders)..." ) REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" REUSE_STORE_SKIPPED_MESSAGE: str = ( diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py index b4763717d1..12d6acfc49 100644 --- a/src/sqlbuild/cli/compile_reuse/models.py +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -131,6 +131,8 @@ class CompileReuseAttempt: restamped: frozenset[str] = frozenset() changed_paths: frozenset[str] | None = None render_state_path: Path | None = None + prior_entry: bool = False + prior_modules: tuple[tuple[str, int, int], ...] = () @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py index 701464d137..f011a76c43 100644 --- a/src/sqlbuild/compiler/compile/classes/render_reuse_session.py +++ b/src/sqlbuild/compiler/compile/classes/render_reuse_session.py @@ -30,7 +30,6 @@ from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload from sqlbuild.compiler.profiling.main._metric import record_compile_metric -_SERIALIZER_THREAD_NAME: str = "sqlbuild-render-reuse-serializer" _LOAD_ERRORS: tuple[type[Exception], ...] = ( FactCachePayloadError, pickle.UnpicklingError, @@ -41,6 +40,9 @@ KeyError, TypeError, ValueError, + RecursionError, + MemoryError, + OverflowError, ) @@ -48,7 +50,12 @@ class CompileRenderReuseSession: """Serve stored renders of inputs a change set cannot affect, and record every render.""" def __init__( - self, *, prior: RenderReuseState | None, changed_paths: frozenset[str] | None + self, + *, + prior: RenderReuseState | None, + changed_paths: frozenset[str] | None, + retained_models: frozenset[str] = frozenset(), + retained_groups: frozenset[str] = frozenset(), ) -> None: self._lock: threading.Lock = threading.Lock() self._claimed: bool = False @@ -56,13 +63,16 @@ def __init__( self._changed_paths: frozenset[str] = changed_paths or frozenset() self._reusable: bool = False self._model_paths: tuple[str, ...] | None = None - self._model_records: dict[str, memoryview] = {} - self._group_records: dict[str, memoryview] = {} + self._retained_models: frozenset[str] = retained_models + self._retained_groups: frozenset[str] = retained_groups + self._model_records: dict[str, memoryview | None] = {} + self._group_records: dict[str, memoryview | None] = {} + self._decoded_models: dict[str, StoredRender] = {} + self._decoded_groups: dict[str, StoredRender] = {} self._pending_models: list[tuple[str, StoredRender]] = [] self._pending_groups: list[tuple[str, StoredRender]] = [] self._serialized_models: dict[str, memoryview] = {} self._serialized_groups: dict[str, memoryview] = {} - self._serializer: threading.Thread | None = None self._reused_paths: set[str] = set() def claim(self) -> bool: @@ -102,12 +112,13 @@ def reused_model(self, *, model_file: DiscoveredSqlModelFile) -> CompileModelInp if not self.has_reusable_model(model_file=model_file) or self._prior is None: return None path: str = str(model_file.relative_path) - payload: memoryview = self._prior.model_payloads[path] - stored: StoredRender | None = _loaded_render(payload) + payload: memoryview | None = self._prior.model_payloads[path] + stored: StoredRender | None = None if payload is None else _loaded_render(payload) if stored is None or not isinstance(stored.value, CompileModelInput): return None _replay(stored=stored) self._model_records[path] = payload + self._decoded_models[path] = stored self._reused_paths.add(path) record_compile_metric(metric="render_reuse_hits", value=1) return replace(stored.value, model_file=model_file) @@ -160,6 +171,7 @@ def group[T](self, *, name: str, render: Callable[[], T], reusable: bool = True) if stored is not None and payload is not None: _replay(stored=stored) self._group_records[name] = payload + self._decoded_groups[name] = stored return cast(T, stored.value) with ( COMPILE_INPUT_READS.recording() as reads, @@ -171,24 +183,42 @@ def group[T](self, *, name: str, render: Callable[[], T], reusable: bool = True) self._pending_groups.append((name, rendered)) return value - def renders_complete(self) -> None: - """Serialize this compile's new renders in the background while later phases run.""" + def pending_renders(self) -> int: + """Return how many new renders storing this compile still has to serialize.""" - if self._serializer is None and (self._pending_models or self._pending_groups): - self._serializer = threading.Thread( - target=self._serialize_pending, name=_SERIALIZER_THREAD_NAME, daemon=True - ) - self._serializer.start() + return len(self._pending_models) + len(self._pending_groups) + + def reused_any(self) -> bool: + """Return whether any render was reused, so imports it made were skipped this run.""" + + return bool(self._decoded_models or self._decoded_groups) - def stored_state(self) -> RenderReuseState | None: - """Serialize this compile's renders, keeping stored bytes of reused ones as they were.""" + def release_stored(self) -> None: + """Drop stored base bytes once every render is decoded, recording reused renders as None.""" + + self._prior = None + self._reusable = False + for records, retained in ( + (self._model_records, self._retained_models), + (self._group_records, self._retained_groups), + ): + for key in records.keys() - retained: + records[key] = None + + def stored_state(self, *, complete: bool = False) -> RenderReuseState | None: + """Serialize this compile's renders; with complete, re-serialize released reused ones.""" if self._model_paths is None: return None - if self._serializer is None: - self._serialize_pending() - else: - self._serializer.join() + self.release_stored() + self._serialize_pending() + if complete: + self._model_records = _reserialized( + records=self._model_records, decoded=self._decoded_models + ) + self._group_records = _reserialized( + records=self._group_records, decoded=self._decoded_groups + ) return RenderReuseState( model_paths=self._model_paths, model_payloads={**self._model_records, **self._serialized_models}, @@ -204,6 +234,23 @@ def _serialize_pending(self) -> None: payload: bytes | None = _dumped(stored) if payload is not None: serialized[key] = memoryview(payload) + pending.clear() + + +def _reserialized( + *, records: dict[str, memoryview | None], decoded: dict[str, StoredRender] +) -> dict[str, memoryview | None]: + complete: dict[str, memoryview | None] = {} + key: str + payload: memoryview | None + for key, payload in records.items(): + if payload is None: + dumped: bytes | None = _dumped(decoded[key]) + if dumped is None: + continue + payload = memoryview(dumped) + complete[key] = payload + return complete def _stored_render( diff --git a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py index 55d1fd4393..af15698a4f 100644 --- a/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py +++ b/src/sqlbuild/compiler/compile/main/_build_compile_inputs.py @@ -186,7 +186,7 @@ def build_compile_inputs( render_reuse=render_reuse, ) if render_reuse is not None: - render_reuse.renders_complete() + render_reuse.release_stored() diagnostics: tuple[CompilerDiagnostic, ...] = ( *audit_diagnostics, *model_build.diagnostics, diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 30ba05a4ca..e07afcffe8 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -1592,8 +1592,8 @@ class StoredRender: @dataclass(frozen=True) class RenderReuseState: - """Serialized renders of one compile: each model by project path and project-wide groups.""" + """Serialized renders of one compile by model path and group; None marks released bytes.""" model_paths: tuple[str, ...] - model_payloads: dict[str, memoryview] - group_payloads: dict[str, memoryview] + model_payloads: dict[str, memoryview | None] + group_payloads: dict[str, memoryview | None] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 7083ea496a..a4f6dea82a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -719,3 +719,43 @@ class BrokenChangeDetectionTestCase: description: str edit: Callable[[Path], None] expected_matches_uncached: bool + + +@dataclass(frozen=True) +class RenderStoreNoticeTestCase: + """A render-count notice threshold and whether storing renders announces itself.""" + + description: str + notice_renders: int + expected_notice: bool + + +@dataclass(frozen=True) +class RenderSavePolicyTestCase: + """Compiles after a cold compile, and the render files and reuse each one must leave.""" + + description: str + edits: tuple[Callable[[Path], None], ...] + expected_cold_render_files: int + expected_render_files: tuple[int, ...] + expected_reused: tuple[bool, ...] + + +@dataclass(frozen=True) +class ExternalModuleEditTestCase: + """An outside module imported only while rendering, rewritten after its render is reused.""" + + description: str + initial_value: str + edited_value: str + expected_matches_uncached: bool + expected_compiled_value: str + + +@dataclass(frozen=True) +class RenderLoadNoticeTestCase: + """A stored-render size threshold and whether loading the store announces itself.""" + + description: str + notice_bytes: int + expected_notice: bool diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 744742c9a0..a5709dcc3f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -26,6 +26,7 @@ import pytest import sqlbuild.cli.commands.main.project._compile as compile_command_module +import sqlbuild.cli.compile_render_reuse._helpers.load_notice as render_load_notice import sqlbuild.cli.compile_reuse._helpers.attempt as reuse_attempt import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files import sqlbuild.cli.compile_reuse._helpers.store as reuse_store @@ -39,7 +40,11 @@ read_entry_stdout, write_entry, ) -from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.cli.compile_reuse.constants import ( + REUSE_DISABLE_ENV_VAR, + REUSE_ENTRY_DIRECTORY_PARTS, + REUSE_RENDER_STATE_SUFFIX, +) from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, @@ -3739,3 +3744,89 @@ def disable_change_detection(monkeypatch: pytest.MonkeyPatch) -> None: """Make compile reuse believe no project file changed since the stored compile.""" monkeypatch.setattr(reuse_attempt, "changed_project_paths", lambda **_kwargs: (frozenset(), {})) + + +def set_store_notice_renders(monkeypatch: pytest.MonkeyPatch, renders: int) -> None: + """Announce storing a compile once it has at least this many new renders to record.""" + + monkeypatch.setattr(reuse_store, "REUSE_STORE_NOTICE_RENDERS", renders) + + +def prime_render_store(project_dir: Path) -> CompileReuseRun: + """After a cold compile, compile one leaf edit so the stored compile also holds renders.""" + + fact_comment(project_dir) + return run_reuse_compile(project_dir=project_dir) + + +def prime_render_store_in_process(project_dir: Path) -> int: + """In this process: compile cold, then compile one leaf edit so renders are stored.""" + + cold: int = compile_in_process(project_dir=project_dir) + fact_comment(project_dir) + return max(cold, compile_in_process(project_dir=project_dir)) + + +def render_store_files(project_dir: Path) -> int: + """Return how many stored render files the project's compile reuse folder holds.""" + + return len( + list( + project_dir.joinpath(*REUSE_ENTRY_DIRECTORY_PARTS).glob(f"*{REUSE_RENDER_STATE_SUFFIX}") + ) + ) + + +def edit_and_compile(*, project_dir: Path, edit: Callable[[Path], None]) -> CompileReuseRun: + """Apply one edit, compile with reuse enabled, and return the run.""" + + edit(project_dir) + return run_reuse_compile(project_dir=project_dir) + + +EXTERNAL_FLAVOR_MODULE: str = "extflavor" + + +def write_external_flavor(extlib: Path, value: str) -> None: + """Write, or rewrite in place, an outside module that a macro imports only while rendering.""" + + extlib.mkdir(parents=True, exist_ok=True) + (extlib / f"{EXTERNAL_FLAVOR_MODULE}.py").write_text(f"VALUE = {value!r}\n", encoding="utf-8") + + +def add_external_flavor_macro(*, project_dir: Path, extlib: Path, value: str) -> dict[str, str]: + """Make the fact orders render import an outside module; return the compile environment.""" + + write_external_flavor(extlib, value) + write_project_file( + project_dir, + "models/marts/_sqlbuild/_macros/flavor.py", + '"""Flavor macros backed by an outside module."""\n\n\n' + "def flavor() -> str:\n" + ' """Return the configured flavor literal."""\n' + f" import {EXTERNAL_FLAVOR_MODULE}\n\n" + f" return {EXTERNAL_FLAVOR_MODULE}.VALUE\n", + ) + replace_project_text( + project_dir, FACT_ORDERS_MODEL, " o.quantity,\n", " o.quantity,\n @flavor() AS flavor,\n" + ) + return {"PYTHONPATH": str(extlib), "PYTHONDONTWRITEBYTECODE": "1"} + + +def compiled_text(*, run: CompileReuseRun, suffix: str) -> str: + """Return the compiled artifact whose path ends with the suffix.""" + + path: str = next(filter(lambda path: path.endswith(suffix), sorted(run.compiled))) + return run.compiled[path].decode("utf-8") + + +def set_render_load_notice_bytes(monkeypatch: pytest.MonkeyPatch, stored_bytes: int) -> None: + """Announce loading stored renders once their files hold at least this many bytes.""" + + monkeypatch.setattr(render_load_notice, "RENDER_LOAD_NOTICE_BYTES", stored_bytes) + + +def json_report_keys(stdout: str) -> tuple[str, ...]: + """Parse a JSON compile report and return its top-level keys.""" + + return tuple(cast(dict[str, object], json.loads(stdout))) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index 5bd3cab376..fc0550161c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -8,16 +8,24 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( BrokenChangeDetectionTestCase, + ExternalModuleEditTestCase, IncrementalEditSequenceTestCase, RandomEditChainTestCase, + RenderLoadNoticeTestCase, + RenderSavePolicyTestCase, + RenderStoreNoticeTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( CompileReuseRun, IncrementalEditComparison, RandomEditChain, + add_external_flavor_macro, compare_incremental_compile, compile_in_process, + compile_in_process_output, + compiled_text, disable_change_detection, + edit_and_compile, enable_compile_reuse, fact_audit_added, fact_comment, @@ -26,16 +34,23 @@ full_edit_step, in_process_reuse_run, is_model_only_edit, + json_report_keys, model_edit_step, move_project_file, + prime_render_store, + prime_render_store_in_process, random_edit_plan, + render_store_files, replace_project_text, run_reuse_compile, + set_render_load_notice_bytes, + set_store_notice_renders, staging_comment, staging_contract_change, staging_header_change, staging_new_column, staging_type_change, + write_external_flavor, write_generated_edit_models, write_project_file, ) @@ -201,7 +216,8 @@ def test_given_edit_sequence_when_compiling_incrementally_then_each_step_matches ) -> None: project_dir: Path = compile_reuse_project cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - assert cold.returncode == 0, cold.stderr + primed: CompileReuseRun = prime_render_store(project_dir) + assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr for step in test_case.steps: step.edit(project_dir) @@ -232,7 +248,8 @@ def test_given_random_edit_chain_when_compiling_incrementally_then_each_step_mat project_dir=project_dir, model_count=test_case.model_count, seed=test_case.seed ) cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) - assert cold.returncode == 0, cold.stderr + primed: CompileReuseRun = prime_render_store(project_dir) + assert (cold.returncode, primed.returncode) == (0, 0), primed.stderr chain: RandomEditChain = RandomEditChain(project_dir=project_dir, seed=test_case.seed) for step, kind in enumerate( @@ -269,7 +286,7 @@ def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_re ) -> None: project_dir: Path = compile_reuse_project enable_compile_reuse(monkeypatch) - assert compile_in_process(project_dir=project_dir) == 0 + assert prime_render_store_in_process(project_dir) == 0 test_case.edit(project_dir) disable_change_detection(monkeypatch) @@ -281,3 +298,141 @@ def test_given_broken_change_detection_when_compiling_an_edit_then_the_oracle_re assert comparison.reused_renders > 0 assert comparison.matches is test_case.expected_matches_uncached + + +@pytest.mark.parametrize( + "test_case", + [ + RenderStoreNoticeTestCase( + description="few_renders_store_quietly", notice_renders=100_000, expected_notice=False + ), + RenderStoreNoticeTestCase( + description="many_renders_announce_recording", notice_renders=1, expected_notice=True + ), + ], + ids=lambda case: case.description, +) +def test_given_new_renders_when_storing_after_an_edit_then_slow_recording_is_announced( + compile_reuse_project: Path, + test_case: RenderStoreNoticeTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + enable_compile_reuse(monkeypatch) + set_store_notice_renders(monkeypatch, test_case.notice_renders) + assert compile_in_process(project_dir=compile_reuse_project) == 0 + fact_comment(compile_reuse_project) + + code, _, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) + + assert code == 0 + assert ("renders)..." in err) is test_case.expected_notice + assert ("Recorded compile for reuse" in err) is test_case.expected_notice + + +@pytest.mark.parametrize( + "test_case", + [ + RenderSavePolicyTestCase( + description="cold_then_two_model_edits", + edits=(fact_comment, staging_comment), + expected_cold_render_files=0, + expected_render_files=(1, 2), + expected_reused=(False, True), + ), + ], + ids=lambda case: case.description, +) +def test_given_cold_compile_when_editing_then_renders_are_stored_only_once_a_compile_exists( + compile_reuse_project: Path, test_case: RenderSavePolicyTestCase +) -> None: + cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project) + cold_render_files: int = render_store_files(compile_reuse_project) + render_files: list[int] = [] + reused: list[bool] = [] + + for edit in test_case.edits: + run: CompileReuseRun = edit_and_compile(project_dir=compile_reuse_project, edit=edit) + assert run.returncode == 0, run.stderr + render_files.append(render_store_files(compile_reuse_project)) + reused.append(run.timings.get("render_reuse_hits", 0) > 0) + + assert cold.returncode == 0, cold.stderr + assert cold_render_files == test_case.expected_cold_render_files + assert tuple(render_files) == test_case.expected_render_files + assert tuple(reused) == test_case.expected_reused + + +@pytest.mark.parametrize( + "test_case", + [ + ExternalModuleEditTestCase( + description="module_rewritten_in_place_after_reuse", + initial_value="'vanilla'", + edited_value="'choco'", + expected_matches_uncached=True, + expected_compiled_value="'choco' AS flavor", + ), + ], + ids=lambda case: case.description, +) +def test_given_module_imported_only_while_rendering_when_it_changes_after_reuse_then_output_is_fresh( + compile_reuse_project: Path, tmp_path: Path, test_case: ExternalModuleEditTestCase +) -> None: + extlib: Path = tmp_path / "extlib" + env: dict[str, str] = add_external_flavor_macro( + project_dir=compile_reuse_project, extlib=extlib, value=test_case.initial_value + ) + cold: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + staging_comment(compile_reuse_project) + recorded: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + staging_comment(compile_reuse_project) + reused: CompileReuseRun = run_reuse_compile(project_dir=compile_reuse_project, env=env) + + write_external_flavor(extlib, test_case.edited_value) + comparison: IncrementalEditComparison = IncrementalEditComparison( + incremental=run_reuse_compile(project_dir=compile_reuse_project, env=env), + reference=run_reuse_compile( + project_dir=compile_reuse_project, env=env, args=("--no-cache",) + ), + ) + + assert (cold.returncode, recorded.returncode, reused.returncode) == (0, 0, 0), reused.stderr + assert reused.timings.get("render_reuse_hits", 0) > 0 + assert comparison.matches is test_case.expected_matches_uncached, ( + comparison.mismatched_artifacts + ) + assert test_case.expected_compiled_value in compiled_text( + run=comparison.incremental, suffix="fact_orders.sql" + ) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderLoadNoticeTestCase( + description="small_store_loads_quietly", notice_bytes=1 << 40, expected_notice=False + ), + RenderLoadNoticeTestCase( + description="large_store_announces_loading", notice_bytes=1, expected_notice=True + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_renders_when_loading_them_then_large_loads_are_announced_on_stderr( + compile_reuse_project: Path, + test_case: RenderLoadNoticeTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + enable_compile_reuse(monkeypatch) + set_render_load_notice_bytes(monkeypatch, test_case.notice_bytes) + assert prime_render_store_in_process(compile_reuse_project) == 0 + staging_comment(compile_reuse_project) + + code, out, err = compile_in_process_output(project_dir=compile_reuse_project, capsys=capsys) + + assert code == 0 + assert "compile_timings" in json_report_keys(out) + assert ("Loading stored renders" in err) is test_case.expected_notice + assert ("Loaded stored renders" in err) is test_case.expected_notice diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py index e59823a920..c47ea6f4db 100644 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py @@ -9,13 +9,12 @@ @dataclass(frozen=True) class RenderLayerTestCase: - """Renders changed after a stored compile, and whether they fit in an overlay.""" + """Renders changed after a stored compile, and whether an overlay can hold them.""" description: str changed_models: tuple[str, ...] removed_models: tuple[str, ...] expected_overlay: bool - expected_overlay_models: frozenset[str] @dataclass(frozen=True) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py index 590026c840..c215b98d84 100644 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/helpers.py @@ -15,9 +15,10 @@ MODEL_PATHS: tuple[str, ...] = tuple(f"models/orders_{index:02d}.sql" for index in range(20)) -def publish_render_state(*, directory: Path, name: str, layer: RenderStateLayer) -> Path: +def publish_render_state(*, directory: Path, name: str, layer: RenderStateLayer | None) -> Path: """Write one render layer as the store would publish it.""" + assert layer is not None path: Path = directory / name path.write_bytes(b"".join(bytes(chunk) for chunk in layer.chunks)) return path @@ -50,25 +51,39 @@ def stored_base(directory: Path) -> tuple[Path, StoredRenderState]: def next_render_state( *, stored: StoredRenderState, edited: tuple[str, ...], removed: tuple[str, ...] = () ) -> RenderReuseState: - """Return the renders of the next compile: some models edited, some stored renders dropped.""" + """Return the next compile's renders as a session stores them after releasing reused bytes. - kept: dict[str, memoryview] = dict( - filter(lambda item: item[0] not in removed, stored.state.model_payloads.items()) - ) + Renders reused from the stored base are None; reused overlay renders keep their bytes. + """ + + kept: dict[str, memoryview | None] = { + path: retained_payload(stored=stored, path=path, payload=payload) + for path, payload in filter( + lambda item: item[0] not in removed, stored.state.model_payloads.items() + ) + } return RenderReuseState( model_paths=MODEL_PATHS, model_payloads={ **kept, **{path: memoryview(f"edited {path}".encode() * 20) for path in edited}, }, - group_payloads=dict(stored.state.group_payloads), + group_payloads=dict.fromkeys(stored.state.group_payloads), ) +def retained_payload( + *, stored: StoredRenderState, path: str, payload: memoryview | None +) -> memoryview | None: + """Keep a reused render's bytes only when it lives in the stored overlay.""" + + return {True: payload, False: None}[path in stored.layers.overlay_models] + + def payload_bytes(state: RenderReuseState) -> dict[str, bytes]: """Return every model payload as bytes for comparison.""" - return {path: bytes(payload) for path, payload in state.model_payloads.items()} + return {path: bytes(payload or b"") for path, payload in state.model_payloads.items()} def store_edit_chain(*, directory: Path, edits: tuple[str, ...]) -> StoredRenderState: @@ -81,7 +96,7 @@ def store_edit_chain(*, directory: Path, edits: tuple[str, ...]) -> StoredRender directory=directory, name=f"orders-edit-{index}.render", layer=render_state_layer( - state=next_render_state(stored=stored, edited=(edited,)), stored=stored + state=next_render_state(stored=stored, edited=(edited,)), stored=stored.layers ), ) stored = read_stored(path) @@ -131,7 +146,7 @@ def overlay_without_base(directory: Path) -> tuple[Path, frozenset[str]]: directory=directory, name="orders-overlay.render", layer=render_state_layer( - state=next_render_state(stored=stored, edited=(MODEL_PATHS[0],)), stored=stored + state=next_render_state(stored=stored, edited=(MODEL_PATHS[0],)), stored=stored.layers ), ) base_path.unlink() diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py index 71bccd8da9..bca44fbe3e 100644 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_render_state_file.py @@ -28,8 +28,6 @@ next_render_state, overlay_without_base, payload_bytes, - publish_render_state, - read_stored, store_edit_chain, stored_base, truncated, @@ -44,26 +42,23 @@ changed_models=(MODEL_PATHS[3],), removed_models=(), expected_overlay=True, - expected_overlay_models=frozenset({MODEL_PATHS[3]}), ), RenderLayerTestCase( description="most_models_changed", changed_models=MODEL_PATHS[:15], removed_models=(), expected_overlay=False, - expected_overlay_models=frozenset(), ), RenderLayerTestCase( description="stored_render_dropped", changed_models=(MODEL_PATHS[3],), removed_models=(MODEL_PATHS[5],), expected_overlay=False, - expected_overlay_models=frozenset(), ), ], ids=lambda case: case.description, ) -def test_given_stored_renders_when_storing_the_next_compile_then_reading_returns_its_renders( +def test_given_released_reused_renders_when_storing_then_only_a_small_overlay_needs_no_rewrite( test_case: RenderLayerTestCase, tmp_path: Path ) -> None: _, stored = stored_base(tmp_path) @@ -71,15 +66,9 @@ def test_given_stored_renders_when_storing_the_next_compile_then_reading_returns stored=stored, edited=test_case.changed_models, removed=test_case.removed_models ) - layer: RenderStateLayer = render_state_layer(state=current, stored=stored) - reread: StoredRenderState = read_stored( - publish_render_state(directory=tmp_path, name="orders-next.render", layer=layer) - ) + layer: RenderStateLayer | None = render_state_layer(state=current, stored=stored.layers) - assert (layer.base_file is not None) is test_case.expected_overlay - assert payload_bytes(reread.state) == payload_bytes(current) - assert reread.state.group_payloads.keys() == current.group_payloads.keys() - assert reread.layers.overlay_models == test_case.expected_overlay_models + assert (layer is not None) is test_case.expected_overlay @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py index 324c935c91..23fea2a02f 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/_test_types.py @@ -36,3 +36,13 @@ class CorruptRenderTestCase: description: str model_name: str expected_rendered_again: bool + + +@dataclass(frozen=True) +class ReleasedRenderTestCase: + """A reused and an edited render, and which stored bytes storing the compile releases.""" + + description: str + retained_models: frozenset[str] + expected_released: dict[str, bool] + expected_query_sqls: dict[str, str] diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py index c6c0b114b1..265ff3b7fc 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/helpers.py @@ -5,6 +5,7 @@ from collections.abc import Callable from dataclasses import replace from pathlib import Path +from typing import cast from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( collect_compile_diagnostics, @@ -16,9 +17,11 @@ CompileModelInput, CompilerDiagnostic, RenderReuseState, + StoredRender, ) from sqlbuild.compiler.compile.types import CompileContextKey, DiagnosticPhase, DiagnosticSeverity from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.main._load_fact_payload import loaded_fact_payload REGION_ENV_VAR: str = "ORDERS_REGION" @@ -80,7 +83,6 @@ def recorded_state( item.file_path.stem in run_id_readers ] _ = session.rendered_model(model_file=item, render=lambda item=item, r=render: r(item)) - session.renders_complete() state: RenderReuseState | None = session.stored_state() assert state is not None return state @@ -115,3 +117,42 @@ def corrupted_session(item: DiscoveredSqlModelFile) -> CompileRenderReuseSession ) session.plan_models(model_files=(item,)) return session + + +def edited_session(*, retained_models: frozenset[str]) -> CompileRenderReuseSession: + """Reuse the stored orders render and render an edited customers model.""" + + orders: DiscoveredSqlModelFile = model_file("orders") + customers: DiscoveredSqlModelFile = model_file("customers", "SELECT 2 AS customer_id") + session: CompileRenderReuseSession = CompileRenderReuseSession( + prior=recorded_state((orders, model_file("customers"))), + changed_paths=frozenset({str(customers.relative_path)}), + retained_models=retained_models, + ) + session.plan_models(model_files=(orders, customers)) + with collect_compile_diagnostics(): + _ = session.reused_model(model_file=orders) + _ = session.rendered_model(model_file=customers, render=lambda: _render(customers)) + return session + + +def released_paths(state: RenderReuseState | None) -> dict[str, bool]: + """Return, per stored model, whether its bytes were released.""" + + assert state is not None + return {path: payload is None for path, payload in state.model_payloads.items()} + + +def stored_query_sqls(state: RenderReuseState | None) -> dict[str, str]: + """Decode every stored model render and return its query SQL.""" + + assert state is not None + return { + path: _stored_query_sql(cast(memoryview, payload)) + for path, payload in filter(lambda item: item[1] is not None, state.model_payloads.items()) + } + + +def _stored_query_sql(payload: memoryview) -> str: + stored: StoredRender = cast(StoredRender, loaded_fact_payload(payload)) + return cast(CompileModelInput, stored.value).query_sql diff --git a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py index 45c765ab18..a906fdd2db 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py +++ b/tests/unit/src/sqlbuild/compiler/compile/classes/test_render_reuse_session.py @@ -11,15 +11,19 @@ from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile from tests.unit.src.sqlbuild.compiler.compile.classes._test_types import ( CorruptRenderTestCase, + ReleasedRenderTestCase, RenderReplayTestCase, ReusableModelsTestCase, ) from tests.unit.src.sqlbuild.compiler.compile.classes.helpers import ( REGION_ENV_VAR, corrupted_session, + edited_session, missing_description, model_file, planned_session, + released_paths, + stored_query_sqls, ) @@ -144,5 +148,41 @@ def test_given_corrupt_stored_render_when_reusing_then_the_model_is_rendered_aga assert (reused is None) is test_case.expected_rendered_again +@pytest.mark.parametrize( + "test_case", + [ + ReleasedRenderTestCase( + description="reused_from_base", + retained_models=frozenset(), + expected_released={"models/orders.sql": True, "models/customers.sql": False}, + expected_query_sqls={ + "models/orders.sql": "SELECT 1 AS order_id", + "models/customers.sql": "SELECT 2 AS customer_id", + }, + ), + ReleasedRenderTestCase( + description="reused_from_overlay", + retained_models=frozenset({"models/orders.sql"}), + expected_released={"models/orders.sql": False, "models/customers.sql": False}, + expected_query_sqls={ + "models/orders.sql": "SELECT 1 AS order_id", + "models/customers.sql": "SELECT 2 AS customer_id", + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_reused_render_when_storing_then_base_bytes_are_released_and_recoverable( + test_case: ReleasedRenderTestCase, +) -> None: + session: CompileRenderReuseSession = edited_session(retained_models=test_case.retained_models) + + released: dict[str, bool] = released_paths(session.stored_state()) + complete: dict[str, str] = stored_query_sqls(session.stored_state(complete=True)) + + assert released == test_case.expected_released + assert complete == test_case.expected_query_sqls + + if __name__ == "__main__": raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index a71bdf5482..affbda9659 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -108,8 +108,9 @@ and when a provider's settings use sources SQLBuild cannot list exactly, such as SQLBuild keeps one stored result per project and target under `target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is ignored and compile runs in full. Storing a result normally takes a fraction of a second after the -report is ready. When it has to hash a lot of data or the project has very many files, compile -prints `Recording compile for reuse...` and a completion line to stderr before the report. +report is ready. When it has to hash a lot of data, the project has very many files, or it records +thousands of new model renders, compile prints `Recording compile for reuse...` and a completion +line to stderr before the report. ### Compiling after model-only edits @@ -123,7 +124,15 @@ byte-identical to `sqb compile --no-cache`. Every other change renders the whole project: adding, removing, or renaming a model, and editing macros or the Python modules they import, configuration, `path_defaults`, enums, constants, scope folders, seeds, sources, tests, or audits. A model whose render read `run.id` or provider settings -is always rendered again. Stored renders are kept for the two newest compiles in the same folder. +is always rendered again. + +Renders are stored only when a compile result for the same project and target is already stored, +so a compile in a fresh checkout or after deleting `target/` stores none and costs nothing extra. +The next full compile stores them, and edits after that reuse them. Stored renders are compressed +and kept for the two newest compiles in the same folder. When the stored renders are large, compile prints +`Loading stored renders...` and a completion line to stderr before it starts. While renders are +reused, the Python modules recorded by the previous compile stay tracked even if this compile did +not import them, so editing such a module runs the next compile in full. Known limits: From c608990de5026a86a84e927eb9137c5f35184f8b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 22:49:08 +0100 Subject: [PATCH 4/4] perf(compile): tolerate concurrent removal when sizing stored renders --- .../_helpers/load_notice.py | 11 ++- .../cli/compile_render_reuse/constants.py | 2 +- .../_helpers/_test_types.py | 11 +++ .../_helpers/test_load_notice.py | 68 +++++++++++++++++++ 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py diff --git a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py index 5e330aaa68..b89b7a1c87 100644 --- a/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py +++ b/src/sqlbuild/cli/compile_render_reuse/_helpers/load_notice.py @@ -19,7 +19,7 @@ def start_load_notice(*, entry_path: Path) -> float | None: """Announce loading on stderr when the slot's stored render files are large.""" stored_bytes: int = sum( - path.stat().st_size + _file_size(path=path) for path in entry_path.parent.glob(f"{entry_path.stem}-*{REUSE_RENDER_STATE_SUFFIX}") ) if stored_bytes < RENDER_LOAD_NOTICE_BYTES: @@ -31,6 +31,15 @@ def start_load_notice(*, entry_path: Path) -> float | None: return time.monotonic() +def _file_size(*, path: Path) -> int: + """Size a stored render file, treating one removed by a concurrent compile as empty.""" + + try: + return path.stat().st_size + except OSError: + return 0 + + def finish_load_notice(*, started: float | None, loaded: bool) -> None: """Report whether an announced load produced usable renders.""" diff --git a/src/sqlbuild/cli/compile_render_reuse/constants.py b/src/sqlbuild/cli/compile_render_reuse/constants.py index f77740df1d..be5e190f0a 100644 --- a/src/sqlbuild/cli/compile_render_reuse/constants.py +++ b/src/sqlbuild/cli/compile_render_reuse/constants.py @@ -10,5 +10,5 @@ RENDER_LOAD_START_MESSAGE: str = "Loading stored renders ({mebibytes:.0f} MiB)..." RENDER_LOAD_DONE_MESSAGE: str = "Loaded stored renders ({seconds:.1f} s)" RENDER_LOAD_FAILED_MESSAGE: str = ( - "Stored renders are unusable; rendering every model again ({seconds:.1f} s)" + "Stored renders not reused; rendering every model again ({seconds:.1f} s)" ) diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py index c47ea6f4db..913e647485 100644 --- a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/_test_types.py @@ -33,3 +33,14 @@ class StoredRenderReadTestCase: description: str prepare: Callable[[Path], tuple[Path, frozenset[str]]] expected_readable: bool + + +@dataclass(frozen=True) +class RenderLoadNoticeTestCase: + """Stored render files, some removed while being sized, and the expected notice.""" + + description: str + file_sizes: tuple[int, ...] + removed_while_sizing: int + notice_bytes: int + expected_notice: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py new file mode 100644 index 0000000000..3feb8dc648 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_render_reuse/_helpers/test_load_notice.py @@ -0,0 +1,68 @@ +"""Sizing the render store for the load notice must tolerate files removed by a concurrent compile.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_render_reuse._helpers import load_notice +from sqlbuild.cli.compile_reuse.constants import REUSE_RENDER_STATE_SUFFIX +from tests.unit.src.sqlbuild.cli.compile_render_reuse._helpers._test_types import ( + RenderLoadNoticeTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + RenderLoadNoticeTestCase( + description="large_store_announced", + file_sizes=(64, 64), + removed_while_sizing=0, + notice_bytes=100, + expected_notice=True, + ), + RenderLoadNoticeTestCase( + description="small_store_silent", + file_sizes=(16, 16), + removed_while_sizing=0, + notice_bytes=100, + expected_notice=False, + ), + RenderLoadNoticeTestCase( + description="file_removed_while_sizing_counts_as_empty", + file_sizes=(64, 64), + removed_while_sizing=1, + notice_bytes=100, + expected_notice=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_render_files_when_starting_load_notice_then_sizes_without_failing( + test_case: RenderLoadNoticeTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + # Arrange + entry_path: Path = tmp_path / "dev.entry" + stored_paths: list[Path] = [ + tmp_path / f"dev-{index}{REUSE_RENDER_STATE_SUFFIX}" + for index in range(len(test_case.file_sizes)) + ] + for path, size in zip(stored_paths, test_case.file_sizes, strict=True): + path.write_bytes(b"x" * size) + listed: list[Path] = sorted(stored_paths) + monkeypatch.setattr(load_notice, "RENDER_LOAD_NOTICE_BYTES", test_case.notice_bytes) + monkeypatch.setattr(Path, "glob", lambda self, pattern: iter(listed)) + for path in listed[: test_case.removed_while_sizing]: + path.unlink() + + # Act + started: float | None = load_notice.start_load_notice(entry_path=entry_path) + + # Assert + assert (started is not None) is test_case.expected_notice + assert capsys.readouterr().out == ""