From b23b2ce82410d3f1b3045ffa82233327e1d879fa Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:17:41 +0100 Subject: [PATCH 1/2] perf(compile): reuse the previous compile when no input changed Replay the stored report, diagnostics, and exit code when the project files, invocation, tracked environment, runtime, and written artifacts all match the previous compile. Misses, bypasses, and damaged entries fall back to a full compile. --- .../cli/commands/_helpers/compile/pipeline.py | 17 +- .../_helpers/compile/target_writer.py | 13 + .../cli/commands/_helpers/entry/dispatch.py | 1 + .../cli/commands/main/project/_compile.py | 92 +- src/sqlbuild/cli/compile/main/run.py | 43 +- src/sqlbuild/cli/compile/models.py | 10 + src/sqlbuild/cli/compile_reuse/__init__.py | 1 + .../cli/compile_reuse/_helpers/__init__.py | 0 .../cli/compile_reuse/_helpers/attempt.py | 208 +++++ .../cli/compile_reuse/_helpers/entry_file.py | 363 ++++++++ .../compile_reuse/_helpers/project_files.py | 286 +++++++ .../_helpers/provider_settings.py | 86 ++ .../_helpers/runtime_identity.py | 180 ++++ .../cli/compile_reuse/_helpers/store.py | 191 +++++ .../compile_reuse/_helpers/target_files.py | 113 +++ .../cli/compile_reuse/_helpers/timings.py | 59 ++ .../cli/compile_reuse/classes/__init__.py | 0 .../compile_artifact_write_registry.py | 69 ++ .../classes/compile_artifact_writes.py | 53 ++ .../classes/recorded_compile_output.py | 43 + src/sqlbuild/cli/compile_reuse/constants.py | 92 ++ src/sqlbuild/cli/compile_reuse/exceptions.py | 7 + .../cli/compile_reuse/main/__init__.py | 0 .../main/_attempt_compile_reuse.py | 12 + .../main/_compile_reuse_timings.py | 12 + .../main/_write_reusable_compile.py | 38 + src/sqlbuild/cli/compile_reuse/models.py | 130 +++ src/sqlbuild/cli/compile_reuse/types.py | 25 + src/sqlbuild/cli/entry/models.py | 1 + .../compile/_helpers/render/sql_vars.py | 3 + .../compile/_helpers/render/templating.py | 3 + .../classes/compile_input_read_registry.py | 51 ++ .../compile/classes/compile_input_reads.py | 54 ++ src/sqlbuild/compiler/compile/constants.py | 4 + .../discovery/_helpers/filesystem/core.py | 4 + tests/conftest.py | 16 + .../main/build/test_compile_json_behavior.py | 4 + .../cli/commands/main/compile/_test_types.py | 123 +++ .../cli/commands/main/compile/conftest.py | 8 + .../cli/commands/main/compile/helpers.py | 513 +++++++++++ .../main/compile/test_compile_reuse.py | 805 ++++++++++++++++++ .../sqlbuild/cli/commands/main/_test_types.py | 11 + .../main/test_startup_import_footprint.py | 60 +- .../sqlbuild/cli/compile_reuse/__init__.py | 0 .../cli/compile_reuse/_helpers/__init__.py | 0 .../cli/compile_reuse/_helpers/_test_types.py | 80 ++ .../cli/compile_reuse/_helpers/helpers.py | 262 ++++++ .../_helpers/test_project_files.py | 286 +++++++ .../_helpers/test_provider_settings.py | 176 ++++ .../cli/compile_reuse/main/__init__.py | 0 .../cli/compile_reuse/main/_test_types.py | 15 + .../cli/compile_reuse/main/helpers.py | 41 + .../main/test_attempt_import_footprint.py | 55 ++ 53 files changed, 4695 insertions(+), 24 deletions(-) create mode 100644 src/sqlbuild/cli/compile_reuse/__init__.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/__init__.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/attempt.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/project_files.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/provider_settings.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/store.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/target_files.py create mode 100644 src/sqlbuild/cli/compile_reuse/_helpers/timings.py create mode 100644 src/sqlbuild/cli/compile_reuse/classes/__init__.py create mode 100644 src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py create mode 100644 src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py create mode 100644 src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py create mode 100644 src/sqlbuild/cli/compile_reuse/constants.py create mode 100644 src/sqlbuild/cli/compile_reuse/exceptions.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/__init__.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py create mode 100644 src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py create mode 100644 src/sqlbuild/cli/compile_reuse/models.py create mode 100644 src/sqlbuild/cli/compile_reuse/types.py create mode 100644 src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py create mode 100644 src/sqlbuild/compiler/compile/classes/compile_input_reads.py create mode 100644 tests/conftest.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/__init__.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/__init__.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/main/__init__.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/main/_test_types.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index 6b76577da5..f722548eee 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import time from dataclasses import replace from functools import partial @@ -28,6 +29,7 @@ CompileAnalysis, CompileWriteResult, ) +from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.entry.models import CompileProfileFlags from sqlbuild.cli.output.models import ( WrittenTarget, @@ -319,13 +321,14 @@ def write_compile_dag_artifact( dag_path=dag_path, ) resolved_dag_path.parent.mkdir(parents=True, exist_ok=True) - resolved_dag_path.write_text( - build_dag_json( - graph=analysis.graph, - project_name=analysis.discovered_inputs.project_config.name, - python_graph=python_graph, - ), - encoding="utf-8", + dag_json: str = build_dag_json( + graph=analysis.graph, + project_name=analysis.discovered_inputs.project_config.name, + python_graph=python_graph, + ) + _ = resolved_dag_path.write_text(dag_json, encoding="utf-8") + COMPILE_ARTIFACT_WRITES.written( + path=resolved_dag_path, contents=dag_json.replace("\n", os.linesep).encode("utf-8") ) _ = complete_compile_phase( status=status, message=f"Wrote DAG artifact. ({time.monotonic() - dag_start:.2f}s)" diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index ecfadd0214..6be15c5435 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -27,6 +27,7 @@ SqlTestArtifactCacheRecord, SqlTestArtifactIdentityContext, ) +from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES from sqlbuild.cli.output.models import ( WrittenTarget, ) @@ -235,6 +236,7 @@ def _move_staged_tree(*, staged_dir: Path, path: Path) -> bool: os.rename(staged_dir, path) except OSError: return False + COMPILE_ARTIFACT_WRITES.moved_tree(source=staged_dir, destination=path) return True @@ -250,6 +252,7 @@ def _staged_files(*, staged_dir: Path) -> list[tuple[Path, str]]: def _publish_staged_file(*, source: Path, path: Path, check_existing: bool) -> None: with record_compile_timing("physical_write_ms"): + COMPILE_ARTIFACT_WRITES.moved(source=source, destination=path) if check_existing and path.is_file(): contents: bytes = source.read_bytes() existing: bytes = path.read_bytes() @@ -542,6 +545,11 @@ def _partition_cached_static_tests( identity=artifact_identity, ) if cached_path is not None: + COMPILE_ARTIFACT_WRITES.kept( + path=cached_path, + size=cached_record.size, + mtime_ns=cached_record.mtime_ns, + ) managed_paths.add(cached_path) current_records[record_key] = cached_record continue @@ -664,6 +672,10 @@ def _write_sql(*, path: Path, sql: str, check_existing: bool = True) -> None: def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): + COMPILE_ARTIFACT_WRITES.written( + path=path, + contents=contents.replace(_POSIX_LINE_SEPARATOR, os.linesep).encode("utf-8"), + ) if check_existing and path.is_file() and path.read_text(encoding="utf-8") == contents: return try: @@ -675,6 +687,7 @@ def _write_text_if_changed(*, path: Path, contents: str, check_existing: bool = def _write_bytes_if_changed(*, path: Path, contents: bytes, check_existing: bool = True) -> None: with record_compile_timing("physical_write_ms"): + COMPILE_ARTIFACT_WRITES.written(path=path, contents=contents) if check_existing: existing: bytes | None = _read_existing_file(path=path) if existing == contents: diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index f5de4c5148..8e8698cf27 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -501,6 +501,7 @@ def _dispatch_compile_or_contract( skip_contracts=args.profile_skip_contracts, skip_write=args.profile_skip_write, ), + debug=args.debug, ) ) return None diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 635188cb63..aa35dbc92c 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -2,12 +2,12 @@ from __future__ import annotations -import sys import time from dataclasses import replace from pathlib import Path from sqlbuild.cli.commands._helpers.compile.collection import paused_cyclic_collection +from sqlbuild.cli.commands._helpers.compile.dag import resolve_compile_dag_path from sqlbuild.cli.commands._helpers.compile.output import ( format_compile_json, format_compile_text, @@ -25,9 +25,17 @@ from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import ( CompileAnalysis, + CompileCommandResult, CompileWriteResult, ) +from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput +from sqlbuild.cli.compile_reuse.constants import COMPILE_ARTIFACT_WRITES +from sqlbuild.cli.compile_reuse.main._compile_reuse_timings import compile_reuse_timings +from sqlbuild.cli.compile_reuse.main._write_reusable_compile import write_reusable_compile +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome from sqlbuild.cli.entry.models import CompileCommandRequest +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic from sqlbuild.compiler.compile.types import DiagnosticPhase from sqlbuild.compiler.profiling.main.collect import collect_compile_timings @@ -37,13 +45,25 @@ from sqlbuild.rule_engine.main.render_skipped_rules import format_skipped_type_proof_rules -def run_compile(request: CompileCommandRequest) -> int: +def run_compile( + *, request: CompileCommandRequest, reuse_attempt: CompileReuseAttempt | None = None +) -> int: """Execute the compile command.""" - total_start: float = time.monotonic() + total_start: float = time.monotonic() if reuse_attempt is None else reuse_attempt.started effective_request: CompileCommandRequest = ( request if request.project_dir is not None else replace(request, project_dir=Path.cwd()) ) + attempt: CompileReuseAttempt = ( + CompileReuseAttempt( + outcome=CompileReuseOutcome.BYPASS, + project_dir=Path(effective_request.project_dir or Path.cwd()), + started=total_start, + ) + if reuse_attempt is None + else reuse_attempt + ) + output: RecordedCompileOutput = RecordedCompileOutput() status: TransientStatusReporter | None = start_compile_status( json_output=request.json_output, no_color=request.no_color, @@ -52,20 +72,40 @@ def run_compile(request: CompileCommandRequest) -> int: with ( paused_cyclic_collection(), collect_compile_timings() as detailed_timings, + COMPILE_INPUT_READS.recording() as input_reads, + COMPILE_ARTIFACT_WRITES.recording() as artifact_writes, PreparedCompileArtifacts( enabled=not request.profile_flags.skip_write ) as prepared_artifacts, ): - return _run_compile_with_status( + result: CompileCommandResult = _run_compile_with_status( request=effective_request, total_start=total_start, status=status, detailed_timings=detailed_timings, prepared_artifacts=prepared_artifacts, + output=output, + reuse_timings_ms=compile_reuse_timings(attempt=attempt), ) finally: if status is not None: status.close() + try: + with paused_cyclic_collection(): + write_reusable_compile( + attempt=attempt, + output=output, + exit_code=result.exit_code, + input_reads=input_reads, + artifact_writes=artifact_writes, + compile_cache_enabled=result.compile_cache_enabled, + artifacts_written=result.artifacts_written, + dag_artifact_path=result.dag_artifact_path, + json_output=request.json_output, + ) + finally: + output.print_stdout() + return result.exit_code def _run_compile_with_status( @@ -75,7 +115,9 @@ def _run_compile_with_status( status: TransientStatusReporter | None, detailed_timings: CompileTimingCollector, prepared_artifacts: PreparedCompileArtifacts, -) -> int: + output: RecordedCompileOutput, + reuse_timings_ms: dict[str, int], +) -> CompileCommandResult: """Execute compile after the optional interactive status reporter is initialized.""" project_dir: Path = request.project_dir if request.project_dir is not None else Path.cwd() @@ -132,6 +174,7 @@ def _run_compile_with_status( "rule_cache_misses": analysis.rule_cache_misses, "write_ms": write_result.write_ms, **detailed_timings.as_milliseconds(), + **reuse_timings_ms, "total_ms": elapsed_ms(total_start), } withheld_test_diagnostics: tuple[CompilerDiagnostic, ...] = ( @@ -154,7 +197,14 @@ def _run_compile_with_status( codes=analysis.skipped_type_proof_rules ) if skipped_rules_note is not None: - print(f"note: {skipped_rules_note}", file=sys.stderr) + output.print_stderr_line(f"note: {skipped_rules_note}") + result: CompileCommandResult = _command_result( + request=request, + project_dir=project_dir, + exit_code=exit_code, + rules_failed=rules_failed, + analysis=analysis, + ) if json_output: notice: str | None = semantic_coverage_notice( @@ -163,8 +213,8 @@ def _run_compile_with_status( sql_validation_enabled=not request.no_sql_validation, ) if notice: - print(notice, file=sys.stderr) - print( + output.print_stderr_line(notice) + output.record_stdout( format_compile_json( graph=analysis.graph, written=write_result.written, @@ -177,9 +227,9 @@ def _run_compile_with_status( sql_validation_enabled=not request.no_sql_validation, ) ) - return exit_code + return result - print( + output.record_stdout( format_compile_text( graph=analysis.graph, written=write_result.written, @@ -192,4 +242,24 @@ def _run_compile_with_status( sql_validation_enabled=not request.no_sql_validation, ) ) - return exit_code + return result + + +def _command_result( + *, + request: CompileCommandRequest, + project_dir: Path, + exit_code: int, + rules_failed: bool, + analysis: CompileAnalysis, +) -> CompileCommandResult: + return CompileCommandResult( + exit_code=exit_code, + compile_cache_enabled=analysis.graph.project.compile_cache_dir is not None, + artifacts_written=not (rules_failed or request.profile_flags.skip_write), + dag_artifact_path=( + None + if rules_failed or request.dag_path is None + else resolve_compile_dag_path(project_dir=project_dir, dag_path=request.dag_path) + ), + ) diff --git a/src/sqlbuild/cli/compile/main/run.py b/src/sqlbuild/cli/compile/main/run.py index 52db97c08c..b9acae3e44 100644 --- a/src/sqlbuild/cli/compile/main/run.py +++ b/src/sqlbuild/cli/compile/main/run.py @@ -1,10 +1,45 @@ """Public compile command entrypoint.""" -from sqlbuild.cli.commands.main.project._compile import run_compile as _run_compile -from sqlbuild.cli.entry.models import CompileCommandRequest +from __future__ import annotations + +from sqlbuild.cli.compile_reuse.main._attempt_compile_reuse import attempt_compile_reuse +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome +from sqlbuild.cli.entry.models import CompileCommandRequest, CompileProfileFlags def run_compile(request: CompileCommandRequest) -> int: - """Run one compile command request.""" + """Run one compile command request, replaying the previous compile when nothing changed.""" + + attempt: CompileReuseAttempt = attempt_compile_reuse(request=_reuse_request(request=request)) + if attempt.outcome is CompileReuseOutcome.HIT and attempt.exit_code is not None: + return attempt.exit_code + from sqlbuild.cli.commands.main.project._compile import run_compile as run_full_compile + + return run_full_compile(request=request, reuse_attempt=attempt) + - return _run_compile(request) +def _reuse_request(*, request: CompileCommandRequest) -> CompileReuseRequest: + flags: CompileProfileFlags = request.profile_flags + return CompileReuseRequest( + project_dir=request.project_dir, + selected_target=request.selected_target, + json_output=request.json_output, + no_color=request.no_color, + manifest=request.manifest, + dag_path=request.dag_path, + defer_to=request.defer_to, + no_cache=request.no_cache, + no_sql_validation=request.no_sql_validation, + lineage_mode=str(request.lineage_mode.value), + select=request.select, + exclude=request.exclude, + cli_vars=request.cli_vars, + debug=request.debug, + profiling=( + flags.skip_discovery_sql_analysis + or flags.skip_column_inference + or flags.skip_contracts + or flags.skip_write + ), + ) diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index 1bb0b44405..a590ab705e 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -18,6 +18,16 @@ from sqlbuild.compiler.planner.models import NativeSqlTestArtifact +@dataclass(frozen=True) +class CompileCommandResult: + """Outcome of one full compile that decides whether it can be stored for reuse.""" + + exit_code: int + compile_cache_enabled: bool + artifacts_written: bool + dag_artifact_path: Path | None + + @dataclass(frozen=True) class CompileAnalysis: """Compiled project analysis shared by compile output phases.""" diff --git a/src/sqlbuild/cli/compile_reuse/__init__.py b/src/sqlbuild/cli/compile_reuse/__init__.py new file mode 100644 index 0000000000..e7ebc10ef2 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/__init__.py @@ -0,0 +1 @@ +"""Whole-project compile reuse when no compile input changed.""" diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/__init__.py b/src/sqlbuild/cli/compile_reuse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py new file mode 100644 index 0000000000..ccb90f2e46 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -0,0 +1,208 @@ +"""Decide whether a compile can replay the stored result, and replay it on a hit.""" + +from __future__ import annotations + +import os +import sys +import time +from dataclasses import replace +from pathlib import Path + +from sqlbuild.cli.compile_reuse._helpers.entry_file import ( + read_entry_header, + read_entry_stdout, + rewrite_entry_inputs, +) +from sqlbuild.cli.compile_reuse._helpers.project_files import ( + carried_forward_digests, + compare_project_files, + needs_refresh, + restamped_paths, + snapshot_project_files, + stored_project_files, + with_missing_digests, +) +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( + entry_slot_name, + environment_digest, + invocation_digest, + module_stamps_unchanged, + runtime_identity, + search_path_stamps, + settings_inputs_digest, + tracked_environment_names, +) +from sqlbuild.cli.compile_reuse._helpers.target_files import target_files_unchanged +from sqlbuild.cli.compile_reuse._helpers.timings import ( + elapsed_ms, + replace_compile_timings, + reuse_timings, +) +from sqlbuild.cli.compile_reuse.constants import ( + MILLISECONDS_PER_SECOND, + PROJECT_CONFIG_FILENAMES, + REUSE_DISABLE_ENV_VAR, + REUSE_DISABLE_VALUE, + REUSE_ENTRY_DIRECTORY_PARTS, + REUSE_ENTRY_SUFFIX, + REUSE_HIT_MESSAGE, + TOTAL_TIMING, +) +from sqlbuild.cli.compile_reuse.models import ( + CompileReuseAttempt, + CompileReuseRequest, + ProjectFilesComparison, + StoredCompileHeader, + StoredCompileInputs, +) +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome +from sqlbuild.compiler.compile.constants import ( + COMPILE_CACHE_DISABLE_ENV_VAR, + COMPILE_CACHE_DISABLE_VALUE, +) +from sqlbuild.presentation.main.supports_color import supports_color + + +def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: + """Replay a stored compile when every input matches, otherwise prepare to store one.""" + + started: float = time.monotonic() + project_dir: str = os.path.abspath( + os.getcwd() if request.project_dir is None else request.project_dir + ) + bypass: CompileReuseAttempt = CompileReuseAttempt( + outcome=CompileReuseOutcome.BYPASS, project_dir=Path(project_dir), started=started + ) + if _bypasses_reuse(request=request, project_dir=project_dir): + return bypass + snapshot_ns: int = time.time_ns() + attempt: CompileReuseAttempt = replace( + bypass, + outcome=CompileReuseOutcome.MISS, + entry_path=Path(project_dir, *REUSE_ENTRY_DIRECTORY_PARTS) + / f"{entry_slot_name(selected_target=request.selected_target)}{REUSE_ENTRY_SUFFIX}", + runtime=runtime_identity(), + search_path=search_path_stamps(project_dir=project_dir), + invocation_digest=invocation_digest( + request=request, + project_dir=project_dir, + use_color=(not request.no_color) and supports_color(), + ), + snapshot=snapshot_project_files(project_dir=project_dir), + snapshot_ns=snapshot_ns, + ) + header: StoredCompileHeader | None = ( + None if attempt.entry_path is None else read_entry_header(path=attempt.entry_path) + ) + if header is not None: + attempt = _checked_attempt(attempt=attempt, inputs=header.inputs) + if attempt.outcome is CompileReuseOutcome.HIT: + return _replayed_attempt( + attempt=attempt, header=header, json_output=request.json_output + ) + attempt = replace( + attempt, + restamped=restamped_paths(stored=header.inputs.project_files, current=attempt.snapshot), + ) + return replace( + attempt, + digests=with_missing_digests( + project_dir=project_dir, + snapshot=attempt.snapshot, + digests=attempt.digests, + paths=frozenset(), + snapshot_ns=snapshot_ns, + ), + check_ms=elapsed_ms(started=started), + ) + + +def _bypasses_reuse(*, request: CompileReuseRequest, project_dir: str) -> bool: + return ( + request.no_cache + or request.manifest + or request.profiling + or request.debug + or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE + or os.environ.get(REUSE_DISABLE_ENV_VAR) == REUSE_DISABLE_VALUE + or not any( + os.path.isfile(os.path.join(project_dir, name)) for name in PROJECT_CONFIG_FILENAMES + ) + ) + + +def _checked_attempt( + *, attempt: CompileReuseAttempt, inputs: StoredCompileInputs +) -> CompileReuseAttempt: + project_dir: str = str(attempt.project_dir) + identity_unchanged: bool = ( + inputs.invocation_digest == attempt.invocation_digest + and inputs.runtime == attempt.runtime + and inputs.search_path == attempt.search_path + and tracked_environment_names(template_names=inputs.environment_names) + == inputs.environment_names + and environment_digest(names=inputs.environment_names) == inputs.environment_digest + and settings_inputs_digest(inputs=inputs.settings_inputs) == inputs.settings_digest + and module_stamps_unchanged(stamps=inputs.modules) + ) + comparison: ProjectFilesComparison = ( + compare_project_files( + project_dir=project_dir, stored=inputs.project_files, current=attempt.snapshot + ) + if identity_unchanged + else ProjectFilesComparison(unchanged=False, verified={}) + ) + unchanged: bool = comparison.unchanged and target_files_unchanged( + stored=inputs.target_files, project_dir=project_dir, tree=inputs.target_tree + ) + return replace( + attempt, + outcome=CompileReuseOutcome.HIT if unchanged else CompileReuseOutcome.MISS, + digests=carried_forward_digests( + stored=inputs.project_files, current=attempt.snapshot, verified=comparison.verified + ), + ) + + +def _replayed_attempt( + *, attempt: CompileReuseAttempt, header: StoredCompileHeader, json_output: bool +) -> CompileReuseAttempt: + stdout: str | None = ( + None + if attempt.entry_path is None + else read_entry_stdout(path=attempt.entry_path, header=header) + ) + span: tuple[int, int] | None = header.output.timings_span + if stdout is None or (json_output and span is None): + return replace(attempt, outcome=CompileReuseOutcome.MISS) + check_ms: int = elapsed_ms(started=attempt.started) + if span is not None: + stdout = replace_compile_timings( + stdout=stdout, + span=span, + timings={ + **reuse_timings(outcome=CompileReuseOutcome.HIT, check_ms=check_ms), + TOTAL_TIMING: elapsed_ms(started=attempt.started), + }, + ) + print(REUSE_HIT_MESSAGE.format(seconds=check_ms / MILLISECONDS_PER_SECOND), file=sys.stderr) + for line in header.output.stderr_lines: + print(line, file=sys.stderr) + _ = sys.stdout.write(stdout) + sys.stdout.flush() + if attempt.entry_path is not None and needs_refresh( + stored=header.inputs.project_files, current=attempt.snapshot + ): + rewrite_entry_inputs( + path=attempt.entry_path, + header=header, + inputs=replace( + header.inputs, + project_files=stored_project_files( + snapshot=attempt.snapshot, + digests=attempt.digests, + snapshot_ns=attempt.snapshot_ns, + ), + ), + ) + return replace(attempt, check_ms=check_ms, exit_code=header.output.exit_code) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py new file mode 100644 index 0000000000..f7da96aeb7 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/entry_file.py @@ -0,0 +1,363 @@ +"""Checksummed, atomically replaced storage for one reusable compile result.""" + +from __future__ import annotations + +import contextlib +import json +import os +import tempfile +import uuid +import zlib +from dataclasses import replace +from pathlib import Path +from typing import BinaryIO, cast + +from sqlbuild.cli.compile_reuse.constants import ( + REUSE_ENTRY_BYTE_ORDER, + REUSE_ENTRY_CHECKSUM_BYTES, + REUSE_ENTRY_LENGTH_BYTES, + REUSE_ENTRY_MAGIC, + REUSE_ENTRY_SUFFIX, + REUSE_FORMAT_VERSION, + REUSE_MAX_ENTRY_BYTES, + REUSE_MAX_STORED_ENTRIES, + REUSE_STDOUT_SEPARATOR, + REUSE_STDOUT_SUFFIX, +) +from sqlbuild.cli.compile_reuse.exceptions import CompileReuseEntryError +from sqlbuild.cli.compile_reuse.models import ( + SettingsEnvironmentInputs, + StoredCompileHeader, + StoredCompileInputs, + StoredCompileOutput, + StoredProjectFile, +) +from sqlbuild.cli.compile_reuse.types import FileStamp + +_TIMINGS_SPAN_LENGTH: int = 2 +_STAMP_FIELD_COUNT: int = 6 +_PROJECT_FILE_FIELD_COUNT: int = 8 +_SEARCH_PATH_FIELD_COUNT: int = 2 +_MODULE_FIELD_COUNT: int = 3 + + +def read_entry_header(*, path: Path) -> StoredCompileHeader | None: + """Read and verify the stored inputs, treating every fault as a cache miss.""" + + try: + with open(path, "rb") as handle: + if handle.read(len(REUSE_ENTRY_MAGIC)) != REUSE_ENTRY_MAGIC: + return None + metadata: bytes = _read_section(handle=handle) + if handle.read(1): + return None + payload: object = json.loads(metadata) + return _header(payload=payload) + except (OSError, UnicodeError, ValueError, TypeError, KeyError, IndexError): + return None + + +def read_entry_stdout(*, path: Path, header: StoredCompileHeader) -> str | None: + """Read and verify the stored stdout once every input matched.""" + + stdout_path: Path = path.parent / header.output.stdout_file + try: + stdout: bytes = stdout_path.read_bytes() + if ( + len(stdout) > REUSE_MAX_ENTRY_BYTES + or zlib.crc32(stdout) != header.output.stdout_checksum + ): + return None + text: str = stdout.decode("utf-8", "surrogateescape") + except (OSError, UnicodeError, ValueError): + return None + return text if len(text) == header.output.stdout_length else None + + +def write_entry( + *, + path: Path, + inputs: StoredCompileInputs, + output: StoredCompileOutput, + stdout: str, +) -> None: + """Publish one entry and its stdout atomically, filling in the stdout file and checksum.""" + + encoded_stdout: bytes = stdout.encode("utf-8", "surrogateescape") + stdout_file: str = f"{path.stem}{REUSE_STDOUT_SEPARATOR}{uuid.uuid4().hex}{REUSE_STDOUT_SUFFIX}" + stored_output: StoredCompileOutput = replace( + output, stdout_file=stdout_file, stdout_checksum=zlib.crc32(encoded_stdout) + ) + metadata: bytes = _metadata(inputs=inputs, output=stored_output) + if len(metadata) + len(encoded_stdout) > REUSE_MAX_ENTRY_BYTES: + remove_entry(path=path) + return + try: + path.parent.mkdir(parents=True, exist_ok=True) + _publish_file(path=path.parent / stdout_file, contents=encoded_stdout) + _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) + _remove_stdout_files(entry_path=path, keep=stdout_file) + _prune_entries(directory=path.parent, keep=path) + except OSError: + remove_entry(path=path) + + +def rewrite_entry_inputs( + *, path: Path, header: StoredCompileHeader, inputs: StoredCompileInputs +) -> None: + """Replace only the stored inputs of an entry, keeping its stored stdout file.""" + + metadata: bytes = _metadata(inputs=inputs, output=header.output) + with contextlib.suppress(OSError): + _publish_file(path=path, contents=_framed_metadata(metadata=metadata)) + + +def remove_entry(*, path: Path) -> None: + """Remove one stored entry and its stored stdout when they exist.""" + + with contextlib.suppress(OSError): + path.unlink(missing_ok=True) + _remove_stdout_files(entry_path=path, keep=None) + + +def _publish_file(*, path: Path, contents: bytes) -> None: + descriptor, temporary_path = tempfile.mkstemp( + prefix=f".{path.name}.", suffix=".tmp", dir=path.parent + ) + try: + with os.fdopen(descriptor, "wb") as handle: + _ = handle.write(contents) + os.replace(temporary_path, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(temporary_path) + raise + + +def _metadata(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> bytes: + return json.dumps(_payload(inputs=inputs, output=output), separators=(",", ":")).encode( + "utf-8", "surrogateescape" + ) + + +def _framed_metadata(*, metadata: bytes) -> bytes: + return b"".join( + ( + REUSE_ENTRY_MAGIC, + len(metadata).to_bytes(REUSE_ENTRY_LENGTH_BYTES, REUSE_ENTRY_BYTE_ORDER), + zlib.crc32(metadata).to_bytes(REUSE_ENTRY_CHECKSUM_BYTES, REUSE_ENTRY_BYTE_ORDER), + metadata, + ) + ) + + +def _remove_stdout_files(*, entry_path: Path, keep: str | None) -> None: + prefix: str = f"{entry_path.stem}{REUSE_STDOUT_SEPARATOR}" + try: + candidates: list[Path] = list(entry_path.parent.iterdir()) + except OSError: + return + for candidate in candidates: + if ( + candidate.name.startswith(prefix) + and candidate.suffix == REUSE_STDOUT_SUFFIX + and candidate.name != keep + ): + with contextlib.suppress(OSError): + candidate.unlink() + + +def _read_section(*, handle: BinaryIO) -> bytes: + length: int = int.from_bytes(handle.read(REUSE_ENTRY_LENGTH_BYTES), REUSE_ENTRY_BYTE_ORDER) + checksum: int = int.from_bytes(handle.read(REUSE_ENTRY_CHECKSUM_BYTES), REUSE_ENTRY_BYTE_ORDER) + if length > REUSE_MAX_ENTRY_BYTES: + raise CompileReuseEntryError("stored compile section is too large") + data: bytes = handle.read(length) + if len(data) != length or zlib.crc32(data) != checksum: + raise CompileReuseEntryError("stored compile section is truncated or corrupt") + return data + + +def _prune_entries(*, directory: Path, keep: Path) -> None: + entries: list[tuple[int, Path]] = [] + for candidate in directory.iterdir(): + if candidate.suffix != REUSE_ENTRY_SUFFIX or candidate == keep: + continue + with contextlib.suppress(OSError): + entries.append((candidate.stat().st_mtime_ns, candidate)) + entries.sort(reverse=True) + for _, stale in entries[REUSE_MAX_STORED_ENTRIES - 1 :]: + remove_entry(path=stale) + + +def _payload(*, inputs: StoredCompileInputs, output: StoredCompileOutput) -> dict[str, object]: + return { + "format": REUSE_FORMAT_VERSION, + "invocation_digest": inputs.invocation_digest, + "runtime": inputs.runtime, + "environment_names": list(inputs.environment_names), + "environment_digest": inputs.environment_digest, + "search_path": [list(item) for item in inputs.search_path], + "modules": [list(item) for item in inputs.modules], + "project_files": { + path: [*item.stamp, item.digest, item.racy] + for path, item in inputs.project_files.items() + }, + "target_files": {path: list(stamp) for path, stamp in inputs.target_files.items()}, + "stderr_lines": list(output.stderr_lines), + "exit_code": output.exit_code, + "timings_span": None if output.timings_span is None else list(output.timings_span), + "stdout_length": output.stdout_length, + "stdout_checksum": output.stdout_checksum, + "stdout_file": output.stdout_file, + "target_tree": inputs.target_tree, + "settings_inputs": [ + { + "case_sensitive": item.case_sensitive, + "names": list(item.names), + "prefixes": list(item.prefixes), + "env_files": list(item.env_files), + "secrets_dirs": list(item.secrets_dirs), + } + for item in inputs.settings_inputs + ], + "settings_digest": inputs.settings_digest, + } + + +def _header(*, payload: object) -> StoredCompileHeader: + fields: dict[str, object] = _mapping(payload) + if fields.get("format") != REUSE_FORMAT_VERSION: + raise CompileReuseEntryError("unknown stored compile format") + runtime: dict[str, object] = _mapping(fields["runtime"]) + span: tuple[int, int] | None = _timings_span(fields["timings_span"]) + return StoredCompileHeader( + inputs=StoredCompileInputs( + invocation_digest=_string(fields["invocation_digest"]), + runtime={_string(key): _string(value) for key, value in runtime.items()}, + environment_names=tuple(_string(name) for name in _list(fields["environment_names"])), + environment_digest=_string(fields["environment_digest"]), + search_path=tuple( + (_string(item[0]), _integer(item[1])) + for item in _rows(value=fields["search_path"], width=_SEARCH_PATH_FIELD_COUNT) + ), + modules=tuple( + (_string(item[0]), _integer(item[1]), _integer(item[2])) + for item in _rows(value=fields["modules"], width=_MODULE_FIELD_COUNT) + ), + project_files={ + _string(path): _project_file(item) + for path, item in _mapping(fields["project_files"]).items() + }, + target_files={ + _string(path): _stamp(item) + for path, item in _mapping(fields["target_files"]).items() + }, + target_tree=_boolean(fields["target_tree"]), + settings_inputs=tuple( + _settings_inputs(item) for item in _list(fields["settings_inputs"]) + ), + settings_digest=_string(fields["settings_digest"]), + ), + output=StoredCompileOutput( + stderr_lines=tuple(_string(line) for line in _list(fields["stderr_lines"])), + exit_code=_integer(fields["exit_code"]), + timings_span=span, + stdout_length=_integer(fields["stdout_length"]), + stdout_checksum=_integer(fields["stdout_checksum"]), + stdout_file=_stdout_file(fields["stdout_file"]), + ), + ) + + +def _settings_inputs(value: object) -> SettingsEnvironmentInputs: + fields: dict[str, object] = _mapping(value) + return SettingsEnvironmentInputs( + case_sensitive=_boolean(fields["case_sensitive"]), + names=_strings(fields["names"]), + prefixes=_strings(fields["prefixes"]), + env_files=_strings(fields["env_files"]), + secrets_dirs=_strings(fields["secrets_dirs"]), + ) + + +def _stdout_file(value: object) -> str: + name: str = _string(value) + if os.path.basename(name) != name or not name.endswith(REUSE_STDOUT_SUFFIX): + raise CompileReuseEntryError("invalid stored stdout file name") + return name + + +def _timings_span(value: object) -> tuple[int, int] | None: + if value is None: + return None + row: list[object] = _rows(value=[value], width=_TIMINGS_SPAN_LENGTH)[0] + return _integer(row[0]), _integer(row[1]) + + +def _project_file(value: object) -> StoredProjectFile: + if type(value) is not list or len(value) != _PROJECT_FILE_FIELD_COUNT: + raise CompileReuseEntryError("invalid stored project file") + row: list[object] = cast(list[object], value) + digest: object = row[6] + racy: object = row[7] + if type(racy) is not bool or not (digest is None or type(digest) is str): + raise CompileReuseEntryError("invalid stored project file") + return StoredProjectFile(stamp=_stamp(row[:_STAMP_FIELD_COUNT]), digest=digest, racy=racy) + + +def _stamp(value: object) -> FileStamp: + if type(value) is not list or len(value) != _STAMP_FIELD_COUNT: + raise CompileReuseEntryError("invalid stored file stamp") + kind, size, mtime_ns, ctime_ns, inode, link = cast(list[object], value) + if ( + type(kind) is not str + or type(size) is not int + or type(mtime_ns) is not int + or type(ctime_ns) is not int + or type(inode) is not int + or not (link is None or type(link) is str) + ): + raise CompileReuseEntryError("invalid stored file stamp") + return FileStamp(kind, size, mtime_ns, ctime_ns, inode, link) + + +def _rows(*, value: object, width: int) -> list[list[object]]: + rows: list[list[object]] = [_list(row) for row in _list(value)] + if any(len(row) != width for row in rows): + raise CompileReuseEntryError("invalid stored row") + return rows + + +def _list(value: object) -> list[object]: + if not isinstance(value, list): + raise CompileReuseEntryError("invalid stored list") + return cast(list[object], value) + + +def _mapping(value: object) -> dict[str, object]: + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + raise CompileReuseEntryError("invalid stored mapping") + return cast(dict[str, object], value) + + +def _strings(value: object) -> tuple[str, ...]: + return tuple(_string(item) for item in _list(value)) + + +def _boolean(value: object) -> bool: + if type(value) is not bool: + raise CompileReuseEntryError("invalid stored boolean") + return value + + +def _string(value: object) -> str: + if not isinstance(value, str): + raise CompileReuseEntryError("invalid stored string") + return value + + +def _integer(value: object) -> int: + if not isinstance(value, int) or isinstance(value, bool): + raise CompileReuseEntryError("invalid stored integer") + return value diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py new file mode 100644 index 0000000000..5c0472b33c --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/project_files.py @@ -0,0 +1,286 @@ +"""Stat walk, content digests, and change detection for project input files.""" + +from __future__ import annotations + +import hashlib +import os +import stat + +from sqlbuild.cli.compile_reuse.constants import ( + BROKEN_LINK_KIND, + DIGEST_SIZE_BYTES, + DIRECTORY_KIND, + DIRECTORY_LINK_KIND, + EXCLUDED_DIRECTORIES, + EXCLUDED_ROOT_DIRECTORIES, + FILE_KIND, + FILE_LINK_KIND, + OUTPUT_FILE_DESCRIPTORS, + PRESENCE_ONLY_FILE_SUFFIXES, + PRESENCE_ONLY_KIND, + RACY_WINDOW_NS, + READ_CHUNK_BYTES, + SPECIAL_FILE_KIND, +) +from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile +from sqlbuild.cli.compile_reuse.types import FileStamp + +_DIRECTORY_STAMP: FileStamp = FileStamp(DIRECTORY_KIND, 0, 0, 0, 0, None) +_PRESENCE_STAMP: FileStamp = FileStamp(PRESENCE_ONLY_KIND, 0, 0, 0, 0, None) +_PRESENCE_ONLY_KINDS: frozenset[str] = frozenset( + {DIRECTORY_KIND, DIRECTORY_LINK_KIND, BROKEN_LINK_KIND, PRESENCE_ONLY_KIND} +) +_HASHABLE_KINDS: frozenset[str] = frozenset({FILE_KIND, FILE_LINK_KIND}) + + +def snapshot_project_files(*, project_dir: str) -> dict[str, FileStamp]: + """Stat every compile-relevant path under the project, keyed by relative path.""" + + stamps: dict[str, FileStamp] = {} + prefix_length: int = len(project_dir) + 1 + output_files: frozenset[tuple[int, int]] = _redirected_output_files() + visited_links: set[str] = {os.path.realpath(project_dir)} + pending: list[tuple[str, bool]] = [(project_dir, True)] + while pending: + directory, is_root = pending.pop() + for entry in _directory_entries(directory=directory): + relative_path: str = entry.path[prefix_length:] + stamp: FileStamp | None = _entry_stamp( + entry=entry, is_root=is_root, output_files=output_files + ) + if stamp is None: + continue + stamps[relative_path] = stamp + if stamp.kind == DIRECTORY_KIND: + pending.append((entry.path, False)) + elif stamp.kind == DIRECTORY_LINK_KIND: + resolved: str = os.path.realpath(entry.path) + if resolved not in visited_links: + visited_links.add(resolved) + pending.append((entry.path, False)) + return stamps + + +def file_digest(*, path: str) -> str | None: + """Return the content digest of one file, or None when it cannot be read.""" + + digest: hashlib.blake2b = hashlib.blake2b(digest_size=DIGEST_SIZE_BYTES) + try: + with open(path, "rb") as handle: + while chunk := handle.read(READ_CHUNK_BYTES): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() + + +def compare_project_files( + *, + project_dir: str, + stored: dict[str, StoredProjectFile], + current: dict[str, FileStamp], +) -> ProjectFilesComparison: + """Compare stat identities, hashing content only where stat changed or may be racy.""" + + verified: dict[str, str] = {} + changed: ProjectFilesComparison = ProjectFilesComparison(unchanged=False, verified=verified) + if len(stored) != len(current): + return changed + for relative_path, stamp in current.items(): + previous: StoredProjectFile | None = stored.get(relative_path) + if previous is None or not _same_identity(stamp=stamp, recorded=previous.stamp): + return changed + if stamp.kind in _PRESENCE_ONLY_KINDS or (stamp == previous.stamp and not previous.racy): + continue + if stamp.kind not in _HASHABLE_KINDS or previous.digest is None: + return changed + digest: str | None = file_digest(path=os.path.join(project_dir, relative_path)) + if digest is None or digest != previous.digest: + return changed + verified[relative_path] = digest + return ProjectFilesComparison(unchanged=True, verified=verified) + + +def carried_forward_digests( + *, + stored: dict[str, StoredProjectFile], + current: dict[str, FileStamp], + verified: dict[str, str], +) -> dict[str, str]: + """Return digests still valid for current files, without reading any content.""" + + digests: dict[str, str] = dict(verified) + for relative_path, stamp in current.items(): + previous: StoredProjectFile | None = stored.get(relative_path) + if ( + relative_path not in digests + and previous is not None + and previous.digest is not None + and not previous.racy + and previous.stamp == stamp + ): + digests[relative_path] = previous.digest + return digests + + +def restamped_paths( + *, stored: dict[str, StoredProjectFile], current: dict[str, FileStamp] +) -> frozenset[str]: + """Return hashable files whose stat identity moved since the stored compile.""" + + return frozenset( + relative_path + for relative_path, stamp in current.items() + if stamp.kind in _HASHABLE_KINDS + and (previous := stored.get(relative_path)) is not None + and previous.stamp.kind == stamp.kind + and previous.stamp != stamp + ) + + +def stored_project_files( + *, snapshot: dict[str, FileStamp], digests: dict[str, str], snapshot_ns: int +) -> dict[str, StoredProjectFile]: + """Pair each project path's stat identity with any digest already known for it.""" + + return { + relative_path: StoredProjectFile( + stamp=stamp, + digest=digests.get(relative_path), + racy=is_racy(stamp=stamp, snapshot_ns=snapshot_ns), + ) + for relative_path, stamp in snapshot.items() + } + + +def needs_refresh(*, stored: dict[str, StoredProjectFile], current: dict[str, FileStamp]) -> bool: + """Return whether a hit verified any file by content, so its new stamp is worth storing.""" + + return any( + (previous := stored.get(relative_path)) is not None + and (previous.racy or previous.stamp != stamp) + and stamp.kind in _HASHABLE_KINDS + for relative_path, stamp in current.items() + ) + + +def is_racy(*, stamp: FileStamp, snapshot_ns: int) -> bool: + """Return whether a same-tick edit after the snapshot could leave this stamp unchanged.""" + + return ( + stamp.kind in _HASHABLE_KINDS + and max(stamp.mtime_ns, stamp.ctime_ns) >= snapshot_ns - RACY_WINDOW_NS + ) + + +def with_missing_digests( + *, + project_dir: str, + snapshot: dict[str, FileStamp], + digests: dict[str, str], + paths: frozenset[str], + snapshot_ns: int, +) -> dict[str, str]: + """Return digests extended with racy files, plus the given paths, that still lack one.""" + + completed: dict[str, str] = dict(digests) + for relative_path, stamp in snapshot.items(): + if ( + stamp.kind not in _HASHABLE_KINDS + or relative_path in completed + or not (relative_path in paths or is_racy(stamp=stamp, snapshot_ns=snapshot_ns)) + ): + continue + digest: str | None = file_digest(path=os.path.join(project_dir, relative_path)) + if digest is not None: + completed[relative_path] = digest + return completed + + +def pending_digest_bytes( + *, snapshot: dict[str, FileStamp], digests: dict[str, str], paths: frozenset[str] +) -> int: + """Return how many bytes hashing the given paths that still lack a digest would read.""" + + return sum( + snapshot[relative_path].size + for relative_path in paths + if relative_path in snapshot and relative_path not in digests + ) + + +def _same_identity(*, stamp: FileStamp, recorded: FileStamp) -> bool: + return ( + stamp.kind == recorded.kind + and stamp.link == recorded.link + and (stamp.kind in _PRESENCE_ONLY_KINDS or stamp.size == recorded.size) + ) + + +def _directory_entries(*, directory: str) -> list[os.DirEntry[str]]: + try: + with os.scandir(directory) as entries: + return list(entries) + except OSError: + return [] + + +def _entry_stamp( + *, entry: os.DirEntry[str], is_root: bool, output_files: frozenset[tuple[int, int]] +) -> FileStamp | None: + if entry.is_symlink(): + return _link_stamp(entry=entry, is_root=is_root) + if entry.is_dir(follow_symlinks=False): + return None if _excluded_directory(name=entry.name, is_root=is_root) else _DIRECTORY_STAMP + if entry.name.endswith(PRESENCE_ONLY_FILE_SUFFIXES): + return _PRESENCE_STAMP + try: + status: os.stat_result = entry.stat(follow_symlinks=False) + except OSError: + return None + if (status.st_dev, status.st_ino) in output_files: + return _PRESENCE_STAMP + return _file_stamp(status=status, link=None) + + +def _link_stamp(*, entry: os.DirEntry[str], is_root: bool) -> FileStamp | None: + try: + link: str = os.readlink(entry.path) + except OSError: + return None + try: + status: os.stat_result = os.stat(entry.path) + except OSError: + return FileStamp(BROKEN_LINK_KIND, 0, 0, 0, 0, link) + if not stat.S_ISDIR(status.st_mode): + return _file_stamp(status=status, link=link) + if _excluded_directory(name=entry.name, is_root=is_root): + return None + return FileStamp(DIRECTORY_LINK_KIND, 0, 0, 0, 0, link) + + +def _redirected_output_files() -> frozenset[tuple[int, int]]: + """Identify stdout and stderr files, which hold this command's output, not its inputs.""" + + identities: set[tuple[int, int]] = set() + for descriptor in OUTPUT_FILE_DESCRIPTORS: + try: + status: os.stat_result = os.fstat(descriptor) + except OSError: + continue + if stat.S_ISREG(status.st_mode): + identities.add((status.st_dev, status.st_ino)) + return frozenset(identities) + + +def _excluded_directory(*, name: str, is_root: bool) -> bool: + return name in EXCLUDED_DIRECTORIES or (is_root and name in EXCLUDED_ROOT_DIRECTORIES) + + +def _file_stamp(*, status: os.stat_result, link: str | None) -> FileStamp: + kind: str = SPECIAL_FILE_KIND + if stat.S_ISREG(status.st_mode): + kind = FILE_KIND if link is None else FILE_LINK_KIND + return FileStamp( + kind, status.st_size, status.st_mtime_ns, status.st_ctime_ns, status.st_ino, link + ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/provider_settings.py b/src/sqlbuild/cli/compile_reuse/_helpers/provider_settings.py new file mode 100644 index 0000000000..9af0523ed5 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/provider_settings.py @@ -0,0 +1,86 @@ +"""Enumerate where provider settings classes read values outside the project files.""" + +from __future__ import annotations + +import os +from collections.abc import Iterable +from typing import TYPE_CHECKING, cast + +from sqlbuild.cli.compile_reuse.models import SettingsEnvironmentInputs, SettingsInputsResult + +if TYPE_CHECKING: + from pydantic_settings import BaseSettings + +_CLI_CONFIG_KEYS: tuple[str, ...] = ("cli_parse_args", "cli_settings_source") + + +def provider_settings_inputs(*, settings_classes: tuple[type, ...]) -> SettingsInputsResult: + """Return every environment name, env file, and secrets dir the settings classes read.""" + + if not settings_classes: + return SettingsInputsResult(inputs=()) + from pydantic_settings import BaseSettings + + inputs: list[SettingsEnvironmentInputs] = [] + for settings_class in settings_classes: + if not issubclass(settings_class, BaseSettings): + continue + reason: str | None = _unsupported_reason(settings_class=settings_class) + if reason is not None: + return SettingsInputsResult(inputs=(), unsupported_reason=reason) + try: + inputs.append(_settings_inputs(settings_class=settings_class)) + except (AttributeError, TypeError, ValueError) as error: + return SettingsInputsResult( + inputs=(), + unsupported_reason=( + f"settings of {settings_class.__qualname__} could not be enumerated: {error}" + ), + ) + return SettingsInputsResult(inputs=tuple(inputs)) + + +def _unsupported_reason(*, settings_class: type[BaseSettings]) -> str | None: + from pydantic_settings import BaseSettings + + if ( + settings_class.settings_customise_sources.__func__ # type: ignore[attr-defined] + is not BaseSettings.settings_customise_sources.__func__ # type: ignore[attr-defined] + ): + return f"{settings_class.__qualname__} customizes its settings sources" + for key in _CLI_CONFIG_KEYS: + if settings_class.model_config.get(key) is not None: + return f"{settings_class.__qualname__} reads settings from command-line arguments" + return None + + +def _settings_inputs(*, settings_class: type[BaseSettings]) -> SettingsEnvironmentInputs: + from pydantic_settings import EnvSettingsSource + + source: EnvSettingsSource = EnvSettingsSource(settings_class) + nested_delimiter: str | None = source.env_nested_delimiter or None + names: set[str] = set() + for field_name, field in settings_class.model_fields.items(): + names.update(env_name for _, env_name, _ in source._extract_field_info(field, field_name)) + return SettingsEnvironmentInputs( + case_sensitive=bool(source.case_sensitive), + names=tuple(sorted(names)), + prefixes=() + if nested_delimiter is None + else tuple(sorted(f"{name}{nested_delimiter}" for name in names)), + env_files=_absolute_paths(settings_class.model_config.get("env_file")), + secrets_dirs=_absolute_paths(settings_class.model_config.get("secrets_dir")), + ) + + +def _absolute_paths(value: object) -> tuple[str, ...]: + if value is None: + return () + items: tuple[object, ...] = ( + (value,) if isinstance(value, (str, os.PathLike)) else tuple(cast(Iterable[object], value)) + ) + return tuple( + os.path.abspath(os.path.expanduser(os.fspath(item))) + for item in items + if isinstance(item, (str, os.PathLike)) + ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py new file mode 100644 index 0000000000..bad781b93b --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -0,0 +1,180 @@ +"""Interpreter, installed code, environment, and invocation identity for compile reuse.""" + +from __future__ import annotations + +import hashlib +import json +import os +import sys + +import sqlbuild._native as native +from sqlbuild.cli.compile_reuse._helpers.project_files import file_digest +from sqlbuild.cli.compile_reuse.constants import ( + DIGEST_SIZE_BYTES, + MISSING_ENVIRONMENT_VALUE, + MISSING_FILE_DIGEST, + MISSING_PATH_MTIME_NS, + PROJECT_ROOT_PATH_MTIME_NS, + REUSE_FORMAT_VERSION, + TRACKED_ENVIRONMENT_PREFIXES, +) +from sqlbuild.cli.compile_reuse.models import CompileReuseRequest, SettingsEnvironmentInputs + + +def runtime_identity() -> dict[str, str]: + """Return the interpreter and native build identity that produced a compile.""" + + return { + "format": str(REUSE_FORMAT_VERSION), + "python": sys.version, + "executable": sys.executable, + "platform": sys.platform, + "cache_tag": str(sys.implementation.cache_tag), + "native_build": str(native.BUILD_IDENTITY), + } + + +def search_path_stamps(*, project_dir: str) -> tuple[tuple[str, int], ...]: + """Stat import search path entries, leaving the project root to the project walk.""" + + project_root: str = os.path.realpath(project_dir) + stamps: list[tuple[str, int]] = [] + for entry in sys.path: + path: str = entry or "." + try: + mtime_ns: int = ( + PROJECT_ROOT_PATH_MTIME_NS + if os.path.realpath(path) == project_root + else os.stat(path).st_mtime_ns + ) + except OSError: + mtime_ns = MISSING_PATH_MTIME_NS + stamps.append((entry, mtime_ns)) + return tuple(stamps) + + +def loaded_module_stamps(*, covered_paths: frozenset[str]) -> tuple[tuple[str, int, int], ...]: + """Stat every loaded module file that the project walk does not already cover.""" + + stamps: dict[str, tuple[str, int, int]] = {} + for module in tuple(sys.modules.values()): + path: object = getattr(module, "__file__", None) + if not isinstance(path, str) or path in stamps or path in covered_paths: + continue + try: + status: os.stat_result = os.stat(path) + except OSError: + continue + stamps[path] = (path, status.st_mtime_ns, status.st_size) + return tuple(sorted(stamps.values())) + + +def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool: + """Return whether every recorded module file still has the same stat identity.""" + + for path, mtime_ns, size in stamps: + try: + status: os.stat_result = os.stat(path) + except OSError: + return False + if status.st_mtime_ns != mtime_ns or status.st_size != size: + return False + return True + + +def tracked_environment_names(*, template_names: tuple[str, ...]) -> tuple[str, ...]: + """Return SQLBuild-prefixed variables plus the variables templates read.""" + + prefixed: set[str] = { + name for name in os.environ if name.startswith(TRACKED_ENVIRONMENT_PREFIXES) + } + return tuple(sorted(prefixed.union(template_names))) + + +def environment_digest(*, names: tuple[str, ...]) -> str: + """Hash the current values of environment variables without retaining them.""" + + digest: hashlib.blake2b = hashlib.blake2b(digest_size=DIGEST_SIZE_BYTES) + for name in names: + digest.update(name.encode("utf-8", "surrogateescape")) + digest.update(b"\0") + digest.update( + os.environ.get(name, MISSING_ENVIRONMENT_VALUE).encode("utf-8", "surrogateescape") + ) + digest.update(b"\0") + return digest.hexdigest() + + +def settings_inputs_digest(*, inputs: tuple[SettingsEnvironmentInputs, ...]) -> str: + """Hash the provider settings values in the environment, env files, and secrets dirs.""" + + fields: list[str] = [] + for item in inputs: + for name, value in _settings_environment(inputs=item): + fields.extend(("env", name, value)) + for path in item.env_files: + fields.extend(("env_file", path, _file_contents_digest(path))) + for directory in item.secrets_dirs: + fields.extend(("secrets_dir", directory)) + for relative_path, contents_digest in _directory_contents(directory=directory): + fields.extend((relative_path, contents_digest)) + encoded: bytes = "\0".join(fields).encode("utf-8", "surrogateescape") + return hashlib.blake2b(encoded, digest_size=DIGEST_SIZE_BYTES).hexdigest() + + +def invocation_digest(*, request: CompileReuseRequest, project_dir: str, use_color: bool) -> str: + """Hash every command-line choice and location that can change compile output.""" + + payload: dict[str, object] = { + "project_dir": project_dir, + "requested_project_dir": None if request.project_dir is None else str(request.project_dir), + "cwd": os.getcwd(), + "no_sql_validation": request.no_sql_validation, + "defer_to": request.defer_to, + "selected_target": request.selected_target, + "json_output": request.json_output, + "manifest": request.manifest, + "dag_path": request.dag_path, + "no_color": request.no_color, + "use_color": use_color, + "lineage_mode": request.lineage_mode, + "select": list(request.select), + "exclude": list(request.exclude), + "cli_vars": request.cli_vars, + "sys_path": list(sys.path), + } + encoded: bytes = json.dumps( + payload, sort_keys=True, separators=(",", ":"), default=repr + ).encode("utf-8", "surrogateescape") + return hashlib.blake2b(encoded, digest_size=DIGEST_SIZE_BYTES).hexdigest() + + +def _settings_environment(*, inputs: SettingsEnvironmentInputs) -> list[tuple[str, str]]: + names: frozenset[str] = frozenset(inputs.names) + matched: list[tuple[str, str]] = [] + for name, value in os.environ.items(): + key: str = name if inputs.case_sensitive else name.lower() + if key in names or key.startswith(inputs.prefixes): + matched.append((name, value)) + return sorted(matched) + + +def _file_contents_digest(path: str) -> str: + return file_digest(path=path) or MISSING_FILE_DIGEST + + +def _directory_contents(*, directory: str) -> list[tuple[str, str]]: + contents: list[tuple[str, str]] = [] + for root, directories, filenames in os.walk(directory): + directories.sort() + for filename in sorted(filenames): + path: str = os.path.join(root, filename) + contents.append((os.path.relpath(path, directory), _file_contents_digest(path))) + return contents + + +def entry_slot_name(*, selected_target: str | None) -> str: + """Name the single stored entry kept for one selected target.""" + + label: str = "" if selected_target is None else f"target:{selected_target}" + return hashlib.blake2b(label.encode("utf-8"), digest_size=DIGEST_SIZE_BYTES).hexdigest() diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/store.py b/src/sqlbuild/cli/compile_reuse/_helpers/store.py new file mode 100644 index 0000000000..7bed444a1d --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/store.py @@ -0,0 +1,191 @@ +"""Record a completed full compile so an unchanged rerun can replay it.""" + +from __future__ import annotations + +import logging +import os +import sys +import time +from pathlib import Path + +from sqlbuild.cli.compile_reuse._helpers.entry_file import remove_entry, write_entry +from sqlbuild.cli.compile_reuse._helpers.project_files import ( + pending_digest_bytes, + snapshot_project_files, + stored_project_files, + with_missing_digests, +) +from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( + environment_digest, + loaded_module_stamps, + settings_inputs_digest, + tracked_environment_names, +) +from sqlbuild.cli.compile_reuse._helpers.target_files import verified_target_files +from sqlbuild.cli.compile_reuse._helpers.timings import compile_timings_span +from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites +from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput +from sqlbuild.cli.compile_reuse.constants import ( + BYTES_PER_MEBIBYTE, + REUSE_LOGGER_NAME, + REUSE_STORE_DONE_MESSAGE, + REUSE_STORE_NOTICE_BYTES, + REUSE_STORE_NOTICE_PATHS, + REUSE_STORE_SKIPPED_MESSAGE, + REUSE_STORE_START_MESSAGE, +) +from sqlbuild.cli.compile_reuse.models import ( + CompileReuseAttempt, + SettingsInputsResult, + StoredCompileInputs, + StoredCompileOutput, +) +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads + +_LOGGER: logging.Logger = logging.getLogger(REUSE_LOGGER_NAME) + + +def write_compile_entry( + *, + attempt: CompileReuseAttempt, + output: RecordedCompileOutput, + exit_code: int, + input_reads: CompileInputReads, + artifact_writes: CompileArtifactWrites, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Path | None, + json_output: bool, +) -> None: + """Store a reusable compile, or drop the stored one when this compile cannot be reused.""" + + if attempt.outcome is not CompileReuseOutcome.MISS or attempt.entry_path is None: + return + try: + _write_compile_entry( + attempt=attempt, + entry_path=attempt.entry_path, + output=output, + exit_code=exit_code, + input_reads=input_reads, + artifact_writes=artifact_writes, + compile_cache_enabled=compile_cache_enabled, + artifacts_written=artifacts_written, + dag_artifact_path=dag_artifact_path, + json_output=json_output, + ) + except Exception: + _LOGGER.debug("Compile reuse did not store this compile", exc_info=True) + remove_entry(path=attempt.entry_path) + + +def _write_compile_entry( + *, + attempt: CompileReuseAttempt, + entry_path: Path, + output: RecordedCompileOutput, + exit_code: int, + input_reads: CompileInputReads, + artifact_writes: CompileArtifactWrites, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Path | None, + json_output: bool, +) -> None: + stdout: str | None = output.stdout + settings: SettingsInputsResult = provider_settings_inputs( + settings_classes=input_reads.settings_classes + ) + if settings.unsupported_reason is not None: + _LOGGER.debug("Compile reuse is off for this project: %s", settings.unsupported_reason) + if ( + stdout is None + or not compile_cache_enabled + or input_reads.read_run_id + or settings.unsupported_reason is not None + ): + remove_entry(path=entry_path) + return + project_dir: str = str(attempt.project_dir) + notice_started: float | None = _start_notice(attempt=attempt) + digests: dict[str, str] = with_missing_digests( + project_dir=project_dir, + snapshot=attempt.snapshot, + digests=attempt.digests, + paths=attempt.restamped, + snapshot_ns=attempt.snapshot_ns, + ) + target_files: dict[str, FileStamp] | None = ( + verified_target_files( + project_dir=project_dir, + dag_path=None if dag_artifact_path is None else os.path.abspath(dag_artifact_path), + artifacts=artifact_writes.artifacts, + written=artifacts_written, + since_ns=attempt.snapshot_ns, + ) + if snapshot_project_files(project_dir=project_dir) == attempt.snapshot + else None + ) + if target_files is None: + remove_entry(path=entry_path) + _finish_notice(started=notice_started, message=REUSE_STORE_SKIPPED_MESSAGE) + return + environment_names: tuple[str, ...] = tracked_environment_names( + template_names=input_reads.environment_names + ) + write_entry( + path=entry_path, + inputs=StoredCompileInputs( + invocation_digest=attempt.invocation_digest, + runtime=attempt.runtime, + environment_names=environment_names, + environment_digest=environment_digest(names=environment_names), + search_path=attempt.search_path, + modules=loaded_module_stamps( + covered_paths=frozenset( + os.path.join(project_dir, relative_path) for relative_path in attempt.snapshot + ) + ), + project_files=stored_project_files( + snapshot=attempt.snapshot, digests=digests, snapshot_ns=attempt.snapshot_ns + ), + target_files=target_files, + target_tree=artifacts_written, + settings_inputs=settings.inputs, + settings_digest=settings_inputs_digest(inputs=settings.inputs), + ), + output=StoredCompileOutput( + stderr_lines=output.stderr_lines, + exit_code=exit_code, + timings_span=compile_timings_span(stdout=stdout) if json_output else None, + stdout_length=len(stdout), + stdout_checksum=0, + stdout_file="", + ), + stdout=stdout, + ) + _finish_notice(started=notice_started, message=REUSE_STORE_DONE_MESSAGE) + + +def _start_notice(*, attempt: CompileReuseAttempt) -> float | None: + pending_bytes: int = pending_digest_bytes( + snapshot=attempt.snapshot, digests=attempt.digests, paths=attempt.restamped + ) + if pending_bytes < REUSE_STORE_NOTICE_BYTES and len(attempt.snapshot) < ( + REUSE_STORE_NOTICE_PATHS + ): + return None + print( + REUSE_STORE_START_MESSAGE.format( + paths=len(attempt.snapshot), mebibytes=pending_bytes / BYTES_PER_MEBIBYTE + ), + file=sys.stderr, + ) + return time.monotonic() + + +def _finish_notice(*, started: float | None, message: str) -> None: + if started is not None: + print(message.format(seconds=time.monotonic() - started), file=sys.stderr) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py new file mode 100644 index 0000000000..72d1fd4ecf --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/target_files.py @@ -0,0 +1,113 @@ +"""Stat identity of the artifacts a compile leaves in target/.""" + +from __future__ import annotations + +import os + +from sqlbuild.cli.compile_reuse._helpers.project_files import file_digest +from sqlbuild.cli.compile_reuse.constants import ( + COMPILED_DIRECTORY_NAME, + FILE_KIND, + RACY_WINDOW_NS, + TARGET_DIRECTORY_NAME, +) +from sqlbuild.cli.compile_reuse.models import RecordedArtifact +from sqlbuild.cli.compile_reuse.types import FileStamp + + +def snapshot_target_files( + *, project_dir: str, extra_paths: tuple[str, ...] +) -> dict[str, FileStamp]: + """Stat every compiled artifact plus extra written artifacts, keyed by path.""" + + stamps: dict[str, FileStamp] = {} + pending: list[str] = [_compiled_root(project_dir=project_dir)] + while pending: + directory: str = pending.pop() + try: + entries: list[os.DirEntry[str]] = list(os.scandir(directory)) + except OSError: + continue + for entry in entries: + if entry.is_dir(follow_symlinks=False): + pending.append(entry.path) + continue + stamp: FileStamp | None = _path_stamp(path=entry.path) + if stamp is not None: + stamps[entry.path] = stamp + for path in extra_paths: + extra: FileStamp | None = _path_stamp(path=path) + if extra is not None: + stamps[path] = extra + return stamps + + +def verified_target_files( + *, + project_dir: str, + dag_path: str | None, + artifacts: dict[str, RecordedArtifact], + written: bool, + since_ns: int, +) -> dict[str, FileStamp] | None: + """Stamp this compile's artifacts, or return None when any changed since it wrote them.""" + + compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep + expected: dict[str, RecordedArtifact] = { + path: artifact + for path, artifact in artifacts.items() + if path.startswith(compiled_prefix) or path == dag_path + } + stamps: dict[str, FileStamp] = _target_stamps( + project_dir=project_dir, paths=tuple(expected), tree=written + ) + if stamps.keys() != expected.keys() or not all( + _matches_recorded(path=path, stamp=stamp, artifact=expected[path], since_ns=since_ns) + for path, stamp in stamps.items() + ): + return None + if _target_stamps(project_dir=project_dir, paths=tuple(expected), tree=written) != stamps: + return None + return stamps + + +def target_files_unchanged(*, stored: dict[str, FileStamp], project_dir: str, tree: bool) -> bool: + """Return whether every stored artifact is unchanged and, for a written tree, none was added.""" + + return _target_stamps(project_dir=project_dir, paths=tuple(stored), tree=tree) == stored + + +def _target_stamps(*, project_dir: str, paths: tuple[str, ...], tree: bool) -> dict[str, FileStamp]: + """Stamp the whole compiled tree plus extra paths, or only the given paths.""" + + if not tree: + return {path: stamp for path in paths if (stamp := _path_stamp(path=path)) is not None} + compiled_prefix: str = _compiled_root(project_dir=project_dir) + os.sep + extra_paths: tuple[str, ...] = tuple( + path for path in paths if not path.startswith(compiled_prefix) + ) + return snapshot_target_files(project_dir=project_dir, extra_paths=extra_paths) + + +def _matches_recorded( + *, path: str, stamp: FileStamp, artifact: RecordedArtifact, since_ns: int +) -> bool: + if artifact.digest is None: + return stamp.size == artifact.size and stamp.mtime_ns == artifact.mtime_ns + if max(stamp.mtime_ns, stamp.ctime_ns) < since_ns - RACY_WINDOW_NS: + return True + return file_digest(path=path) == artifact.digest + + +def _compiled_root(*, project_dir: str) -> str: + return os.path.join(project_dir, TARGET_DIRECTORY_NAME, COMPILED_DIRECTORY_NAME) + + +def _path_stamp(*, path: str) -> FileStamp | None: + try: + status: os.stat_result = os.lstat(path) + except OSError: + return None + return FileStamp( + FILE_KIND, status.st_size, status.st_mtime_ns, status.st_ctime_ns, status.st_ino, None + ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/timings.py b/src/sqlbuild/cli/compile_reuse/_helpers/timings.py new file mode 100644 index 0000000000..5ade4ea173 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/_helpers/timings.py @@ -0,0 +1,59 @@ +"""Locate and replace the compile_timings object inside a stored JSON report.""" + +from __future__ import annotations + +import json +import time + +from sqlbuild.cli.compile_reuse.constants import ( + COMPILE_TIMINGS_CLOSING, + COMPILE_TIMINGS_OPENING, + JSON_OBJECT_OPENING, + MILLISECONDS_PER_SECOND, + REUSE_BYPASS_TIMING, + REUSE_CHECK_TIMING, + REUSE_HIT_TIMING, + REUSE_MISS_TIMING, +) +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome + + +def compile_timings_span(*, stdout: str) -> tuple[int, int] | None: + """Return the character span of the top-level compile_timings object, if any.""" + + opening: int = stdout.find(COMPILE_TIMINGS_OPENING) + if opening < 0: + return None + start: int = opening + len(COMPILE_TIMINGS_OPENING) - 1 + closing: int = stdout.find(COMPILE_TIMINGS_CLOSING, start) + if closing < 0 or JSON_OBJECT_OPENING in stdout[start + 1 : closing]: + return None + return start, closing + len(COMPILE_TIMINGS_CLOSING) + + +def replace_compile_timings(*, stdout: str, span: tuple[int, int], timings: dict[str, int]) -> str: + """Splice freshly measured timings into a stored two-space-indented JSON report.""" + + rendered: str = ( + "{\n" + + ",\n".join(f" {json.dumps(name)}: {value}" for name, value in timings.items()) + + COMPILE_TIMINGS_CLOSING + ) + return stdout[: span[0]] + rendered + stdout[span[1] :] + + +def reuse_timings(*, outcome: CompileReuseOutcome, check_ms: int) -> dict[str, int]: + """Return the reuse counters and check time reported in compile_timings.""" + + return { + REUSE_HIT_TIMING: int(outcome is CompileReuseOutcome.HIT), + REUSE_MISS_TIMING: int(outcome is CompileReuseOutcome.MISS), + REUSE_BYPASS_TIMING: int(outcome is CompileReuseOutcome.BYPASS), + REUSE_CHECK_TIMING: check_ms, + } + + +def elapsed_ms(*, started: float) -> int: + """Return whole milliseconds elapsed since a monotonic start time.""" + + return int((time.monotonic() - started) * MILLISECONDS_PER_SECOND) diff --git a/src/sqlbuild/cli/compile_reuse/classes/__init__.py b/src/sqlbuild/cli/compile_reuse/classes/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py new file mode 100644 index 0000000000..069c0f24a9 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_write_registry.py @@ -0,0 +1,69 @@ +"""Process-wide delivery of compile artifact writes to the active compile recorders.""" + +from __future__ import annotations + +import hashlib +import os +import threading +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites +from sqlbuild.cli.compile_reuse.models import RecordedArtifact + + +class CompileArtifactWriteRegistry: + """Deliver artifact writes from every thread, including background staging threads.""" + + def __init__(self, *, digest_size: int) -> None: + self._digest_size: int = digest_size + self._lock: threading.Lock = threading.Lock() + self._recorders: tuple[CompileArtifactWrites, ...] = () + + @contextmanager + def recording(self) -> Iterator[CompileArtifactWrites]: + """Record artifact writes until the block exits.""" + + recorder: CompileArtifactWrites = CompileArtifactWrites() + with self._lock: + self._recorders = (*self._recorders, recorder) + try: + yield recorder + finally: + with self._lock: + self._recorders = tuple(item for item in self._recorders if item is not recorder) + + def written(self, *, path: str | os.PathLike[str], contents: bytes) -> None: + """Note the exact bytes an artifact holds after a write or an unchanged-file check.""" + + recorders: tuple[CompileArtifactWrites, ...] = self._recorders + if not recorders: + return + artifact: RecordedArtifact = RecordedArtifact( + digest=hashlib.blake2b(contents, digest_size=self._digest_size).hexdigest() + ) + for recorder in recorders: + recorder.record(path=os.path.abspath(path), artifact=artifact) + + def kept(self, *, path: str | os.PathLike[str], size: int, mtime_ns: int) -> None: + """Note an earlier artifact kept as-is after its size and mtime were validated.""" + + artifact: RecordedArtifact = RecordedArtifact(digest=None, size=size, mtime_ns=mtime_ns) + for recorder in self._recorders: + recorder.record(path=os.path.abspath(path), artifact=artifact) + + def moved(self, *, source: str | os.PathLike[str], destination: str | os.PathLike[str]) -> None: + """Note that one staged artifact was published at another path.""" + + for recorder in self._recorders: + recorder.move(source=os.path.abspath(source), destination=os.path.abspath(destination)) + + def moved_tree( + self, *, source: str | os.PathLike[str], destination: str | os.PathLike[str] + ) -> None: + """Note that a staged artifact directory was published at another directory.""" + + for recorder in self._recorders: + recorder.move_tree( + source=os.path.abspath(source), destination=os.path.abspath(destination) + ) diff --git a/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py new file mode 100644 index 0000000000..8f797a49c0 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/classes/compile_artifact_writes.py @@ -0,0 +1,53 @@ +"""Artifacts one compile wrote or kept, keyed by absolute path.""" + +from __future__ import annotations + +import os +import threading + +from sqlbuild.cli.compile_reuse.models import RecordedArtifact + + +class CompileArtifactWrites: + """Record what one compile left in each artifact path, following staged moves.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._artifacts: dict[str, RecordedArtifact] = {} + + def record(self, *, path: str, artifact: RecordedArtifact) -> None: + """Record the artifact this compile left at one path.""" + + with self._lock: + self._artifacts[path] = artifact + + def move(self, *, source: str, destination: str) -> None: + """Follow one staged artifact to its published path.""" + + with self._lock: + artifact: RecordedArtifact | None = self._artifacts.pop(source, None) + if artifact is not None: + self._artifacts[destination] = artifact + + def move_tree(self, *, source: str, destination: str) -> None: + """Follow every staged artifact under one directory to its published directory.""" + + prefix: str = source + os.sep + with self._lock: + moved: dict[str, RecordedArtifact] = { + destination + path[len(source) :]: artifact + for path, artifact in self._artifacts.items() + if path.startswith(prefix) + } + self._artifacts = { + path: artifact + for path, artifact in self._artifacts.items() + if not path.startswith(prefix) + } | moved + + @property + def artifacts(self) -> dict[str, RecordedArtifact]: + """Return the recorded artifacts by absolute path.""" + + with self._lock: + return dict(self._artifacts) diff --git a/src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py b/src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py new file mode 100644 index 0000000000..fec9748f59 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/classes/recorded_compile_output.py @@ -0,0 +1,43 @@ +"""Compile command output that is printed and retained for later reuse.""" + +from __future__ import annotations + +import sys + + +class RecordedCompileOutput: + """Print compile notes at once and the report after storing, keeping the exact text.""" + + def __init__(self) -> None: + self._stderr_lines: list[str] = [] + self._stdout: str | None = None + + def print_stderr_line(self, line: str) -> None: + """Print one stderr result line, such as a note, and record it.""" + + print(line, file=sys.stderr) + self._stderr_lines.append(line) + + def record_stdout(self, text: str) -> None: + """Record the compile report; it is printed once the compile has been stored.""" + + self._stdout = f"{text}\n" + + def print_stdout(self) -> None: + """Print the recorded compile report, if any, exactly as it is stored.""" + + if self._stdout is not None: + _ = sys.stdout.write(self._stdout) + sys.stdout.flush() + + @property + def stderr_lines(self) -> tuple[str, ...]: + """Return the recorded stderr result lines in emission order.""" + + return tuple(self._stderr_lines) + + @property + def stdout(self) -> str | None: + """Return the recorded stdout text, or None when no report was printed.""" + + return self._stdout diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py new file mode 100644 index 0000000000..b5af8a4d5b --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -0,0 +1,92 @@ +"""Compile reuse storage, fingerprint, and environment constants.""" + +from __future__ import annotations + +from typing import Literal + +from sqlbuild.cli.compile_reuse.classes.compile_artifact_write_registry import ( + CompileArtifactWriteRegistry, +) + +REUSE_FORMAT_VERSION: int = 1 +REUSE_ENTRY_DIRECTORY_PARTS: tuple[str, ...] = ("target", "cache", "compiler", "project-reuse-v1") +REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" +REUSE_ENTRY_SUFFIX: str = ".entry" +REUSE_STDOUT_SUFFIX: str = ".stdout" +REUSE_STDOUT_SEPARATOR: str = "-" +REUSE_ENTRY_LENGTH_BYTES: int = 8 +REUSE_ENTRY_CHECKSUM_BYTES: int = 4 +REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" +REUSE_MAX_ENTRY_BYTES: int = 512 * 1024 * 1024 +REUSE_MAX_STORED_ENTRIES: int = 8 +REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" +REUSE_DISABLE_VALUE: str = "1" +REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" +TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") +MISSING_ENVIRONMENT_VALUE: str = "\0missing" +MISSING_FILE_DIGEST: str = "\0missing" +PROJECT_CONFIG_FILENAMES: tuple[str, ...] = ("sqlbuild_project.toml", "sqlbuild_project.yml") + +RACY_WINDOW_NS: int = 2_000_000_000 +DIGEST_SIZE_BYTES: int = 16 +READ_CHUNK_BYTES: int = 1024 * 1024 +MISSING_PATH_MTIME_NS: int = -1 +PROJECT_ROOT_PATH_MTIME_NS: int = -2 +OUTPUT_FILE_DESCRIPTORS: tuple[int, ...] = (1, 2) + +TARGET_DIRECTORY_NAME: str = "target" +COMPILED_DIRECTORY_NAME: str = "compiled" +EXCLUDED_ROOT_DIRECTORIES: frozenset[str] = frozenset( + { + "logs", + "target", + "venv", + ".cache", + ".fensu", + ".hg", + ".idea", + ".mypy_cache", + ".nox", + ".pytest_cache", + ".ruff_cache", + ".sqlbuild", + ".svn", + ".tox", + ".venv", + ".vscode", + } +) +EXCLUDED_DIRECTORIES: frozenset[str] = frozenset({".git", "__pycache__"}) +PRESENCE_ONLY_FILE_SUFFIXES: tuple[str, ...] = (".duckdb", ".duckdb.wal") + +FILE_KIND: str = "f" +FILE_LINK_KIND: str = "fl" +DIRECTORY_KIND: str = "d" +DIRECTORY_LINK_KIND: str = "dl" +BROKEN_LINK_KIND: str = "bl" +SPECIAL_FILE_KIND: str = "s" +PRESENCE_ONLY_KIND: str = "p" + +COMPILE_TIMINGS_OPENING: str = '\n "compile_timings": {' +COMPILE_TIMINGS_CLOSING: str = "\n }" +JSON_OBJECT_OPENING: str = "{" +REUSE_HIT_TIMING: str = "project_reuse_hits" +REUSE_MISS_TIMING: str = "project_reuse_misses" +REUSE_BYPASS_TIMING: str = "project_reuse_bypasses" +REUSE_CHECK_TIMING: str = "project_reuse_check_ms" +TOTAL_TIMING: str = "total_ms" +REUSE_HIT_MESSAGE: str = "Inputs unchanged; reused the previous compile ({seconds:.1f} s)" +MILLISECONDS_PER_SECOND: int = 1000 +REUSE_STORE_NOTICE_BYTES: int = 64 * 1024 * 1024 +REUSE_STORE_NOTICE_PATHS: int = 100_000 +BYTES_PER_MEBIBYTE: int = 1024 * 1024 +REUSE_STORE_START_MESSAGE: str = ( + "Recording compile for reuse ({paths} paths, {mebibytes:.0f} MiB to hash)..." +) +REUSE_STORE_DONE_MESSAGE: str = "Recorded compile for reuse ({seconds:.1f} s)" +REUSE_STORE_SKIPPED_MESSAGE: str = ( + "Did not record compile for reuse: files changed while it ran ({seconds:.1f} s)" +) +COMPILE_ARTIFACT_WRITES: CompileArtifactWriteRegistry = CompileArtifactWriteRegistry( + digest_size=DIGEST_SIZE_BYTES +) diff --git a/src/sqlbuild/cli/compile_reuse/exceptions.py b/src/sqlbuild/cli/compile_reuse/exceptions.py new file mode 100644 index 0000000000..7e360b538d --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/exceptions.py @@ -0,0 +1,7 @@ +"""Compile reuse storage errors.""" + +from __future__ import annotations + + +class CompileReuseEntryError(ValueError): + """A stored compile is truncated, corrupt, or written in an unknown format.""" diff --git a/src/sqlbuild/cli/compile_reuse/main/__init__.py b/src/sqlbuild/cli/compile_reuse/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py b/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py new file mode 100644 index 0000000000..587fd42003 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_attempt_compile_reuse.py @@ -0,0 +1,12 @@ +"""Public entry for replaying the previous compile when no compile input changed.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_reuse._helpers.attempt import attempt_reuse +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt, CompileReuseRequest + + +def attempt_compile_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: + """Replay the stored compile on a hit, otherwise return the state a full compile stores.""" + + return attempt_reuse(request=request) diff --git a/src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py b/src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py new file mode 100644 index 0000000000..f98d3d077b --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_compile_reuse_timings.py @@ -0,0 +1,12 @@ +"""Public entry for the compile reuse counters reported in compile_timings.""" + +from __future__ import annotations + +from sqlbuild.cli.compile_reuse._helpers.timings import reuse_timings +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt + + +def compile_reuse_timings(*, attempt: CompileReuseAttempt) -> dict[str, int]: + """Return the hit, miss, and bypass counters plus the time spent checking for reuse.""" + + return reuse_timings(outcome=attempt.outcome, check_ms=attempt.check_ms) diff --git a/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py new file mode 100644 index 0000000000..0736318972 --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/main/_write_reusable_compile.py @@ -0,0 +1,38 @@ +"""Public entry for storing a full compile so an unchanged rerun can replay it.""" + +from __future__ import annotations + +from pathlib import Path + +from sqlbuild.cli.compile_reuse._helpers.store import write_compile_entry +from sqlbuild.cli.compile_reuse.classes.compile_artifact_writes import CompileArtifactWrites +from sqlbuild.cli.compile_reuse.classes.recorded_compile_output import RecordedCompileOutput +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads + + +def write_reusable_compile( + *, + attempt: CompileReuseAttempt, + output: RecordedCompileOutput, + exit_code: int, + input_reads: CompileInputReads, + artifact_writes: CompileArtifactWrites, + compile_cache_enabled: bool, + artifacts_written: bool, + dag_artifact_path: Path | None, + json_output: bool, +) -> None: + """Store a reusable full compile, or drop the stored one when it cannot be reused.""" + + write_compile_entry( + attempt=attempt, + output=output, + exit_code=exit_code, + input_reads=input_reads, + artifact_writes=artifact_writes, + compile_cache_enabled=compile_cache_enabled, + artifacts_written=artifacts_written, + dag_artifact_path=dag_artifact_path, + json_output=json_output, + ) diff --git a/src/sqlbuild/cli/compile_reuse/models.py b/src/sqlbuild/cli/compile_reuse/models.py new file mode 100644 index 0000000000..cc2bd55c4b --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/models.py @@ -0,0 +1,130 @@ +"""Compile reuse requests, stored compile results, and per-invocation reuse state.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path + +from sqlbuild.cli.compile_reuse.types import CompileReuseOutcome, FileStamp + + +@dataclass(frozen=True) +class CompileReuseRequest: + """The compile command choices that decide whether and how a compile is reused.""" + + project_dir: Path | None + selected_target: str | None + json_output: bool + no_color: bool + manifest: bool + dag_path: str | None + defer_to: str | None + no_cache: bool + no_sql_validation: bool + lineage_mode: str + select: tuple[str, ...] + exclude: tuple[str, ...] + cli_vars: dict[str, object] | None + profiling: bool + debug: bool + + +@dataclass(frozen=True) +class SettingsEnvironmentInputs: + """Where one provider settings class can read values outside the project files.""" + + case_sensitive: bool + names: tuple[str, ...] + prefixes: tuple[str, ...] + env_files: tuple[str, ...] + secrets_dirs: tuple[str, ...] + + +@dataclass(frozen=True) +class SettingsInputsResult: + """Enumerated provider settings inputs, or why they cannot be enumerated exactly.""" + + inputs: tuple[SettingsEnvironmentInputs, ...] + unsupported_reason: str | None = None + + +@dataclass(frozen=True) +class RecordedArtifact: + """What one compile wrote to an artifact: its content digest, or the stat it kept.""" + + digest: str | None + size: int | None = None + mtime_ns: int | None = None + + +@dataclass(frozen=True) +class StoredProjectFile: + """One project path recorded before the stored compile ran.""" + + stamp: FileStamp + digest: str | None + racy: bool + + +@dataclass(frozen=True) +class StoredCompileInputs: + """Every compile input identity recorded with one stored compile result.""" + + invocation_digest: str + runtime: dict[str, str] + environment_names: tuple[str, ...] + environment_digest: str + search_path: tuple[tuple[str, int], ...] + modules: tuple[tuple[str, int, int], ...] + project_files: dict[str, StoredProjectFile] + target_files: dict[str, FileStamp] + target_tree: bool + settings_inputs: tuple[SettingsEnvironmentInputs, ...] + settings_digest: str + + +@dataclass(frozen=True) +class StoredCompileOutput: + """The command output of one stored compile, in emission order.""" + + stderr_lines: tuple[str, ...] + exit_code: int + timings_span: tuple[int, int] | None + stdout_length: int + stdout_checksum: int + stdout_file: str + + +@dataclass(frozen=True) +class StoredCompileHeader: + """Inputs and output description read without loading the stored stdout.""" + + inputs: StoredCompileInputs + output: StoredCompileOutput + + +@dataclass(frozen=True) +class ProjectFilesComparison: + """Whether project files are unchanged, with digests verified while comparing.""" + + unchanged: bool + verified: dict[str, str] + + +@dataclass(frozen=True) +class CompileReuseAttempt: + """Result of checking for a reusable compile, carried into a full compile on a miss.""" + + outcome: CompileReuseOutcome + project_dir: Path + started: float + check_ms: int = 0 + exit_code: int | None = None + entry_path: Path | None = None + invocation_digest: str = "" + runtime: dict[str, str] = field(default_factory=dict) + search_path: tuple[tuple[str, int], ...] = () + snapshot: dict[str, FileStamp] = field(default_factory=dict) + snapshot_ns: int = 0 + digests: dict[str, str] = field(default_factory=dict) + restamped: frozenset[str] = frozenset() diff --git a/src/sqlbuild/cli/compile_reuse/types.py b/src/sqlbuild/cli/compile_reuse/types.py new file mode 100644 index 0000000000..e681ba99af --- /dev/null +++ b/src/sqlbuild/cli/compile_reuse/types.py @@ -0,0 +1,25 @@ +"""Compile reuse outcome and file stamp types.""" + +from __future__ import annotations + +from enum import StrEnum +from typing import NamedTuple + + +class CompileReuseOutcome(StrEnum): + """How one compile invocation used the stored previous compile.""" + + HIT = "hit" + MISS = "miss" + BYPASS = "bypass" + + +class FileStamp(NamedTuple): + """Filesystem identity of one project path, as seen by a stat walk.""" + + kind: str + size: int + mtime_ns: int + ctime_ns: int + inode: int + link: str | None diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py index 90ff1071a4..31f59a7118 100644 --- a/src/sqlbuild/cli/entry/models.py +++ b/src/sqlbuild/cli/entry/models.py @@ -48,6 +48,7 @@ class CompileCommandRequest: cli_vars: dict[str, object] | None = None profile_flags: CompileProfileFlags = CompileProfileFlags() no_cache: bool = False + debug: bool = False @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index f621c5c7cc..d3b115feab 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -18,6 +18,7 @@ expand_sql_macros_with_spans, ) from sqlbuild.compiler.compile.constants import ( + COMPILE_INPUT_READS, SQL_CONTEXT_NAME_EXTRA_TOKENS, SQL_IDENTIFIER_EXTRA_TOKEN, SQL_INTERPOLATION_TOKEN, @@ -366,6 +367,7 @@ def _render_interpolation_token( if env_name_end == env_name_start: raise CompileInputError(f"invalid environment interpolation token in '{file_path}'") env_name: str = sql[env_name_start:env_name_end] + COMPILE_INPUT_READS.environment_read(env_name) if env_name not in os.environ: raise CompileInputError( f"unknown environment variable '@@ENV:{env_name}' in '{file_path}'" @@ -387,6 +389,7 @@ def _render_interpolation_token( context_values=context_values, ) context_name = sql[context_name_start:context_name_end] + COMPILE_INPUT_READS.context_read(context_name) if context_name not in context_values: raise CompileInputError( f"SQL text in '{file_path}' references unknown CTX key '{context_name}'" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 3bd6eb68c0..54dd2dd0eb 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -9,6 +9,7 @@ from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text from sqlbuild.compiler.compile.constants import ( + COMPILE_INPUT_READS, MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, MISSING_TEMPLATE_VALUE_MESSAGE_PARTS, @@ -521,6 +522,7 @@ def _lookup_variable(*, name: str, variables: dict[str, object], context_label: def _lookup_environment_variable(*, name: str, context_label: str) -> str: + COMPILE_INPUT_READS.environment_read(name) if name not in os.environ: raise CompileInputError(f"{context_label} references missing ENV variable '{name}'") return os.environ[name] @@ -533,6 +535,7 @@ def _lookup_context_value( context_label: str, preserve_unknown_context: bool, ) -> str: + COMPILE_INPUT_READS.context_read(name) if name not in context_values: if preserve_unknown_context: return f"${{{TemplateNamespace.CTX}:{name}}}" diff --git a/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py b/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py new file mode 100644 index 0000000000..0b2a8370bf --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/compile_input_read_registry.py @@ -0,0 +1,51 @@ +"""Process-wide delivery of template input reads to the active compile recorders.""" + +from __future__ import annotations + +import threading +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlbuild.compiler.compile.classes.compile_input_reads import CompileInputReads +from sqlbuild.compiler.compile.types import CompileContextKey + + +class CompileInputReadRegistry: + """Deliver template reads from every thread, including ones without copied context.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._recorders: tuple[CompileInputReads, ...] = () + + @contextmanager + def recording(self) -> Iterator[CompileInputReads]: + """Record environment variables and run-identity reads until the block exits.""" + + recorder: CompileInputReads = CompileInputReads() + with self._lock: + self._recorders = (*self._recorders, recorder) + try: + yield recorder + finally: + with self._lock: + self._recorders = tuple(item for item in self._recorders if item is not recorder) + + def environment_read(self, name: str) -> None: + """Note that a template looked up one environment variable, present or not.""" + + for recorder in self._recorders: + recorder.add_environment_name(name) + + def settings_class_read(self, settings_class: type) -> None: + """Note a provider settings class instantiated from the environment and files.""" + + for recorder in self._recorders: + recorder.add_settings_class(settings_class) + + def context_read(self, name: str) -> None: + """Note a CTX lookup; only the per-invocation run identity is volatile.""" + + if name != CompileContextKey.RUN_ID: + return + for recorder in self._recorders: + recorder.mark_run_id_read() diff --git a/src/sqlbuild/compiler/compile/classes/compile_input_reads.py b/src/sqlbuild/compiler/compile/classes/compile_input_reads.py new file mode 100644 index 0000000000..62841a8eef --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/compile_input_reads.py @@ -0,0 +1,54 @@ +"""Volatile inputs that templates read while one compile renders the project.""" + +from __future__ import annotations + +import threading + + +class CompileInputReads: + """Environment, settings, and run-identity reads observed during one compile.""" + + def __init__(self) -> None: + self._lock: threading.Lock = threading.Lock() + self._environment_names: set[str] = set() + self._read_run_id: bool = False + self._settings_classes: dict[type, None] = {} + + def add_environment_name(self, name: str) -> None: + """Record one environment variable a template looked up, present or not.""" + + with self._lock: + self._environment_names.add(name) + + def add_settings_class(self, settings_class: type) -> None: + """Record one settings class whose values come from the environment or files.""" + + with self._lock: + self._settings_classes[settings_class] = None + + def mark_run_id_read(self) -> None: + """Record that a template read the per-invocation run identity.""" + + with self._lock: + self._read_run_id = True + + @property + def environment_names(self) -> tuple[str, ...]: + """Return the recorded environment variable names in sorted order.""" + + with self._lock: + return tuple(sorted(self._environment_names)) + + @property + def settings_classes(self) -> tuple[type, ...]: + """Return the recorded settings classes in discovery order.""" + + with self._lock: + return tuple(self._settings_classes) + + @property + def read_run_id(self) -> bool: + """Return whether any template read the per-invocation run identity.""" + + with self._lock: + return self._read_run_id diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 793916ddc0..cabdcb1316 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -4,6 +4,9 @@ import re +from sqlbuild.compiler.compile.classes.compile_input_read_registry import ( + CompileInputReadRegistry, +) from sqlbuild.compiler.compile.types import SqlTestMode from sqlbuild.compiler.references.types import SqlReferenceKind @@ -167,6 +170,7 @@ ) COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" +COMPILE_INPUT_READS: CompileInputReadRegistry = CompileInputReadRegistry() SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v2" SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 8197740900..a4ec31738f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -13,6 +13,7 @@ from types import ModuleType from typing import TYPE_CHECKING, cast, get_type_hints +from sqlbuild.compiler.compile.constants import COMPILE_INPUT_READS from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( parse_source_file_with_cache, ) @@ -1004,6 +1005,8 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, | None ) = cached_event_exporter_extensions(project_dir=project_dir) if cached is not None: + for cached_provider in cached[0]: + COMPILE_INPUT_READS.settings_class_read(cached_provider.provider_class) return cached[0] providers_root: Path = project_dir / "providers" if not providers_root.is_dir(): @@ -1034,6 +1037,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, f"{file_path.relative_to(project_dir)}" ) seen_names[provider_name] = file_path + COMPILE_INPUT_READS.settings_class_read(provider_class) discovered_providers.append( DiscoveredProvider( file_path=file_path, diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000000..2427d418f2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +"""Suite-wide test configuration.""" + +from __future__ import annotations + +import os + +import pytest + +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE + + +def pytest_configure(config: pytest.Config) -> None: + """Exercise full compiles by default; compile reuse tests opt back in explicitly.""" + + del config + os.environ.setdefault(REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index a79d284bc6..47e87b2de1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -90,6 +90,10 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "project_reuse_hits", + "project_reuse_misses", + "project_reuse_bypasses", + "project_reuse_check_ms", "total_ms", } assert all(isinstance(value, int) and value >= 0 for value in timings.values()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index e0ece67ea9..3ca4ed3d78 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -558,3 +558,126 @@ class EarlyLintCompileTestCase: order_totals_sql: str expected_exit_code: int expected_diagnostics: tuple[tuple[str, str, int, int], ...] + + +@dataclass(frozen=True) +class CompileReuseInvalidationTestCase: + """One compile input change that must make the stored compile miss.""" + + description: str + edit: Callable[[Path], None] = lambda _project_dir: None + edited_env: dict[str, str] = field(default_factory=dict) + edited_args: tuple[str, ...] = () + expected_returncode: int = 0 + expected_rewarm_hit: bool = True + + +@dataclass(frozen=True) +class CompileReuseHitTestCase: + """One change outside the compile inputs that must still reuse the stored compile.""" + + description: str + edit: Callable[[Path], None] + edited_env: dict[str, str] = field(default_factory=dict) + expected_reused: bool = True + + +@dataclass(frozen=True) +class CompileReuseBypassTestCase: + """One control or input that must always run the full compile.""" + + description: str + edit: Callable[[Path], None] = lambda _project_dir: None + compile_args: tuple[str, ...] = () + global_args: tuple[str, ...] = () + env: dict[str, str] = field(default_factory=dict) + expected_stored_entries: int = 0 + + +@dataclass(frozen=True) +class CompileReuseCorruptEntryTestCase: + """One way a stored compile can be damaged on disk.""" + + description: str + corrupt: Callable[[Path], None] + expected_fallback_reused: bool = False + expected_restored_reused: bool = True + + +@dataclass(frozen=True) +class CompileReuseReplayTestCase: + """One compile whose replay must reproduce output, notes, and exit code.""" + + description: str + compile_args: tuple[str, ...] + expected_returncode: int + expected_note_prefix: str = "note:" + + +@dataclass(frozen=True) +class CompileReuseRedirectTestCase: + """A compile whose stdout is redirected into a file inside the project.""" + + description: str + report_name: str + expected_hit_counts: tuple[int, int] + + +@dataclass(frozen=True) +class CompileReuseEventTestCase: + """A lifecycle sink that must still receive invocation events on a reuse hit.""" + + description: str + sink_name: str + expected_event_types: tuple[str, ...] + + +@dataclass(frozen=True) +class CompileReuseTimingsTestCase: + """The compile_timings object a reuse hit reports in place of the stored timings.""" + + description: str + expected_timing_names: frozenset[str] + + +@dataclass(frozen=True) +class CompileReuseProviderSettingsTestCase: + """A provider settings input change that must miss and match a full compile.""" + + description: str + edited_env: dict[str, str] + edit: Callable[[Path], None] = lambda _project_dir: None + expected_returncode: int = 0 + expected_rewarm_hit: bool = True + + +@dataclass(frozen=True) +class CompileReuseConcurrentWriterTestCase: + """Another writer touching compiled artifacts between this compile's write and store.""" + + description: str + other_write: Callable[[Path], None] + expected_stored_entries: int + expected_rerun_reused: bool + + +@dataclass(frozen=True) +class CompileReuseLargeFileTestCase: + """A large data file that reuse must not read unless its stat identity moves.""" + + description: str + size_bytes: int + relative_path: str + expected_reads: tuple[int, int, int, int, int] + expected_reused: tuple[bool, bool, bool, bool, bool] + + +@dataclass(frozen=True) +class CompileReuseStoreFailureTestCase: + """An unexpected error while storing a finished compile.""" + + description: str + error: BaseException + expected_returncode: int = 0 + expected_stored_entries: int = 0 + expected_raised: type[BaseException] | None = None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py index 2679e022f6..60b8811e59 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/conftest.py @@ -13,6 +13,7 @@ InspectionCommandMeasurement, prepare_build_benchmark_project, prepare_compile_cache_invalidation_project, + prepare_compile_reuse_project, prepare_inspection_benchmark_project, run_fresh_process_inspection_command, ) @@ -41,6 +42,13 @@ def cache_invalidation_project(tmp_path: Path) -> Path: return project_dir +@pytest.fixture +def compile_reuse_project(tmp_path: Path) -> Path: + project_dir: Path = tmp_path / "orders_project" + prepare_compile_reuse_project(project_dir=project_dir) + return project_dir + + @pytest.fixture(scope="module") def inspection_benchmark_project(tmp_path_factory: pytest.TempPathFactory) -> Path: assert_required_cgroup_memory_limit() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index e203868234..29858c0293 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -14,6 +14,7 @@ from bisect import bisect_left from collections.abc import Callable, Iterator from contextlib import contextmanager, redirect_stdout +from dataclasses import replace from io import StringIO from pathlib import Path from types import FrameType @@ -22,11 +23,21 @@ import duckdb import pytest +import sqlbuild.cli.commands.main.project._compile as compile_command_module +import sqlbuild.cli.compile_reuse._helpers.project_files as reuse_project_files +import sqlbuild.cli.compile_reuse._helpers.store as reuse_store from scripts.cold_compile_performance.main.read_compile_measurement import read_compile_measurement from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, ) from sqlbuild.cli.commands.main.entrypoint.entry import main +from sqlbuild.cli.compile_reuse._helpers.entry_file import ( + read_entry_header, + read_entry_stdout, + write_entry, +) +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.cli.compile_reuse.models import StoredCompileHeader, StoredCompileInputs from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FreshProcessCompileCachePerformanceGuardTestCase, SemanticCorpusCase, @@ -2920,3 +2931,505 @@ def diagnostic_location_keys(payload: dict[str, Any]) -> tuple[tuple[str, str, i for diagnostic in payload["diagnostics"] ) ) + + +COMPILE_REUSE_REGION_ENV_VAR: str = "ORDERS_REGION" +COMPILE_REUSE_ENV: dict[str, str] = { + REUSE_DISABLE_ENV_VAR: "0", + COMPILE_CACHE_REGION_ENV_VAR: "east", + COMPILE_REUSE_REGION_ENV_VAR: "north", +} +COMPILE_REUSE_HIT_LINE: str = "Inputs unchanged; reused the previous compile" +_COMPILE_REUSE_RULES_CONFIG: str = ( + '\n[rules]\nselect = ["XSQBR"]\n\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n' +) +_FUTURE_MTIME_OFFSET_NS: int = 5_000_000_000 +_PROGRESS_LINE: re.Pattern[str] = re.compile(r".+ (START|OK \(\d+\.\d+s\))") +_COMPILE_TIMINGS_PATTERN: re.Pattern[str] = re.compile(r'\n "compile_timings": \{[^{}]*\n \}') +_COMPILE_REUSE_EXTRA_PROJECT_FILES: dict[str, str] = { + "macros/_rounding.py": ( + "_SCALE: int = 2\n\n\ndef scale() -> int:\n" + ' """Return the rounding scale."""\n return _SCALE\n' + ), + "macros/currency.py": ( + '"""Project-wide currency macros."""\n\nfrom macros._rounding import scale\n\n\n' + "def line_total_cents(price_cents: str, quantity: str) -> str:\n" + ' """Calculate line total cents from a unit price and quantity."""\n' + ' return f"ROUND({price_cents} * {quantity}, {scale()})"\n' + ), + "models/marts/regional_orders.sql": ( + "MODEL (description 'Orders tagged with the configured region.');\n\n" + f"SELECT order_id, '@@ENV:{COMPILE_REUSE_REGION_ENV_VAR}' AS region " + 'FROM __ref("stg_orders")\n' + ), + "rules/limits.py": "MAX_NAME_LENGTH: int = 40\n", + "rules/naming.py": ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n" + "from rules import limits\n\n\n" + '@rule(code="XSQBRNAME001", message="Model names stay short", ' + 'remediation="Rename the model.")\n' + "def short_names(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + " if len(model.name) <= limits.MAX_NAME_LENGTH:\n" + " return []\n" + " return [ctx.finding(subject=model)]\n" + ), +} + + +class CompileReuseRun(NamedTuple): + """Comparable result of one fresh-process JSON compile with compile reuse enabled.""" + + returncode: int + report: str + stderr: str + timings: dict[str, int] + compiled: dict[str, bytes] + + @property + def reused(self) -> bool: + """Return whether the run replayed the stored compile.""" + + return COMPILE_REUSE_HIT_LINE in self.stderr + + +def prepare_compile_reuse_project(*, project_dir: Path) -> None: + """Create a project whose inputs cover every kind compile reuse fingerprints.""" + + prepare_compile_cache_invalidation_project(project_dir=project_dir) + for relative_path, contents in _COMPILE_REUSE_EXTRA_PROJECT_FILES.items(): + write_project_file(project_dir, relative_path, contents) + config_path: Path = project_dir / "sqlbuild_project.toml" + config_path.write_text( + config_path.read_text(encoding="utf-8") + _COMPILE_REUSE_RULES_CONFIG, encoding="utf-8" + ) + (project_dir / "waffle_shop_control.duckdb").write_bytes(b"original database pages") + + +def run_reuse_compile( + *, + project_dir: Path, + env: dict[str, str] | None = None, + args: tuple[str, ...] = (), + global_args: tuple[str, ...] = (), +) -> CompileReuseRun: + """Compile with --json in a fresh process with reuse enabled and capture comparable output.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, + args=(*global_args, "compile", "--json", *args), + env={**COMPILE_REUSE_ENV, **(env or {})}, + ) + payload: dict[str, object] = cast(dict[str, object], json.loads(result.stdout)) + return CompileReuseRun( + returncode=result.returncode, + report=_COMPILE_TIMINGS_PATTERN.sub("", result.stdout), + stderr=result.stderr, + timings=cast(dict[str, int], payload.get("compile_timings", {})), + compiled=compiled_artifacts(project_dir=project_dir), + ) + + +def run_reuse_text_compile( + *, project_dir: Path, args: tuple[str, ...] +) -> subprocess.CompletedProcess[str]: + """Compile with the text report in a fresh process with reuse enabled.""" + + return run_installed_sqb( + project_dir=project_dir, args=("compile", *args), env=COMPILE_REUSE_ENV + ) + + +def run_reuse_compile_into_file( + *, project_dir: Path, report_path: Path +) -> subprocess.CompletedProcess[str]: + """Compile with --json while the shell-style stdout redirect targets a project file.""" + + with report_path.open("w", encoding="utf-8") as report: + return subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--project-dir", + str(project_dir), + "--no-color", + "compile", + "--json", + ], + stdout=report, + stderr=subprocess.PIPE, + text=True, + env={**os.environ, **COMPILE_REUSE_ENV}, + check=False, + ) + + +def compile_reuse_hit_count(*, report_path: Path) -> int: + """Return the reuse hit counter of a JSON report written to a file.""" + + payload: dict[str, object] = cast( + dict[str, object], json.loads(report_path.read_text(encoding="utf-8")) + ) + return cast(dict[str, int], payload["compile_timings"])["project_reuse_hits"] + + +def compile_notes(*, stderr: str) -> list[str]: + """Return the note lines a compile printed to stderr.""" + + return re.findall(r"^note:.*$", stderr, flags=re.MULTILINE) + + +def progress_only(*, text: str) -> bool: + """Return whether text consists only of per-phase progress lines.""" + + return all(_PROGRESS_LINE.fullmatch(line) is not None for line in text.splitlines()) + + +def write_recording_sink(*, project_dir: Path, sink_name: str) -> None: + """Add a lifecycle sink that appends invocation events to ORDERS_EVENT_PATH.""" + + write_project_file( + project_dir, + f"sinks/{sink_name}.py", + "import os\nfrom pathlib import Path\n\n" + "from sqlbuild.sinks import (\n LifecycleEvent,\n LifecycleEventKind,\n" + " lifecycle_event_sink,\n lifecycle_event_to_json,\n)\n\n\n" + f'@lifecycle_event_sink(name="{sink_name}", ' + "event_kinds={LifecycleEventKind.INVOCATION})\n" + "def record_event(event: LifecycleEvent) -> None:\n" + ' with Path(os.environ["ORDERS_EVENT_PATH"]).open("a", encoding="utf-8") as stream:\n' + ' stream.write(lifecycle_event_to_json(event) + "\\n")\n', + ) + + +def recorded_events(*, path: Path) -> list[dict[str, object]]: + """Read the lifecycle events a recording sink appended.""" + + return [ + cast(dict[str, object], json.loads(line)) + for line in path.read_text(encoding="utf-8").splitlines() + ] + + +def compile_reuse_entry_paths(*, project_dir: Path) -> tuple[Path, ...]: + """Return every stored compile reuse entry of a project.""" + + return tuple( + sorted((project_dir / "target" / "cache" / "compiler" / "project-reuse-v1").glob("*.entry")) + ) + + +def _rewrite_compile_reuse_entries( + *, project_dir: Path, rewrite: Callable[[StoredCompileInputs], StoredCompileInputs] +) -> None: + for path in compile_reuse_entry_paths(project_dir=project_dir): + header: StoredCompileHeader | None = read_entry_header(path=path) + assert header is not None + stdout: str | None = read_entry_stdout(path=path, header=header) + assert stdout is not None + write_entry(path=path, inputs=rewrite(header.inputs), output=header.output, stdout=stdout) + + +def simulate_native_build_change(project_dir: Path) -> None: + """Simulate an upgraded SQLBuild version and native build in the stored identity.""" + + _rewrite_compile_reuse_entries( + project_dir=project_dir, + rewrite=lambda inputs: replace( + inputs, runtime={**inputs.runtime, "native_build": "0.0.0+simulated"} + ), + ) + + +def simulate_python_version_change(project_dir: Path) -> None: + """Simulate a different Python interpreter in the stored identity.""" + + _rewrite_compile_reuse_entries( + project_dir=project_dir, + rewrite=lambda inputs: replace( + inputs, runtime={**inputs.runtime, "python": "3.0.0 (simulated)"} + ), + ) + + +def simulate_installed_module_change(project_dir: Path) -> None: + """Simulate reinstalled packages by aging every recorded module file stamp.""" + + _rewrite_compile_reuse_entries( + project_dir=project_dir, + rewrite=lambda inputs: replace( + inputs, + modules=tuple((path, mtime_ns - 1, size) for path, mtime_ns, size in inputs.modules), + ), + ) + + +def touch_model_without_change(project_dir: Path) -> None: + """Move a model's mtime forward without changing its content.""" + + path: Path = project_dir / "models/staging/stg_orders.sql" + mtime_ns: int = path.stat().st_mtime_ns + _FUTURE_MTIME_OFFSET_NS + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def rewrite_macro_without_change(project_dir: Path) -> None: + """Rewrite a macro file with identical bytes.""" + + path: Path = project_dir / "macros/currency.py" + path.write_bytes(path.read_bytes()) + + +def delete_compiled_model(project_dir: Path) -> None: + """Delete one compiled artifact by hand.""" + + (project_dir / "target/compiled/models/marts/regional_orders.sql").unlink() + + +def edit_compiled_model(project_dir: Path) -> None: + """Edit one compiled artifact by hand.""" + + path: Path = project_dir / "target/compiled/models/marts/regional_orders.sql" + path.write_text(path.read_text(encoding="utf-8") + "-- edited by hand\n", encoding="utf-8") + + +def add_stale_compiled_file(project_dir: Path) -> None: + """Add a compiled artifact that no model produces.""" + + write_project_file(project_dir, "target/compiled/models/stale_model.sql", "SELECT 1\n") + + +def truncate_file(path: Path) -> None: + """Drop the last bytes of a stored entry.""" + + path.write_bytes(path.read_bytes()[:-7]) + + +def flip_trailing_bytes(path: Path) -> None: + """Corrupt the stored stdout section without changing its length.""" + + path.write_bytes(path.read_bytes()[:-3] + b"xyz") + + +def flip_header_byte(path: Path) -> None: + """Corrupt one byte inside the stored inputs section.""" + + contents: bytes = path.read_bytes() + path.write_bytes(contents[:40] + b"!" + contents[41:]) + + +def empty_file(path: Path) -> None: + """Leave an empty stored entry, as an interrupted copy would.""" + + path.write_bytes(b"") + + +def garbage_file(path: Path) -> None: + """Replace a stored entry with unrelated bytes.""" + + path.write_bytes(b"not a stored compile") + + +ORDERS_API_TOKEN_ENV_VAR: str = "ORDERS_API_TOKEN" +ORDERS_API_TIMEOUT_ENV_VAR: str = "ORDERS_API_TIMEOUT_SECONDS" +_ORDERS_API_ENV_FILE_NAME: str = "orders_api.env" + + +def orders_api_env_file(project_dir: Path) -> Path: + """Return the provider env file kept beside, not inside, the project.""" + + return project_dir.parent / _ORDERS_API_ENV_FILE_NAME + + +def write_orders_api_provider(*, project_dir: Path) -> None: + """Add a provider whose required settings come from the environment and an env file.""" + + write_project_file( + project_dir, + "providers/orders_api.py", + "from pydantic_settings import SettingsConfigDict\n" + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersApi(Provider):\n" + ' """Orders API connection settings."""\n\n' + " model_config = SettingsConfigDict(\n" + f" env_file={str(orders_api_env_file(project_dir))!r}\n" + " )\n\n" + " orders_api_token: str\n" + " orders_api_timeout_seconds: int = 30\n", + ) + + +def write_custom_source_provider(project_dir: Path) -> None: + """Add a provider whose settings sources reuse cannot enumerate.""" + + write_project_file( + project_dir, + "providers/orders_vault.py", + "from pydantic_settings import BaseSettings, PydanticBaseSettingsSource\n" + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersVault(Provider):\n" + ' """Orders vault settings read from a custom source."""\n\n' + ' vault_path: str = "orders"\n\n' + " @classmethod\n" + " def settings_customise_sources(\n" + " cls,\n" + " settings_cls: type[BaseSettings],\n" + " init_settings: PydanticBaseSettingsSource,\n" + " env_settings: PydanticBaseSettingsSource,\n" + " dotenv_settings: PydanticBaseSettingsSource,\n" + " file_secret_settings: PydanticBaseSettingsSource,\n" + " ) -> tuple[PydanticBaseSettingsSource, ...]:\n" + " return (init_settings,)\n", + ) + + +def write_orders_api_timeout_file(project_dir: Path) -> None: + """Change a provider setting through its env file only.""" + + orders_api_env_file(project_dir).write_text( + f"{ORDERS_API_TIMEOUT_ENV_VAR}=45\n", encoding="utf-8" + ) + + +def stored_stdout_path(entry_path: Path) -> Path: + """Return the stored stdout file that one entry references.""" + + header: StoredCompileHeader | None = read_entry_header(path=entry_path) + assert header is not None + return entry_path.parent / header.output.stdout_file + + +def flip_stored_stdout_bytes(entry_path: Path) -> None: + """Corrupt the stored stdout without changing its length.""" + + path: Path = stored_stdout_path(entry_path) + path.write_bytes(path.read_bytes()[:-3] + b"xyz") + + +def remove_stored_stdout(entry_path: Path) -> None: + """Delete the stored stdout while keeping the entry that references it.""" + + stored_stdout_path(entry_path).unlink() + + +def enable_compile_reuse(monkeypatch: pytest.MonkeyPatch) -> None: + """Enable compile reuse and the fixture environment for in-process compiles.""" + + for name, value in COMPILE_REUSE_ENV.items(): + monkeypatch.setenv(name, value) + + +def compile_in_process(*, project_dir: Path) -> int: + """Run one JSON compile through the CLI entry point in this process.""" + + with redirect_stdout(StringIO()): + return main(["--project-dir", str(project_dir), "--no-color", "compile", "--json"]) + + +def write_before_reuse_store( + *, monkeypatch: pytest.MonkeyPatch, project_dir: Path, write: Callable[[Path], None] +) -> None: + """Let another writer change target/ once, after a compile wrote it and before it stores.""" + + original: Callable[..., None] = compile_command_module.write_reusable_compile + pending: list[Callable[[Path], None]] = [write] + + def write_then_store(**kwargs: Any) -> None: + for action in tuple(pending): + action(project_dir) + pending.clear() + original(**kwargs) + + monkeypatch.setattr(compile_command_module, "write_reusable_compile", write_then_store) + + +def rewrite_compiled_model_unchanged(project_dir: Path) -> None: + """Rewrite one compiled artifact with the bytes it already holds.""" + + path: Path = project_dir / "target/compiled/models/marts/regional_orders.sql" + path.write_bytes(path.read_bytes()) + + +def record_digested_paths(*, monkeypatch: pytest.MonkeyPatch) -> list[str]: + """Record every project file compile reuse reads to compute a content digest.""" + + paths: list[str] = [] + original: Callable[..., str | None] = reuse_project_files.file_digest + + def recording_digest(*, path: str) -> str | None: + paths.append(path) + return original(path=path) + + monkeypatch.setattr(reuse_project_files, "file_digest", recording_digest) + return paths + + +def settle_racy_window(*, monkeypatch: pytest.MonkeyPatch) -> None: + """Treat files written before a compile starts as settled instead of waiting two seconds.""" + + monkeypatch.setattr(reuse_project_files, "RACY_WINDOW_NS", 0) + + +def write_large_file(*, project_dir: Path, relative_path: str, size_bytes: int) -> Path: + """Write a large data file that no compile reads.""" + + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"0" * size_bytes) + return path + + +def touch_back(path: Path, *, seconds: int) -> None: + """Set a file's mtime to a past time without changing its content, as a checkout can.""" + + mtime_ns: int = time.time_ns() - seconds * 1_000_000_000 + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def compiled_artifacts(*, project_dir: Path) -> dict[str, bytes]: + """Return every compiled SQL artifact by path relative to target/compiled.""" + + compiled_dir: Path = project_dir / "target" / "compiled" + return { + path.relative_to(compiled_dir).as_posix(): path.read_bytes() + for path in sorted(compiled_dir.rglob("*.sql")) + } + + +def compile_in_process_reused( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, bool]: + """Compile in this process and return its exit code and whether it reused.""" + + _ = capsys.readouterr() + code: int = compile_in_process(project_dir=project_dir) + return code, COMPILE_REUSE_HIT_LINE in capsys.readouterr().err + + +def in_process_compile_reads( + *, project_dir: Path, digested: list[str], path: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, int, bool]: + """Compile in this process; return its exit code, reads of path, and whether it reused.""" + + start: int = len(digested) + _ = capsys.readouterr() + code: int = compile_in_process(project_dir=project_dir) + reused: bool = COMPILE_REUSE_HIT_LINE in capsys.readouterr().err + return code, digested[start:].count(str(path)), reused + + +def fail_reuse_store(*, monkeypatch: pytest.MonkeyPatch, error: BaseException) -> None: + """Make storing a finished compile raise while enumerating provider settings.""" + + def raise_error(**_kwargs: Any) -> None: + raise error + + monkeypatch.setattr(reuse_store, "provider_settings_inputs", raise_error) + + +def compile_in_process_output( + *, project_dir: Path, capsys: pytest.CaptureFixture[str] +) -> tuple[int, str, str]: + """Run one JSON compile in this process and return its exit code, stdout, and stderr.""" + + _ = capsys.readouterr() + code: int = main(["--project-dir", str(project_dir), "--no-color", "compile", "--json"]) + out, err = capsys.readouterr() + return code, out, err diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py new file mode 100644 index 0000000000..01bd824022 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py @@ -0,0 +1,805 @@ +"""Whole-project compile reuse must replay exactly what a full compile would produce.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import cast + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + CompileReuseBypassTestCase, + CompileReuseConcurrentWriterTestCase, + CompileReuseCorruptEntryTestCase, + CompileReuseEventTestCase, + CompileReuseHitTestCase, + CompileReuseInvalidationTestCase, + CompileReuseLargeFileTestCase, + CompileReuseProviderSettingsTestCase, + CompileReuseRedirectTestCase, + CompileReuseReplayTestCase, + CompileReuseStoreFailureTestCase, + CompileReuseTimingsTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + COMPILE_REUSE_HIT_LINE, + COMPILE_REUSE_REGION_ENV_VAR, + ORDERS_API_TIMEOUT_ENV_VAR, + ORDERS_API_TOKEN_ENV_VAR, + CompileReuseRun, + add_stale_compiled_file, + compile_in_process, + compile_in_process_output, + compile_in_process_reused, + compile_notes, + compile_reuse_entry_paths, + compile_reuse_hit_count, + compiled_artifacts, + delete_compiled_model, + edit_compiled_model, + empty_file, + enable_compile_reuse, + fail_reuse_store, + flip_header_byte, + flip_stored_stdout_bytes, + flip_trailing_bytes, + garbage_file, + in_process_compile_reads, + progress_only, + record_digested_paths, + recorded_events, + remove_stored_stdout, + replace_project_text, + rewrite_compiled_model_unchanged, + rewrite_macro_without_change, + run_reuse_compile, + run_reuse_compile_into_file, + run_reuse_text_compile, + settle_racy_window, + simulate_installed_module_change, + simulate_native_build_change, + simulate_python_version_change, + touch_back, + touch_model_without_change, + truncate_file, + write_before_reuse_store, + write_custom_source_provider, + write_large_file, + write_orders_api_provider, + write_orders_api_timeout_file, + write_project_file, + write_recording_sink, +) + +_CONTRACT_ERROR_MODEL: str = ( + "MODEL (description 'Order contract.', columns (missing_col (type INTEGER)));\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseInvalidationTestCase( + description="model_sql_edit", + edit=lambda root: replace_project_text( + root, "models/staging/stg_orders.sql", " quantity,", " quantity + 0 AS quantity," + ), + ), + CompileReuseInvalidationTestCase( + description="model_added", + edit=lambda root: write_project_file( + root, + "models/marts/order_quantities.sql", + "MODEL (description 'Order quantities.');\n\n" + 'SELECT order_id, quantity FROM __ref("stg_orders")\n', + ), + ), + CompileReuseInvalidationTestCase( + description="model_deleted", + edit=lambda root: (root / "models/marts/regional_orders.sql").unlink(), + ), + CompileReuseInvalidationTestCase( + description="source_yaml_edit", + edit=lambda root: replace_project_text( + root, + "sources/raw.yml", + " - name: quantity\n type: INTEGER", + " - name: quantity\n type: BIGINT", + ), + ), + CompileReuseInvalidationTestCase( + description="seed_csv_edit", + edit=lambda root: replace_project_text( + root, "seeds/waffle_types.csv", "Classic Belgian", "Classic Brussels" + ), + ), + CompileReuseInvalidationTestCase( + description="macro_file_edit", + edit=lambda root: replace_project_text( + root, + "macros/currency.py", + "{price_cents} * {quantity}", + "{quantity} * {price_cents}", + ), + ), + CompileReuseInvalidationTestCase( + description="module_imported_by_macro_edit", + edit=lambda root: replace_project_text( + root, "macros/_rounding.py", "_SCALE: int = 2", "_SCALE: int = 3" + ), + ), + CompileReuseInvalidationTestCase( + description="scoped_macro_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_macros/currency.py", + 'return f"ROUND(({column}) / 100.0, 2)"', + 'return f"ROUND(({column}) / 100.0, 3)"', + ), + ), + CompileReuseInvalidationTestCase( + description="enum_declaration_edit", + edit=lambda root: replace_project_text( + root, "models/marts/_sqlbuild/_enums/order_channel.sql", 'WEB "web"', 'WEB "online"' + ), + ), + CompileReuseInvalidationTestCase( + description="project_toml_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'quantity_multiplier = "2"', + 'quantity_multiplier = "5"', + ), + ), + CompileReuseInvalidationTestCase( + description="hidden_root_directory_file_added", + edit=lambda root: write_project_file(root, ".orders_rules/limits.txt", "10\n"), + ), + CompileReuseInvalidationTestCase( + description="local_toml_added", + edit=lambda root: write_project_file(root, "sqlbuild_local.toml", 'target = "prod"\n'), + ), + CompileReuseInvalidationTestCase( + description="cli_vars_change", + edited_args=("--vars", json.dumps({"quantity_multiplier": "7"})), + ), + CompileReuseInvalidationTestCase( + description="template_environment_variable_change", + edited_env={COMPILE_REUSE_REGION_ENV_VAR: "south"}, + ), + CompileReuseInvalidationTestCase( + description="sqlbuild_prefixed_environment_variable_added", + edited_env={"SQB_ORDERS_UNUSED_SETTING": "1"}, + ), + CompileReuseInvalidationTestCase( + description="custom_rule_file_edit", + edit=lambda root: replace_project_text( + root, + "rules/naming.py", + "<= limits.MAX_NAME_LENGTH:", + "<= limits.MAX_NAME_LENGTH - 30:", + ), + expected_returncode=1, + ), + CompileReuseInvalidationTestCase( + description="module_imported_by_custom_rule_edit", + edit=lambda root: replace_project_text( + root, "rules/limits.py", "MAX_NAME_LENGTH: int = 40", "MAX_NAME_LENGTH: int = 10" + ), + expected_returncode=1, + ), + CompileReuseInvalidationTestCase( + description="contract_error_replayed_as_failure", + edit=lambda root: write_project_file( + root, "models/marts/order_contract.sql", _CONTRACT_ERROR_MODEL + ), + expected_returncode=1, + ), + CompileReuseInvalidationTestCase( + description="stopped_compile_is_never_replayed", + edit=lambda root: replace_project_text( + root, "models/staging/stg_orders.sql", "materialized view,", "materialized view,,((" + ), + expected_returncode=1, + expected_rewarm_hit=False, + ), + CompileReuseInvalidationTestCase( + description="compiled_artifact_deleted", edit=delete_compiled_model + ), + CompileReuseInvalidationTestCase( + description="compiled_artifact_edited", edit=edit_compiled_model + ), + CompileReuseInvalidationTestCase( + description="stale_compiled_artifact_added", edit=add_stale_compiled_file + ), + CompileReuseInvalidationTestCase( + description="native_build_and_version_simulated", edit=simulate_native_build_change + ), + CompileReuseInvalidationTestCase( + description="python_version_simulated", edit=simulate_python_version_change + ), + CompileReuseInvalidationTestCase( + description="installed_module_simulated", edit=simulate_installed_module_change + ), + ], + ids=lambda case: case.description, +) +def test_given_reused_compile_when_input_changes_then_it_misses_and_matches_uncached_compile( + compile_reuse_project: Path, test_case: CompileReuseInvalidationTestCase +) -> None: + project_dir: Path = compile_reuse_project + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + warm: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + assert cold.returncode == 0, cold.stderr + assert (cold.reused, cold.timings["project_reuse_misses"]) == (False, 1) + assert (warm.reused, warm.timings["project_reuse_hits"]) == (True, 1) + assert (warm.returncode, warm.report, warm.compiled) == ( + cold.returncode, + cold.report, + cold.compiled, + ) + + test_case.edit(project_dir) + edited: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=test_case.edited_env, args=test_case.edited_args + ) + reference: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, + env=test_case.edited_env, + args=(*test_case.edited_args, "--no-cache"), + ) + rewarmed: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=test_case.edited_env, args=test_case.edited_args + ) + + assert (edited.reused, reference.reused) == (False, False) + assert edited.returncode == test_case.expected_returncode, edited.stderr + assert (edited.returncode, edited.report, edited.compiled) == ( + reference.returncode, + reference.report, + reference.compiled, + ) + assert rewarmed.reused is test_case.expected_rewarm_hit + assert (rewarmed.returncode, rewarmed.report, rewarmed.compiled) == ( + reference.returncode, + reference.report, + reference.compiled, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseHitTestCase( + description="unrelated_environment_variable", + edit=lambda _root: None, + edited_env={"ORDERS_UNRELATED_SETTING": "1"}, + ), + CompileReuseHitTestCase( + description="editor_settings_added", + edit=lambda root: write_project_file(root, ".vscode/settings.json", "{}\n"), + ), + CompileReuseHitTestCase( + description="database_file_content_change", + edit=lambda root: (root / "waffle_shop_control.duckdb").write_bytes(b"changed pages"), + ), + ], + ids=lambda case: case.description, +) +def test_given_reused_compile_when_no_input_changes_then_rerun_reuses_it( + compile_reuse_project: Path, test_case: CompileReuseHitTestCase +) -> None: + project_dir: Path = compile_reuse_project + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + + test_case.edit(project_dir) + rerun: CompileReuseRun = run_reuse_compile(project_dir=project_dir, env=test_case.edited_env) + + assert rerun.reused is test_case.expected_reused, rerun.stderr + assert (rerun.returncode, rerun.report, rerun.compiled) == ( + cold.returncode, + cold.report, + cold.compiled, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseHitTestCase( + description="touch_without_change", edit=touch_model_without_change + ), + CompileReuseHitTestCase( + description="rewrite_without_change", edit=rewrite_macro_without_change + ), + ], + ids=lambda case: case.description, +) +def test_given_timestamp_only_change_when_it_repeats_then_the_known_content_is_reused( + compile_reuse_project: Path, test_case: CompileReuseHitTestCase +) -> None: + project_dir: Path = compile_reuse_project + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + test_case.edit(project_dir) + first: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + + test_case.edit(project_dir) + second: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + + assert second.reused is test_case.expected_reused, second.stderr + assert (first.returncode, first.report, first.compiled) == ( + cold.returncode, + cold.report, + cold.compiled, + ) + assert (second.returncode, second.report, second.compiled) == ( + cold.returncode, + cold.report, + cold.compiled, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseBypassTestCase(description="no_cache_flag", compile_args=("--no-cache",)), + CompileReuseBypassTestCase( + description="reuse_disabled_environment_variable", + env={"SQLBUILD_DISABLE_COMPILE_REUSE": "1"}, + ), + CompileReuseBypassTestCase( + description="compile_cache_disabled_environment_variable", + env={"SQLBUILD_DISABLE_COMPILE_CACHE": "1"}, + ), + CompileReuseBypassTestCase( + description="target_compile_cache_false", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'schema = "dev"', + 'schema = "dev"\ncompile_cache = false', + ), + ), + CompileReuseBypassTestCase(description="manifest_flag", compile_args=("--manifest",)), + CompileReuseBypassTestCase(description="debug_flag", global_args=("--debug",)), + CompileReuseBypassTestCase( + description="provider_with_custom_settings_sources", + edit=write_custom_source_provider, + ), + CompileReuseBypassTestCase( + description="model_reads_run_id", + edit=lambda root: write_project_file( + root, + "models/marts/stamped_orders.sql", + "MODEL (description 'Orders stamped by run ${CTX:run.id}.');\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n', + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_reuse_bypass_when_compiling_twice_then_both_compiles_run_in_full( + compile_reuse_project: Path, test_case: CompileReuseBypassTestCase +) -> None: + project_dir: Path = compile_reuse_project + test_case.edit(project_dir) + + first: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, + env=test_case.env, + args=test_case.compile_args, + global_args=test_case.global_args, + ) + second: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, + env=test_case.env, + args=test_case.compile_args, + global_args=test_case.global_args, + ) + + assert (first.returncode, second.returncode) == (0, 0), first.stderr + second.stderr + assert (first.reused, second.reused, second.timings["project_reuse_hits"]) == (False, False, 0) + assert len(compile_reuse_entry_paths(project_dir=project_dir)) == ( + test_case.expected_stored_entries + ) + assert second.report == first.report + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseCorruptEntryTestCase(description="truncated", corrupt=truncate_file), + CompileReuseCorruptEntryTestCase(description="header_tail", corrupt=flip_trailing_bytes), + CompileReuseCorruptEntryTestCase( + description="stdout_bytes", corrupt=flip_stored_stdout_bytes + ), + CompileReuseCorruptEntryTestCase( + description="stdout_missing", corrupt=remove_stored_stdout + ), + CompileReuseCorruptEntryTestCase(description="header_bytes", corrupt=flip_header_byte), + CompileReuseCorruptEntryTestCase(description="empty", corrupt=empty_file), + CompileReuseCorruptEntryTestCase(description="garbage", corrupt=garbage_file), + ], + ids=lambda case: case.description, +) +def test_given_corrupt_stored_compile_when_compiling_then_it_falls_back_and_restores_it( + compile_reuse_project: Path, test_case: CompileReuseCorruptEntryTestCase +) -> None: + project_dir: Path = compile_reuse_project + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + entries: tuple[Path, ...] = compile_reuse_entry_paths(project_dir=project_dir) + assert len(entries) == 1 + test_case.corrupt(entries[0]) + + fallback: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + restored: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + + assert fallback.reused is test_case.expected_fallback_reused + assert restored.reused is test_case.expected_restored_reused + assert (fallback.returncode, fallback.report, fallback.compiled) == ( + cold.returncode, + cold.report, + cold.compiled, + ) + assert (restored.returncode, restored.report, restored.compiled) == ( + cold.returncode, + cold.report, + cold.compiled, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseReplayTestCase( + description="failed_text_compile_without_sql_analysis", + compile_args=("--no-sql-analysis",), + expected_returncode=1, + ) + ], + ids=lambda case: case.description, +) +def test_given_failed_text_compile_when_rerun_then_report_notes_and_exit_code_are_replayed( + compile_reuse_project: Path, test_case: CompileReuseReplayTestCase +) -> None: + project_dir: Path = compile_reuse_project + replace_project_text( + project_dir, + "sqlbuild_project.toml", + 'select = ["XSQBR"]', + 'select = ["XSQBR", "SQBRCONTRACT105"]', + ) + replace_project_text( + project_dir, "rules/limits.py", "MAX_NAME_LENGTH: int = 40", "MAX_NAME_LENGTH: int = 10" + ) + + cold: subprocess.CompletedProcess[str] = run_reuse_text_compile( + project_dir=project_dir, args=test_case.compile_args + ) + warm: subprocess.CompletedProcess[str] = run_reuse_text_compile( + project_dir=project_dir, args=test_case.compile_args + ) + + assert (cold.returncode, warm.returncode) == ( + test_case.expected_returncode, + test_case.expected_returncode, + ) + assert COMPILE_REUSE_HIT_LINE not in cold.stderr + assert COMPILE_REUSE_HIT_LINE in warm.stderr + assert warm.stdout.strip() + assert cold.stdout.endswith(warm.stdout) + assert progress_only(text=cold.stdout.removesuffix(warm.stdout)) + assert compile_notes(stderr=cold.stderr)[0].startswith(test_case.expected_note_prefix) + assert compile_notes(stderr=warm.stderr) == compile_notes(stderr=cold.stderr) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseRedirectTestCase( + description="json_report_redirected_into_project_root", + report_name="compile-report.json", + expected_hit_counts=(0, 1), + ) + ], + ids=lambda case: case.description, +) +def test_given_stdout_redirected_into_project_when_rerun_then_it_reuses_the_compile( + compile_reuse_project: Path, test_case: CompileReuseRedirectTestCase +) -> None: + project_dir: Path = compile_reuse_project + report_path: Path = project_dir / test_case.report_name + + first: subprocess.CompletedProcess[str] = run_reuse_compile_into_file( + project_dir=project_dir, report_path=report_path + ) + first_hits: int = compile_reuse_hit_count(report_path=report_path) + second: subprocess.CompletedProcess[str] = run_reuse_compile_into_file( + project_dir=project_dir, report_path=report_path + ) + second_hits: int = compile_reuse_hit_count(report_path=report_path) + + assert (first.returncode, second.returncode) == (0, 0), first.stderr + second.stderr + assert (first_hits, second_hits) == test_case.expected_hit_counts + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseEventTestCase( + description="invocation_events_on_reuse_hit", + sink_name="orders_events", + expected_event_types=("invocation_started", "invocation_completed"), + ) + ], + ids=lambda case: case.description, +) +def test_given_lifecycle_sink_when_compile_is_reused_then_invocation_events_are_exported( + compile_reuse_project: Path, tmp_path: Path, test_case: CompileReuseEventTestCase +) -> None: + project_dir: Path = compile_reuse_project + write_recording_sink(project_dir=project_dir, sink_name=test_case.sink_name) + cold_events: Path = tmp_path / "cold-events.jsonl" + warm_events: Path = tmp_path / "warm-events.jsonl" + + cold: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env={"ORDERS_EVENT_PATH": str(cold_events)} + ) + warm: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env={"ORDERS_EVENT_PATH": str(warm_events)} + ) + events: list[dict[str, object]] = recorded_events(path=warm_events) + + assert cold.returncode == 0, cold.stderr + assert warm.reused, warm.stderr + assert tuple(event["event_type"] for event in events) == test_case.expected_event_types + assert [cast(dict[str, object], event["payload"])["command"] for event in events] == [ + "compile", + "compile", + ] + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseTimingsTestCase( + description="reuse_hit_timings", + expected_timing_names=frozenset( + { + "project_reuse_hits", + "project_reuse_misses", + "project_reuse_bypasses", + "project_reuse_check_ms", + "total_ms", + } + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_reused_compile_when_replayed_then_compile_timings_describe_the_reuse( + compile_reuse_project: Path, test_case: CompileReuseTimingsTestCase +) -> None: + project_dir: Path = compile_reuse_project + _ = run_reuse_compile(project_dir=project_dir) + + replay: CompileReuseRun = run_reuse_compile(project_dir=project_dir) + + assert replay.reused + assert frozenset(replay.timings) == test_case.expected_timing_names + assert replay.timings["project_reuse_hits"] == 1 + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseProviderSettingsTestCase( + description="required_setting_unset", + edited_env={}, + expected_returncode=1, + expected_rewarm_hit=False, + ), + CompileReuseProviderSettingsTestCase( + description="setting_value_changed", + edited_env={ORDERS_API_TOKEN_ENV_VAR: "beta"}, + ), + CompileReuseProviderSettingsTestCase( + description="setting_value_invalid", + edited_env={ORDERS_API_TOKEN_ENV_VAR: "alpha", ORDERS_API_TIMEOUT_ENV_VAR: "soon"}, + expected_returncode=1, + expected_rewarm_hit=False, + ), + CompileReuseProviderSettingsTestCase( + description="env_file_changed", + edited_env={ORDERS_API_TOKEN_ENV_VAR: "alpha"}, + edit=write_orders_api_timeout_file, + ), + ], + ids=lambda case: case.description, +) +def test_given_provider_settings_when_their_inputs_change_then_reuse_misses_like_full_compile( + compile_reuse_project: Path, test_case: CompileReuseProviderSettingsTestCase +) -> None: + project_dir: Path = compile_reuse_project + write_orders_api_provider(project_dir=project_dir) + cold_env: dict[str, str] = {ORDERS_API_TOKEN_ENV_VAR: "alpha"} + cold: CompileReuseRun = run_reuse_compile(project_dir=project_dir, env=cold_env) + warm: CompileReuseRun = run_reuse_compile(project_dir=project_dir, env=cold_env) + assert (cold.returncode, warm.reused) == (0, True), cold.stderr + + test_case.edit(project_dir) + edited: CompileReuseRun = run_reuse_compile(project_dir=project_dir, env=test_case.edited_env) + reference: CompileReuseRun = run_reuse_compile( + project_dir=project_dir, env=test_case.edited_env, args=("--no-cache",) + ) + rewarmed: CompileReuseRun = run_reuse_compile(project_dir=project_dir, env=test_case.edited_env) + + assert edited.reused is False + assert edited.returncode == test_case.expected_returncode, edited.stderr + assert (edited.returncode, edited.report) == (reference.returncode, reference.report) + assert rewarmed.reused is test_case.expected_rewarm_hit + assert (rewarmed.returncode, rewarmed.report) == (reference.returncode, reference.report) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseConcurrentWriterTestCase( + description="other_compile_edits_artifact", + other_write=edit_compiled_model, + expected_stored_entries=0, + expected_rerun_reused=False, + ), + CompileReuseConcurrentWriterTestCase( + description="other_compile_adds_artifact", + other_write=add_stale_compiled_file, + expected_stored_entries=0, + expected_rerun_reused=False, + ), + CompileReuseConcurrentWriterTestCase( + description="other_compile_rewrites_same_bytes", + other_write=rewrite_compiled_model_unchanged, + expected_stored_entries=1, + expected_rerun_reused=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_other_writer_before_store_when_compiling_then_only_own_artifacts_are_stored( + compile_reuse_project: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + test_case: CompileReuseConcurrentWriterTestCase, +) -> None: + project_dir: Path = compile_reuse_project + enable_compile_reuse(monkeypatch) + write_before_reuse_store( + monkeypatch=monkeypatch, project_dir=project_dir, write=test_case.other_write + ) + + code: int = compile_in_process(project_dir=project_dir) + stored: int = len(compile_reuse_entry_paths(project_dir=project_dir)) + rerun: tuple[int, bool] = compile_in_process_reused(project_dir=project_dir, capsys=capsys) + rerun_compiled: dict[str, bytes] = compiled_artifacts(project_dir=project_dir) + reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + + assert (code, stored) == (0, test_case.expected_stored_entries) + assert rerun == (reference.returncode, test_case.expected_rerun_reused) + assert rerun_compiled == reference.compiled + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseLargeFileTestCase( + description="untouched_then_restamped_data_file", + size_bytes=8 * 1024 * 1024, + relative_path="data/orders_export.bin", + expected_reads=(0, 0, 1, 1, 0), + expected_reused=(False, True, False, True, True), + ) + ], + ids=lambda case: case.description, +) +def test_given_large_data_file_when_storing_and_reusing_then_it_is_read_only_after_restamp( + compile_reuse_project: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + test_case: CompileReuseLargeFileTestCase, +) -> None: + project_dir: Path = compile_reuse_project + data_file: Path = write_large_file( + project_dir=project_dir, + relative_path=test_case.relative_path, + size_bytes=test_case.size_bytes, + ) + enable_compile_reuse(monkeypatch) + settle_racy_window(monkeypatch=monkeypatch) + digested: list[str] = record_digested_paths(monkeypatch=monkeypatch) + + stored: tuple[int, int, bool] = in_process_compile_reads( + project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + ) + reused: tuple[int, int, bool] = in_process_compile_reads( + project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + ) + touch_back(data_file, seconds=3600) + restamped: tuple[int, int, bool] = in_process_compile_reads( + project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + ) + touch_back(data_file, seconds=7200) + verified: tuple[int, int, bool] = in_process_compile_reads( + project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + ) + refreshed: tuple[int, int, bool] = in_process_compile_reads( + project_dir=project_dir, digested=digested, path=data_file, capsys=capsys + ) + runs: tuple[tuple[int, int, bool], ...] = (stored, reused, restamped, verified, refreshed) + + assert tuple(run[0] for run in runs) == (0, 0, 0, 0, 0) + assert tuple(run[1] for run in runs) == test_case.expected_reads + assert tuple(run[2] for run in runs) == test_case.expected_reused + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseStoreFailureTestCase( + description="unexpected_store_error", error=RuntimeError("settings went away") + ) + ], + ids=lambda case: case.description, +) +def test_given_store_raises_when_compiling_then_report_and_exit_code_are_kept( + compile_reuse_project: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + test_case: CompileReuseStoreFailureTestCase, +) -> None: + project_dir: Path = compile_reuse_project + enable_compile_reuse(monkeypatch) + fail_reuse_store(monkeypatch=monkeypatch, error=test_case.error) + + code, stdout, stderr = compile_in_process_output(project_dir=project_dir, capsys=capsys) + reference: CompileReuseRun = run_reuse_compile(project_dir=project_dir, args=("--no-cache",)) + + assert code == test_case.expected_returncode == reference.returncode + assert json.loads(stdout)["command"] == "compile" + assert "Traceback" not in stderr + assert len(compile_reuse_entry_paths(project_dir=project_dir)) == ( + test_case.expected_stored_entries + ) + + +@pytest.mark.parametrize( + "test_case", + [ + CompileReuseStoreFailureTestCase( + description="interrupted_store", + error=KeyboardInterrupt(), + expected_raised=KeyboardInterrupt, + ) + ], + ids=lambda case: case.description, +) +def test_given_store_is_interrupted_when_compiling_then_the_interrupt_propagates( + compile_reuse_project: Path, + monkeypatch: pytest.MonkeyPatch, + test_case: CompileReuseStoreFailureTestCase, +) -> None: + project_dir: Path = compile_reuse_project + enable_compile_reuse(monkeypatch) + fail_reuse_store(monkeypatch=monkeypatch, error=test_case.error) + + with pytest.raises(BaseException) as raised: + _ = compile_in_process(project_dir=project_dir) + + assert raised.type is test_case.expected_raised + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index d1d3de4b52..99cfcd2840 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1018,3 +1018,14 @@ class StartupImportFootprintTestCase: argv: tuple[str, ...] expected_exit_code: int forbidden_modules: tuple[str, ...] + + +@dataclass(frozen=True) +class ReusedCompileImportFootprintTestCase: + """One reused compile in a fresh process and the compile modules it must not load.""" + + description: str + argv: tuple[str, ...] + expected_exit_code: int + expected_reuse_message: str + forbidden_modules: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py index 8f7093562a..475159e14b 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import subprocess import sys from pathlib import Path @@ -10,7 +11,9 @@ import pytest +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + ReusedCompileImportFootprintTestCase, StartupImportFootprintTestCase, ) from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( @@ -60,6 +63,21 @@ *WAREHOUSE_AND_INTEGRATION_MODULES, *PRESENTATION_AND_PROVIDER_MODULES, ) +FULL_COMPILE_MODULES: tuple[str, ...] = ( + "polyglot_sql", + "pydantic", + "sqlbuild.adapter.contract.classes.base_adapter", + "sqlbuild.cli.commands.main.project._compile", + "sqlbuild.cli.compile.models", + "sqlbuild.compiler.compile.models", + "sqlbuild.compiler.pipeline", + "sqlbuild.compiler.planner.models", +) +REUSED_COMPILE_FORBIDDEN: tuple[str, ...] = ( + *WAREHOUSE_AND_INTEGRATION_MODULES, + *FULL_COMPILE_MODULES, + *PRESENTATION_AND_PROVIDER_MODULES, +) _FOOTPRINT_SCRIPT: str = """ import contextlib, io, json, sys @@ -67,13 +85,14 @@ argv = json.loads(sys.argv[1]) forbidden = json.loads(sys.argv[2]) -with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): +stderr = io.StringIO() +with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(stderr): code = main(argv) loaded = sorted( name for name in forbidden if any(module == name or module.startswith(name + ".") for module in sys.modules) ) -print(json.dumps({"code": code, "loaded": loaded})) +print(json.dumps({"code": code, "loaded": loaded, "stderr": stderr.getvalue()})) """ @@ -138,5 +157,42 @@ def test_given_fresh_interpreter_when_running_cli_then_heavy_modules_stay_unload assert cast(list[str], outcome["loaded"]) == [] +@pytest.mark.parametrize( + "test_case", + [ + ReusedCompileImportFootprintTestCase( + description="reused compile", + argv=("compile", "--json"), + expected_exit_code=0, + expected_reuse_message="Inputs unchanged; reused the previous compile", + forbidden_modules=REUSED_COMPILE_FORBIDDEN, + ), + ], + ids=lambda case: case.description, +) +def test_given_unchanged_project_when_compile_is_reused_then_compiler_stays_unloaded( + test_case: ReusedCompileImportFootprintTestCase, tmp_path: Path +) -> None: + write_compile_startup_project(tmp_path) + command: list[str] = [ + sys.executable, + "-c", + _FOOTPRINT_SCRIPT, + json.dumps(["--project-dir", str(tmp_path), *test_case.argv]), + json.dumps(test_case.forbidden_modules), + ] + environment: dict[str, str] = {**os.environ, REUSE_DISABLE_ENV_VAR: "0"} + + _ = subprocess.run(command, check=True, capture_output=True, cwd=tmp_path, env=environment) + result: subprocess.CompletedProcess[str] = subprocess.run( + command, check=True, capture_output=True, text=True, cwd=tmp_path, env=environment + ) + outcome: dict[str, object] = json.loads(result.stdout.strip().splitlines()[-1]) + + assert outcome["code"] == test_case.expected_exit_code + assert test_case.expected_reuse_message in cast(str, outcome["stderr"]) + assert cast(list[str], outcome["loaded"]) == [] + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/__init__.py b/tests/unit/src/sqlbuild/cli/compile_reuse/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/__init__.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py new file mode 100644 index 0000000000..77b9011266 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -0,0 +1,80 @@ +"""Test case types for project input fingerprinting.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import pytest + + +@dataclass(frozen=True) +class ProjectFingerprintTestCase: + """One filesystem change and whether the project fingerprint must report it.""" + + description: str + change: Callable[[Path], None] + expected_unchanged: bool + + +@dataclass(frozen=True) +class DigestCarryForwardTestCase: + """A same-content change whose digest must be verified once and then carried forward.""" + + description: str + change: Callable[[Path], None] + expected_verified: frozenset[str] + expected_carried: frozenset[str] + + +@dataclass(frozen=True) +class RacyRewriteTestCase: + """A rewrite inside one timestamp tick that leaves every stat field unchanged.""" + + description: str + racy: bool + expected_unchanged: bool + + +@dataclass(frozen=True) +class LinkCycleTestCase: + """A directory link that points back into the project.""" + + description: str + link_path: str + expected_present: tuple[str, ...] + expected_descended_prefix: str + + +@dataclass(frozen=True) +class RestampTestCase: + """A stat change to a file stored without a digest, which must miss and queue hashing.""" + + description: str + change: Callable[[Path], None] + expected_unchanged: bool + expected_restamped: frozenset[str] + + +@dataclass(frozen=True) +class ProviderSettingsInputsTestCase: + """One provider settings class and the external inputs its settings can read.""" + + description: str + settings_class: type + expected_case_sensitive: bool = False + expected_names: tuple[str, ...] = () + expected_prefixes: tuple[str, ...] = () + expected_env_file_names: tuple[str, ...] = () + expected_secrets_dir_names: tuple[str, ...] = () + expected_unsupported: bool = False + + +@dataclass(frozen=True) +class SettingsDigestTestCase: + """One change to the environment or settings files and whether the digest must move.""" + + description: str + change: Callable[[pytest.MonkeyPatch, Path], None] + expected_changed: bool diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py new file mode 100644 index 0000000000..3dcf51f238 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -0,0 +1,262 @@ +"""Project builders and filesystem changes for project input fingerprint tests.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import ClassVar + +import pytest +from pydantic import AliasChoices, BaseModel, Field +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, SettingsConfigDict + +from sqlbuild.cli.compile_reuse._helpers.project_files import ( + file_digest, + is_racy, + snapshot_project_files, +) +from sqlbuild.cli.compile_reuse.constants import RACY_WINDOW_NS +from sqlbuild.cli.compile_reuse.models import ( + SettingsEnvironmentInputs, + SettingsInputsResult, + StoredProjectFile, +) +from sqlbuild.cli.compile_reuse.types import FileStamp + +_LATER_SNAPSHOT_NS: int = 10 * RACY_WINDOW_NS + + +def write_file(path: Path, contents: str) -> None: + """Write one file, creating parent folders.""" + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(contents, encoding="utf-8") + + +def write_fingerprint_project(root: Path) -> Path: + """Create a project with links, excluded folders, and presence-only database files.""" + + project_dir: Path = root / "project" + write_file(project_dir / "sqlbuild_project.toml", 'name = "orders"\nadapter = "duckdb"\n') + write_file(project_dir / "models/orders.sql", "SELECT 1 AS order_id\n") + write_file(project_dir / "models/.drafts/returns.sql", "SELECT 2 AS return_id\n") + write_file(project_dir / "macros/currency.py", "def cents() -> str:\n return 'cents'\n") + write_file(project_dir / "macros/__pycache__/currency.cpython-312.pyc", "bytecode") + write_file(project_dir / "target/compiled/models/orders.sql", "SELECT 1 AS order_id\n") + write_file(project_dir / "logs/compile.log", "compile log\n") + write_file(project_dir / ".venv/lib/vendored.py", "VALUE = 1\n") + write_file(project_dir / ".git/HEAD", "ref: refs/heads/main\n") + write_file(project_dir / ".vscode/settings.json", "{}\n") + write_file(project_dir / ".orders_rules/limits.txt", "40\n") + write_file(project_dir / "warehouse.duckdb", "database pages") + write_file(root / "shared/customers.sql", "SELECT 3 AS customer_id\n") + write_file(root / "shared_dir/products.sql", "SELECT 4 AS product_id\n") + (project_dir / "models/customers.sql").symlink_to(root / "shared/customers.sql") + (project_dir / "models/shared").symlink_to(root / "shared_dir", target_is_directory=True) + return project_dir + + +def stored_project_files(*, project_dir: Path, snapshot_ns: int) -> dict[str, StoredProjectFile]: + """Record the project as a stored compile would, with digests for every readable file.""" + + snapshot: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) + return { + relative_path: StoredProjectFile( + stamp=stamp, + digest=file_digest(path=str(project_dir / relative_path)), + racy=is_racy(stamp=stamp, snapshot_ns=snapshot_ns), + ) + for relative_path, stamp in snapshot.items() + } + + +def later_snapshot_ns() -> int: + """Return a snapshot time after which no current file counts as racy.""" + + return time.time_ns() + _LATER_SNAPSHOT_NS + + +def touch_without_change(project_dir: Path) -> None: + """Move a file's mtime forward without changing its content.""" + + path: Path = project_dir / "models/orders.sql" + mtime_ns: int = path.stat().st_mtime_ns + RACY_WINDOW_NS + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def rewrite_same_size_keeping_mtime(project_dir: Path) -> None: + """Change content without changing size, then restore the previous mtime.""" + + path: Path = project_dir / "models/orders.sql" + mtime_ns: int = path.stat().st_mtime_ns + path.write_text("SELECT 9 AS order_id\n", encoding="utf-8") + os.utime(path, ns=(mtime_ns, mtime_ns)) + + +def edit_link_target(project_dir: Path) -> None: + """Edit the file a project link points to, outside the project.""" + + (project_dir.parent / "shared/customers.sql").write_text( + "SELECT 30 AS customer_id\n", encoding="utf-8" + ) + + +def retarget_link(project_dir: Path) -> None: + """Point a project link at another file with identical content.""" + + link: Path = project_dir / "models/customers.sql" + replacement: Path = project_dir.parent / "shared/customers_copy.sql" + replacement.write_text("SELECT 3 AS customer_id\n", encoding="utf-8") + link.unlink() + link.symlink_to(replacement) + + +def add_file_in_linked_directory(project_dir: Path) -> None: + """Add a file inside a directory the project links to.""" + + write_file(project_dir.parent / "shared_dir/suppliers.sql", "SELECT 5 AS supplier_id\n") + + +def break_link(project_dir: Path) -> None: + """Delete the target of a project link.""" + + (project_dir.parent / "shared/customers.sql").unlink() + + +def stat_only_project_files(*, project_dir: Path) -> dict[str, StoredProjectFile]: + """Record the project as a stored compile does by default: stat stamps without digests.""" + + return { + relative_path: StoredProjectFile(stamp=stamp, digest=None, racy=False) + for relative_path, stamp in snapshot_project_files(project_dir=str(project_dir)).items() + } + + +SETTINGS_TOKEN_ENV_VAR: str = "ORDERS_API_TOKEN" +SETTINGS_ENV_FILE_NAME: str = "orders_api.env" +SETTINGS_SECRETS_DIR_NAME: str = "orders_secrets" + + +class OrdersApiSettings(BaseSettings): + """Plain provider settings read from one environment variable per field.""" + + orders_api_token: str = "" + + +class PrefixedOrdersSettings(BaseSettings): + """Provider settings read through an environment prefix.""" + + model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(env_prefix="ORDERS_") + + token: str = "" + + +class AliasedOrdersSettings(BaseSettings): + """Provider settings read through alias choices.""" + + token: str = Field(default="", validation_alias=AliasChoices("ORDERS_KEY", "LEGACY_KEY")) + + +class OrdersEndpoint(BaseModel): + """Nested endpoint settings.""" + + host: str = "localhost" + + +class NestedOrdersSettings(BaseSettings): + """Provider settings with nested values split by a delimiter.""" + + model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(env_nested_delimiter="__") + + endpoint: OrdersEndpoint = OrdersEndpoint() + + +class CaseSensitiveOrdersSettings(BaseSettings): + """Provider settings matched case-sensitively.""" + + model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(case_sensitive=True) + + OrdersToken: str = "" + + +class FileOrdersSettings(BaseSettings): + """Provider settings that also read an env file and a secrets directory.""" + + model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict( + env_file=SETTINGS_ENV_FILE_NAME, secrets_dir=SETTINGS_SECRETS_DIR_NAME + ) + + orders_api_token: str = "" + + +class CustomSourceOrdersSettings(BaseSettings): + """Provider settings whose sources cannot be enumerated.""" + + orders_api_token: str = "" + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Read only the environment.""" + + return (env_settings,) + + +class CommandLineOrdersSettings(BaseSettings): + """Provider settings that would parse command-line arguments.""" + + model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(cli_parse_args=False) + + orders_api_token: str = "" + + +def file_settings_inputs(*, root: Path) -> SettingsEnvironmentInputs: + """Describe FileOrdersSettings as if read from root, the process working directory.""" + + return SettingsEnvironmentInputs( + case_sensitive=False, + names=("orders_api_token",), + prefixes=(), + env_files=(str(root / SETTINGS_ENV_FILE_NAME),), + secrets_dirs=(str(root / SETTINGS_SECRETS_DIR_NAME),), + ) + + +def set_token(monkeypatch: pytest.MonkeyPatch, value: str) -> None: + """Set the tracked token variable.""" + + monkeypatch.setenv(SETTINGS_TOKEN_ENV_VAR, value) + + +def write_settings_file(root: Path, relative_path: str, contents: str) -> None: + """Write one env file or secrets file below root.""" + + write_file(root / relative_path, contents) + + +def described_settings_inputs( + *, result: SettingsInputsResult +) -> tuple[tuple[bool, tuple[str, ...], tuple[str, ...], tuple[str, ...], tuple[str, ...]], ...]: + """Describe enumerated settings inputs with file and directory paths reduced to names.""" + + return tuple( + ( + item.case_sensitive, + item.names, + item.prefixes, + _path_names(item.env_files), + _path_names(item.secrets_dirs), + ) + for item in result.inputs + ) + + +def _path_names(paths: tuple[str, ...]) -> tuple[str, ...]: + return tuple(Path(path).name for path in paths) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py new file mode 100644 index 0000000000..9f0eeb019f --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_project_files.py @@ -0,0 +1,286 @@ +"""Project input fingerprint edge cases for whole-project compile reuse.""" + +from __future__ import annotations + +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_reuse._helpers.project_files import ( + carried_forward_digests, + compare_project_files, + restamped_paths, + snapshot_project_files, +) +from sqlbuild.cli.compile_reuse.models import ProjectFilesComparison, StoredProjectFile +from sqlbuild.cli.compile_reuse.types import FileStamp +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + DigestCarryForwardTestCase, + LinkCycleTestCase, + ProjectFingerprintTestCase, + RacyRewriteTestCase, + RestampTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import ( + add_file_in_linked_directory, + break_link, + edit_link_target, + later_snapshot_ns, + retarget_link, + rewrite_same_size_keeping_mtime, + stat_only_project_files, + stored_project_files, + touch_without_change, + write_file, + write_fingerprint_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectFingerprintTestCase( + description="no_change", change=lambda _root: None, expected_unchanged=True + ), + ProjectFingerprintTestCase( + description="mtime_only_change_same_content", + change=touch_without_change, + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="same_size_and_mtime_new_content", + change=rewrite_same_size_keeping_mtime, + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="file_added", + change=lambda root: write_file(root / "models/returns.sql", "SELECT 1\n"), + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="file_removed", + change=lambda root: (root / "macros/currency.py").unlink(), + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="empty_directory_added", + change=lambda root: (root / "models/_sqlbuild").mkdir(), + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="nested_hidden_directory_edit", + change=lambda root: write_file(root / "models/.drafts/returns.sql", "SELECT 22\n"), + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="file_link_target_edit", change=edit_link_target, expected_unchanged=False + ), + ProjectFingerprintTestCase( + description="file_link_retarget", change=retarget_link, expected_unchanged=False + ), + ProjectFingerprintTestCase( + description="file_added_in_linked_directory", + change=add_file_in_linked_directory, + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="file_link_broken", change=break_link, expected_unchanged=False + ), + ProjectFingerprintTestCase( + description="database_file_removed", + change=lambda root: (root / "warehouse.duckdb").unlink(), + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="database_file_content_change", + change=lambda root: write_file(root / "warehouse.duckdb", "rewritten database pages"), + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="target_directory_change", + change=lambda root: write_file( + root / "target/compiled/models/orders.sql", "SELECT 2\n" + ), + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="compute_log_change", + change=lambda root: write_file(root / "logs/compile-2.log", "next compile log\n"), + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="bytecode_cache_change", + change=lambda root: write_file(root / "macros/__pycache__/currency.pyc", "bytecode 2"), + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="root_hidden_directory_change", + change=lambda root: write_file(root / ".venv/lib/vendored.py", "VALUE = 2\n"), + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="root_editor_settings_change", + change=lambda root: write_file(root / ".vscode/settings.json", '{"tabs": 2}\n'), + expected_unchanged=True, + ), + ProjectFingerprintTestCase( + description="root_hidden_project_directory_change", + change=lambda root: write_file(root / ".orders_rules/limits.txt", "10\n"), + expected_unchanged=False, + ), + ProjectFingerprintTestCase( + description="version_control_metadata_change", + change=lambda root: write_file(root / ".git/HEAD", "ref: refs/heads/feature\n"), + expected_unchanged=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_stored_fingerprint_when_project_changes_then_change_is_detected_exactly( + test_case: ProjectFingerprintTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + stored: dict[str, StoredProjectFile] = stored_project_files( + project_dir=project_dir, snapshot_ns=later_snapshot_ns() + ) + + test_case.change(project_dir) + comparison: ProjectFilesComparison = compare_project_files( + project_dir=str(project_dir), + stored=stored, + current=snapshot_project_files(project_dir=str(project_dir)), + ) + + assert comparison.unchanged is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + DigestCarryForwardTestCase( + description="touched_model", + change=touch_without_change, + expected_verified=frozenset({"models/orders.sql"}), + expected_carried=frozenset({"models/orders.sql", "macros/currency.py"}), + ) + ], + ids=lambda case: case.description, +) +def test_given_touched_file_with_same_content_when_checking_then_its_digest_carries_forward( + test_case: DigestCarryForwardTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + stored: dict[str, StoredProjectFile] = stored_project_files( + project_dir=project_dir, snapshot_ns=later_snapshot_ns() + ) + test_case.change(project_dir) + current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) + + comparison: ProjectFilesComparison = compare_project_files( + project_dir=str(project_dir), stored=stored, current=current + ) + digests: dict[str, str] = carried_forward_digests( + stored=stored, current=current, verified=comparison.verified + ) + + assert comparison.unchanged + assert frozenset(comparison.verified) == test_case.expected_verified + assert test_case.expected_carried <= frozenset(digests) + assert {path: digests[path] for path in test_case.expected_carried} == { + path: stored[path].digest for path in test_case.expected_carried + } + + +@pytest.mark.parametrize( + "test_case", + [ + RestampTestCase( + description="touched_file_without_digest", + change=touch_without_change, + expected_unchanged=False, + expected_restamped=frozenset({"models/orders.sql"}), + ), + RestampTestCase( + description="unchanged_file_without_digest", + change=lambda _root: None, + expected_unchanged=True, + expected_restamped=frozenset(), + ), + ], + ids=lambda case: case.description, +) +def test_given_stat_only_stored_files_when_a_stamp_moves_then_it_misses_and_is_queued_for_hashing( + test_case: RestampTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + stored: dict[str, StoredProjectFile] = stat_only_project_files(project_dir=project_dir) + + test_case.change(project_dir) + current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) + comparison: ProjectFilesComparison = compare_project_files( + project_dir=str(project_dir), stored=stored, current=current + ) + + assert comparison.unchanged is test_case.expected_unchanged + assert comparison.verified == {} + assert restamped_paths(stored=stored, current=current) == test_case.expected_restamped + + +@pytest.mark.parametrize( + "test_case", + [ + RacyRewriteTestCase(description="racy_stamp_rehashed", racy=True, expected_unchanged=False), + RacyRewriteTestCase( + description="settled_stamp_trusted", racy=False, expected_unchanged=True + ), + ], + ids=lambda case: case.description, +) +def test_given_rewrite_within_one_timestamp_tick_when_checking_then_only_racy_stamps_rehash( + test_case: RacyRewriteTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + stored: dict[str, StoredProjectFile] = stored_project_files( + project_dir=project_dir, snapshot_ns=time.time_ns() + ) + current: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) + (project_dir / "models/orders.sql").write_text("SELECT 9 AS order_id\n", encoding="utf-8") + stored["models/orders.sql"] = replace( + stored["models/orders.sql"], stamp=current["models/orders.sql"], racy=test_case.racy + ) + + comparison: ProjectFilesComparison = compare_project_files( + project_dir=str(project_dir), stored=stored, current=current + ) + + assert comparison.unchanged is test_case.expected_unchanged + + +@pytest.mark.parametrize( + "test_case", + [ + LinkCycleTestCase( + description="link_to_project_root", + link_path="models/loop", + expected_present=("models/loop", "models/orders.sql"), + expected_descended_prefix="models/loop/", + ) + ], + ids=lambda case: case.description, +) +def test_given_directory_link_cycle_when_snapshotting_then_walk_terminates( + test_case: LinkCycleTestCase, tmp_path: Path +) -> None: + project_dir: Path = write_fingerprint_project(tmp_path) + (project_dir / test_case.link_path).symlink_to(project_dir, target_is_directory=True) + + snapshot: dict[str, FileStamp] = snapshot_project_files(project_dir=str(project_dir)) + + assert frozenset(test_case.expected_present) <= frozenset(snapshot) + assert not any(path.startswith(test_case.expected_descended_prefix) for path in snapshot) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py new file mode 100644 index 0000000000..eb01b8c105 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_provider_settings.py @@ -0,0 +1,176 @@ +"""Provider settings inputs must be enumerated exactly, or reuse must be refused.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_reuse._helpers.provider_settings import provider_settings_inputs +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import settings_inputs_digest +from sqlbuild.cli.compile_reuse.models import SettingsEnvironmentInputs, SettingsInputsResult +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + ProviderSettingsInputsTestCase, + SettingsDigestTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import ( + SETTINGS_ENV_FILE_NAME, + SETTINGS_SECRETS_DIR_NAME, + SETTINGS_TOKEN_ENV_VAR, + AliasedOrdersSettings, + CaseSensitiveOrdersSettings, + CommandLineOrdersSettings, + CustomSourceOrdersSettings, + FileOrdersSettings, + NestedOrdersSettings, + OrdersApiSettings, + PrefixedOrdersSettings, + described_settings_inputs, + file_settings_inputs, + set_token, + write_settings_file, +) + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderSettingsInputsTestCase( + description="field_name", + settings_class=OrdersApiSettings, + expected_names=("orders_api_token",), + ), + ProviderSettingsInputsTestCase( + description="env_prefix", + settings_class=PrefixedOrdersSettings, + expected_names=("orders_token",), + ), + ProviderSettingsInputsTestCase( + description="alias_choices", + settings_class=AliasedOrdersSettings, + expected_names=("legacy_key", "orders_key"), + ), + ProviderSettingsInputsTestCase( + description="nested_delimiter", + settings_class=NestedOrdersSettings, + expected_names=("endpoint",), + expected_prefixes=("endpoint__",), + ), + ProviderSettingsInputsTestCase( + description="case_sensitive", + settings_class=CaseSensitiveOrdersSettings, + expected_case_sensitive=True, + expected_names=("OrdersToken",), + ), + ProviderSettingsInputsTestCase( + description="env_file_and_secrets_dir", + settings_class=FileOrdersSettings, + expected_names=("orders_api_token",), + expected_env_file_names=(SETTINGS_ENV_FILE_NAME,), + expected_secrets_dir_names=(SETTINGS_SECRETS_DIR_NAME,), + ), + ], + ids=lambda case: case.description, +) +def test_given_provider_settings_class_when_enumerating_then_every_external_input_is_listed( + test_case: ProviderSettingsInputsTestCase, +) -> None: + result: SettingsInputsResult = provider_settings_inputs( + settings_classes=(test_case.settings_class,) + ) + assert result.unsupported_reason is None + assert described_settings_inputs(result=result) == ( + ( + test_case.expected_case_sensitive, + test_case.expected_names, + test_case.expected_prefixes, + test_case.expected_env_file_names, + test_case.expected_secrets_dir_names, + ), + ) + + +@pytest.mark.parametrize( + "test_case", + [ + ProviderSettingsInputsTestCase( + description="customized_sources", + settings_class=CustomSourceOrdersSettings, + expected_unsupported=True, + ), + ProviderSettingsInputsTestCase( + description="command_line_source", + settings_class=CommandLineOrdersSettings, + expected_unsupported=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_settings_with_unlisted_sources_when_enumerating_then_reuse_is_refused( + test_case: ProviderSettingsInputsTestCase, +) -> None: + result: SettingsInputsResult = provider_settings_inputs( + settings_classes=(OrdersApiSettings, test_case.settings_class) + ) + + assert (result.inputs, result.unsupported_reason is not None) == ( + (), + test_case.expected_unsupported, + ) + + +@pytest.mark.parametrize( + "test_case", + [ + SettingsDigestTestCase( + description="unrelated_variable", + change=lambda mp, _root: mp.setenv("OTHER", "1"), + expected_changed=False, + ), + SettingsDigestTestCase( + description="token_unset", + change=lambda mp, _root: mp.delenv(SETTINGS_TOKEN_ENV_VAR), + expected_changed=True, + ), + SettingsDigestTestCase( + description="token_changed", + change=lambda mp, _root: set_token(mp, "beta"), + expected_changed=True, + ), + SettingsDigestTestCase( + description="token_set_in_other_case", + change=lambda mp, _root: mp.setenv(SETTINGS_TOKEN_ENV_VAR.lower(), "gamma"), + expected_changed=True, + ), + SettingsDigestTestCase( + description="env_file_changed", + change=lambda _mp, root: write_settings_file( + root, SETTINGS_ENV_FILE_NAME, "ORDERS_API_TOKEN=delta\n" + ), + expected_changed=True, + ), + SettingsDigestTestCase( + description="secret_added", + change=lambda _mp, root: write_settings_file( + root, f"{SETTINGS_SECRETS_DIR_NAME}/orders_api_token", "epsilon" + ), + expected_changed=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_settings_inputs_when_their_values_change_then_the_digest_moves( + test_case: SettingsDigestTestCase, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + set_token(monkeypatch, "alpha") + write_settings_file(tmp_path, SETTINGS_ENV_FILE_NAME, "ORDERS_API_TOKEN=alpha\n") + inputs: tuple[SettingsEnvironmentInputs, ...] = (file_settings_inputs(root=tmp_path),) + before: str = settings_inputs_digest(inputs=inputs) + + test_case.change(monkeypatch, tmp_path) + + assert (settings_inputs_digest(inputs=inputs) != before) is test_case.expected_changed + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/__init__.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/_test_types.py new file mode 100644 index 0000000000..47c040e443 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/_test_types.py @@ -0,0 +1,15 @@ +"""Test case types for the compile reuse import footprint.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ReuseImportFootprintTestCase: + """Modules a full compile loads and a reused compile must leave unloaded.""" + + description: str + checked_modules: tuple[str, ...] + expected_full_loaded: tuple[str, ...] + expected_reused_loaded: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py new file mode 100644 index 0000000000..b322238284 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/helpers.py @@ -0,0 +1,41 @@ +"""Fresh-interpreter compile runs for compile reuse import checks.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import cast + +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR + + +def compile_in_fresh_process( + *, project_dir: Path, checked_modules: tuple[str, ...] +) -> tuple[int, tuple[str, ...], str]: + """Compile once with reuse enabled and report which checked modules were imported.""" + + script: str = ( + "import contextlib, io, json, sys\n" + "from sqlbuild.cli.entry.main.entry import main\n" + "with contextlib.redirect_stdout(io.StringIO()):\n" + f" code = main(['--project-dir', {str(project_dir)!r}, 'compile', '--json'])\n" + f"names = {checked_modules!r}\n" + "print(json.dumps({'code': code, " + "'loaded': [name for name in names if name in sys.modules]}))\n" + ) + result: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", script], + check=True, + capture_output=True, + text=True, + env={**os.environ, REUSE_DISABLE_ENV_VAR: "0"}, + ) + outcome: dict[str, object] = json.loads(result.stdout.strip().splitlines()[-1]) + return ( + cast(int, outcome["code"]), + tuple(cast(list[str], outcome["loaded"])), + result.stderr, + ) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py new file mode 100644 index 0000000000..02ba9206a4 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/main/test_attempt_import_footprint.py @@ -0,0 +1,55 @@ +"""A reused compile must not import the full compile pipeline.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.unit.src.sqlbuild.cli.commands.main.compile.helpers import ( + prepare_static_compile_project, +) +from tests.unit.src.sqlbuild.cli.compile_reuse.main._test_types import ( + ReuseImportFootprintTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_reuse.main.helpers import compile_in_fresh_process + +_FULL_COMPILE_MODULES: tuple[str, ...] = ( + "sqlbuild.cli.commands.main.project._compile", + "sqlbuild.cli.commands._helpers.compile.pipeline", + "sqlbuild.compiler.pipeline.main.compiled_project", +) + + +@pytest.mark.parametrize( + "test_case", + [ + ReuseImportFootprintTestCase( + description="reuse_hit_skips_full_compile_modules", + checked_modules=_FULL_COMPILE_MODULES, + expected_full_loaded=_FULL_COMPILE_MODULES, + expected_reused_loaded=(), + ) + ], + ids=lambda case: case.description, +) +def test_given_unchanged_project_when_compile_is_reused_then_full_compile_stays_unloaded( + test_case: ReuseImportFootprintTestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_static_compile_project(tmp_path) + + full_code, full_loaded, _ = compile_in_fresh_process( + project_dir=project_dir, checked_modules=test_case.checked_modules + ) + reused_code, reused_loaded, reused_stderr = compile_in_fresh_process( + project_dir=project_dir, checked_modules=test_case.checked_modules + ) + + assert (full_code, reused_code) == (0, 0) + assert full_loaded == test_case.expected_full_loaded + assert "Inputs unchanged; reused the previous compile" in reused_stderr + assert reused_loaded == test_case.expected_reused_loaded + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From d6b254c22329e24ccb43bcfe2e49fccce2e0e13e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:17:41 +0100 Subject: [PATCH 2/2] docs: describe compile reuse of unchanged inputs --- .../sqlbuild/references/docs/cli/compile.md | 73 ++++++++++++++++++ website/src/content/docs/docs/cli/compile.mdx | 74 +++++++++++++++++++ 2 files changed, 147 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index edf93ffdb7..f4ca9887b3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -12,6 +12,7 @@ Online: https://sqlbuild.com/docs/cli/compile/ - Flags - What compile does - Focused compilation +- Reusing an unchanged compile - Static analysis - Output - Column lineage modes @@ -69,6 +70,78 @@ with selected model, seed, and function counts. Manifest and DAG outputs remain full-project artifacts. Focused compilation changes analysis and reporting scope; it does not produce a partial project graph. +## Reusing an unchanged compile + +When nothing that can change the result has changed since the last compile, `sqb compile` replays +that result instead of compiling again. It prints one line to stderr, for example +`Inputs unchanged; reused the previous compile (0.1 s)`, then the same report and exit code as the +previous run. Failed compiles are replayed too, with the same diagnostics. + +A compile is reused only when all of these match the stored run: + +- Every file and directory under the project, compared by type, size, modification and change + times, and inode. When those differ, SQLBuild compares content only for files whose content + digest it already knows; otherwise the compile runs in full. Digests are recorded for files that + changed within two seconds of a compile and for files whose timestamps moved since the previous + compile, so large unchanged files are never read. After a checkout that only moves timestamps, + the next compile runs in full once, and later timestamp-only changes are reused. +- The command line: project directory, working directory, flags, selectors, `--vars`, target, and + whether color output is active. +- Environment variables that SQLBuild reads: every `SQLBUILD_*` and `SQB_*` variable, every + variable that a template or `@@ENV` read during the stored compile, and every variable a + provider's settings can read, including `env_prefix`, aliases, and nested-delimiter variables. + Values are stored only as hashes. +- The contents of each provider settings `env_file` and `secrets_dir`. +- The Python interpreter, the SQLBuild version and native extension, `sys.path` entries, and every + loaded Python module file outside the project. +- Every artifact the stored compile wrote under `target/compiled/`, and the `--dag` file. If one is + missing or changed, compile runs in full and rewrites it. A compile is stored only when these + artifacts still hold the bytes it wrote, so a concurrent compile cannot leave another compile's + artifacts behind a reused result. + +These project paths are not inputs: `target/`, `logs/`, `venv/`, `.git/` and `__pycache__/` +anywhere, and these folders at the project root: `.cache`, `.fensu`, `.hg`, `.idea`, +`.mypy_cache`, `.nox`, `.pytest_cache`, `.ruff_cache`, `.sqlbuild`, `.svn`, `.tox`, `.venv`, and +`.vscode`. Other hidden folders are inputs, because custom rules can read them. DuckDB database +files count only by presence. + +Reuse assumes macros, hooks, providers, adapters, and custom rules are deterministic: the same inputs +must produce the same output. A compile whose templates read `run.id` is never reused. + +Compile always runs in full with `--no-cache`, `--manifest`, `--debug`, or a profiling flag, when +`SQLBUILD_DISABLE_COMPILE_CACHE=1` or `SQLBUILD_DISABLE_COMPILE_REUSE=1` is set, when the project has +no `sqlbuild_project.toml` or `sqlbuild_project.yml`, when the target sets `compile_cache = false`, +and when a provider's settings use sources SQLBuild cannot list exactly, such as a custom +`settings_customise_sources` or command-line parsing. The reason is logged at debug level. + +SQLBuild keeps one stored result per project and target under +`target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is +ignored and compile runs in full. Storing a result normally takes a fraction of a second after the +report is ready. When it has to hash a lot of data or the project has very many files, compile +prints `Recording compile for reuse...` and a completion line to stderr before the report. + +Known limits: + +- Python packages outside the project are checked by the stamps of the module files the compile + loaded. A newly added submodule or package data file that the compile reads without importing it + first is not tracked. +- On network filesystems that cache file attributes, or when clocks differ between machines, + timestamps may not reflect a recent edit. +- On Windows, a file's change time is its creation time. An in-place rewrite that keeps the same + size and restores the previous modification time, such as `touch -r`, `rsync -t`, or + `robocopy /COPY:T`, is not detected unless the file changed within two seconds of the stored + compile. The same applies to filesystems without a real change time, such as some FUSE, SMB, and + exFAT mounts. +- In these cases, or whenever a result looks stale, run `sqb compile --no-cache` or set + `SQLBUILD_DISABLE_COMPILE_REUSE=1`. + +A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which +holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, +`project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside +the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` +events are not emitted because no compile phase runs. + ## Static analysis When SQL analysis is enabled (default), compile performs static analysis on your models without connecting to the warehouse: diff --git a/website/src/content/docs/docs/cli/compile.mdx b/website/src/content/docs/docs/cli/compile.mdx index a7dd474af3..b039cbc86d 100644 --- a/website/src/content/docs/docs/cli/compile.mdx +++ b/website/src/content/docs/docs/cli/compile.mdx @@ -61,6 +61,80 @@ Manifest and DAG outputs remain full-project artifacts. Focused compilation chan reporting scope; it does not produce a partial project graph. +## Reusing an unchanged compile + +When nothing that can change the result has changed since the last compile, `sqb compile` replays +that result instead of compiling again. It prints one line to stderr, for example +`Inputs unchanged; reused the previous compile (0.1 s)`, then the same report and exit code as the +previous run. Failed compiles are replayed too, with the same diagnostics. + +A compile is reused only when all of these match the stored run: + +- Every file and directory under the project, compared by type, size, modification and change + times, and inode. When those differ, SQLBuild compares content only for files whose content + digest it already knows; otherwise the compile runs in full. Digests are recorded for files that + changed within two seconds of a compile and for files whose timestamps moved since the previous + compile, so large unchanged files are never read. After a checkout that only moves timestamps, + the next compile runs in full once, and later timestamp-only changes are reused. +- The command line: project directory, working directory, flags, selectors, `--vars`, target, and + whether color output is active. +- Environment variables that SQLBuild reads: every `SQLBUILD_*` and `SQB_*` variable, every + variable that a template or `@@ENV` read during the stored compile, and every variable a + provider's settings can read, including `env_prefix`, aliases, and nested-delimiter variables. + Values are stored only as hashes. +- The contents of each provider settings `env_file` and `secrets_dir`. +- The Python interpreter, the SQLBuild version and native extension, `sys.path` entries, and every + loaded Python module file outside the project. +- Every artifact the stored compile wrote under `target/compiled/`, and the `--dag` file. If one is + missing or changed, compile runs in full and rewrites it. A compile is stored only when these + artifacts still hold the bytes it wrote, so a concurrent compile cannot leave another compile's + artifacts behind a reused result. + +These project paths are not inputs: `target/`, `logs/`, `venv/`, `.git/` and `__pycache__/` +anywhere, and these folders at the project root: `.cache`, `.fensu`, `.hg`, `.idea`, +`.mypy_cache`, `.nox`, `.pytest_cache`, `.ruff_cache`, `.sqlbuild`, `.svn`, `.tox`, `.venv`, and +`.vscode`. Other hidden folders are inputs, because custom rules can read them. DuckDB database +files count only by presence. + +Reuse assumes macros, hooks, providers, adapters, and custom rules are deterministic: the same inputs +must produce the same output. A compile whose templates read `run.id` is never reused. + +Compile always runs in full with `--no-cache`, `--manifest`, `--debug`, or a profiling flag, when +`SQLBUILD_DISABLE_COMPILE_CACHE=1` or `SQLBUILD_DISABLE_COMPILE_REUSE=1` is set, when the project has +no `sqlbuild_project.toml` or `sqlbuild_project.yml`, when the target sets `compile_cache = false`, +and when a provider's settings use sources SQLBuild cannot list exactly, such as a custom +`settings_customise_sources` or command-line parsing. The reason is logged at debug level. + +SQLBuild keeps one stored result per project and target under +`target/cache/compiler/project-reuse-v1/`. It is replaced atomically; a damaged or unreadable entry is +ignored and compile runs in full. Storing a result normally takes a fraction of a second after the +report is ready. When it has to hash a lot of data or the project has very many files, compile +prints `Recording compile for reuse...` and a completion line to stderr before the report. + +Known limits: + +- Python packages outside the project are checked by the stamps of the module files the compile + loaded. A newly added submodule or package data file that the compile reads without importing it + first is not tracked. +- On network filesystems that cache file attributes, or when clocks differ between machines, + timestamps may not reflect a recent edit. +- On Windows, a file's change time is its creation time. An in-place rewrite that keeps the same + size and restores the previous modification time, such as `touch -r`, `rsync -t`, or + `robocopy /COPY:T`, is not detected unless the file changed within two seconds of the stored + compile. The same applies to filesystems without a real change time, such as some FUSE, SMB, and + exFAT mounts. +- In these cases, or whenever a result looks stale, run `sqb compile --no-cache` or set + `SQLBUILD_DISABLE_COMPILE_REUSE=1`. + + +A replayed JSON report is byte-identical to the previous one except for `compile_timings`, which +holds `project_reuse_hits`, `project_reuse_misses`, `project_reuse_bypasses`, +`project_reuse_check_ms`, and `total_ms`. Full compiles report the same reuse counters alongside +the phase timings. In text mode, a replayed compile does not print the per-phase progress lines. +Lifecycle sinks receive only the invocation events for a replayed compile; per-phase `operation` +events are not emitted because no compile phase runs. + + ## Static analysis When SQL analysis is enabled (default), compile performs static analysis on your models without connecting to the warehouse: