From 479a23125305a832a580fe42033e6f9c17b6a0ca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 03:10:03 +0100 Subject: [PATCH 1/3] perf(compile): reuse model attachment on warm and one-edit compiles Model attachment (config layering, declaration and macro expansion, hook expansion, header columns and reference extraction) was recomputed for every model on every compile. Each model's result is now stored in the compile cache under a key covering the model file and path, its matched path defaults, project defaults, vars, target, adapter and dialect, settings, the shared macro and declaration sources, hook and audit-factory definitions, project-local adapter sources, and the installed code identity. Editing any macro or declaration re-attaches every model. Models whose config, hooks or SQL read an environment variable or run.id are always recomputed; environment values referenced by the target config are part of the key instead. Diagnostics reported during attachment are stored and replayed on a hit, and project-wide reference checks re-run so adding or removing a model only attaches that model. compile_cache = false and --no-cache disable the cache. Macros are documented as deterministic, along with which project files the reuse check covers. Refs CHI-442 --- .../references/docs/concepts/macros.md | 22 + .../compile/_helpers/attachment/core.py | 710 ++++++++++-------- .../compile/_helpers/diagnostics/collector.py | 18 + .../compile/_helpers/model_reuse/keys.py | 235 ++++++ .../compile/_helpers/render/sql_vars.py | 6 + .../compile/_helpers/render/templating.py | 6 + .../compile/_helpers/render/volatile_reads.py | 47 ++ .../classes/collected_compile_diagnostics.py | 6 + .../compile/classes/model_attachment_cache.py | 137 ++++ src/sqlbuild/compiler/compile/constants.py | 2 + src/sqlbuild/compiler/compile/models.py | 19 + src/sqlbuild/compiler/profiling/constants.py | 3 + src/sqlbuild/compiler/profiling/types.py | 3 + .../main/build/test_compile_json_behavior.py | 3 + .../cli/commands/main/compile/_test_types.py | 47 ++ .../cli/commands/main/compile/helpers.py | 184 ++++- .../compile/test_compile_attachment_cache.py | 264 +++++++ .../test_compile_cache_invalidation.py | 177 ++++- .../compile/test_dense_compile_performance.py | 4 + .../compiler/compile/_helpers/helpers.py | 11 + .../compile/_helpers/render/_test_types.py | 8 + .../_helpers/render/test_volatile_reads.py | 72 ++ .../compile/_helpers/test_analysis_cache.py | 5 +- .../src/content/docs/docs/concepts/macros.mdx | 21 + 24 files changed, 1699 insertions(+), 311 deletions(-) create mode 100644 src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py create mode 100644 src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py create mode 100644 src/sqlbuild/compiler/compile/classes/model_attachment_cache.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py create mode 100644 tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 7cf958daae..dfb97c7bfb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -13,6 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/ - Arguments - Typed reference arguments - Keep implementation details private +- Keep macros deterministic - Use macros in tests - Use macros in hooks - Where macros work @@ -154,6 +155,27 @@ Python implementation details. Imported functions are not re-exported as new macros from the importing file. +## Keep macros deterministic + +A macro must return the same SQL every time it receives the same arguments and macro context. Do +not read environment variables, the clock, random values, files, or network services inside a +macro. Pass such values in through [project variables](interpolation.md#project-variables) +or the macro context instead. + +SQLBuild relies on this when it reuses compiled models between compile runs. A model is compiled +again when its file, its configuration and defaults, project variables, the target, the adapter, +any project macro or declaration, or the installed SQLBuild version changes. Models whose +configuration, hooks, or SQL read an environment variable or `run.id` are compiled again on every +run. `sqb compile --no-cache` compiles everything from scratch. + +The reuse check covers model files, project and local configuration, macro, enum, constant, schema, +and SQL hook files, the source files that define Python hooks and audit factories, and custom +adapter files under `adapters/`, along with the target, project variables, and command-line flags. +It does not cover code those files import from elsewhere: third-party libraries, and project +modules outside those locations that a hook or adapter imports. Keep macro, hook, and adapter code +independent of such modules, or run `sqb compile --no-cache` (or delete `target/cache/compiler/`) +after upgrading or editing one that changes their output. + ## Use macros in tests Tests are SQL, so they can use macros as reusable fixture generators: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c2260c1862..62d6599705 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -48,6 +48,10 @@ validate_preserved_logical_namespace, ) from sqlbuild.compiler.compile._helpers.config.table_type import resolve_storage_policies +from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( + capture_compile_diagnostics, + report_compile_diagnostic, +) from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( rejected_sql_analysis_opt_out, ) @@ -55,6 +59,10 @@ merge_call_site_references, resource_references, ) +from sqlbuild.compiler.compile._helpers.model_reuse.keys import ( + model_attachment_environment, + model_attachment_key, +) from sqlbuild.compiler.compile._helpers.named_declarations.core import ( named_declaration_usages, ) @@ -95,6 +103,8 @@ expand_effective_vars, expand_template_data, ) +from sqlbuild.compiler.compile._helpers.render.volatile_reads import record_volatile_reads +from sqlbuild.compiler.compile.classes.model_attachment_cache import ModelAttachmentCache from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_FULL_REFRESH_CONFIG_KEY, @@ -116,6 +126,8 @@ LoadedMacro, MacroContext, MacroExpansionResult, + ModelAttachmentEnvironment, + ModelAttachmentFact, ModelConfigBuildRequest, ModelConfigScanCache, ModelHeaderColumnCache, @@ -379,6 +391,19 @@ def _is_reusable(self, matched_path_default: str | None) -> bool: return reusable +@dataclass(frozen=True) +class _ModelAttachmentState: + discovered_inputs: DiscoveredProjectInputs + context: ModelInputBuildContext + validation_context: _ModelValidationContext + sql_hook_definitions: dict[str, DiscoveredSqlHookFile] + header_column_cache: ModelHeaderColumnCache + config_scan_cache: ModelConfigScanCache + reusable_config_cache: _ReusableModelConfigCache + declaration_cache: _VisibleModelDeclarationCache + legacy_schema_files: tuple[DiscoveredSchemaFile, ...] + + def _contains_mutable_nested_config(values: dict[str, object]) -> bool: if any(key in values for key in _MODEL_HOOK_KEYS): return True @@ -431,6 +456,7 @@ def build_model_inputs( external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, legacy_schema_files=legacy_schema_files, + cache_root=reference_cache_dir, ) @@ -443,341 +469,419 @@ def _build_model_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, extract_references: Callable[[str], tuple[CompileSqlReference, ...]], legacy_schema_files: tuple[DiscoveredSchemaFile, ...], + cache_root: Path | None, ) -> tuple[CompileModelInput, ...]: - effective_vars: dict[str, object] = context.effective_vars - effective_settings: SettingsConfig = context.effective_settings - target_config: TargetConfig | None = context.target_config - effective_target_name: str | None = context.effective_target_name - run_id: str = context.run_id - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros known_model_names: set[str] = build_known_ref_names(discovered_inputs) - known_seed_names: set[str] = build_known_seed_names(discovered_inputs) - known_source_names: set[str] = build_known_source_names(discovered_inputs) - known_function_names: set[str] = build_known_function_names(discovered_inputs) - known_table_function_names: set[str] = build_known_table_function_names(discovered_inputs) if external_sql_reference_resolver is not None: external_sql_reference_resolver.validate_model_names(known_model_names=known_model_names) - custom_materialization_names: frozenset[str] = frozenset( - mf.name for mf in discovered_inputs.materialization_files - ) - validation_context: _ModelValidationContext = _ModelValidationContext( - effective_settings=effective_settings, - no_sql_validation=no_sql_validation, - project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - extract_references=extract_references, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - custom_materialization_names=custom_materialization_names, - ) - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( - discovered_inputs.sql_hook_files + state: _ModelAttachmentState = _ModelAttachmentState( + discovered_inputs=discovered_inputs, + context=context, + validation_context=_ModelValidationContext( + effective_settings=context.effective_settings, + no_sql_validation=no_sql_validation, + project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, + defer_model_sql_validation=defer_model_sql_validation, + external_sql_reference_resolver=external_sql_reference_resolver, + extract_references=extract_references, + known_model_names=known_model_names, + known_seed_names=build_known_seed_names(discovered_inputs), + known_source_names=build_known_source_names(discovered_inputs), + known_function_names=build_known_function_names(discovered_inputs), + known_table_function_names=build_known_table_function_names(discovered_inputs), + custom_materialization_names=frozenset( + mf.name for mf in discovered_inputs.materialization_files + ), + ), + sql_hook_definitions=_index_sql_hook_definitions(discovered_inputs.sql_hook_files), + header_column_cache=ModelHeaderColumnCache(), + config_scan_cache=ModelConfigScanCache(), + reusable_config_cache=_ReusableModelConfigCache( + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, + target_config=context.target_config, + ), + declaration_cache=_VisibleModelDeclarationCache.build(context), + legacy_schema_files=legacy_schema_files, ) - model_inputs: list[CompileModelInput] = [] - model_header_column_cache: ModelHeaderColumnCache = ModelHeaderColumnCache() - config_scan_cache: ModelConfigScanCache = ModelConfigScanCache() - reusable_config_cache: _ReusableModelConfigCache = _ReusableModelConfigCache( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - target_config=target_config, + environment: ModelAttachmentEnvironment | None = ( + None + if cache_root is None + else model_attachment_environment( + discovered_inputs=discovered_inputs, + context=context, + no_sql_validation=no_sql_validation, + defer_model_sql_validation=defer_model_sql_validation, + ) ) - declaration_cache: _VisibleModelDeclarationCache = _VisibleModelDeclarationCache.build(context) - model_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files - prepared_var_substituted_sqls: tuple[str | None, ...] = prepare_static_project_vars_batch( - sqls=tuple(model_file.query_sql for model_file in model_files), - effective_vars=effective_vars, + with ModelAttachmentCache(root=cache_root, environment=environment) as attachment_cache: + return _attach_models(state=state, attachment_cache=attachment_cache) + + +def _attach_models( + *, state: _ModelAttachmentState, attachment_cache: ModelAttachmentCache +) -> tuple[CompileModelInput, ...]: + model_files: tuple[DiscoveredSqlModelFile, ...] = state.discovered_inputs.model_files + path_defaults: dict[str, dict[str, object]] = ( + state.discovered_inputs.project_config.path_defaults ) - model_file: DiscoveredSqlModelFile - prepared_var_substituted_sql: str | None - for model_file, prepared_var_substituted_sql in zip( - model_files, prepared_var_substituted_sqls, strict=True - ): - model_identity: ResourceIdentity = ResourceIdentity( - ResourceKind.MODEL, model_file.file_path.stem - ) - declarations: _VisibleModelDeclarations = declaration_cache.for_model( - model_file=model_file, consumer=model_identity - ) - matched_path_default: str | None = find_matching_path_default( + matched_path_defaults: tuple[str | None, ...] = tuple( + find_matching_path_default(model_file=model_file, path_defaults=path_defaults) + for model_file in model_files + ) + environment: ModelAttachmentEnvironment | None = attachment_cache.environment + keys: tuple[str | None, ...] = tuple( + None + if environment is None + else model_attachment_key( + store=attachment_cache.store, + environment=environment, model_file=model_file, - path_defaults=discovered_inputs.project_config.path_defaults, + path_default=( + matched, + None if matched is None else path_defaults.get(matched), + ), + resolver=state.context.declaration_resolver, ) - effective_config: CompileModelConfig | None = reusable_config_cache.get( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, + for model_file, matched in zip(model_files, matched_path_defaults, strict=True) + ) + facts: dict[str, ModelAttachmentFact] = attachment_cache.read( + tuple( + (model_file.relative_path.as_posix(), key) + for model_file, key in zip(model_files, keys, strict=True) + if key is not None ) - if effective_config is None: - effective_config = build_model_config( - request=ModelConfigBuildRequest( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - effective_vars=effective_vars, - target_config=target_config, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - materialization_defaults=( - discovered_inputs.project_config.materialization_defaults - ), - scan_cache=config_scan_cache, - ) + ) + reused: tuple[CompileModelInput | None, ...] = tuple( + attachment_cache.reuse(fact=None if key is None else facts.get(key), model_file=model_file) + for model_file, key in zip(model_files, keys, strict=True) + ) + miss_indexes: tuple[int, ...] = tuple( + index for index, model_input in enumerate(reused) if model_input is None + ) + prepared_sqls: dict[int, str | None] = ( + dict( + zip( + miss_indexes, + prepare_static_project_vars_batch( + sqls=tuple(model_files[index].query_sql for index in miss_indexes), + effective_vars=state.context.effective_vars, + ), + strict=True, ) - reusable_config_cache.remember( + ) + if miss_indexes + else {} + ) + model_inputs: list[CompileModelInput] = [] + for index, model_file in enumerate(model_files): + key: str | None = keys[index] + cached_input: CompileModelInput | None = reused[index] + if cached_input is not None and key is not None: + _replay_model_attachment(state=state, fact=facts[key], model_input=cached_input) + model_inputs.append(cached_input) + continue + hook_references: tuple[CompileSqlReference, ...] + with record_volatile_reads() as reads, capture_compile_diagnostics() as diagnostics: + model_input, hook_references = _attach_model( + state=state, + model_file=model_file, + matched_path_default=matched_path_defaults[index], + prepared_var_substituted_sql=prepared_sqls[index], + ) + if key is not None: + attachment_cache.stage( + slot=model_file.relative_path.as_posix(), + key=key, + model_input=model_input, + hook_references=hook_references, + reads=reads, + diagnostics=diagnostics, + ) + model_inputs.append(model_input) + + validate_declared_schema_models_are_attached( + model_inputs=tuple(model_inputs), + schema_files=state.discovered_inputs.schema_files, + ) + return tuple(model_inputs) + + +def _replay_model_attachment( + *, + state: _ModelAttachmentState, + fact: ModelAttachmentFact, + model_input: CompileModelInput, +) -> None: + for diagnostic_key, diagnostic in fact.diagnostics: + report_compile_diagnostic(key=diagnostic_key, diagnostic=diagnostic) + validation_context: _ModelValidationContext = state.validation_context + references: tuple[CompileSqlReference, ...] + for references in (model_input.references, fact.hook_references): + validate_model_references( + references=references, + model_file=model_input.model_file, + known_model_names=validation_context.known_model_names, + known_seed_names=validation_context.known_seed_names, + known_source_names=validation_context.known_source_names, + known_function_names=validation_context.known_function_names, + known_table_function_names=validation_context.known_table_function_names, + external_sql_reference_resolver=validation_context.external_sql_reference_resolver, + ) + _reject_legacy_schema_match( + model_file=model_input.model_file, schema_files=state.legacy_schema_files + ) + + +def _attach_model( + *, + state: _ModelAttachmentState, + model_file: DiscoveredSqlModelFile, + matched_path_default: str | None, + prepared_var_substituted_sql: str | None, +) -> tuple[CompileModelInput, tuple[CompileSqlReference, ...]]: + context: ModelInputBuildContext = state.context + model_identity: ResourceIdentity = ResourceIdentity( + ResourceKind.MODEL, model_file.file_path.stem + ) + declarations: _VisibleModelDeclarations = state.declaration_cache.for_model( + model_file=model_file, consumer=model_identity + ) + effective_config: CompileModelConfig | None = state.reusable_config_cache.get( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + ) + if effective_config is None: + effective_config = build_model_config( + request=ModelConfigBuildRequest( + defaults=state.discovered_inputs.project_config.defaults, + path_defaults=state.discovered_inputs.project_config.path_defaults, matched_path_default=matched_path_default, model_header_values=model_file.header_values, - config=effective_config, + effective_vars=context.effective_vars, + target_config=context.target_config, + model_name=model_file.file_path.stem, + effective_target_name=context.effective_target_name, + run_id=context.run_id, + materialization_defaults=( + state.discovered_inputs.project_config.materialization_defaults + ), + scan_cache=state.config_scan_cache, ) - model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( - values=effective_config.values, - model_name=model_file.file_path.stem, - public_model_schemas=context.public_model_schemas, - ) - model_schema_columns: tuple[SchemaColumn, ...] | None = ( - model_schema.columns if model_schema is not None else None ) - validate_python_hook_config( - values=effective_config.values, - model_name=model_file.file_path.stem, - hook_functions=discovered_inputs.hook_functions, - provider_names=frozenset(provider.name for provider in discovered_inputs.providers), + state.reusable_config_cache.remember( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + config=effective_config, ) - named_usages: tuple[UsageRecord, ...] = ( - *( - named_declaration_usages( - resolver=context.declaration_resolver, - kind=DeclarationKind.SCHEMA, - name=model_schema.name, - consumer=model_identity, - consumer_path=model_file.relative_path, - ) - if model_schema is not None - else () - ), - *_python_hook_usages( - values=effective_config.values, + model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( + values=effective_config.values, + model_name=model_file.file_path.stem, + public_model_schemas=context.public_model_schemas, + ) + model_schema_columns: tuple[SchemaColumn, ...] | None = ( + model_schema.columns if model_schema is not None else None + ) + validate_python_hook_config( + values=effective_config.values, + model_name=model_file.file_path.stem, + hook_functions=state.discovered_inputs.hook_functions, + provider_names=frozenset(provider.name for provider in state.discovered_inputs.providers), + ) + named_usages: tuple[UsageRecord, ...] = ( + *( + named_declaration_usages( resolver=context.declaration_resolver, + kind=DeclarationKind.SCHEMA, + name=model_schema.name, consumer=model_identity, consumer_path=model_file.relative_path, - ), - ) - var_substituted_sql: str = ( - prepared_var_substituted_sql - if prepared_var_substituted_sql is not None - else substitute_sql_vars( - sql=model_file.query_sql, - file_path=model_file.file_path, - effective_vars=effective_vars, ) - ) - declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - enum_visibility=declarations.enum_visibility, - constant_visibility=declarations.constant_visibility, - macros=declarations.macros, - macro_records=declarations.macro_records, - inaccessible_macros=declarations.inaccessible_macros, + if model_schema is not None + else () + ), + *_python_hook_usages( + values=effective_config.values, + resolver=context.declaration_resolver, consumer=model_identity, - ) - declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( - sql=var_substituted_sql, - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) - declaration_expanded_sql: str = declaration_expansion.sql - macro_expansion: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_expanded_sql, + consumer_path=model_file.relative_path, + ), + ) + var_substituted_sql: str = ( + prepared_var_substituted_sql + if prepared_var_substituted_sql is not None + else substitute_sql_vars( + sql=model_file.query_sql, file_path=model_file.file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=context.declaration_resolver, - declarations=( - declaration_context if context.declaration_resolver is not None else None - ), - consumer=model_identity, - ) - expanded_query_sql: str = macro_expansion.sql - expanded_query_sql = get_validated_model_cursor_intrinsics( - sql=expanded_query_sql, - config_values=effective_config.values, - model_name=model_file.file_path.stem, - ) - raw_placeholders: object | None = effective_config.values.get("placeholders") - sql_validation_placeholders: dict[str, str] | None = ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - sql_validation_enabled, references, rejected_opt_out = _validate_model_input( - context=validation_context, - model_file=model_file, - config=effective_config, - expanded_query_sql=expanded_query_sql, - sql_validation_placeholders=sql_validation_placeholders, - model_schema_columns=model_schema_columns, - argument_references=macro_expansion.argument_references, + effective_vars=context.effective_vars, ) - hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + ) + declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, + enum_visibility=declarations.enum_visibility, + constant_visibility=declarations.constant_visibility, + macros=declarations.macros, + macro_records=declarations.macro_records, + inaccessible_macros=declarations.inaccessible_macros, + consumer=model_identity, + ) + declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( + sql=var_substituted_sql, + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + ) + declaration_expanded_sql: str = declaration_expansion.sql + macro_expansion: MacroExpansionResult = expand_sql_macros_result( + sql=declaration_expanded_sql, + file_path=model_file.file_path, + loaded_macros=context.loaded_macros, + macro_context=context.macro_context, + declaration_resolver=context.declaration_resolver, + declarations=(declaration_context if context.declaration_resolver is not None else None), + consumer=model_identity, + ) + expanded_query_sql: str = macro_expansion.sql + expanded_query_sql = get_validated_model_cursor_intrinsics( + sql=expanded_query_sql, + config_values=effective_config.values, + model_name=model_file.file_path.stem, + ) + raw_placeholders: object | None = effective_config.values.get("placeholders") + sql_validation_placeholders: dict[str, str] | None = ( + {str(k): str(v) for k, v in raw_placeholders.items()} + if isinstance(raw_placeholders, dict) + else None + ) + sql_validation_enabled, references, rejected_opt_out = _validate_model_input( + context=state.validation_context, + model_file=model_file, + config=effective_config, + expanded_query_sql=expanded_query_sql, + sql_validation_placeholders=sql_validation_placeholders, + model_schema_columns=model_schema_columns, + argument_references=macro_expansion.argument_references, + ) + hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + values=effective_config.values, + file_path=model_file.file_path, + effective_vars=context.effective_vars, + context_values=build_model_context_values( values=effective_config.values, - file_path=model_file.file_path, - effective_vars=effective_vars, - context_values=build_model_context_values( - values=effective_config.values, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=True, - ), - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - ), - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - resolver=context.declaration_resolver, - ), - sql_lexical_syntax=context.sql_lexical_syntax, - sql_hook_definitions=sql_hook_definitions, - consumer=model_identity, - ) - expanded_config: CompileModelConfig = _hook_expanded_config( - config=effective_config, - hook_expansion=hook_expansion, - model_file=model_file, - validation_context=validation_context, - ) - hook_name: str - for hook_name in ("pre_hooks", "post_hooks"): - hook_value: object | None = expanded_config.values.get(hook_name) - if isinstance(hook_value, list | tuple): - hook_entry: object - for hook_entry in hook_value: - if isinstance(hook_entry, SqlHookEntry): - reject_cursor_intrinsics( - sql=hook_entry.statement, - context=f"Model '{model_file.file_path.stem}' {hook_name}", - ) - header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( model_name=model_file.file_path.stem, - model_header_values=expanded_config.values, - file_path=model_file.relative_path, - column_locations=model_file.header_column_locations, - model_schema_columns=model_schema_columns, - model_schema_name=model_schema.name if model_schema is not None else None, - model_schema_description=model_schema.description if model_schema is not None else None, - audit_factories=discovered_inputs.audit_factories, - column_cache=model_header_column_cache, - ) - model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) - header_schema_entry, enum_columns = resolve_enum_contract_columns( - schema_entry=header_schema_entry, - config_values=model_config.values, - enums=( - declarations.enums - if model_schema is None - else { - **declarations.enums, - **model_schema_enum_declarations( - schema=model_schema, resolver=context.declaration_resolver - ), - } + effective_target_name=context.effective_target_name, + run_id=context.run_id, + include_target_values=True, + ), + loaded_macros=context.loaded_macros, + macro_context=context.macro_context, + declaration_expansion=DeclarationExpansionContext( + declarations=DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, ), - ) - generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( - enum_columns=enum_columns, - declarations=declarations, - consumer=model_identity, - ) - model_declaration_usages: tuple[UsageRecord, ...] = tuple( - dict.fromkeys( - ( - *declaration_expansion.usages, - *generated_usages, - *hook_expansion.usages, - *named_usages, - ) - ) - ) - _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) - if header_schema_entry is None: - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None - ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, - ) - ) - continue - - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + resolver=context.declaration_resolver, + ), + sql_lexical_syntax=context.sql_lexical_syntax, + sql_hook_definitions=state.sql_hook_definitions, + consumer=model_identity, + ) + expanded_config: CompileModelConfig = _hook_expanded_config( + config=effective_config, + hook_expansion=hook_expansion, + model_file=model_file, + validation_context=state.validation_context, + ) + hook_name: str + for hook_name in ("pre_hooks", "post_hooks"): + hook_value: object | None = expanded_config.values.get(hook_name) + if isinstance(hook_value, list | tuple): + hook_entry: object + for hook_entry in hook_value: + if isinstance(hook_entry, SqlHookEntry): + reject_cursor_intrinsics( + sql=hook_entry.statement, + context=f"Model '{model_file.file_path.stem}' {hook_name}", ) - if var_substituted_sql == model_file.query_sql - else None + header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( + model_name=model_file.file_path.stem, + model_header_values=expanded_config.values, + file_path=model_file.relative_path, + column_locations=model_file.header_column_locations, + model_schema_columns=model_schema_columns, + model_schema_name=model_schema.name if model_schema is not None else None, + model_schema_description=model_schema.description if model_schema is not None else None, + audit_factories=state.discovered_inputs.audit_factories, + column_cache=state.header_column_cache, + ) + model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) + header_schema_entry, enum_columns = resolve_enum_contract_columns( + schema_entry=header_schema_entry, + config_values=model_config.values, + enums=( + declarations.enums + if model_schema is None + else { + **declarations.enums, + **model_schema_enum_declarations( + schema=model_schema, resolver=context.declaration_resolver ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - schema_entry=header_schema_entry, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, + } + ), + ) + generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( + enum_columns=enum_columns, + declarations=declarations, + consumer=model_identity, + ) + model_declaration_usages: tuple[UsageRecord, ...] = tuple( + dict.fromkeys( + ( + *declaration_expansion.usages, + *generated_usages, + *hook_expansion.usages, + *named_usages, ) ) - - validate_declared_schema_models_are_attached( - model_inputs=tuple(model_inputs), - schema_files=discovered_inputs.schema_files, ) - return tuple(model_inputs) + _reject_legacy_schema_match(model_file=model_file, schema_files=state.legacy_schema_files) + return ( + CompileModelInput( + sql_expansion=( + CompiledSqlExpansion( + authored_sql=model_file.query_sql, + expanded_sql=macro_expansion.sql, + passes=(declaration_expansion.spans, macro_expansion.spans), + ) + if var_substituted_sql == model_file.query_sql + else None + ), + model_file=model_file, + config=model_config, + query_sql=expanded_query_sql, + macro_source_sql=declaration_expanded_sql, + references=references, + schema_entry=header_schema_entry, + sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, + enum_declarations=tuple(declarations.local_enums.values()), + constant_declarations=tuple(declarations.local_constants.values()), + enum_columns=enum_columns, + macro_deps=tuple(item.name for item in macro_expansion.dependencies), + macro_usages=macro_expansion.usages, + declaration_usages=model_declaration_usages, + ), + hook_expansion.references, + ) def _hook_expanded_config( diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index ae6080acb7..9489de6b13 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -35,6 +35,24 @@ def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: _ACTIVE_COLLECTOR.reset(token) +@contextmanager +def capture_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics | None]: + """Record diagnostics reported inside the block and forward them to the active collector.""" + + parent: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() + if parent is None: + yield None + return + captured: CollectedCompileDiagnostics = CollectedCompileDiagnostics() + token: Token[CollectedCompileDiagnostics | None] = _ACTIVE_COLLECTOR.set(captured) + try: + yield captured + finally: + _ACTIVE_COLLECTOR.reset(token) + for key, diagnostic in captured.entries: + parent.add(key=key, diagnostic=diagnostic) + + def with_collected_compile_diagnostics[**P, R: (CompileProjectInputs, CompiledProject)]( build: Callable[P, R], ) -> Callable[P, R]: diff --git a/src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py b/src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py new file mode 100644 index 0000000000..4cf5357660 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py @@ -0,0 +1,235 @@ +"""Cache identity for one model's attachment result.""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import replace +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.render.volatile_reads import environment_read_label +from sqlbuild.compiler.compile.models import ( + DeclarationScopeResolver, + ModelAttachmentEnvironment, + ModelInputBuildContext, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ResourceRecord +from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind + +_TARGET_ENVIRONMENT_REFERENCE: re.Pattern[str] = re.compile(r"\bENV:\s*([A-Za-z0-9_]+)") +_PROCESS_LOCAL_REPR_MARKER: str = " at 0x" +_PROJECT_ADAPTERS_DIRECTORY: str = "adapters" +_ENVIRONMENT_DIGEST_BYTES: int = 32 +_ATTACHMENT_DECLARATION_KINDS: frozenset[DeclarationKind] = frozenset( + { + DeclarationKind.MACRO, + DeclarationKind.ENUM, + DeclarationKind.CONSTANT, + DeclarationKind.SCHEMA, + DeclarationKind.SQL_HOOK, + DeclarationKind.PYTHON_HOOK, + } +) + + +def model_attachment_environment( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + no_sql_validation: bool, + defer_model_sql_validation: bool, +) -> ModelAttachmentEnvironment | None: + """Return the project-wide key part, or None when an input has no stable identity.""" + + target_text: str = repr( + None if context.target_config is None else replace(context.target_config, connection={}) + ) + target_environment_names: tuple[str, ...] = tuple( + sorted(set(_TARGET_ENVIRONMENT_REFERENCE.findall(target_text))) + ) + renderer_type: type = type(context.value_renderer) + material: str = "\0".join( + ( + compiled_code_identity(), + str(discovered_inputs.project_dir.resolve() if discovered_inputs.project_dir else ""), + context.macro_context.adapter_name, + f"{renderer_type.__module__}.{renderer_type.__qualname__}", + str(context.collection_rendering), + context.sql_lexical_syntax.cache_key, + _project_adapter_sources(discovered_inputs), + repr(context.effective_target_name), + target_text, + repr(tuple((name, os.environ.get(name)) for name in target_environment_names)), + repr(context.effective_vars), + repr(context.effective_settings), + repr( + ( + context.macro_context.sql_analysis_enabled, + context.macro_context.target_name, + discovered_inputs.project_config.references.enforce_explicit, + no_sql_validation, + defer_model_sql_validation, + ) + ), + repr(discovered_inputs.project_config.defaults), + repr(discovered_inputs.project_config.materialization_defaults), + repr(sorted(item.name for item in discovered_inputs.materialization_files)), + repr(sorted(item.name for item in discovered_inputs.providers)), + _declaration_surface( + discovered_inputs=discovered_inputs, resolver=context.declaration_resolver + ), + ) + ) + if _PROCESS_LOCAL_REPR_MARKER in material: + return None + return ModelAttachmentEnvironment( + digest=hashlib.blake2b( + material.encode("utf-8", "surrogatepass"), digest_size=_ENVIRONMENT_DIGEST_BYTES + ).hexdigest(), + keyed_reads=frozenset(environment_read_label(name) for name in target_environment_names), + ) + + +def model_attachment_key( + *, + store: FactCacheStore, + environment: ModelAttachmentEnvironment, + model_file: DiscoveredSqlModelFile, + path_default: tuple[str | None, object], + resolver: DeclarationScopeResolver | None, +) -> str: + """Return the exact identity of one model's attachment inputs.""" + + relative_path: str = model_file.relative_path.as_posix() + return store.key( + environment.digest, + relative_path, + str(model_file.file_path), + model_file.contents, + repr(model_file.extract_implicit_alias_columns), + repr(path_default), + "" if resolver is None else _resource_scope(resolver=resolver, model_file=model_file), + ) + + +def _resource_scope( + *, resolver: DeclarationScopeResolver, model_file: DiscoveredSqlModelFile +) -> str: + records: tuple[ResourceRecord, ...] = resolver.lookup.resources_by_path.get( + model_file.relative_path.as_posix(), () + ) + identities: tuple[ResourceIdentity, ...] = ( + ResourceIdentity(ResourceKind.MODEL, model_file.file_path.stem), + *(record.identity for record in records), + ) + return repr( + ( + records, + tuple( + resolver.lookup.grants_by_resource.get(identity, ()) + for identity in dict.fromkeys(identities) + ), + ) + ) + + +def _declaration_surface( + *, discovered_inputs: DiscoveredProjectInputs, resolver: DeclarationScopeResolver | None +) -> str: + """Return every shared macro and declaration source; editing one re-attaches every model.""" + + parts: list[str] = [ + repr( + ( + type(item).__name__, + item.relative_path.as_posix(), + item.contents, + item.scope_kind, + item.ownership_root, + item.owning_path, + item.declaration_root, + ) + ) + for item in ( + *discovered_inputs.macro_files, + *discovered_inputs.enum_files, + *discovered_inputs.constant_files, + *discovered_inputs.model_schema_files, + *discovered_inputs.sql_hook_files, + ) + ] + parts.extend( + repr( + ( + hook.name, + hook.relative_path.as_posix(), + hook.description, + hook.reads, + hook.provider_usages, + hook.scope_kind, + hook.ownership_root, + hook.owning_path, + hook.declaration_root, + _source_text(hook.file_path), + ) + ) + for hook in discovered_inputs.hook_functions + ) + parts.extend( + repr( + ( + factory.name, + factory.relative_path.as_posix(), + factory.line, + factory.cases, + _source_text(factory.file_path), + ) + ) + for factory in discovered_inputs.audit_factories + ) + if resolver is not None: + parts.extend( + repr(record) + for record in resolver.lookup.index.declarations + if _is_shared_attachment_declaration(record) + ) + return "\0".join(parts) + + +def _is_shared_attachment_declaration(record: DeclarationRecord) -> bool: + owner: ResourceIdentity | None = record.identity.owner + return record.identity.kind in _ATTACHMENT_DECLARATION_KINDS and ( + owner is None or owner.kind is not ResourceKind.MODEL + ) + + +def _project_adapter_sources(discovered_inputs: DiscoveredProjectInputs) -> str: + project_dir: Path | None = discovered_inputs.project_dir + adapter_paths: tuple[Path, ...] = ( + () + if project_dir is None + else tuple(sorted((project_dir / _PROJECT_ADAPTERS_DIRECTORY).rglob("*.py"))) + ) + return repr( + ( + _source_text( + None + if discovered_inputs.adapter_file is None + else discovered_inputs.adapter_file.file_path + ), + tuple((path.as_posix(), _source_text(path)) for path in adapter_paths), + ) + ) + + +def _source_text(path: Path | None) -> str: + if path is None: + return "" + try: + return path.read_text(encoding="utf-8", errors="surrogateescape") + except OSError: + return "" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index f621c5c7cc..78ed54e748 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -17,6 +17,10 @@ expand_sql_macros_result, expand_sql_macros_with_spans, ) +from sqlbuild.compiler.compile._helpers.render.volatile_reads import ( + note_context_read, + note_environment_read, +) from sqlbuild.compiler.compile.constants import ( SQL_CONTEXT_NAME_EXTRA_TOKENS, SQL_IDENTIFIER_EXTRA_TOKEN, @@ -366,6 +370,7 @@ def _render_interpolation_token( if env_name_end == env_name_start: raise CompileInputError(f"invalid environment interpolation token in '{file_path}'") env_name: str = sql[env_name_start:env_name_end] + note_environment_read(env_name) if env_name not in os.environ: raise CompileInputError( f"unknown environment variable '@@ENV:{env_name}' in '{file_path}'" @@ -387,6 +392,7 @@ def _render_interpolation_token( context_values=context_values, ) context_name = sql[context_name_start:context_name_end] + note_context_read(context_name) if context_name not in context_values: raise CompileInputError( f"SQL text in '{file_path}' references unknown CTX key '{context_name}'" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 3bd6eb68c0..1eeabd4dcc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -8,6 +8,10 @@ from dataclasses import dataclass from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text +from sqlbuild.compiler.compile._helpers.render.volatile_reads import ( + note_context_read, + note_environment_read, +) from sqlbuild.compiler.compile.constants import ( MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, @@ -521,6 +525,7 @@ def _lookup_variable(*, name: str, variables: dict[str, object], context_label: def _lookup_environment_variable(*, name: str, context_label: str) -> str: + note_environment_read(name) if name not in os.environ: raise CompileInputError(f"{context_label} references missing ENV variable '{name}'") return os.environ[name] @@ -537,6 +542,7 @@ def _lookup_context_value( if preserve_unknown_context: return f"${{{TemplateNamespace.CTX}:{name}}}" raise CompileInputError(f"{context_label} references unknown CTX key '{name}'") + note_context_read(name) context_value: str | None = context_values.get(name) if context_value is None: raise CompileInputError( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py b/src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py new file mode 100644 index 0000000000..ee74175f14 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py @@ -0,0 +1,47 @@ +"""Record per-invocation values that a reused attachment result cannot reproduce.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token + +from sqlbuild.compiler.compile.types import CompileContextKey, TemplateNamespace + +_ACTIVE_READS: ContextVar[set[str] | None] = ContextVar( + "sqlbuild_volatile_compile_reads", default=None +) + + +def environment_read_label(name: str) -> str: + """Return the recorded label for one environment variable read.""" + + return f"{TemplateNamespace.ENV}:{name}" + + +def note_environment_read(name: str) -> None: + """Record that template or SQL interpolation read one environment variable.""" + + reads: set[str] | None = _ACTIVE_READS.get() + if reads is not None: + reads.add(environment_read_label(name)) + + +def note_context_read(name: str) -> None: + """Record a CTX read whose value differs between compile invocations.""" + + reads: set[str] | None = _ACTIVE_READS.get() + if reads is not None and name == CompileContextKey.RUN_ID: + reads.add(f"{TemplateNamespace.CTX}:{name}") + + +@contextmanager +def record_volatile_reads() -> Iterator[set[str]]: + """Collect environment and run-id reads made while the block runs.""" + + reads: set[str] = set() + token: Token[set[str] | None] = _ACTIVE_READS.set(reads) + try: + yield reads + finally: + _ACTIVE_READS.reset(token) diff --git a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py index 9a1a32e725..40e12b7038 100644 --- a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py +++ b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py @@ -16,6 +16,12 @@ def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: self._by_key.setdefault(key, diagnostic) + @property + def entries(self) -> tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...]: + """Return each recorded key and diagnostic in report order.""" + + return tuple(self._by_key.items()) + @property def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: """Return the recorded diagnostics in deterministic key order.""" diff --git a/src/sqlbuild/compiler/compile/classes/model_attachment_cache.py b/src/sqlbuild/compiler/compile/classes/model_attachment_cache.py new file mode 100644 index 0000000000..9f08d14b69 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/model_attachment_cache.py @@ -0,0 +1,137 @@ +"""Reuse model attachment results across compile invocations.""" + +from __future__ import annotations + +import gc +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path +from types import TracebackType + +from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( + CollectedCompileDiagnostics, +) +from sqlbuild.compiler.compile.constants import ( + MODEL_ATTACHMENT_FACT_ALGORITHM, + MODEL_ATTACHMENT_FACT_CACHE_NAMESPACE, +) +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompileSqlReference, + ModelAttachmentEnvironment, + ModelAttachmentFact, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.profiling.main._metric import record_compile_metric + +_DETACHED_MODEL_FILE: DiscoveredSqlModelFile = DiscoveredSqlModelFile( + file_path=Path(), + relative_path=Path(), + contents="", + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql="", +) + + +@contextmanager +def _collection_paused() -> Iterator[None]: + """Pause cyclic garbage collection while thousands of acyclic cached facts are unpickled.""" + + collecting: bool = gc.isenabled() + gc.disable() + try: + yield + finally: + if collecting: + gc.enable() + + +class ModelAttachmentCache: + """Store one attachment fact per model; models that read per-run values are recomputed.""" + + def __init__( + self, *, root: Path | None, environment: ModelAttachmentEnvironment | None + ) -> None: + self.environment: ModelAttachmentEnvironment | None = environment + self.store: FactCacheStore = FactCacheStore( + root=root if environment is not None else None, + namespace=MODEL_ATTACHMENT_FACT_CACHE_NAMESPACE, + algorithm=MODEL_ATTACHMENT_FACT_ALGORITHM, + ) + self._hits: int = 0 + self._misses: int = 0 + self._bypasses: int = 0 + + def __enter__(self) -> ModelAttachmentCache: + _ = self.store.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + self.store.__exit__(exc_type, exc_value, traceback) + finally: + if self.store.enabled: + record_compile_metric(metric="attachment_cache_hits", value=self._hits) + record_compile_metric(metric="attachment_cache_misses", value=self._misses) + record_compile_metric(metric="attachment_cache_bypasses", value=self._bypasses) + + def read(self, entries: Sequence[tuple[str, str]]) -> dict[str, ModelAttachmentFact]: + """Return verified facts by key for requested (slot, key) pairs.""" + + with _collection_paused(): + found: dict[str, object] = self.store.read_many(entries) + return { + key: value for key, value in found.items() if isinstance(value, ModelAttachmentFact) + } + + def reuse( + self, *, fact: ModelAttachmentFact | None, model_file: DiscoveredSqlModelFile + ) -> CompileModelInput | None: + """Return the cached model input rebound to the current file, or None to recompute.""" + + if fact is None or fact.model_input is None: + if fact is not None: + self._bypasses += 1 + elif self.store.enabled: + self._misses += 1 + return None + self._hits += 1 + return replace(fact.model_input, model_file=model_file) + + def stage( + self, + *, + slot: str, + key: str, + model_input: CompileModelInput, + hook_references: tuple[CompileSqlReference, ...], + reads: set[str], + diagnostics: CollectedCompileDiagnostics | None, + ) -> None: + """Queue one freshly attached model for publication after a successful compile.""" + + if self.environment is None: + return + volatile: bool = not reads <= self.environment.keyed_reads + self.store.stage( + key=key, + slot=slot, + value=( + ModelAttachmentFact(model_input=None) + if volatile + else ModelAttachmentFact( + model_input=replace(model_input, model_file=_DETACHED_MODEL_FILE), + hook_references=hook_references, + diagnostics=() if diagnostics is None else diagnostics.entries, + ) + ), + ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 063b1f129f..4188f04341 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -170,6 +170,8 @@ SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v2" SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" +MODEL_ATTACHMENT_FACT_CACHE_NAMESPACE: str = "model-attachment" +MODEL_ATTACHMENT_FACT_ALGORITHM: str = "model-attachment-v1" COMPACT_ANALYSIS_RESPONSE_LENGTH: int = 2 COMPACT_RELATION_STUB_PREFIX: str = "__sqlbuild_project_input_" COMPACT_REFERENCE_MARKER_PATTERN: re.Pattern[str] = re.compile( diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 93bfc4495e..e797ab76ab 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -763,6 +763,25 @@ class CompileModelInput: sql_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) +@dataclass(frozen=True) +class ModelAttachmentFact: + """One model's reusable attachment result, or None when it read per-run values.""" + + model_input: CompileModelInput | None + hook_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) + diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] = field( + default_factory=tuple + ) + + +@dataclass(frozen=True) +class ModelAttachmentEnvironment: + """Project-wide identity shared by every model attachment cache key.""" + + digest: str + keyed_reads: frozenset[str] = frozenset() + + @dataclass(frozen=True) class ModelSqlAnalysis: """Completed model analysis and its in-memory binding proof.""" diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 6ca434213c..3aed4d58f8 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -24,4 +24,7 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "attachment_cache_hits", + "attachment_cache_misses", + "attachment_cache_bypasses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index abb9028ae4..cb394933c6 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -24,4 +24,7 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "attachment_cache_hits", + "attachment_cache_misses", + "attachment_cache_bypasses", ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index a79d284bc6..e07cd48cda 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -90,6 +90,9 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "attachment_cache_hits", + "attachment_cache_misses", + "attachment_cache_bypasses", "total_ms", } assert all(isinstance(value, int) and value >= 0 for value in timings.values()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index d6166840c1..2c6535289e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -378,12 +378,59 @@ class CompileCacheInvalidationTestCase: description: str edit: Callable[[Path], None] + setup: Callable[[Path], None] = lambda _root: None edited_env: dict[str, str] = field(default_factory=dict) + edited_compile_args: tuple[str, ...] = () expected_failure: bool = False expected_output_change: bool = True expected_rewarmed_fact_cache_misses: int = 0 +@dataclass(frozen=True) +class AttachmentCacheReuseTestCase: + """One project change after a warm compile and the expected model attachment reuse.""" + + description: str + edit: Callable[[Path], None] + expected_attachment_counts: tuple[int, int, int] + setup: Callable[[Path], None] = lambda _root: None + initial_env: dict[str, str] = field(default_factory=dict) + edited_env: dict[str, str] = field(default_factory=dict) + expected_failure: bool = False + + +@dataclass(frozen=True) +class AttachmentRunIdTestCase: + """One model whose hook renders the per-invocation run id.""" + + description: str + model_path: str + model_sql: str + expected_warm_attachment_counts: tuple[int, int, int] + + +@dataclass(frozen=True) +class AttachmentDiagnosticReplayTestCase: + """Project files whose attachment reports diagnostics a cache hit must replay.""" + + description: str + files: dict[str, str] + expected_diagnostic_codes: tuple[str, ...] + expected_warm_attachment_counts: tuple[int, int, int] + + +@dataclass(frozen=True) +class AttachmentReferencePolicyTestCase: + """A replayed diagnostic whose presence follows the explicit-reference policy.""" + + description: str + files: dict[str, str] + relax: Callable[[Path], None] + restore: Callable[[Path], None] + expected_strict_codes: tuple[str, ...] + expected_relaxed_codes: tuple[str, ...] + + @dataclass(frozen=True) class CompileCacheDisabledTestCase: """One supported control that disables compile-cache reads and writes.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 40ad9bd6cd..21112d946f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import os import re @@ -2424,6 +2425,9 @@ def write_relation_stub_project( COMPILE_CACHE_REGION_ENV_VAR: str = "SQB_CACHE_INVALIDATION_REGION" +DESCRIBED_HOOK_DECORATOR: str = '@hook(description="Notify that orders completed")' +COMPILE_CACHE_SCHEMA_ENV_VAR: str = "SQB_CACHE_INVALIDATION_SCHEMA" +_HOOK_RUN_ID_PATTERN: re.Pattern[str] = re.compile(r"'(\d{8}T\d{6}Z_[0-9a-f]{12})' AS run_id") _COMPILE_CACHE_DIAGNOSTIC_PATTERN: re.Pattern[str] = re.compile( r"^(?:error|warning)\[.*$", re.MULTILINE ) @@ -2480,9 +2484,72 @@ def write_relation_stub_project( ")\n" "SELECT 1\n" ), + "models/marts/_sqlbuild/_macros/scaling.py": ( + '"""Marts scaling macros."""\n\n' + "from macros.currency import line_total_cents\n\n\n" + "def scaled_quantity(column: str) -> str:\n" + ' """Scale an order quantity column."""\n' + ' return line_total_cents(column, "2")\n' + ), + "models/marts/_sqlbuild/_schemas/order_metric.sql": ( + "SCHEMA (\n" + " name order_metric,\n" + ' description "Scaled order quantities",\n' + " columns (\n" + " order_id (nullable false),\n" + " scaled_quantity (nullable true),\n" + " ),\n" + ");\n" + ), + "models/marts/_sqlbuild/_hooks/sql/record_orders.sql": ( + 'HOOK (\n description "Count the rows written to a relation"\n);\n\n' + "SELECT COUNT(*) AS row_count FROM @relation\n" + ), + "models/marts/_sqlbuild/_hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n\n" + '@hook(description="Notify that orders completed")\n' + 'def notify_complete(ctx, channel="#orders"):\n' + ' ctx.log(f"Notify {channel}: {ctx.model_name} completed")\n' + ), + "python/audits/order_quality.py": ( + "from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory\n\n\n" + "@audit_factory\n" + "def order_quality():\n" + " return [\n" + " AuditCase(\n" + ' name="non_negative_quantity",\n' + ' definition="expression_is_true",\n' + ' arguments={"expression": "scaled_quantity >= 0"},\n' + " severity=AuditSeverity.WARN,\n" + " )\n" + " ]\n" + ), + "models/marts/order_metrics.sql": ( + "MODEL (\n" + " materialized table,\n" + " model_schema order_metric,\n" + " audit_factories [order_quality],\n" + " post_hooks [\n" + ' sql("record_orders", relation: "@@CTX:destination.qualified"),\n' + ' python("notify_complete", channel: "#orders"),\n' + " ],\n" + ");\n\n" + 'SELECT o.order_id, @scaled_quantity("o.quantity") AS scaled_quantity\n' + 'FROM __ref("stg_orders") o\n' + ), } +_CUSTOM_DUCKDB_ADAPTER_SOURCE: str = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class DuckDbPlusAdapter(DuckDbAdapter):\n" + ' adapter_name = "duckdb_plus"\n\n' + " def render_typed_scalar(self, *, value):\n" + " rendered = super().render_typed_scalar(value=value)\n" + ' return f"{rendered}"\n' +) + + class CompileCacheOutcome(NamedTuple): """Comparable result of one fresh-process compile.""" @@ -2491,6 +2558,9 @@ class CompileCacheOutcome(NamedTuple): fingerprint: str fact_cache_hits: int fact_cache_misses: int + attachment_cache_hits: int = 0 + attachment_cache_misses: int = 0 + attachment_cache_bypasses: int = 0 def run_installed_sqb( @@ -2516,21 +2586,129 @@ def run_installed_sqb( def compile_cache_outcome( *, project_dir: Path, env: dict[str, str], compile_args: tuple[str, ...] = () ) -> CompileCacheOutcome: - """Compile in a fresh process and return its semantic fingerprint and fact-cache counts.""" + """Compile in a fresh process and return its output fingerprint and fact-cache counts.""" + manifest_path: Path = project_dir / "target" / "manifest.json" + manifest_path.unlink(missing_ok=True) result: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir, args=("compile", "--json", *compile_args), env=env + project_dir=project_dir, args=("compile", "--json", "--manifest", *compile_args), env=env ) values: dict[str, object] = cast(dict[str, object], json.loads(result.stdout or "{}")) timings: dict[str, int] = cast(dict[str, int], values.get("compile_timings", {})) + manifest_digest: str = "".join( + _invocation_independent_manifest_digest(path) + for path in manifest_path.parent.glob(manifest_path.name) + ) return CompileCacheOutcome( returncode=result.returncode, diagnostics=tuple(_COMPILE_CACHE_DIAGNOSTIC_PATTERN.findall(result.stderr)), fingerprint=semantic_compile_fingerprint( payload=values, compiled_dir=project_dir / "target" / "compiled" - ), + ) + + manifest_digest, fact_cache_hits=timings.get("fact_cache_hits", 0), fact_cache_misses=timings.get("fact_cache_misses", 0), + attachment_cache_hits=timings.get("attachment_cache_hits", 0), + attachment_cache_misses=timings.get("attachment_cache_misses", 0), + attachment_cache_bypasses=timings.get("attachment_cache_bypasses", 0), + ) + + +def _invocation_independent_manifest_digest(manifest_path: Path) -> str: + manifest: dict[str, Any] = cast(dict[str, Any], json.loads(manifest_path.read_text())) + metadata: dict[str, object] = manifest["metadata"] + for volatile_key in ("generated_at", "invocation_id"): + _ = metadata.pop(volatile_key) + return hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest() + + +def compile_manifest_run_ids(*, project_dir: Path, env: dict[str, str]) -> frozenset[str]: + """Compile with a manifest and return the run ids rendered into compiled hooks.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=("compile", "--json", "--manifest"), env=env + ) + assert result.returncode == 0, result.stderr + manifest: str = (project_dir / "target" / "manifest.json").read_text(encoding="utf-8") + return frozenset(_HOOK_RUN_ID_PATTERN.findall(manifest)) + + +def compile_json_payload(*, project_dir: Path, env: dict[str, str]) -> dict[str, Any]: + """Compile in a fresh process and return the parsed JSON payload.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=("compile", "--json"), env=env + ) + return cast(dict[str, Any], json.loads(result.stdout)) + + +def attachment_cache_counts(timings: dict[str, int]) -> tuple[int, int, int]: + """Return model attachment cache hits, misses, and bypasses from compile timings.""" + + return ( + timings["attachment_cache_hits"], + timings["attachment_cache_misses"], + timings["attachment_cache_bypasses"], + ) + + +def outcome_attachment_counts(outcome: CompileCacheOutcome) -> tuple[int, int, int]: + """Return model attachment cache hits, misses, and bypasses from one compile outcome.""" + + return ( + outcome.attachment_cache_hits, + outcome.attachment_cache_misses, + outcome.attachment_cache_bypasses, + ) + + +def write_project_files(project_dir: Path, files: dict[str, str]) -> None: + """Write several authored project files.""" + + for relative_path, contents in files.items(): + write_project_file(project_dir, relative_path, contents) + + +def install_custom_duckdb_adapter(project_dir: Path) -> None: + """Switch the project to a project-local adapter and add a model rendering a constant.""" + + write_project_file(project_dir, "adapters/duckdb_plus.py", _CUSTOM_DUCKDB_ADAPTER_SOURCE) + replace_project_text( + project_dir, "sqlbuild_project.toml", 'adapter = "duckdb"', 'adapter = "duckdb_plus"' + ) + write_project_file( + project_dir, + "models/marts/minimum_orders.sql", + 'MODEL (description "Orders meeting the minimum");\n\n' + 'SELECT order_id\nFROM __ref("stg_orders")\n' + 'WHERE quantity >= @const("min_quantity")\n', + ) + + +def add_project_provider(project_dir: Path, provider_name: str) -> None: + """Add one project provider with an explicit runtime name.""" + + write_project_file( + project_dir, + f"providers/{provider_name}_client.py", + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersClient(Provider):\n" + ' """Order notification client."""\n\n' + f' provider_name = "{provider_name}"\n', + ) + + +def rewrite_python_hook(project_dir: Path, *, decorator: str, parameters: str) -> None: + """Rewrite the order notification hook with a new decorator and parameter list.""" + + write_project_file( + project_dir, + "models/marts/_sqlbuild/_hooks/python/notifications.py", + "from sqlbuild.hooks import hook\n" + "from sqlbuild.refs import model\n\n\n" + f"{decorator}\n" + f"def notify_complete({parameters}):\n" + ' ctx.log(f"{ctx.model_name} completed")\n', ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py new file mode 100644 index 0000000000..dd41389a37 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py @@ -0,0 +1,264 @@ +"""Reused model attachment must match a cache-disabled compile through the real CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + AttachmentCacheReuseTestCase, + AttachmentDiagnosticReplayTestCase, + AttachmentReferencePolicyTestCase, + AttachmentRunIdTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + COMPILE_CACHE_REGION_ENV_VAR, + COMPILE_CACHE_SCHEMA_ENV_VAR, + CompileCacheOutcome, + attachment_cache_counts, + compile_cache_outcome, + compile_json_payload, + compile_manifest_run_ids, + outcome_attachment_counts, + replace_project_text, + write_project_file, + write_project_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentCacheReuseTestCase( + description="model_added_attaches_only_the_new_model", + edit=lambda root: write_project_file( + root, + "models/marts/order_quantities.sql", + 'MODEL (description "Order quantities");\n\n' + 'SELECT order_id, quantity FROM __ref("stg_orders")\n', + ), + expected_attachment_counts=(13, 1, 1), + ), + AttachmentCacheReuseTestCase( + description="unreferenced_model_removed_reuses_the_rest", + edit=lambda root: (root / "models/marts/dim_customers.sql").unlink(), + expected_attachment_counts=(12, 0, 1), + ), + AttachmentCacheReuseTestCase( + description="referenced_model_removed_fails_after_reuse", + edit=lambda root: (root / "models/staging/stg_orders.sql").unlink(), + expected_attachment_counts=(0, 0, 0), + expected_failure=True, + ), + AttachmentCacheReuseTestCase( + description="project_macro_edit_reattaches_every_model", + edit=lambda root: replace_project_text( + root, + "macros/currency.py", + 'return f"{price_cents} * {quantity}"', + 'return f"({price_cents}) * ({quantity})"', + ), + expected_attachment_counts=(0, 14, 0), + ), + AttachmentCacheReuseTestCase( + description="target_schema_environment_value_unchanged", + setup=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'schema = "dev"', + f'schema = "${{ENV:{COMPILE_CACHE_SCHEMA_ENV_VAR}}}"', + ), + edit=lambda root: None, + initial_env={COMPILE_CACHE_SCHEMA_ENV_VAR: "analytics_east"}, + expected_attachment_counts=(13, 0, 1), + ), + AttachmentCacheReuseTestCase( + description="target_schema_environment_value_changed", + setup=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'schema = "dev"', + f'schema = "${{ENV:{COMPILE_CACHE_SCHEMA_ENV_VAR}}}"', + ), + edit=lambda root: None, + initial_env={COMPILE_CACHE_SCHEMA_ENV_VAR: "analytics_east"}, + edited_env={COMPILE_CACHE_SCHEMA_ENV_VAR: "analytics_west"}, + expected_attachment_counts=(0, 14, 0), + ), + ], + ids=lambda case: case.description, +) +def test_given_warm_attachment_cache_when_project_changes_then_matches_uncached_reference( + cache_invalidation_project: Path, test_case: AttachmentCacheReuseTestCase +) -> None: + project_dir: Path = cache_invalidation_project + test_case.setup(project_dir) + initial_env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east", **test_case.initial_env} + cold: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) + + test_case.edit(project_dir) + edited_env: dict[str, str] = {**initial_env, **test_case.edited_env} + edited: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=edited_env) + reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=edited_env, compile_args=("--no-cache",) + ) + + assert cold.returncode == 0 + assert outcome_attachment_counts(cold) == (0, 14, 0) + assert edited[:3] == reference[:3] + assert (edited.returncode != 0) is test_case.expected_failure + assert outcome_attachment_counts(edited) == test_case.expected_attachment_counts + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentRunIdTestCase( + description="sql_hook_renders_run_id", + model_path="models/marts/run_stamp.sql", + model_sql=( + "MODEL (\n" + " materialized table,\n" + ' description "Run stamp",\n' + ' post_hooks [inline_sql("CREATE TABLE IF NOT EXISTS run_log AS SELECT ' + "'@@CTX:run.id' AS run_id\")],\n" + ");\n\n" + "SELECT 1 AS order_id\n" + ), + expected_warm_attachment_counts=(13, 0, 2), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_reading_run_id_when_compiling_warm_then_each_compile_renders_its_own_run_id( + cache_invalidation_project: Path, test_case: AttachmentRunIdTestCase +) -> None: + project_dir: Path = cache_invalidation_project + write_project_file(project_dir, test_case.model_path, test_case.model_sql) + env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + + first_run_ids: frozenset[str] = compile_manifest_run_ids(project_dir=project_dir, env=env) + second_run_ids: frozenset[str] = compile_manifest_run_ids(project_dir=project_dir, env=env) + warm: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + + assert (len(first_run_ids), len(second_run_ids)) == (1, 1) + assert first_run_ids != second_run_ids + assert outcome_attachment_counts(warm) == test_case.expected_warm_attachment_counts + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentDiagnosticReplayTestCase( + description="macro_generated_reference", + files={ + "models/marts/_sqlbuild/_macros/orders_base.py": ( + "def orders_base() -> str:\n" + " return 'SELECT order_id FROM __ref(\"stg_orders\")'\n" + ), + "models/marts/order_summary.sql": ( + 'MODEL (description "Order summary");\n\n@orders_base()\n' + ), + }, + expected_diagnostic_codes=("P006",), + expected_warm_attachment_counts=(14, 0, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_attachment_diagnostics_when_compiling_warm_then_cache_hit_replays_them( + cache_invalidation_project: Path, test_case: AttachmentDiagnosticReplayTestCase +) -> None: + project_dir: Path = cache_invalidation_project + write_project_files(project_dir, test_case.files) + env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + + cold: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + warm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + warm_outcome: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=env, compile_args=("--no-cache",) + ) + + assert tuple(item["code"] for item in warm["diagnostics"]) == ( + test_case.expected_diagnostic_codes + ) + assert warm["diagnostics"] == cold["diagnostics"] + assert attachment_cache_counts(warm["compile_timings"]) == ( + test_case.expected_warm_attachment_counts + ) + assert warm_outcome[:3] == reference[:3] + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentReferencePolicyTestCase( + description="macro_generated_reference_policy_toggle", + files={ + "models/marts/_sqlbuild/_macros/orders_base.py": ( + "def orders_base() -> str:\n" + " return 'SELECT order_id FROM __ref(\"stg_orders\")'\n" + ), + "models/marts/order_summary.sql": ( + 'MODEL (description "Order summary");\n\n@orders_base()\n' + ), + }, + relax=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + "[settings]", + "[references]\nenforce_explicit = false\n\n[settings]", + ), + restore=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + "[references]\nenforce_explicit = false\n\n[settings]", + "[settings]", + ), + expected_strict_codes=("P006",), + expected_relaxed_codes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_replayed_diagnostic_when_reference_policy_toggles_then_matches_uncached_reference( + cache_invalidation_project: Path, test_case: AttachmentReferencePolicyTestCase +) -> None: + project_dir: Path = cache_invalidation_project + write_project_files(project_dir, test_case.files) + env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + _ = compile_json_payload(project_dir=project_dir, env=env) + strict_warm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + + test_case.relax(project_dir) + relaxed: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + relaxed_reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=env, compile_args=("--no-cache",) + ) + relaxed_rewarm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + test_case.restore(project_dir) + restored: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + restored_reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=env, compile_args=("--no-cache",) + ) + restored_rewarm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + + assert tuple(item["code"] for item in strict_warm["diagnostics"]) == ( + test_case.expected_strict_codes + ) + assert tuple(item["code"] for item in relaxed_rewarm["diagnostics"]) == ( + test_case.expected_relaxed_codes + ) + assert tuple(item["code"] for item in restored_rewarm["diagnostics"]) == ( + test_case.expected_strict_codes + ) + assert relaxed[:3] == relaxed_reference[:3] + assert restored[:3] == restored_reference[:3] + assert attachment_cache_counts(restored_rewarm["compile_timings"])[1] == 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py index ddf3c7d886..da9961bb3b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py @@ -14,10 +14,14 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( COMPILE_CACHE_REGION_ENV_VAR, + DESCRIBED_HOOK_DECORATOR, CompileCacheOutcome, + add_project_provider, compile_cache_outcome, + install_custom_duckdb_adapter, move_project_file, replace_project_text, + rewrite_python_hook, run_installed_sqb, write_project_file, ) @@ -75,7 +79,6 @@ " - name: quantity\n type: INTEGER", " - name: quantity\n type: BIGINT", ), - expected_output_change=False, ), CompileCacheInvalidationTestCase( description="source_yaml_column_rename", @@ -183,6 +186,162 @@ edit=lambda root: None, edited_env={COMPILE_CACHE_REGION_ENV_VAR: "west"}, ), + CompileCacheInvalidationTestCase( + description="project_defaults_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + '[defaults]\nmaterialized = "table"', + '[defaults]\nmaterialized = "table"\ntags = ["nightly"]', + ), + ), + CompileCacheInvalidationTestCase( + description="path_defaults_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + '[path_defaults.staging]\nmaterialized = "view"', + '[path_defaults.staging]\nmaterialized = "view"\nalias = "${CTX:model.name}_v2"', + ), + ), + CompileCacheInvalidationTestCase( + description="cli_vars_edit", + edit=lambda root: None, + edited_compile_args=("--vars", '{"quantity_multiplier": "7"}'), + ), + CompileCacheInvalidationTestCase( + description="project_settings_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'default_audit_severity = "warn"', + 'default_audit_severity = "warn"\nsql_analysis = false', + ), + ), + CompileCacheInvalidationTestCase( + description="adapter_dialect_edit", + edit=lambda root: replace_project_text( + root, "sqlbuild_project.toml", 'adapter = "duckdb"', 'adapter = "postgres"' + ), + ), + CompileCacheInvalidationTestCase( + description="scoped_macro_import_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_macros/scaling.py", + 'line_total_cents(column, "2")', + 'line_total_cents(column, "3")', + ), + ), + CompileCacheInvalidationTestCase( + description="enum_member_added", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_enums/order_channel.sql", + 'PARTNER "partner"', + 'PARTNER "partner", STORE "store"', + ), + expected_output_change=False, + ), + CompileCacheInvalidationTestCase( + description="model_schema_declaration_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_schemas/order_metric.sql", + 'description "Scaled order quantities"', + 'description "Order quantities scaled for reporting"', + ), + ), + CompileCacheInvalidationTestCase( + description="named_sql_hook_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_hooks/sql/record_orders.sql", + "COUNT(*) AS row_count", + "COUNT(*) AS written_rows", + ), + ), + CompileCacheInvalidationTestCase( + description="python_hook_signature_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_hooks/python/notifications.py", + 'def notify_complete(ctx, channel="#orders"):', + 'def notify_complete(ctx, channel="#orders", *, priority):', + ), + expected_failure=True, + ), + CompileCacheInvalidationTestCase( + description="audit_factory_edit", + edit=lambda root: replace_project_text( + root, + "python/audits/order_quality.py", + "scaled_quantity >= 0", + "scaled_quantity >= 1", + ), + ), + CompileCacheInvalidationTestCase( + description="no_sql_analysis_flag", + edit=lambda root: None, + edited_compile_args=("--no-sql-analysis",), + ), + CompileCacheInvalidationTestCase( + description="project_sql_analysis_reenabled", + setup=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'default_audit_severity = "warn"', + 'default_audit_severity = "warn"\nsql_analysis = false', + ), + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'default_audit_severity = "warn"\nsql_analysis = false', + 'default_audit_severity = "warn"', + ), + ), + CompileCacheInvalidationTestCase( + description="custom_materialization_removed", + edit=lambda root: (root / "materializations/partition_tracked.py").unlink(), + expected_failure=True, + ), + CompileCacheInvalidationTestCase( + description="provider_name_conflicts_with_hook_argument", + setup=lambda root: rewrite_python_hook( + root, decorator=DESCRIBED_HOOK_DECORATOR, parameters="ctx, **kwargs" + ), + edit=lambda root: add_project_provider(root, "channel"), + expected_failure=True, + ), + CompileCacheInvalidationTestCase( + description="project_adapter_rendering_edit", + setup=install_custom_duckdb_adapter, + edit=lambda root: replace_project_text( + root, + "adapters/duckdb_plus.py", + 'return f"{rendered}"', + 'return f"({rendered})"', + ), + ), + CompileCacheInvalidationTestCase( + description="python_hook_reads_edit", + edit=lambda root: rewrite_python_hook( + root, + decorator=( + '@hook(reads=model("stg_customers"), description="Notify that orders completed")' + ), + parameters='ctx, channel="#orders"', + ), + ), + CompileCacheInvalidationTestCase( + description="python_hook_provider_usage_edit", + setup=lambda root: add_project_provider(root, "order_notifier"), + edit=lambda root: rewrite_python_hook( + root, + decorator=DESCRIBED_HOOK_DECORATOR, + parameters='ctx, order_notifier, channel="#orders"', + ), + ), ], ids=lambda case: case.description, ) @@ -190,6 +349,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d cache_invalidation_project: Path, test_case: CompileCacheInvalidationTestCase ) -> None: project_dir: Path = cache_invalidation_project + test_case.setup(project_dir) initial_env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} cold: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) warm: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) @@ -201,15 +361,23 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d fingerprint=cold.fingerprint, fact_cache_hits=cold.fact_cache_misses, fact_cache_misses=0, + attachment_cache_hits=cold.attachment_cache_misses - 1, + attachment_cache_misses=0, + attachment_cache_bypasses=1, ) test_case.edit(project_dir) edited_env: dict[str, str] = {**initial_env, **test_case.edited_env} - edited: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=edited_env) + edited_args: tuple[str, ...] = test_case.edited_compile_args + edited: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=edited_env, compile_args=edited_args + ) reference: CompileCacheOutcome = compile_cache_outcome( - project_dir=project_dir, env=edited_env, compile_args=("--no-cache",) + project_dir=project_dir, env=edited_env, compile_args=("--no-cache", *edited_args) + ) + rewarmed: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=edited_env, compile_args=edited_args ) - rewarmed: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=edited_env) assert (edited.returncode != 0) is test_case.expected_failure assert ( @@ -218,6 +386,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d assert edited[:3] == reference[:3] assert rewarmed[:3] == reference[:3] assert rewarmed.fact_cache_misses == test_case.expected_rewarmed_fact_cache_misses + assert rewarmed.attachment_cache_misses == 0 @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 27cb152395..8f84c4c630 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -24,6 +24,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, + attachment_cache_counts, fresh_process_compile_cache_metrics, measure_model_sql_bytes, run_dense_warm_edit_benchmark, @@ -203,14 +204,17 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm) assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0) assert warm_timings["rule_cache_misses"] == 0 + assert attachment_cache_counts(warm_timings) == (test_case.model_count, 0, 0) batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit) assert 0 < misses < test_case.model_count assert batch_hits + entry_hits + misses == test_case.model_count assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses + assert attachment_cache_counts(edit_timings) == (test_case.model_count - 1, 1, 0) oracle_timings: dict[str, int] = cast( dict[str, int], measurements["oracle"].payload["compile_timings"] ) assert oracle_timings["rule_cache_hits"] == 0 + assert attachment_cache_counts(oracle_timings) == (0, 0, 0) assert warm.elapsed_seconds < test_case.expected_warm_max_seconds assert edit.elapsed_seconds < test_case.expected_edit_max_seconds diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 18a979f4fb..fbd3a85ab0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -66,6 +66,7 @@ DiscoveredSqlScenarioFile, DiscoveredTaskFunction, ) +from sqlbuild.compiler.fact_cache._helpers.publication import await_fact_publication from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -761,3 +762,13 @@ def required_description_diagnostics( source_inputs=inputs.source_inputs, function_inputs=inputs.function_inputs, ) + + +def clear_model_attachment_cache(*, project_dir: Path) -> None: + """Remove reusable model attachment facts so the next compile re-attaches every model.""" + + await_fact_publication() + for database in (project_dir / "target" / "cache" / "compiler").rglob( + "model-attachment.sqlite3" + ): + database.unlink() diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py index 191fdad6dc..7b434be23d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py @@ -43,3 +43,11 @@ class ParameterizedSqlRenderErrorTestCase: sql: str arguments: dict[str, object] expected_error_fragment: str + + +@dataclass(frozen=True) +class VolatileReadTestCase: + description: str + template: str + sql: str + expected_reads: frozenset[str] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py new file mode 100644 index 0000000000..fba8804a25 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py @@ -0,0 +1,72 @@ +"""Per-run reads recorded while attachment renders templates and SQL.""" + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile._helpers.render.sql_vars import substitute_sql_vars +from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data +from sqlbuild.compiler.compile._helpers.render.volatile_reads import record_volatile_reads +from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( + VolatileReadTestCase, +) + +_CONTEXT_VALUES: dict[str, str | None] = { + "run.id": "20260101T000000Z_000000000000", + "run.target": "dev", + "model.name": "orders", +} + + +@pytest.mark.parametrize( + "test_case", + [ + VolatileReadTestCase( + description="deterministic_context_and_vars", + template="${CTX:model.name}_${CTX:run.target}_${suffix}", + sql="SELECT '@@CTX:model.name' AS name, @@suffix AS suffix", + expected_reads=frozenset(), + ), + VolatileReadTestCase( + description="run_id_in_template_and_sql", + template="${CTX:run.id}", + sql="SELECT '@@CTX:run.id' AS run_id", + expected_reads=frozenset({"CTX:run.id"}), + ), + VolatileReadTestCase( + description="environment_in_template_and_sql", + template="${coalesce(ENV:ORDERS_SCHEMA, 'fallback')}", + sql="SELECT '@@ENV:ORDERS_REGION' AS region", + expected_reads=frozenset({"ENV:ORDERS_SCHEMA", "ENV:ORDERS_REGION"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_rendering_when_recording_then_only_per_run_values_are_reported( + test_case: VolatileReadTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ORDERS_REGION", "east") + monkeypatch.setenv("ORDERS_SCHEMA", "sales") + + with record_volatile_reads() as reads: + _ = expand_template_data( + value={"schema": test_case.template}, + variables={"suffix": "v2"}, + context_values=_CONTEXT_VALUES, + context_label="model config", + allow_context=True, + preserve_context_tokens=False, + preserve_unknown_context=False, + ) + _ = substitute_sql_vars( + sql=test_case.sql, + file_path=Path("models/orders.sql"), + effective_vars={"suffix": "1"}, + context_values=_CONTEXT_VALUES, + ) + + assert reads == test_case.expected_reads + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index 30ab6049fb..fac3821fa7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -44,6 +44,7 @@ CompactLineageCacheEncodingTestCase, ) from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + clear_model_attachment_cache, compile_project_with_cache, stored_analysis_contents, ) @@ -146,7 +147,7 @@ def test_given_successful_analysis_when_compiling_again_then_reuses_identical_ca signature_builder.assert_not_called() reference_scanner.assert_not_called() assert len(tuple((tmp_path / "target" / "cache" / "compiler").rglob("*.sqlite3"))) == ( - test_case.expected_count + 1 + test_case.expected_count + 2 ) @@ -396,6 +397,7 @@ def test_given_corrupt_reference_cache_when_compiling_then_rescans_and_repairs_t "UPDATE sql_reference SET payload = ?", (corrupt_contents,), ) + clear_model_attachment_cache(project_dir=tmp_path) scanner: Mock = Mock(wraps=reference_cache.extract_sql_references) monkeypatch.setattr(reference_cache, "extract_sql_references", scanner) @@ -431,6 +433,7 @@ def test_given_non_text_reference_cache_when_compiling_then_rescans_safely( "UPDATE sql_reference SET payload = ?", (sqlite3.Binary(b"broken"),), ) + clear_model_attachment_cache(project_dir=tmp_path) scanner: Mock = Mock(wraps=reference_cache.extract_sql_references) monkeypatch.setattr(reference_cache, "extract_sql_references", scanner) diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 9eccc3bc64..0a6eb60330 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -145,6 +145,27 @@ Python implementation details. Imported functions are not re-exported as new macros from the importing file. +## Keep macros deterministic + +A macro must return the same SQL every time it receives the same arguments and macro context. Do +not read environment variables, the clock, random values, files, or network services inside a +macro. Pass such values in through [project variables](/docs/concepts/interpolation#project-variables) +or the macro context instead. + +SQLBuild relies on this when it reuses compiled models between compile runs. A model is compiled +again when its file, its configuration and defaults, project variables, the target, the adapter, +any project macro or declaration, or the installed SQLBuild version changes. Models whose +configuration, hooks, or SQL read an environment variable or `run.id` are compiled again on every +run. `sqb compile --no-cache` compiles everything from scratch. + +The reuse check covers model files, project and local configuration, macro, enum, constant, schema, +and SQL hook files, the source files that define Python hooks and audit factories, and custom +adapter files under `adapters/`, along with the target, project variables, and command-line flags. +It does not cover code those files import from elsewhere: third-party libraries, and project +modules outside those locations that a hook or adapter imports. Keep macro, hook, and adapter code +independent of such modules, or run `sqb compile --no-cache` (or delete `target/cache/compiler/`) +after upgrading or editing one that changes their output. + ## Use macros in tests Tests are SQL, so they can use macros as reusable fixture generators: From 89eb40c2bc211f28c9eff29e66df19c77aed1ba0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:22:47 +0100 Subject: [PATCH 2/3] test(e2e): count the model attachment cache among compiler caches --- .../commands/main/build/test_runtime_artifact_preservation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 79487b7949..80728a5aad 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -32,7 +32,7 @@ "target/run/functions/python/is_completed_order_py.sql", ), expected_exit_code=0, - expected_compiler_cache_count=4, + expected_compiler_cache_count=5, expected_local_history=False, expected_compiled_paths=( "target/compiled/models/marts/fact_orders.sql", From 4ffa6d45d4d60bcbcd9578766e151d92ac91ef0e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 3 Oct 2026 06:40:11 +0100 Subject: [PATCH 3/3] test(e2e): rename the attachment cache JSON helper to avoid shadowing --- .../sqlbuild/cli/commands/main/compile/helpers.py | 2 +- .../main/compile/test_compile_attachment_cache.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 21112d946f..57283bf828 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2633,7 +2633,7 @@ def compile_manifest_run_ids(*, project_dir: Path, env: dict[str, str]) -> froze return frozenset(_HOOK_RUN_ID_PATTERN.findall(manifest)) -def compile_json_payload(*, project_dir: Path, env: dict[str, str]) -> dict[str, Any]: +def fresh_compile_json_payload(*, project_dir: Path, env: dict[str, str]) -> dict[str, Any]: """Compile in a fresh process and return the parsed JSON payload.""" result: subprocess.CompletedProcess[str] = run_installed_sqb( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py index dd41389a37..70ed21efb0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py @@ -19,8 +19,8 @@ CompileCacheOutcome, attachment_cache_counts, compile_cache_outcome, - compile_json_payload, compile_manifest_run_ids, + fresh_compile_json_payload, outcome_attachment_counts, replace_project_text, write_project_file, @@ -175,8 +175,8 @@ def test_given_attachment_diagnostics_when_compiling_warm_then_cache_hit_replays write_project_files(project_dir, test_case.files) env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} - cold: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) - warm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + cold: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) + warm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) warm_outcome: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) reference: CompileCacheOutcome = compile_cache_outcome( project_dir=project_dir, env=env, compile_args=("--no-cache",) @@ -230,21 +230,21 @@ def test_given_replayed_diagnostic_when_reference_policy_toggles_then_matches_un project_dir: Path = cache_invalidation_project write_project_files(project_dir, test_case.files) env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} - _ = compile_json_payload(project_dir=project_dir, env=env) - strict_warm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + _ = fresh_compile_json_payload(project_dir=project_dir, env=env) + strict_warm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) test_case.relax(project_dir) relaxed: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) relaxed_reference: CompileCacheOutcome = compile_cache_outcome( project_dir=project_dir, env=env, compile_args=("--no-cache",) ) - relaxed_rewarm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + relaxed_rewarm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) test_case.restore(project_dir) restored: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) restored_reference: CompileCacheOutcome = compile_cache_outcome( project_dir=project_dir, env=env, compile_args=("--no-cache",) ) - restored_rewarm: dict[str, Any] = compile_json_payload(project_dir=project_dir, env=env) + restored_rewarm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) assert tuple(item["code"] for item in strict_warm["diagnostics"]) == ( test_case.expected_strict_codes