diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md index 7cf958daae..dfb97c7bfb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/macros.md @@ -13,6 +13,7 @@ Online: https://sqlbuild.com/docs/concepts/macros/ - Arguments - Typed reference arguments - Keep implementation details private +- Keep macros deterministic - Use macros in tests - Use macros in hooks - Where macros work @@ -154,6 +155,27 @@ Python implementation details. Imported functions are not re-exported as new macros from the importing file. +## Keep macros deterministic + +A macro must return the same SQL every time it receives the same arguments and macro context. Do +not read environment variables, the clock, random values, files, or network services inside a +macro. Pass such values in through [project variables](interpolation.md#project-variables) +or the macro context instead. + +SQLBuild relies on this when it reuses compiled models between compile runs. A model is compiled +again when its file, its configuration and defaults, project variables, the target, the adapter, +any project macro or declaration, or the installed SQLBuild version changes. Models whose +configuration, hooks, or SQL read an environment variable or `run.id` are compiled again on every +run. `sqb compile --no-cache` compiles everything from scratch. + +The reuse check covers model files, project and local configuration, macro, enum, constant, schema, +and SQL hook files, the source files that define Python hooks and audit factories, and custom +adapter files under `adapters/`, along with the target, project variables, and command-line flags. +It does not cover code those files import from elsewhere: third-party libraries, and project +modules outside those locations that a hook or adapter imports. Keep macro, hook, and adapter code +independent of such modules, or run `sqb compile --no-cache` (or delete `target/cache/compiler/`) +after upgrading or editing one that changes their output. + ## Use macros in tests Tests are SQL, so they can use macros as reusable fixture generators: diff --git a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py index c2260c1862..62d6599705 100644 --- a/src/sqlbuild/compiler/compile/_helpers/attachment/core.py +++ b/src/sqlbuild/compiler/compile/_helpers/attachment/core.py @@ -48,6 +48,10 @@ validate_preserved_logical_namespace, ) from sqlbuild.compiler.compile._helpers.config.table_type import resolve_storage_policies +from sqlbuild.compiler.compile._helpers.diagnostics.collector import ( + capture_compile_diagnostics, + report_compile_diagnostic, +) from sqlbuild.compiler.compile._helpers.diagnostics.sql_analysis_opt_outs import ( rejected_sql_analysis_opt_out, ) @@ -55,6 +59,10 @@ merge_call_site_references, resource_references, ) +from sqlbuild.compiler.compile._helpers.model_reuse.keys import ( + model_attachment_environment, + model_attachment_key, +) from sqlbuild.compiler.compile._helpers.named_declarations.core import ( named_declaration_usages, ) @@ -95,6 +103,8 @@ expand_effective_vars, expand_template_data, ) +from sqlbuild.compiler.compile._helpers.render.volatile_reads import record_volatile_reads +from sqlbuild.compiler.compile.classes.model_attachment_cache import ModelAttachmentCache from sqlbuild.compiler.compile.constants import ( MACRO_CALL_PATTERN, MODEL_FULL_REFRESH_CONFIG_KEY, @@ -116,6 +126,8 @@ LoadedMacro, MacroContext, MacroExpansionResult, + ModelAttachmentEnvironment, + ModelAttachmentFact, ModelConfigBuildRequest, ModelConfigScanCache, ModelHeaderColumnCache, @@ -379,6 +391,19 @@ def _is_reusable(self, matched_path_default: str | None) -> bool: return reusable +@dataclass(frozen=True) +class _ModelAttachmentState: + discovered_inputs: DiscoveredProjectInputs + context: ModelInputBuildContext + validation_context: _ModelValidationContext + sql_hook_definitions: dict[str, DiscoveredSqlHookFile] + header_column_cache: ModelHeaderColumnCache + config_scan_cache: ModelConfigScanCache + reusable_config_cache: _ReusableModelConfigCache + declaration_cache: _VisibleModelDeclarationCache + legacy_schema_files: tuple[DiscoveredSchemaFile, ...] + + def _contains_mutable_nested_config(values: dict[str, object]) -> bool: if any(key in values for key in _MODEL_HOOK_KEYS): return True @@ -431,6 +456,7 @@ def build_model_inputs( external_sql_reference_resolver=external_sql_reference_resolver, extract_references=extract_references, legacy_schema_files=legacy_schema_files, + cache_root=reference_cache_dir, ) @@ -443,341 +469,419 @@ def _build_model_inputs( external_sql_reference_resolver: ExternalSqlReferenceResolver | None, extract_references: Callable[[str], tuple[CompileSqlReference, ...]], legacy_schema_files: tuple[DiscoveredSchemaFile, ...], + cache_root: Path | None, ) -> tuple[CompileModelInput, ...]: - effective_vars: dict[str, object] = context.effective_vars - effective_settings: SettingsConfig = context.effective_settings - target_config: TargetConfig | None = context.target_config - effective_target_name: str | None = context.effective_target_name - run_id: str = context.run_id - macro_context: MacroContext = context.macro_context - loaded_macros: dict[str, LoadedMacro] = context.loaded_macros known_model_names: set[str] = build_known_ref_names(discovered_inputs) - known_seed_names: set[str] = build_known_seed_names(discovered_inputs) - known_source_names: set[str] = build_known_source_names(discovered_inputs) - known_function_names: set[str] = build_known_function_names(discovered_inputs) - known_table_function_names: set[str] = build_known_table_function_names(discovered_inputs) if external_sql_reference_resolver is not None: external_sql_reference_resolver.validate_model_names(known_model_names=known_model_names) - custom_materialization_names: frozenset[str] = frozenset( - mf.name for mf in discovered_inputs.materialization_files - ) - validation_context: _ModelValidationContext = _ModelValidationContext( - effective_settings=effective_settings, - no_sql_validation=no_sql_validation, - project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, - defer_model_sql_validation=defer_model_sql_validation, - external_sql_reference_resolver=external_sql_reference_resolver, - extract_references=extract_references, - known_model_names=known_model_names, - known_seed_names=known_seed_names, - known_source_names=known_source_names, - known_function_names=known_function_names, - known_table_function_names=known_table_function_names, - custom_materialization_names=custom_materialization_names, - ) - sql_hook_definitions: dict[str, DiscoveredSqlHookFile] = _index_sql_hook_definitions( - discovered_inputs.sql_hook_files + state: _ModelAttachmentState = _ModelAttachmentState( + discovered_inputs=discovered_inputs, + context=context, + validation_context=_ModelValidationContext( + effective_settings=context.effective_settings, + no_sql_validation=no_sql_validation, + project_config_path=(discovered_inputs.project_dir or Path()) / PROJECT_CONFIG_FILENAME, + defer_model_sql_validation=defer_model_sql_validation, + external_sql_reference_resolver=external_sql_reference_resolver, + extract_references=extract_references, + known_model_names=known_model_names, + known_seed_names=build_known_seed_names(discovered_inputs), + known_source_names=build_known_source_names(discovered_inputs), + known_function_names=build_known_function_names(discovered_inputs), + known_table_function_names=build_known_table_function_names(discovered_inputs), + custom_materialization_names=frozenset( + mf.name for mf in discovered_inputs.materialization_files + ), + ), + sql_hook_definitions=_index_sql_hook_definitions(discovered_inputs.sql_hook_files), + header_column_cache=ModelHeaderColumnCache(), + config_scan_cache=ModelConfigScanCache(), + reusable_config_cache=_ReusableModelConfigCache( + defaults=discovered_inputs.project_config.defaults, + path_defaults=discovered_inputs.project_config.path_defaults, + target_config=context.target_config, + ), + declaration_cache=_VisibleModelDeclarationCache.build(context), + legacy_schema_files=legacy_schema_files, ) - model_inputs: list[CompileModelInput] = [] - model_header_column_cache: ModelHeaderColumnCache = ModelHeaderColumnCache() - config_scan_cache: ModelConfigScanCache = ModelConfigScanCache() - reusable_config_cache: _ReusableModelConfigCache = _ReusableModelConfigCache( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - target_config=target_config, + environment: ModelAttachmentEnvironment | None = ( + None + if cache_root is None + else model_attachment_environment( + discovered_inputs=discovered_inputs, + context=context, + no_sql_validation=no_sql_validation, + defer_model_sql_validation=defer_model_sql_validation, + ) ) - declaration_cache: _VisibleModelDeclarationCache = _VisibleModelDeclarationCache.build(context) - model_files: tuple[DiscoveredSqlModelFile, ...] = discovered_inputs.model_files - prepared_var_substituted_sqls: tuple[str | None, ...] = prepare_static_project_vars_batch( - sqls=tuple(model_file.query_sql for model_file in model_files), - effective_vars=effective_vars, + with ModelAttachmentCache(root=cache_root, environment=environment) as attachment_cache: + return _attach_models(state=state, attachment_cache=attachment_cache) + + +def _attach_models( + *, state: _ModelAttachmentState, attachment_cache: ModelAttachmentCache +) -> tuple[CompileModelInput, ...]: + model_files: tuple[DiscoveredSqlModelFile, ...] = state.discovered_inputs.model_files + path_defaults: dict[str, dict[str, object]] = ( + state.discovered_inputs.project_config.path_defaults ) - model_file: DiscoveredSqlModelFile - prepared_var_substituted_sql: str | None - for model_file, prepared_var_substituted_sql in zip( - model_files, prepared_var_substituted_sqls, strict=True - ): - model_identity: ResourceIdentity = ResourceIdentity( - ResourceKind.MODEL, model_file.file_path.stem - ) - declarations: _VisibleModelDeclarations = declaration_cache.for_model( - model_file=model_file, consumer=model_identity - ) - matched_path_default: str | None = find_matching_path_default( + matched_path_defaults: tuple[str | None, ...] = tuple( + find_matching_path_default(model_file=model_file, path_defaults=path_defaults) + for model_file in model_files + ) + environment: ModelAttachmentEnvironment | None = attachment_cache.environment + keys: tuple[str | None, ...] = tuple( + None + if environment is None + else model_attachment_key( + store=attachment_cache.store, + environment=environment, model_file=model_file, - path_defaults=discovered_inputs.project_config.path_defaults, + path_default=( + matched, + None if matched is None else path_defaults.get(matched), + ), + resolver=state.context.declaration_resolver, ) - effective_config: CompileModelConfig | None = reusable_config_cache.get( - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, + for model_file, matched in zip(model_files, matched_path_defaults, strict=True) + ) + facts: dict[str, ModelAttachmentFact] = attachment_cache.read( + tuple( + (model_file.relative_path.as_posix(), key) + for model_file, key in zip(model_files, keys, strict=True) + if key is not None ) - if effective_config is None: - effective_config = build_model_config( - request=ModelConfigBuildRequest( - defaults=discovered_inputs.project_config.defaults, - path_defaults=discovered_inputs.project_config.path_defaults, - matched_path_default=matched_path_default, - model_header_values=model_file.header_values, - effective_vars=effective_vars, - target_config=target_config, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - materialization_defaults=( - discovered_inputs.project_config.materialization_defaults - ), - scan_cache=config_scan_cache, - ) + ) + reused: tuple[CompileModelInput | None, ...] = tuple( + attachment_cache.reuse(fact=None if key is None else facts.get(key), model_file=model_file) + for model_file, key in zip(model_files, keys, strict=True) + ) + miss_indexes: tuple[int, ...] = tuple( + index for index, model_input in enumerate(reused) if model_input is None + ) + prepared_sqls: dict[int, str | None] = ( + dict( + zip( + miss_indexes, + prepare_static_project_vars_batch( + sqls=tuple(model_files[index].query_sql for index in miss_indexes), + effective_vars=state.context.effective_vars, + ), + strict=True, ) - reusable_config_cache.remember( + ) + if miss_indexes + else {} + ) + model_inputs: list[CompileModelInput] = [] + for index, model_file in enumerate(model_files): + key: str | None = keys[index] + cached_input: CompileModelInput | None = reused[index] + if cached_input is not None and key is not None: + _replay_model_attachment(state=state, fact=facts[key], model_input=cached_input) + model_inputs.append(cached_input) + continue + hook_references: tuple[CompileSqlReference, ...] + with record_volatile_reads() as reads, capture_compile_diagnostics() as diagnostics: + model_input, hook_references = _attach_model( + state=state, + model_file=model_file, + matched_path_default=matched_path_defaults[index], + prepared_var_substituted_sql=prepared_sqls[index], + ) + if key is not None: + attachment_cache.stage( + slot=model_file.relative_path.as_posix(), + key=key, + model_input=model_input, + hook_references=hook_references, + reads=reads, + diagnostics=diagnostics, + ) + model_inputs.append(model_input) + + validate_declared_schema_models_are_attached( + model_inputs=tuple(model_inputs), + schema_files=state.discovered_inputs.schema_files, + ) + return tuple(model_inputs) + + +def _replay_model_attachment( + *, + state: _ModelAttachmentState, + fact: ModelAttachmentFact, + model_input: CompileModelInput, +) -> None: + for diagnostic_key, diagnostic in fact.diagnostics: + report_compile_diagnostic(key=diagnostic_key, diagnostic=diagnostic) + validation_context: _ModelValidationContext = state.validation_context + references: tuple[CompileSqlReference, ...] + for references in (model_input.references, fact.hook_references): + validate_model_references( + references=references, + model_file=model_input.model_file, + known_model_names=validation_context.known_model_names, + known_seed_names=validation_context.known_seed_names, + known_source_names=validation_context.known_source_names, + known_function_names=validation_context.known_function_names, + known_table_function_names=validation_context.known_table_function_names, + external_sql_reference_resolver=validation_context.external_sql_reference_resolver, + ) + _reject_legacy_schema_match( + model_file=model_input.model_file, schema_files=state.legacy_schema_files + ) + + +def _attach_model( + *, + state: _ModelAttachmentState, + model_file: DiscoveredSqlModelFile, + matched_path_default: str | None, + prepared_var_substituted_sql: str | None, +) -> tuple[CompileModelInput, tuple[CompileSqlReference, ...]]: + context: ModelInputBuildContext = state.context + model_identity: ResourceIdentity = ResourceIdentity( + ResourceKind.MODEL, model_file.file_path.stem + ) + declarations: _VisibleModelDeclarations = state.declaration_cache.for_model( + model_file=model_file, consumer=model_identity + ) + effective_config: CompileModelConfig | None = state.reusable_config_cache.get( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + ) + if effective_config is None: + effective_config = build_model_config( + request=ModelConfigBuildRequest( + defaults=state.discovered_inputs.project_config.defaults, + path_defaults=state.discovered_inputs.project_config.path_defaults, matched_path_default=matched_path_default, model_header_values=model_file.header_values, - config=effective_config, + effective_vars=context.effective_vars, + target_config=context.target_config, + model_name=model_file.file_path.stem, + effective_target_name=context.effective_target_name, + run_id=context.run_id, + materialization_defaults=( + state.discovered_inputs.project_config.materialization_defaults + ), + scan_cache=state.config_scan_cache, ) - model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( - values=effective_config.values, - model_name=model_file.file_path.stem, - public_model_schemas=context.public_model_schemas, - ) - model_schema_columns: tuple[SchemaColumn, ...] | None = ( - model_schema.columns if model_schema is not None else None ) - validate_python_hook_config( - values=effective_config.values, - model_name=model_file.file_path.stem, - hook_functions=discovered_inputs.hook_functions, - provider_names=frozenset(provider.name for provider in discovered_inputs.providers), + state.reusable_config_cache.remember( + matched_path_default=matched_path_default, + model_header_values=model_file.header_values, + config=effective_config, ) - named_usages: tuple[UsageRecord, ...] = ( - *( - named_declaration_usages( - resolver=context.declaration_resolver, - kind=DeclarationKind.SCHEMA, - name=model_schema.name, - consumer=model_identity, - consumer_path=model_file.relative_path, - ) - if model_schema is not None - else () - ), - *_python_hook_usages( - values=effective_config.values, + model_schema: ModelSchemaDeclaration | None = _resolve_model_schema( + values=effective_config.values, + model_name=model_file.file_path.stem, + public_model_schemas=context.public_model_schemas, + ) + model_schema_columns: tuple[SchemaColumn, ...] | None = ( + model_schema.columns if model_schema is not None else None + ) + validate_python_hook_config( + values=effective_config.values, + model_name=model_file.file_path.stem, + hook_functions=state.discovered_inputs.hook_functions, + provider_names=frozenset(provider.name for provider in state.discovered_inputs.providers), + ) + named_usages: tuple[UsageRecord, ...] = ( + *( + named_declaration_usages( resolver=context.declaration_resolver, + kind=DeclarationKind.SCHEMA, + name=model_schema.name, consumer=model_identity, consumer_path=model_file.relative_path, - ), - ) - var_substituted_sql: str = ( - prepared_var_substituted_sql - if prepared_var_substituted_sql is not None - else substitute_sql_vars( - sql=model_file.query_sql, - file_path=model_file.file_path, - effective_vars=effective_vars, ) - ) - declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - enum_visibility=declarations.enum_visibility, - constant_visibility=declarations.constant_visibility, - macros=declarations.macros, - macro_records=declarations.macro_records, - inaccessible_macros=declarations.inaccessible_macros, + if model_schema is not None + else () + ), + *_python_hook_usages( + values=effective_config.values, + resolver=context.declaration_resolver, consumer=model_identity, - ) - declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( - sql=var_substituted_sql, - file_path=model_file.file_path, - declarations=declaration_context, - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - ) - declaration_expanded_sql: str = declaration_expansion.sql - macro_expansion: MacroExpansionResult = expand_sql_macros_result( - sql=declaration_expanded_sql, + consumer_path=model_file.relative_path, + ), + ) + var_substituted_sql: str = ( + prepared_var_substituted_sql + if prepared_var_substituted_sql is not None + else substitute_sql_vars( + sql=model_file.query_sql, file_path=model_file.file_path, - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_resolver=context.declaration_resolver, - declarations=( - declaration_context if context.declaration_resolver is not None else None - ), - consumer=model_identity, - ) - expanded_query_sql: str = macro_expansion.sql - expanded_query_sql = get_validated_model_cursor_intrinsics( - sql=expanded_query_sql, - config_values=effective_config.values, - model_name=model_file.file_path.stem, - ) - raw_placeholders: object | None = effective_config.values.get("placeholders") - sql_validation_placeholders: dict[str, str] | None = ( - {str(k): str(v) for k, v in raw_placeholders.items()} - if isinstance(raw_placeholders, dict) - else None - ) - sql_validation_enabled, references, rejected_opt_out = _validate_model_input( - context=validation_context, - model_file=model_file, - config=effective_config, - expanded_query_sql=expanded_query_sql, - sql_validation_placeholders=sql_validation_placeholders, - model_schema_columns=model_schema_columns, - argument_references=macro_expansion.argument_references, + effective_vars=context.effective_vars, ) - hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + ) + declaration_context: DeclarationResolutionContext = DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, + enum_visibility=declarations.enum_visibility, + constant_visibility=declarations.constant_visibility, + macros=declarations.macros, + macro_records=declarations.macro_records, + inaccessible_macros=declarations.inaccessible_macros, + consumer=model_identity, + ) + declaration_expansion: DeclarationExpansionResult = expand_declaration_references_result( + sql=var_substituted_sql, + file_path=model_file.file_path, + declarations=declaration_context, + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + ) + declaration_expanded_sql: str = declaration_expansion.sql + macro_expansion: MacroExpansionResult = expand_sql_macros_result( + sql=declaration_expanded_sql, + file_path=model_file.file_path, + loaded_macros=context.loaded_macros, + macro_context=context.macro_context, + declaration_resolver=context.declaration_resolver, + declarations=(declaration_context if context.declaration_resolver is not None else None), + consumer=model_identity, + ) + expanded_query_sql: str = macro_expansion.sql + expanded_query_sql = get_validated_model_cursor_intrinsics( + sql=expanded_query_sql, + config_values=effective_config.values, + model_name=model_file.file_path.stem, + ) + raw_placeholders: object | None = effective_config.values.get("placeholders") + sql_validation_placeholders: dict[str, str] | None = ( + {str(k): str(v) for k, v in raw_placeholders.items()} + if isinstance(raw_placeholders, dict) + else None + ) + sql_validation_enabled, references, rejected_opt_out = _validate_model_input( + context=state.validation_context, + model_file=model_file, + config=effective_config, + expanded_query_sql=expanded_query_sql, + sql_validation_placeholders=sql_validation_placeholders, + model_schema_columns=model_schema_columns, + argument_references=macro_expansion.argument_references, + ) + hook_expansion: HookExpansionResult = expand_model_hook_macros_result( + values=effective_config.values, + file_path=model_file.file_path, + effective_vars=context.effective_vars, + context_values=build_model_context_values( values=effective_config.values, - file_path=model_file.file_path, - effective_vars=effective_vars, - context_values=build_model_context_values( - values=effective_config.values, - model_name=model_file.file_path.stem, - effective_target_name=effective_target_name, - run_id=run_id, - include_target_values=True, - ), - loaded_macros=loaded_macros, - macro_context=macro_context, - declaration_expansion=DeclarationExpansionContext( - declarations=DeclarationResolutionContext( - enums=declarations.enums, - constants=declarations.constants, - inaccessible_enums=declarations.inaccessible_enums, - inaccessible_constants=declarations.inaccessible_constants, - ), - value_renderer=context.value_renderer, - collection_rendering=context.collection_rendering, - resolver=context.declaration_resolver, - ), - sql_lexical_syntax=context.sql_lexical_syntax, - sql_hook_definitions=sql_hook_definitions, - consumer=model_identity, - ) - expanded_config: CompileModelConfig = _hook_expanded_config( - config=effective_config, - hook_expansion=hook_expansion, - model_file=model_file, - validation_context=validation_context, - ) - hook_name: str - for hook_name in ("pre_hooks", "post_hooks"): - hook_value: object | None = expanded_config.values.get(hook_name) - if isinstance(hook_value, list | tuple): - hook_entry: object - for hook_entry in hook_value: - if isinstance(hook_entry, SqlHookEntry): - reject_cursor_intrinsics( - sql=hook_entry.statement, - context=f"Model '{model_file.file_path.stem}' {hook_name}", - ) - header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( model_name=model_file.file_path.stem, - model_header_values=expanded_config.values, - file_path=model_file.relative_path, - column_locations=model_file.header_column_locations, - model_schema_columns=model_schema_columns, - model_schema_name=model_schema.name if model_schema is not None else None, - model_schema_description=model_schema.description if model_schema is not None else None, - audit_factories=discovered_inputs.audit_factories, - column_cache=model_header_column_cache, - ) - model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) - header_schema_entry, enum_columns = resolve_enum_contract_columns( - schema_entry=header_schema_entry, - config_values=model_config.values, - enums=( - declarations.enums - if model_schema is None - else { - **declarations.enums, - **model_schema_enum_declarations( - schema=model_schema, resolver=context.declaration_resolver - ), - } + effective_target_name=context.effective_target_name, + run_id=context.run_id, + include_target_values=True, + ), + loaded_macros=context.loaded_macros, + macro_context=context.macro_context, + declaration_expansion=DeclarationExpansionContext( + declarations=DeclarationResolutionContext( + enums=declarations.enums, + constants=declarations.constants, + inaccessible_enums=declarations.inaccessible_enums, + inaccessible_constants=declarations.inaccessible_constants, ), - ) - generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( - enum_columns=enum_columns, - declarations=declarations, - consumer=model_identity, - ) - model_declaration_usages: tuple[UsageRecord, ...] = tuple( - dict.fromkeys( - ( - *declaration_expansion.usages, - *generated_usages, - *hook_expansion.usages, - *named_usages, - ) - ) - ) - _reject_legacy_schema_match(model_file=model_file, schema_files=legacy_schema_files) - if header_schema_entry is None: - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), - ) - if var_substituted_sql == model_file.query_sql - else None - ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, - ) - ) - continue - - model_inputs.append( - CompileModelInput( - sql_expansion=( - CompiledSqlExpansion( - authored_sql=model_file.query_sql, - expanded_sql=macro_expansion.sql, - passes=(declaration_expansion.spans, macro_expansion.spans), + value_renderer=context.value_renderer, + collection_rendering=context.collection_rendering, + resolver=context.declaration_resolver, + ), + sql_lexical_syntax=context.sql_lexical_syntax, + sql_hook_definitions=state.sql_hook_definitions, + consumer=model_identity, + ) + expanded_config: CompileModelConfig = _hook_expanded_config( + config=effective_config, + hook_expansion=hook_expansion, + model_file=model_file, + validation_context=state.validation_context, + ) + hook_name: str + for hook_name in ("pre_hooks", "post_hooks"): + hook_value: object | None = expanded_config.values.get(hook_name) + if isinstance(hook_value, list | tuple): + hook_entry: object + for hook_entry in hook_value: + if isinstance(hook_entry, SqlHookEntry): + reject_cursor_intrinsics( + sql=hook_entry.statement, + context=f"Model '{model_file.file_path.stem}' {hook_name}", ) - if var_substituted_sql == model_file.query_sql - else None + header_schema_entry: SchemaModelEntry | None = build_model_header_schema_entry( + model_name=model_file.file_path.stem, + model_header_values=expanded_config.values, + file_path=model_file.relative_path, + column_locations=model_file.header_column_locations, + model_schema_columns=model_schema_columns, + model_schema_name=model_schema.name if model_schema is not None else None, + model_schema_description=model_schema.description if model_schema is not None else None, + audit_factories=state.discovered_inputs.audit_factories, + column_cache=state.header_column_cache, + ) + model_config: CompileModelConfig = strip_model_header_metadata_from_config(expanded_config) + header_schema_entry, enum_columns = resolve_enum_contract_columns( + schema_entry=header_schema_entry, + config_values=model_config.values, + enums=( + declarations.enums + if model_schema is None + else { + **declarations.enums, + **model_schema_enum_declarations( + schema=model_schema, resolver=context.declaration_resolver ), - model_file=model_file, - config=model_config, - query_sql=expanded_query_sql, - macro_source_sql=declaration_expanded_sql, - references=references, - schema_entry=header_schema_entry, - sql_validation_enabled=sql_validation_enabled, - rejected_sql_analysis_opt_out=rejected_opt_out, - enum_declarations=tuple(declarations.local_enums.values()), - constant_declarations=tuple(declarations.local_constants.values()), - enum_columns=enum_columns, - macro_deps=tuple(item.name for item in macro_expansion.dependencies), - macro_usages=macro_expansion.usages, - declaration_usages=model_declaration_usages, + } + ), + ) + generated_usages: tuple[UsageRecord, ...] = _generated_enum_usages( + enum_columns=enum_columns, + declarations=declarations, + consumer=model_identity, + ) + model_declaration_usages: tuple[UsageRecord, ...] = tuple( + dict.fromkeys( + ( + *declaration_expansion.usages, + *generated_usages, + *hook_expansion.usages, + *named_usages, ) ) - - validate_declared_schema_models_are_attached( - model_inputs=tuple(model_inputs), - schema_files=discovered_inputs.schema_files, ) - return tuple(model_inputs) + _reject_legacy_schema_match(model_file=model_file, schema_files=state.legacy_schema_files) + return ( + CompileModelInput( + sql_expansion=( + CompiledSqlExpansion( + authored_sql=model_file.query_sql, + expanded_sql=macro_expansion.sql, + passes=(declaration_expansion.spans, macro_expansion.spans), + ) + if var_substituted_sql == model_file.query_sql + else None + ), + model_file=model_file, + config=model_config, + query_sql=expanded_query_sql, + macro_source_sql=declaration_expanded_sql, + references=references, + schema_entry=header_schema_entry, + sql_validation_enabled=sql_validation_enabled, + rejected_sql_analysis_opt_out=rejected_opt_out, + enum_declarations=tuple(declarations.local_enums.values()), + constant_declarations=tuple(declarations.local_constants.values()), + enum_columns=enum_columns, + macro_deps=tuple(item.name for item in macro_expansion.dependencies), + macro_usages=macro_expansion.usages, + declaration_usages=model_declaration_usages, + ), + hook_expansion.references, + ) def _hook_expanded_config( diff --git a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py index ae6080acb7..9489de6b13 100644 --- a/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py +++ b/src/sqlbuild/compiler/compile/_helpers/diagnostics/collector.py @@ -35,6 +35,24 @@ def collect_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics]: _ACTIVE_COLLECTOR.reset(token) +@contextmanager +def capture_compile_diagnostics() -> Iterator[CollectedCompileDiagnostics | None]: + """Record diagnostics reported inside the block and forward them to the active collector.""" + + parent: CollectedCompileDiagnostics | None = _ACTIVE_COLLECTOR.get() + if parent is None: + yield None + return + captured: CollectedCompileDiagnostics = CollectedCompileDiagnostics() + token: Token[CollectedCompileDiagnostics | None] = _ACTIVE_COLLECTOR.set(captured) + try: + yield captured + finally: + _ACTIVE_COLLECTOR.reset(token) + for key, diagnostic in captured.entries: + parent.add(key=key, diagnostic=diagnostic) + + def with_collected_compile_diagnostics[**P, R: (CompileProjectInputs, CompiledProject)]( build: Callable[P, R], ) -> Callable[P, R]: diff --git a/src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py b/src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py new file mode 100644 index 0000000000..4cf5357660 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/model_reuse/keys.py @@ -0,0 +1,235 @@ +"""Cache identity for one model's attachment result.""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import replace +from pathlib import Path + +from sqlbuild.compiler.compile._helpers.render.volatile_reads import environment_read_label +from sqlbuild.compiler.compile.models import ( + DeclarationScopeResolver, + ModelAttachmentEnvironment, + ModelInputBuildContext, +) +from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.compiler.scopes.models import DeclarationRecord, ResourceIdentity, ResourceRecord +from sqlbuild.compiler.scopes.types import DeclarationKind, ResourceKind + +_TARGET_ENVIRONMENT_REFERENCE: re.Pattern[str] = re.compile(r"\bENV:\s*([A-Za-z0-9_]+)") +_PROCESS_LOCAL_REPR_MARKER: str = " at 0x" +_PROJECT_ADAPTERS_DIRECTORY: str = "adapters" +_ENVIRONMENT_DIGEST_BYTES: int = 32 +_ATTACHMENT_DECLARATION_KINDS: frozenset[DeclarationKind] = frozenset( + { + DeclarationKind.MACRO, + DeclarationKind.ENUM, + DeclarationKind.CONSTANT, + DeclarationKind.SCHEMA, + DeclarationKind.SQL_HOOK, + DeclarationKind.PYTHON_HOOK, + } +) + + +def model_attachment_environment( + *, + discovered_inputs: DiscoveredProjectInputs, + context: ModelInputBuildContext, + no_sql_validation: bool, + defer_model_sql_validation: bool, +) -> ModelAttachmentEnvironment | None: + """Return the project-wide key part, or None when an input has no stable identity.""" + + target_text: str = repr( + None if context.target_config is None else replace(context.target_config, connection={}) + ) + target_environment_names: tuple[str, ...] = tuple( + sorted(set(_TARGET_ENVIRONMENT_REFERENCE.findall(target_text))) + ) + renderer_type: type = type(context.value_renderer) + material: str = "\0".join( + ( + compiled_code_identity(), + str(discovered_inputs.project_dir.resolve() if discovered_inputs.project_dir else ""), + context.macro_context.adapter_name, + f"{renderer_type.__module__}.{renderer_type.__qualname__}", + str(context.collection_rendering), + context.sql_lexical_syntax.cache_key, + _project_adapter_sources(discovered_inputs), + repr(context.effective_target_name), + target_text, + repr(tuple((name, os.environ.get(name)) for name in target_environment_names)), + repr(context.effective_vars), + repr(context.effective_settings), + repr( + ( + context.macro_context.sql_analysis_enabled, + context.macro_context.target_name, + discovered_inputs.project_config.references.enforce_explicit, + no_sql_validation, + defer_model_sql_validation, + ) + ), + repr(discovered_inputs.project_config.defaults), + repr(discovered_inputs.project_config.materialization_defaults), + repr(sorted(item.name for item in discovered_inputs.materialization_files)), + repr(sorted(item.name for item in discovered_inputs.providers)), + _declaration_surface( + discovered_inputs=discovered_inputs, resolver=context.declaration_resolver + ), + ) + ) + if _PROCESS_LOCAL_REPR_MARKER in material: + return None + return ModelAttachmentEnvironment( + digest=hashlib.blake2b( + material.encode("utf-8", "surrogatepass"), digest_size=_ENVIRONMENT_DIGEST_BYTES + ).hexdigest(), + keyed_reads=frozenset(environment_read_label(name) for name in target_environment_names), + ) + + +def model_attachment_key( + *, + store: FactCacheStore, + environment: ModelAttachmentEnvironment, + model_file: DiscoveredSqlModelFile, + path_default: tuple[str | None, object], + resolver: DeclarationScopeResolver | None, +) -> str: + """Return the exact identity of one model's attachment inputs.""" + + relative_path: str = model_file.relative_path.as_posix() + return store.key( + environment.digest, + relative_path, + str(model_file.file_path), + model_file.contents, + repr(model_file.extract_implicit_alias_columns), + repr(path_default), + "" if resolver is None else _resource_scope(resolver=resolver, model_file=model_file), + ) + + +def _resource_scope( + *, resolver: DeclarationScopeResolver, model_file: DiscoveredSqlModelFile +) -> str: + records: tuple[ResourceRecord, ...] = resolver.lookup.resources_by_path.get( + model_file.relative_path.as_posix(), () + ) + identities: tuple[ResourceIdentity, ...] = ( + ResourceIdentity(ResourceKind.MODEL, model_file.file_path.stem), + *(record.identity for record in records), + ) + return repr( + ( + records, + tuple( + resolver.lookup.grants_by_resource.get(identity, ()) + for identity in dict.fromkeys(identities) + ), + ) + ) + + +def _declaration_surface( + *, discovered_inputs: DiscoveredProjectInputs, resolver: DeclarationScopeResolver | None +) -> str: + """Return every shared macro and declaration source; editing one re-attaches every model.""" + + parts: list[str] = [ + repr( + ( + type(item).__name__, + item.relative_path.as_posix(), + item.contents, + item.scope_kind, + item.ownership_root, + item.owning_path, + item.declaration_root, + ) + ) + for item in ( + *discovered_inputs.macro_files, + *discovered_inputs.enum_files, + *discovered_inputs.constant_files, + *discovered_inputs.model_schema_files, + *discovered_inputs.sql_hook_files, + ) + ] + parts.extend( + repr( + ( + hook.name, + hook.relative_path.as_posix(), + hook.description, + hook.reads, + hook.provider_usages, + hook.scope_kind, + hook.ownership_root, + hook.owning_path, + hook.declaration_root, + _source_text(hook.file_path), + ) + ) + for hook in discovered_inputs.hook_functions + ) + parts.extend( + repr( + ( + factory.name, + factory.relative_path.as_posix(), + factory.line, + factory.cases, + _source_text(factory.file_path), + ) + ) + for factory in discovered_inputs.audit_factories + ) + if resolver is not None: + parts.extend( + repr(record) + for record in resolver.lookup.index.declarations + if _is_shared_attachment_declaration(record) + ) + return "\0".join(parts) + + +def _is_shared_attachment_declaration(record: DeclarationRecord) -> bool: + owner: ResourceIdentity | None = record.identity.owner + return record.identity.kind in _ATTACHMENT_DECLARATION_KINDS and ( + owner is None or owner.kind is not ResourceKind.MODEL + ) + + +def _project_adapter_sources(discovered_inputs: DiscoveredProjectInputs) -> str: + project_dir: Path | None = discovered_inputs.project_dir + adapter_paths: tuple[Path, ...] = ( + () + if project_dir is None + else tuple(sorted((project_dir / _PROJECT_ADAPTERS_DIRECTORY).rglob("*.py"))) + ) + return repr( + ( + _source_text( + None + if discovered_inputs.adapter_file is None + else discovered_inputs.adapter_file.file_path + ), + tuple((path.as_posix(), _source_text(path)) for path in adapter_paths), + ) + ) + + +def _source_text(path: Path | None) -> str: + if path is None: + return "" + try: + return path.read_text(encoding="utf-8", errors="surrogateescape") + except OSError: + return "" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py index f621c5c7cc..78ed54e748 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/sql_vars.py @@ -17,6 +17,10 @@ expand_sql_macros_result, expand_sql_macros_with_spans, ) +from sqlbuild.compiler.compile._helpers.render.volatile_reads import ( + note_context_read, + note_environment_read, +) from sqlbuild.compiler.compile.constants import ( SQL_CONTEXT_NAME_EXTRA_TOKENS, SQL_IDENTIFIER_EXTRA_TOKEN, @@ -366,6 +370,7 @@ def _render_interpolation_token( if env_name_end == env_name_start: raise CompileInputError(f"invalid environment interpolation token in '{file_path}'") env_name: str = sql[env_name_start:env_name_end] + note_environment_read(env_name) if env_name not in os.environ: raise CompileInputError( f"unknown environment variable '@@ENV:{env_name}' in '{file_path}'" @@ -387,6 +392,7 @@ def _render_interpolation_token( context_values=context_values, ) context_name = sql[context_name_start:context_name_end] + note_context_read(context_name) if context_name not in context_values: raise CompileInputError( f"SQL text in '{file_path}' references unknown CTX key '{context_name}'" diff --git a/src/sqlbuild/compiler/compile/_helpers/render/templating.py b/src/sqlbuild/compiler/compile/_helpers/render/templating.py index 3bd6eb68c0..1eeabd4dcc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/templating.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/templating.py @@ -8,6 +8,10 @@ from dataclasses import dataclass from sqlbuild.compiler.authored_values.main._project_var_values import render_project_var_text +from sqlbuild.compiler.compile._helpers.render.volatile_reads import ( + note_context_read, + note_environment_read, +) from sqlbuild.compiler.compile.constants import ( MISSING_TEMPLATE_CONTEXT_MESSAGE_PART, MISSING_TEMPLATE_CONTEXT_VALUE_MESSAGE_PART, @@ -521,6 +525,7 @@ def _lookup_variable(*, name: str, variables: dict[str, object], context_label: def _lookup_environment_variable(*, name: str, context_label: str) -> str: + note_environment_read(name) if name not in os.environ: raise CompileInputError(f"{context_label} references missing ENV variable '{name}'") return os.environ[name] @@ -537,6 +542,7 @@ def _lookup_context_value( if preserve_unknown_context: return f"${{{TemplateNamespace.CTX}:{name}}}" raise CompileInputError(f"{context_label} references unknown CTX key '{name}'") + note_context_read(name) context_value: str | None = context_values.get(name) if context_value is None: raise CompileInputError( diff --git a/src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py b/src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py new file mode 100644 index 0000000000..ee74175f14 --- /dev/null +++ b/src/sqlbuild/compiler/compile/_helpers/render/volatile_reads.py @@ -0,0 +1,47 @@ +"""Record per-invocation values that a reused attachment result cannot reproduce.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token + +from sqlbuild.compiler.compile.types import CompileContextKey, TemplateNamespace + +_ACTIVE_READS: ContextVar[set[str] | None] = ContextVar( + "sqlbuild_volatile_compile_reads", default=None +) + + +def environment_read_label(name: str) -> str: + """Return the recorded label for one environment variable read.""" + + return f"{TemplateNamespace.ENV}:{name}" + + +def note_environment_read(name: str) -> None: + """Record that template or SQL interpolation read one environment variable.""" + + reads: set[str] | None = _ACTIVE_READS.get() + if reads is not None: + reads.add(environment_read_label(name)) + + +def note_context_read(name: str) -> None: + """Record a CTX read whose value differs between compile invocations.""" + + reads: set[str] | None = _ACTIVE_READS.get() + if reads is not None and name == CompileContextKey.RUN_ID: + reads.add(f"{TemplateNamespace.CTX}:{name}") + + +@contextmanager +def record_volatile_reads() -> Iterator[set[str]]: + """Collect environment and run-id reads made while the block runs.""" + + reads: set[str] = set() + token: Token[set[str] | None] = _ACTIVE_READS.set(reads) + try: + yield reads + finally: + _ACTIVE_READS.reset(token) diff --git a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py index 9a1a32e725..40e12b7038 100644 --- a/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py +++ b/src/sqlbuild/compiler/compile/classes/collected_compile_diagnostics.py @@ -16,6 +16,12 @@ def add(self, *, key: tuple[str, ...], diagnostic: CompilerDiagnostic) -> None: self._by_key.setdefault(key, diagnostic) + @property + def entries(self) -> tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...]: + """Return each recorded key and diagnostic in report order.""" + + return tuple(self._by_key.items()) + @property def diagnostics(self) -> tuple[CompilerDiagnostic, ...]: """Return the recorded diagnostics in deterministic key order.""" diff --git a/src/sqlbuild/compiler/compile/classes/model_attachment_cache.py b/src/sqlbuild/compiler/compile/classes/model_attachment_cache.py new file mode 100644 index 0000000000..9f08d14b69 --- /dev/null +++ b/src/sqlbuild/compiler/compile/classes/model_attachment_cache.py @@ -0,0 +1,137 @@ +"""Reuse model attachment results across compile invocations.""" + +from __future__ import annotations + +import gc +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path +from types import TracebackType + +from sqlbuild.compiler.compile.classes.collected_compile_diagnostics import ( + CollectedCompileDiagnostics, +) +from sqlbuild.compiler.compile.constants import ( + MODEL_ATTACHMENT_FACT_ALGORITHM, + MODEL_ATTACHMENT_FACT_CACHE_NAMESPACE, +) +from sqlbuild.compiler.compile.models import ( + CompileModelInput, + CompileSqlReference, + ModelAttachmentEnvironment, + ModelAttachmentFact, +) +from sqlbuild.compiler.discovery.models import DiscoveredSqlModelFile +from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.profiling.main._metric import record_compile_metric + +_DETACHED_MODEL_FILE: DiscoveredSqlModelFile = DiscoveredSqlModelFile( + file_path=Path(), + relative_path=Path(), + contents="", + header_values={}, + header_column_locations={}, + output_column_locations={}, + query_sql="", +) + + +@contextmanager +def _collection_paused() -> Iterator[None]: + """Pause cyclic garbage collection while thousands of acyclic cached facts are unpickled.""" + + collecting: bool = gc.isenabled() + gc.disable() + try: + yield + finally: + if collecting: + gc.enable() + + +class ModelAttachmentCache: + """Store one attachment fact per model; models that read per-run values are recomputed.""" + + def __init__( + self, *, root: Path | None, environment: ModelAttachmentEnvironment | None + ) -> None: + self.environment: ModelAttachmentEnvironment | None = environment + self.store: FactCacheStore = FactCacheStore( + root=root if environment is not None else None, + namespace=MODEL_ATTACHMENT_FACT_CACHE_NAMESPACE, + algorithm=MODEL_ATTACHMENT_FACT_ALGORITHM, + ) + self._hits: int = 0 + self._misses: int = 0 + self._bypasses: int = 0 + + def __enter__(self) -> ModelAttachmentCache: + _ = self.store.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + self.store.__exit__(exc_type, exc_value, traceback) + finally: + if self.store.enabled: + record_compile_metric(metric="attachment_cache_hits", value=self._hits) + record_compile_metric(metric="attachment_cache_misses", value=self._misses) + record_compile_metric(metric="attachment_cache_bypasses", value=self._bypasses) + + def read(self, entries: Sequence[tuple[str, str]]) -> dict[str, ModelAttachmentFact]: + """Return verified facts by key for requested (slot, key) pairs.""" + + with _collection_paused(): + found: dict[str, object] = self.store.read_many(entries) + return { + key: value for key, value in found.items() if isinstance(value, ModelAttachmentFact) + } + + def reuse( + self, *, fact: ModelAttachmentFact | None, model_file: DiscoveredSqlModelFile + ) -> CompileModelInput | None: + """Return the cached model input rebound to the current file, or None to recompute.""" + + if fact is None or fact.model_input is None: + if fact is not None: + self._bypasses += 1 + elif self.store.enabled: + self._misses += 1 + return None + self._hits += 1 + return replace(fact.model_input, model_file=model_file) + + def stage( + self, + *, + slot: str, + key: str, + model_input: CompileModelInput, + hook_references: tuple[CompileSqlReference, ...], + reads: set[str], + diagnostics: CollectedCompileDiagnostics | None, + ) -> None: + """Queue one freshly attached model for publication after a successful compile.""" + + if self.environment is None: + return + volatile: bool = not reads <= self.environment.keyed_reads + self.store.stage( + key=key, + slot=slot, + value=( + ModelAttachmentFact(model_input=None) + if volatile + else ModelAttachmentFact( + model_input=replace(model_input, model_file=_DETACHED_MODEL_FILE), + hook_references=hook_references, + diagnostics=() if diagnostics is None else diagnostics.entries, + ) + ), + ) diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 063b1f129f..4188f04341 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -170,6 +170,8 @@ SQL_TEST_FACT_CACHE_NAMESPACE: str = "sql-tests" SQL_TEST_FACT_CACHE_ALGORITHM: str = "expanded-sql-test-ctes-v2" SQL_TEST_EXPECTED_MODELS_FACT_ALGORITHM: str = "sql-test-expected-models-v2" +MODEL_ATTACHMENT_FACT_CACHE_NAMESPACE: str = "model-attachment" +MODEL_ATTACHMENT_FACT_ALGORITHM: str = "model-attachment-v1" COMPACT_ANALYSIS_RESPONSE_LENGTH: int = 2 COMPACT_RELATION_STUB_PREFIX: str = "__sqlbuild_project_input_" COMPACT_REFERENCE_MARKER_PATTERN: re.Pattern[str] = re.compile( diff --git a/src/sqlbuild/compiler/compile/models.py b/src/sqlbuild/compiler/compile/models.py index 93bfc4495e..e797ab76ab 100644 --- a/src/sqlbuild/compiler/compile/models.py +++ b/src/sqlbuild/compiler/compile/models.py @@ -763,6 +763,25 @@ class CompileModelInput: sql_expansion: CompiledSqlExpansion | None = field(default=None, compare=False, repr=False) +@dataclass(frozen=True) +class ModelAttachmentFact: + """One model's reusable attachment result, or None when it read per-run values.""" + + model_input: CompileModelInput | None + hook_references: tuple[CompileSqlReference, ...] = field(default_factory=tuple) + diagnostics: tuple[tuple[tuple[str, ...], CompilerDiagnostic], ...] = field( + default_factory=tuple + ) + + +@dataclass(frozen=True) +class ModelAttachmentEnvironment: + """Project-wide identity shared by every model attachment cache key.""" + + digest: str + keyed_reads: frozenset[str] = frozenset() + + @dataclass(frozen=True) class ModelSqlAnalysis: """Completed model analysis and its in-memory binding proof.""" diff --git a/src/sqlbuild/compiler/profiling/constants.py b/src/sqlbuild/compiler/profiling/constants.py index 6ca434213c..3aed4d58f8 100644 --- a/src/sqlbuild/compiler/profiling/constants.py +++ b/src/sqlbuild/compiler/profiling/constants.py @@ -24,4 +24,7 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "attachment_cache_hits", + "attachment_cache_misses", + "attachment_cache_bypasses", ) diff --git a/src/sqlbuild/compiler/profiling/types.py b/src/sqlbuild/compiler/profiling/types.py index abb9028ae4..cb394933c6 100644 --- a/src/sqlbuild/compiler/profiling/types.py +++ b/src/sqlbuild/compiler/profiling/types.py @@ -24,4 +24,7 @@ "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "attachment_cache_hits", + "attachment_cache_misses", + "attachment_cache_bypasses", ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py index a79d284bc6..e07cd48cda 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_compile_json_behavior.py @@ -90,6 +90,9 @@ def test_given_waffle_shop_when_running_compile_json_then_it_reports_offline_que "analysis_cache_bypasses", "fact_cache_hits", "fact_cache_misses", + "attachment_cache_hits", + "attachment_cache_misses", + "attachment_cache_bypasses", "total_ms", } assert all(isinstance(value, int) and value >= 0 for value in timings.values()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py index 79487b7949..80728a5aad 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_runtime_artifact_preservation.py @@ -32,7 +32,7 @@ "target/run/functions/python/is_completed_order_py.sql", ), expected_exit_code=0, - expected_compiler_cache_count=4, + expected_compiler_cache_count=5, expected_local_history=False, expected_compiled_paths=( "target/compiled/models/marts/fact_orders.sql", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index d6166840c1..2c6535289e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -378,12 +378,59 @@ class CompileCacheInvalidationTestCase: description: str edit: Callable[[Path], None] + setup: Callable[[Path], None] = lambda _root: None edited_env: dict[str, str] = field(default_factory=dict) + edited_compile_args: tuple[str, ...] = () expected_failure: bool = False expected_output_change: bool = True expected_rewarmed_fact_cache_misses: int = 0 +@dataclass(frozen=True) +class AttachmentCacheReuseTestCase: + """One project change after a warm compile and the expected model attachment reuse.""" + + description: str + edit: Callable[[Path], None] + expected_attachment_counts: tuple[int, int, int] + setup: Callable[[Path], None] = lambda _root: None + initial_env: dict[str, str] = field(default_factory=dict) + edited_env: dict[str, str] = field(default_factory=dict) + expected_failure: bool = False + + +@dataclass(frozen=True) +class AttachmentRunIdTestCase: + """One model whose hook renders the per-invocation run id.""" + + description: str + model_path: str + model_sql: str + expected_warm_attachment_counts: tuple[int, int, int] + + +@dataclass(frozen=True) +class AttachmentDiagnosticReplayTestCase: + """Project files whose attachment reports diagnostics a cache hit must replay.""" + + description: str + files: dict[str, str] + expected_diagnostic_codes: tuple[str, ...] + expected_warm_attachment_counts: tuple[int, int, int] + + +@dataclass(frozen=True) +class AttachmentReferencePolicyTestCase: + """A replayed diagnostic whose presence follows the explicit-reference policy.""" + + description: str + files: dict[str, str] + relax: Callable[[Path], None] + restore: Callable[[Path], None] + expected_strict_codes: tuple[str, ...] + expected_relaxed_codes: tuple[str, ...] + + @dataclass(frozen=True) class CompileCacheDisabledTestCase: """One supported control that disables compile-cache reads and writes.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 40ad9bd6cd..57283bf828 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json import os import re @@ -2424,6 +2425,9 @@ def write_relation_stub_project( COMPILE_CACHE_REGION_ENV_VAR: str = "SQB_CACHE_INVALIDATION_REGION" +DESCRIBED_HOOK_DECORATOR: str = '@hook(description="Notify that orders completed")' +COMPILE_CACHE_SCHEMA_ENV_VAR: str = "SQB_CACHE_INVALIDATION_SCHEMA" +_HOOK_RUN_ID_PATTERN: re.Pattern[str] = re.compile(r"'(\d{8}T\d{6}Z_[0-9a-f]{12})' AS run_id") _COMPILE_CACHE_DIAGNOSTIC_PATTERN: re.Pattern[str] = re.compile( r"^(?:error|warning)\[.*$", re.MULTILINE ) @@ -2480,9 +2484,72 @@ def write_relation_stub_project( ")\n" "SELECT 1\n" ), + "models/marts/_sqlbuild/_macros/scaling.py": ( + '"""Marts scaling macros."""\n\n' + "from macros.currency import line_total_cents\n\n\n" + "def scaled_quantity(column: str) -> str:\n" + ' """Scale an order quantity column."""\n' + ' return line_total_cents(column, "2")\n' + ), + "models/marts/_sqlbuild/_schemas/order_metric.sql": ( + "SCHEMA (\n" + " name order_metric,\n" + ' description "Scaled order quantities",\n' + " columns (\n" + " order_id (nullable false),\n" + " scaled_quantity (nullable true),\n" + " ),\n" + ");\n" + ), + "models/marts/_sqlbuild/_hooks/sql/record_orders.sql": ( + 'HOOK (\n description "Count the rows written to a relation"\n);\n\n' + "SELECT COUNT(*) AS row_count FROM @relation\n" + ), + "models/marts/_sqlbuild/_hooks/python/notifications.py": ( + "from sqlbuild.hooks import hook\n\n\n" + '@hook(description="Notify that orders completed")\n' + 'def notify_complete(ctx, channel="#orders"):\n' + ' ctx.log(f"Notify {channel}: {ctx.model_name} completed")\n' + ), + "python/audits/order_quality.py": ( + "from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory\n\n\n" + "@audit_factory\n" + "def order_quality():\n" + " return [\n" + " AuditCase(\n" + ' name="non_negative_quantity",\n' + ' definition="expression_is_true",\n' + ' arguments={"expression": "scaled_quantity >= 0"},\n' + " severity=AuditSeverity.WARN,\n" + " )\n" + " ]\n" + ), + "models/marts/order_metrics.sql": ( + "MODEL (\n" + " materialized table,\n" + " model_schema order_metric,\n" + " audit_factories [order_quality],\n" + " post_hooks [\n" + ' sql("record_orders", relation: "@@CTX:destination.qualified"),\n' + ' python("notify_complete", channel: "#orders"),\n' + " ],\n" + ");\n\n" + 'SELECT o.order_id, @scaled_quantity("o.quantity") AS scaled_quantity\n' + 'FROM __ref("stg_orders") o\n' + ), } +_CUSTOM_DUCKDB_ADAPTER_SOURCE: str = ( + "from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter\n\n\n" + "class DuckDbPlusAdapter(DuckDbAdapter):\n" + ' adapter_name = "duckdb_plus"\n\n' + " def render_typed_scalar(self, *, value):\n" + " rendered = super().render_typed_scalar(value=value)\n" + ' return f"{rendered}"\n' +) + + class CompileCacheOutcome(NamedTuple): """Comparable result of one fresh-process compile.""" @@ -2491,6 +2558,9 @@ class CompileCacheOutcome(NamedTuple): fingerprint: str fact_cache_hits: int fact_cache_misses: int + attachment_cache_hits: int = 0 + attachment_cache_misses: int = 0 + attachment_cache_bypasses: int = 0 def run_installed_sqb( @@ -2516,21 +2586,129 @@ def run_installed_sqb( def compile_cache_outcome( *, project_dir: Path, env: dict[str, str], compile_args: tuple[str, ...] = () ) -> CompileCacheOutcome: - """Compile in a fresh process and return its semantic fingerprint and fact-cache counts.""" + """Compile in a fresh process and return its output fingerprint and fact-cache counts.""" + manifest_path: Path = project_dir / "target" / "manifest.json" + manifest_path.unlink(missing_ok=True) result: subprocess.CompletedProcess[str] = run_installed_sqb( - project_dir=project_dir, args=("compile", "--json", *compile_args), env=env + project_dir=project_dir, args=("compile", "--json", "--manifest", *compile_args), env=env ) values: dict[str, object] = cast(dict[str, object], json.loads(result.stdout or "{}")) timings: dict[str, int] = cast(dict[str, int], values.get("compile_timings", {})) + manifest_digest: str = "".join( + _invocation_independent_manifest_digest(path) + for path in manifest_path.parent.glob(manifest_path.name) + ) return CompileCacheOutcome( returncode=result.returncode, diagnostics=tuple(_COMPILE_CACHE_DIAGNOSTIC_PATTERN.findall(result.stderr)), fingerprint=semantic_compile_fingerprint( payload=values, compiled_dir=project_dir / "target" / "compiled" - ), + ) + + manifest_digest, fact_cache_hits=timings.get("fact_cache_hits", 0), fact_cache_misses=timings.get("fact_cache_misses", 0), + attachment_cache_hits=timings.get("attachment_cache_hits", 0), + attachment_cache_misses=timings.get("attachment_cache_misses", 0), + attachment_cache_bypasses=timings.get("attachment_cache_bypasses", 0), + ) + + +def _invocation_independent_manifest_digest(manifest_path: Path) -> str: + manifest: dict[str, Any] = cast(dict[str, Any], json.loads(manifest_path.read_text())) + metadata: dict[str, object] = manifest["metadata"] + for volatile_key in ("generated_at", "invocation_id"): + _ = metadata.pop(volatile_key) + return hashlib.sha256(json.dumps(manifest, sort_keys=True).encode()).hexdigest() + + +def compile_manifest_run_ids(*, project_dir: Path, env: dict[str, str]) -> frozenset[str]: + """Compile with a manifest and return the run ids rendered into compiled hooks.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=("compile", "--json", "--manifest"), env=env + ) + assert result.returncode == 0, result.stderr + manifest: str = (project_dir / "target" / "manifest.json").read_text(encoding="utf-8") + return frozenset(_HOOK_RUN_ID_PATTERN.findall(manifest)) + + +def fresh_compile_json_payload(*, project_dir: Path, env: dict[str, str]) -> dict[str, Any]: + """Compile in a fresh process and return the parsed JSON payload.""" + + result: subprocess.CompletedProcess[str] = run_installed_sqb( + project_dir=project_dir, args=("compile", "--json"), env=env + ) + return cast(dict[str, Any], json.loads(result.stdout)) + + +def attachment_cache_counts(timings: dict[str, int]) -> tuple[int, int, int]: + """Return model attachment cache hits, misses, and bypasses from compile timings.""" + + return ( + timings["attachment_cache_hits"], + timings["attachment_cache_misses"], + timings["attachment_cache_bypasses"], + ) + + +def outcome_attachment_counts(outcome: CompileCacheOutcome) -> tuple[int, int, int]: + """Return model attachment cache hits, misses, and bypasses from one compile outcome.""" + + return ( + outcome.attachment_cache_hits, + outcome.attachment_cache_misses, + outcome.attachment_cache_bypasses, + ) + + +def write_project_files(project_dir: Path, files: dict[str, str]) -> None: + """Write several authored project files.""" + + for relative_path, contents in files.items(): + write_project_file(project_dir, relative_path, contents) + + +def install_custom_duckdb_adapter(project_dir: Path) -> None: + """Switch the project to a project-local adapter and add a model rendering a constant.""" + + write_project_file(project_dir, "adapters/duckdb_plus.py", _CUSTOM_DUCKDB_ADAPTER_SOURCE) + replace_project_text( + project_dir, "sqlbuild_project.toml", 'adapter = "duckdb"', 'adapter = "duckdb_plus"' + ) + write_project_file( + project_dir, + "models/marts/minimum_orders.sql", + 'MODEL (description "Orders meeting the minimum");\n\n' + 'SELECT order_id\nFROM __ref("stg_orders")\n' + 'WHERE quantity >= @const("min_quantity")\n', + ) + + +def add_project_provider(project_dir: Path, provider_name: str) -> None: + """Add one project provider with an explicit runtime name.""" + + write_project_file( + project_dir, + f"providers/{provider_name}_client.py", + "from sqlbuild.providers import Provider\n\n\n" + "class OrdersClient(Provider):\n" + ' """Order notification client."""\n\n' + f' provider_name = "{provider_name}"\n', + ) + + +def rewrite_python_hook(project_dir: Path, *, decorator: str, parameters: str) -> None: + """Rewrite the order notification hook with a new decorator and parameter list.""" + + write_project_file( + project_dir, + "models/marts/_sqlbuild/_hooks/python/notifications.py", + "from sqlbuild.hooks import hook\n" + "from sqlbuild.refs import model\n\n\n" + f"{decorator}\n" + f"def notify_complete({parameters}):\n" + ' ctx.log(f"{ctx.model_name} completed")\n', ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py new file mode 100644 index 0000000000..70ed21efb0 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_attachment_cache.py @@ -0,0 +1,264 @@ +"""Reused model attachment must match a cache-disabled compile through the real CLI.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + AttachmentCacheReuseTestCase, + AttachmentDiagnosticReplayTestCase, + AttachmentReferencePolicyTestCase, + AttachmentRunIdTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + COMPILE_CACHE_REGION_ENV_VAR, + COMPILE_CACHE_SCHEMA_ENV_VAR, + CompileCacheOutcome, + attachment_cache_counts, + compile_cache_outcome, + compile_manifest_run_ids, + fresh_compile_json_payload, + outcome_attachment_counts, + replace_project_text, + write_project_file, + write_project_files, +) + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentCacheReuseTestCase( + description="model_added_attaches_only_the_new_model", + edit=lambda root: write_project_file( + root, + "models/marts/order_quantities.sql", + 'MODEL (description "Order quantities");\n\n' + 'SELECT order_id, quantity FROM __ref("stg_orders")\n', + ), + expected_attachment_counts=(13, 1, 1), + ), + AttachmentCacheReuseTestCase( + description="unreferenced_model_removed_reuses_the_rest", + edit=lambda root: (root / "models/marts/dim_customers.sql").unlink(), + expected_attachment_counts=(12, 0, 1), + ), + AttachmentCacheReuseTestCase( + description="referenced_model_removed_fails_after_reuse", + edit=lambda root: (root / "models/staging/stg_orders.sql").unlink(), + expected_attachment_counts=(0, 0, 0), + expected_failure=True, + ), + AttachmentCacheReuseTestCase( + description="project_macro_edit_reattaches_every_model", + edit=lambda root: replace_project_text( + root, + "macros/currency.py", + 'return f"{price_cents} * {quantity}"', + 'return f"({price_cents}) * ({quantity})"', + ), + expected_attachment_counts=(0, 14, 0), + ), + AttachmentCacheReuseTestCase( + description="target_schema_environment_value_unchanged", + setup=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'schema = "dev"', + f'schema = "${{ENV:{COMPILE_CACHE_SCHEMA_ENV_VAR}}}"', + ), + edit=lambda root: None, + initial_env={COMPILE_CACHE_SCHEMA_ENV_VAR: "analytics_east"}, + expected_attachment_counts=(13, 0, 1), + ), + AttachmentCacheReuseTestCase( + description="target_schema_environment_value_changed", + setup=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'schema = "dev"', + f'schema = "${{ENV:{COMPILE_CACHE_SCHEMA_ENV_VAR}}}"', + ), + edit=lambda root: None, + initial_env={COMPILE_CACHE_SCHEMA_ENV_VAR: "analytics_east"}, + edited_env={COMPILE_CACHE_SCHEMA_ENV_VAR: "analytics_west"}, + expected_attachment_counts=(0, 14, 0), + ), + ], + ids=lambda case: case.description, +) +def test_given_warm_attachment_cache_when_project_changes_then_matches_uncached_reference( + cache_invalidation_project: Path, test_case: AttachmentCacheReuseTestCase +) -> None: + project_dir: Path = cache_invalidation_project + test_case.setup(project_dir) + initial_env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east", **test_case.initial_env} + cold: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) + + test_case.edit(project_dir) + edited_env: dict[str, str] = {**initial_env, **test_case.edited_env} + edited: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=edited_env) + reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=edited_env, compile_args=("--no-cache",) + ) + + assert cold.returncode == 0 + assert outcome_attachment_counts(cold) == (0, 14, 0) + assert edited[:3] == reference[:3] + assert (edited.returncode != 0) is test_case.expected_failure + assert outcome_attachment_counts(edited) == test_case.expected_attachment_counts + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentRunIdTestCase( + description="sql_hook_renders_run_id", + model_path="models/marts/run_stamp.sql", + model_sql=( + "MODEL (\n" + " materialized table,\n" + ' description "Run stamp",\n' + ' post_hooks [inline_sql("CREATE TABLE IF NOT EXISTS run_log AS SELECT ' + "'@@CTX:run.id' AS run_id\")],\n" + ");\n\n" + "SELECT 1 AS order_id\n" + ), + expected_warm_attachment_counts=(13, 0, 2), + ), + ], + ids=lambda case: case.description, +) +def test_given_model_reading_run_id_when_compiling_warm_then_each_compile_renders_its_own_run_id( + cache_invalidation_project: Path, test_case: AttachmentRunIdTestCase +) -> None: + project_dir: Path = cache_invalidation_project + write_project_file(project_dir, test_case.model_path, test_case.model_sql) + env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + + first_run_ids: frozenset[str] = compile_manifest_run_ids(project_dir=project_dir, env=env) + second_run_ids: frozenset[str] = compile_manifest_run_ids(project_dir=project_dir, env=env) + warm: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + + assert (len(first_run_ids), len(second_run_ids)) == (1, 1) + assert first_run_ids != second_run_ids + assert outcome_attachment_counts(warm) == test_case.expected_warm_attachment_counts + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentDiagnosticReplayTestCase( + description="macro_generated_reference", + files={ + "models/marts/_sqlbuild/_macros/orders_base.py": ( + "def orders_base() -> str:\n" + " return 'SELECT order_id FROM __ref(\"stg_orders\")'\n" + ), + "models/marts/order_summary.sql": ( + 'MODEL (description "Order summary");\n\n@orders_base()\n' + ), + }, + expected_diagnostic_codes=("P006",), + expected_warm_attachment_counts=(14, 0, 1), + ), + ], + ids=lambda case: case.description, +) +def test_given_attachment_diagnostics_when_compiling_warm_then_cache_hit_replays_them( + cache_invalidation_project: Path, test_case: AttachmentDiagnosticReplayTestCase +) -> None: + project_dir: Path = cache_invalidation_project + write_project_files(project_dir, test_case.files) + env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + + cold: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) + warm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) + warm_outcome: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=env, compile_args=("--no-cache",) + ) + + assert tuple(item["code"] for item in warm["diagnostics"]) == ( + test_case.expected_diagnostic_codes + ) + assert warm["diagnostics"] == cold["diagnostics"] + assert attachment_cache_counts(warm["compile_timings"]) == ( + test_case.expected_warm_attachment_counts + ) + assert warm_outcome[:3] == reference[:3] + + +@pytest.mark.parametrize( + "test_case", + [ + AttachmentReferencePolicyTestCase( + description="macro_generated_reference_policy_toggle", + files={ + "models/marts/_sqlbuild/_macros/orders_base.py": ( + "def orders_base() -> str:\n" + " return 'SELECT order_id FROM __ref(\"stg_orders\")'\n" + ), + "models/marts/order_summary.sql": ( + 'MODEL (description "Order summary");\n\n@orders_base()\n' + ), + }, + relax=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + "[settings]", + "[references]\nenforce_explicit = false\n\n[settings]", + ), + restore=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + "[references]\nenforce_explicit = false\n\n[settings]", + "[settings]", + ), + expected_strict_codes=("P006",), + expected_relaxed_codes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_replayed_diagnostic_when_reference_policy_toggles_then_matches_uncached_reference( + cache_invalidation_project: Path, test_case: AttachmentReferencePolicyTestCase +) -> None: + project_dir: Path = cache_invalidation_project + write_project_files(project_dir, test_case.files) + env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} + _ = fresh_compile_json_payload(project_dir=project_dir, env=env) + strict_warm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) + + test_case.relax(project_dir) + relaxed: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + relaxed_reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=env, compile_args=("--no-cache",) + ) + relaxed_rewarm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) + test_case.restore(project_dir) + restored: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=env) + restored_reference: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=env, compile_args=("--no-cache",) + ) + restored_rewarm: dict[str, Any] = fresh_compile_json_payload(project_dir=project_dir, env=env) + + assert tuple(item["code"] for item in strict_warm["diagnostics"]) == ( + test_case.expected_strict_codes + ) + assert tuple(item["code"] for item in relaxed_rewarm["diagnostics"]) == ( + test_case.expected_relaxed_codes + ) + assert tuple(item["code"] for item in restored_rewarm["diagnostics"]) == ( + test_case.expected_strict_codes + ) + assert relaxed[:3] == relaxed_reference[:3] + assert restored[:3] == restored_reference[:3] + assert attachment_cache_counts(restored_rewarm["compile_timings"])[1] == 0 + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-n", "auto", "--dist", "loadfile"])) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py index ddf3c7d886..da9961bb3b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_cache_invalidation.py @@ -14,10 +14,14 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( COMPILE_CACHE_REGION_ENV_VAR, + DESCRIBED_HOOK_DECORATOR, CompileCacheOutcome, + add_project_provider, compile_cache_outcome, + install_custom_duckdb_adapter, move_project_file, replace_project_text, + rewrite_python_hook, run_installed_sqb, write_project_file, ) @@ -75,7 +79,6 @@ " - name: quantity\n type: INTEGER", " - name: quantity\n type: BIGINT", ), - expected_output_change=False, ), CompileCacheInvalidationTestCase( description="source_yaml_column_rename", @@ -183,6 +186,162 @@ edit=lambda root: None, edited_env={COMPILE_CACHE_REGION_ENV_VAR: "west"}, ), + CompileCacheInvalidationTestCase( + description="project_defaults_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + '[defaults]\nmaterialized = "table"', + '[defaults]\nmaterialized = "table"\ntags = ["nightly"]', + ), + ), + CompileCacheInvalidationTestCase( + description="path_defaults_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + '[path_defaults.staging]\nmaterialized = "view"', + '[path_defaults.staging]\nmaterialized = "view"\nalias = "${CTX:model.name}_v2"', + ), + ), + CompileCacheInvalidationTestCase( + description="cli_vars_edit", + edit=lambda root: None, + edited_compile_args=("--vars", '{"quantity_multiplier": "7"}'), + ), + CompileCacheInvalidationTestCase( + description="project_settings_edit", + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'default_audit_severity = "warn"', + 'default_audit_severity = "warn"\nsql_analysis = false', + ), + ), + CompileCacheInvalidationTestCase( + description="adapter_dialect_edit", + edit=lambda root: replace_project_text( + root, "sqlbuild_project.toml", 'adapter = "duckdb"', 'adapter = "postgres"' + ), + ), + CompileCacheInvalidationTestCase( + description="scoped_macro_import_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_macros/scaling.py", + 'line_total_cents(column, "2")', + 'line_total_cents(column, "3")', + ), + ), + CompileCacheInvalidationTestCase( + description="enum_member_added", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_enums/order_channel.sql", + 'PARTNER "partner"', + 'PARTNER "partner", STORE "store"', + ), + expected_output_change=False, + ), + CompileCacheInvalidationTestCase( + description="model_schema_declaration_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_schemas/order_metric.sql", + 'description "Scaled order quantities"', + 'description "Order quantities scaled for reporting"', + ), + ), + CompileCacheInvalidationTestCase( + description="named_sql_hook_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_hooks/sql/record_orders.sql", + "COUNT(*) AS row_count", + "COUNT(*) AS written_rows", + ), + ), + CompileCacheInvalidationTestCase( + description="python_hook_signature_edit", + edit=lambda root: replace_project_text( + root, + "models/marts/_sqlbuild/_hooks/python/notifications.py", + 'def notify_complete(ctx, channel="#orders"):', + 'def notify_complete(ctx, channel="#orders", *, priority):', + ), + expected_failure=True, + ), + CompileCacheInvalidationTestCase( + description="audit_factory_edit", + edit=lambda root: replace_project_text( + root, + "python/audits/order_quality.py", + "scaled_quantity >= 0", + "scaled_quantity >= 1", + ), + ), + CompileCacheInvalidationTestCase( + description="no_sql_analysis_flag", + edit=lambda root: None, + edited_compile_args=("--no-sql-analysis",), + ), + CompileCacheInvalidationTestCase( + description="project_sql_analysis_reenabled", + setup=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'default_audit_severity = "warn"', + 'default_audit_severity = "warn"\nsql_analysis = false', + ), + edit=lambda root: replace_project_text( + root, + "sqlbuild_project.toml", + 'default_audit_severity = "warn"\nsql_analysis = false', + 'default_audit_severity = "warn"', + ), + ), + CompileCacheInvalidationTestCase( + description="custom_materialization_removed", + edit=lambda root: (root / "materializations/partition_tracked.py").unlink(), + expected_failure=True, + ), + CompileCacheInvalidationTestCase( + description="provider_name_conflicts_with_hook_argument", + setup=lambda root: rewrite_python_hook( + root, decorator=DESCRIBED_HOOK_DECORATOR, parameters="ctx, **kwargs" + ), + edit=lambda root: add_project_provider(root, "channel"), + expected_failure=True, + ), + CompileCacheInvalidationTestCase( + description="project_adapter_rendering_edit", + setup=install_custom_duckdb_adapter, + edit=lambda root: replace_project_text( + root, + "adapters/duckdb_plus.py", + 'return f"{rendered}"', + 'return f"({rendered})"', + ), + ), + CompileCacheInvalidationTestCase( + description="python_hook_reads_edit", + edit=lambda root: rewrite_python_hook( + root, + decorator=( + '@hook(reads=model("stg_customers"), description="Notify that orders completed")' + ), + parameters='ctx, channel="#orders"', + ), + ), + CompileCacheInvalidationTestCase( + description="python_hook_provider_usage_edit", + setup=lambda root: add_project_provider(root, "order_notifier"), + edit=lambda root: rewrite_python_hook( + root, + decorator=DESCRIBED_HOOK_DECORATOR, + parameters='ctx, order_notifier, channel="#orders"', + ), + ), ], ids=lambda case: case.description, ) @@ -190,6 +349,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d cache_invalidation_project: Path, test_case: CompileCacheInvalidationTestCase ) -> None: project_dir: Path = cache_invalidation_project + test_case.setup(project_dir) initial_env: dict[str, str] = {COMPILE_CACHE_REGION_ENV_VAR: "east"} cold: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) warm: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=initial_env) @@ -201,15 +361,23 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d fingerprint=cold.fingerprint, fact_cache_hits=cold.fact_cache_misses, fact_cache_misses=0, + attachment_cache_hits=cold.attachment_cache_misses - 1, + attachment_cache_misses=0, + attachment_cache_bypasses=1, ) test_case.edit(project_dir) edited_env: dict[str, str] = {**initial_env, **test_case.edited_env} - edited: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=edited_env) + edited_args: tuple[str, ...] = test_case.edited_compile_args + edited: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=edited_env, compile_args=edited_args + ) reference: CompileCacheOutcome = compile_cache_outcome( - project_dir=project_dir, env=edited_env, compile_args=("--no-cache",) + project_dir=project_dir, env=edited_env, compile_args=("--no-cache", *edited_args) + ) + rewarmed: CompileCacheOutcome = compile_cache_outcome( + project_dir=project_dir, env=edited_env, compile_args=edited_args ) - rewarmed: CompileCacheOutcome = compile_cache_outcome(project_dir=project_dir, env=edited_env) assert (edited.returncode != 0) is test_case.expected_failure assert ( @@ -218,6 +386,7 @@ def test_given_warm_compile_cache_when_input_changes_then_output_matches_cache_d assert edited[:3] == reference[:3] assert rewarmed[:3] == reference[:3] assert rewarmed.fact_cache_misses == test_case.expected_rewarmed_fact_cache_misses + assert rewarmed.attachment_cache_misses == 0 @pytest.mark.parametrize( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 27cb152395..8f84c4c630 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -24,6 +24,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, + attachment_cache_counts, fresh_process_compile_cache_metrics, measure_model_sql_bytes, run_dense_warm_edit_benchmark, @@ -203,14 +204,17 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm) assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0) assert warm_timings["rule_cache_misses"] == 0 + assert attachment_cache_counts(warm_timings) == (test_case.model_count, 0, 0) batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit) assert 0 < misses < test_case.model_count assert batch_hits + entry_hits + misses == test_case.model_count assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses + assert attachment_cache_counts(edit_timings) == (test_case.model_count - 1, 1, 0) oracle_timings: dict[str, int] = cast( dict[str, int], measurements["oracle"].payload["compile_timings"] ) assert oracle_timings["rule_cache_hits"] == 0 + assert attachment_cache_counts(oracle_timings) == (0, 0, 0) assert warm.elapsed_seconds < test_case.expected_warm_max_seconds assert edit.elapsed_seconds < test_case.expected_edit_max_seconds diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py index 18a979f4fb..fbd3a85ab0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/helpers.py @@ -66,6 +66,7 @@ DiscoveredSqlScenarioFile, DiscoveredTaskFunction, ) +from sqlbuild.compiler.fact_cache._helpers.publication import await_fact_publication from sqlbuild.compiler.graph.main._build_lineage_upstream_deps import build_lineage_upstream_deps from sqlbuild.compiler.lineage.types import ColumnLineageConfidence, ColumnTransformKind from sqlbuild.compiler.pipeline.main.compiled_project import build_compiled_project @@ -761,3 +762,13 @@ def required_description_diagnostics( source_inputs=inputs.source_inputs, function_inputs=inputs.function_inputs, ) + + +def clear_model_attachment_cache(*, project_dir: Path) -> None: + """Remove reusable model attachment facts so the next compile re-attaches every model.""" + + await_fact_publication() + for database in (project_dir / "target" / "cache" / "compiler").rglob( + "model-attachment.sqlite3" + ): + database.unlink() diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py index 191fdad6dc..7b434be23d 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/_test_types.py @@ -43,3 +43,11 @@ class ParameterizedSqlRenderErrorTestCase: sql: str arguments: dict[str, object] expected_error_fragment: str + + +@dataclass(frozen=True) +class VolatileReadTestCase: + description: str + template: str + sql: str + expected_reads: frozenset[str] diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py new file mode 100644 index 0000000000..fba8804a25 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/render/test_volatile_reads.py @@ -0,0 +1,72 @@ +"""Per-run reads recorded while attachment renders templates and SQL.""" + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.compile._helpers.render.sql_vars import substitute_sql_vars +from sqlbuild.compiler.compile._helpers.render.templating import expand_template_data +from sqlbuild.compiler.compile._helpers.render.volatile_reads import record_volatile_reads +from tests.unit.src.sqlbuild.compiler.compile._helpers.render._test_types import ( + VolatileReadTestCase, +) + +_CONTEXT_VALUES: dict[str, str | None] = { + "run.id": "20260101T000000Z_000000000000", + "run.target": "dev", + "model.name": "orders", +} + + +@pytest.mark.parametrize( + "test_case", + [ + VolatileReadTestCase( + description="deterministic_context_and_vars", + template="${CTX:model.name}_${CTX:run.target}_${suffix}", + sql="SELECT '@@CTX:model.name' AS name, @@suffix AS suffix", + expected_reads=frozenset(), + ), + VolatileReadTestCase( + description="run_id_in_template_and_sql", + template="${CTX:run.id}", + sql="SELECT '@@CTX:run.id' AS run_id", + expected_reads=frozenset({"CTX:run.id"}), + ), + VolatileReadTestCase( + description="environment_in_template_and_sql", + template="${coalesce(ENV:ORDERS_SCHEMA, 'fallback')}", + sql="SELECT '@@ENV:ORDERS_REGION' AS region", + expected_reads=frozenset({"ENV:ORDERS_SCHEMA", "ENV:ORDERS_REGION"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_rendering_when_recording_then_only_per_run_values_are_reported( + test_case: VolatileReadTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("ORDERS_REGION", "east") + monkeypatch.setenv("ORDERS_SCHEMA", "sales") + + with record_volatile_reads() as reads: + _ = expand_template_data( + value={"schema": test_case.template}, + variables={"suffix": "v2"}, + context_values=_CONTEXT_VALUES, + context_label="model config", + allow_context=True, + preserve_context_tokens=False, + preserve_unknown_context=False, + ) + _ = substitute_sql_vars( + sql=test_case.sql, + file_path=Path("models/orders.sql"), + effective_vars={"suffix": "1"}, + context_values=_CONTEXT_VALUES, + ) + + assert reads == test_case.expected_reads + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-vv"])) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index 30ab6049fb..fac3821fa7 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -44,6 +44,7 @@ CompactLineageCacheEncodingTestCase, ) from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( + clear_model_attachment_cache, compile_project_with_cache, stored_analysis_contents, ) @@ -146,7 +147,7 @@ def test_given_successful_analysis_when_compiling_again_then_reuses_identical_ca signature_builder.assert_not_called() reference_scanner.assert_not_called() assert len(tuple((tmp_path / "target" / "cache" / "compiler").rglob("*.sqlite3"))) == ( - test_case.expected_count + 1 + test_case.expected_count + 2 ) @@ -396,6 +397,7 @@ def test_given_corrupt_reference_cache_when_compiling_then_rescans_and_repairs_t "UPDATE sql_reference SET payload = ?", (corrupt_contents,), ) + clear_model_attachment_cache(project_dir=tmp_path) scanner: Mock = Mock(wraps=reference_cache.extract_sql_references) monkeypatch.setattr(reference_cache, "extract_sql_references", scanner) @@ -431,6 +433,7 @@ def test_given_non_text_reference_cache_when_compiling_then_rescans_safely( "UPDATE sql_reference SET payload = ?", (sqlite3.Binary(b"broken"),), ) + clear_model_attachment_cache(project_dir=tmp_path) scanner: Mock = Mock(wraps=reference_cache.extract_sql_references) monkeypatch.setattr(reference_cache, "extract_sql_references", scanner) diff --git a/website/src/content/docs/docs/concepts/macros.mdx b/website/src/content/docs/docs/concepts/macros.mdx index 9eccc3bc64..0a6eb60330 100644 --- a/website/src/content/docs/docs/concepts/macros.mdx +++ b/website/src/content/docs/docs/concepts/macros.mdx @@ -145,6 +145,27 @@ Python implementation details. Imported functions are not re-exported as new macros from the importing file. +## Keep macros deterministic + +A macro must return the same SQL every time it receives the same arguments and macro context. Do +not read environment variables, the clock, random values, files, or network services inside a +macro. Pass such values in through [project variables](/docs/concepts/interpolation#project-variables) +or the macro context instead. + +SQLBuild relies on this when it reuses compiled models between compile runs. A model is compiled +again when its file, its configuration and defaults, project variables, the target, the adapter, +any project macro or declaration, or the installed SQLBuild version changes. Models whose +configuration, hooks, or SQL read an environment variable or `run.id` are compiled again on every +run. `sqb compile --no-cache` compiles everything from scratch. + +The reuse check covers model files, project and local configuration, macro, enum, constant, schema, +and SQL hook files, the source files that define Python hooks and audit factories, and custom +adapter files under `adapters/`, along with the target, project variables, and command-line flags. +It does not cover code those files import from elsewhere: third-party libraries, and project +modules outside those locations that a hook or adapter imports. Keep macro, hook, and adapter code +independent of such modules, or run `sqb compile --no-cache` (or delete `target/cache/compiler/`) +after upgrading or editing one that changes their output. + ## Use macros in tests Tests are SQL, so they can use macros as reusable fixture generators: