From 1447de43206e8794a36c49539aa061b9e92e3dfe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:18:41 +0100 Subject: [PATCH 1/2] perf(cli): load command code only after argument parsing The CLI entry imported the dispatch, error-projection, compute-log and observability layers before parsing arguments, which pulled in compiler, planner, adapter and output models plus pydantic-settings. `sqb --version` and every `--help` paid about 0.7 s of CPU for roughly 930 modules. Parsing now loads only the parser and its type modules; the dispatch layers load after a command is parsed. The dispatch path itself no longer imports heavy models for annotations: the compile request models move beside the CLI entry models, the warehouse scope context moves out of the warehouse resolution module, provider and output annotations use lazy type aliases, and the provider base class, pydantic and rich load where they are used. Help text, parse errors, exit codes and JSON output are unchanged. A fresh-process test guards that version, help, parse errors and a plain compile do not load warehouse drivers, integrations, pydantic-settings or rich, and that parsing alone loads no compiler or pydantic modules. --- .../main/session_warehouse_support.py | 4 +- .../cli/commands/_helpers/compile/pipeline.py | 2 +- .../cli/commands/_helpers/debug/checks.py | 6 +- .../cli/commands/_helpers/entry/dispatch.py | 24 +-- .../_helpers/refactor/compile_facts.py | 3 +- .../_helpers/runtime/warehouse_scope.py | 43 ++++++ .../commands/_helpers/runtime/warehouses.py | 39 +---- .../cli/commands/main/entrypoint/entry.py | 11 +- .../cli/commands/main/project/_compile.py | 2 +- src/sqlbuild/cli/commands/models.py | 4 +- src/sqlbuild/cli/compile/main/run.py | 2 +- src/sqlbuild/cli/compile/models.py | 31 ---- src/sqlbuild/cli/entry/models.py | 32 +++- src/sqlbuild/cli/output/models.py | 8 +- src/sqlbuild/cli/output/types.py | 9 ++ .../discovery/_helpers/filesystem/core.py | 15 +- src/sqlbuild/compiler/discovery/models.py | 11 +- src/sqlbuild/compiler/discovery/types.py | 7 +- .../classes/transient_status_reporter.py | 17 ++- src/sqlbuild/provider/_helpers/injection.py | 8 +- src/sqlbuild/provider/_helpers/session.py | 2 +- src/sqlbuild/provider/classes/container.py | 2 +- src/sqlbuild/provider/classes/session.py | 5 +- .../sqlbuild/cli/commands/main/_test_types.py | 10 ++ .../main/test_compile_startup_imports.py | 4 +- .../main/test_startup_import_footprint.py | 142 ++++++++++++++++++ .../commands/_helpers/debug/test_checks.py | 2 +- .../_helpers/runtime/test_warehouses.py | 6 +- .../main/compile/test_compile_command.py | 4 +- .../main/entry/test_compute_capture.py | 4 +- .../cli/commands/main/entry/test_main.py | 4 +- 31 files changed, 328 insertions(+), 135 deletions(-) create mode 100644 src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py create mode 100644 tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py diff --git a/src/sqlbuild/adapter/discovery/main/session_warehouse_support.py b/src/sqlbuild/adapter/discovery/main/session_warehouse_support.py index 44465ae76c..7590060210 100644 --- a/src/sqlbuild/adapter/discovery/main/session_warehouse_support.py +++ b/src/sqlbuild/adapter/discovery/main/session_warehouse_support.py @@ -4,12 +4,12 @@ from pathlib import Path -from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter - def adapter_supports_session_warehouse(*, adapter_name: str, project_dir: Path | None) -> bool: """Return whether the adapter, resolved like any project adapter, has a session warehouse.""" + from sqlbuild.adapter.discovery.main.resolve_adapter import resolve_adapter + return resolve_adapter( adapter_name=adapter_name, project_dir=project_dir ).supports_session_warehouse diff --git a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py index ae4817b5db..6b76577da5 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/pipeline.py @@ -26,9 +26,9 @@ from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import ( CompileAnalysis, - CompileProfileFlags, CompileWriteResult, ) +from sqlbuild.cli.entry.models import CompileProfileFlags from sqlbuild.cli.output.models import ( WrittenTarget, ) diff --git a/src/sqlbuild/cli/commands/_helpers/debug/checks.py b/src/sqlbuild/cli/commands/_helpers/debug/checks.py index 36e6e294d0..85cade7924 100644 --- a/src/sqlbuild/cli/commands/_helpers/debug/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/debug/checks.py @@ -12,10 +12,12 @@ from sqlbuild.cli.commands._helpers.runtime.connection import ( resolve_project_connection_config, ) -from sqlbuild.cli.commands._helpers.runtime.warehouses import ( - apply_command_warehouse, +from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import ( command_warehouse_scope, current_command_warehouse_scope, +) +from sqlbuild.cli.commands._helpers.runtime.warehouses import ( + apply_command_warehouse, find_target_group_warehouse, resolve_command_warehouse, ) diff --git a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py index 2ac93f75b8..f5de4c5148 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/dispatch.py @@ -7,10 +7,7 @@ from sqlbuild.cli.commands._helpers.diff.validation import parse_diff_name_range from sqlbuild.cli.commands._helpers.entry.parsing import read_selector_file_inputs from sqlbuild.cli.commands._helpers.lint.positional_paths import format_path_selectors -from sqlbuild.cli.commands._helpers.runtime.warehouses import ( - command_warehouse_scope, - validate_cli_warehouse, -) +from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.constants import ( DBT_INIT_COMMAND, @@ -22,12 +19,10 @@ ) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode -from sqlbuild.cli.compile.models import ( - CompileCommandRequest, - CompileProfileFlags, -) from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, + CompileCommandRequest, + CompileProfileFlags, SelectorInputs, ) from sqlbuild.compiler.lineage.types import ColumnLineageMode @@ -37,10 +32,15 @@ def dispatch_cli_command(*, args: CliNamespace, handlers: CliEntrypointHandlers) -> int: """Route a parsed CLI namespace to its command handler and return its exit code.""" - validate_cli_warehouse( - cli_warehouse=args.warehouse, - project_dir=Path.cwd() if args.project_dir is None else Path(args.project_dir).resolve(), - ) + if args.warehouse is not None: + from sqlbuild.cli.commands._helpers.runtime.warehouses import validate_cli_warehouse + + validate_cli_warehouse( + cli_warehouse=args.warehouse, + project_dir=( + Path.cwd() if args.project_dir is None else Path(args.project_dir).resolve() + ), + ) with command_warehouse_scope( command=args.command, cli_warehouse=args.warehouse, cli_vars=args.vars ): diff --git a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py index 5bc0c1e0a1..6c66786e36 100644 --- a/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py +++ b/src/sqlbuild/cli/commands/_helpers/refactor/compile_facts.py @@ -10,7 +10,8 @@ ) from sqlbuild.cli.commands.models import RefactorCompile from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.compile.models import CompileAnalysis, CompileProfileFlags +from sqlbuild.cli.compile.models import CompileAnalysis +from sqlbuild.cli.entry.models import CompileProfileFlags from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic from sqlbuild.compiler.compile.types import DiagnosticPhase, DiagnosticSeverity diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py b/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py new file mode 100644 index 0000000000..a068f97c5f --- /dev/null +++ b/src/sqlbuild/cli/commands/_helpers/runtime/warehouse_scope.py @@ -0,0 +1,43 @@ +"""Active command warehouse scope for CLI warehouse connections.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar, Token + +from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS +from sqlbuild.cli.commands.types import CliCommand +from sqlbuild.cli.entry.models import CommandWarehouseScope + +_CURRENT_SCOPE: ContextVar[CommandWarehouseScope | None] = ContextVar( + "sqlbuild_command_warehouse_scope", default=None +) + + +@contextmanager +def command_warehouse_scope( + *, + command: CliCommand | str | None, + cli_warehouse: str | None, + cli_vars: dict[str, object] | None = None, +) -> Iterator[CommandWarehouseScope]: + """Install the warehouse group, CLI override, and CLI vars for one command invocation.""" + + scope: CommandWarehouseScope = CommandWarehouseScope( + group=None if command is None else COMMAND_WAREHOUSE_GROUPS.get(CliCommand(command)), + cli_warehouse=cli_warehouse, + cli_vars={} if cli_vars is None else dict(cli_vars), + ) + token: Token[CommandWarehouseScope | None] = _CURRENT_SCOPE.set(scope) + try: + yield scope + finally: + _CURRENT_SCOPE.reset(token) + + +def current_command_warehouse_scope() -> CommandWarehouseScope: + """Return the active command's warehouse scope, or an empty one outside the CLI.""" + + scope: CommandWarehouseScope | None = _CURRENT_SCOPE.get() + return scope if scope is not None else CommandWarehouseScope(group=None) diff --git a/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py b/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py index f2d21c241c..cca799a06b 100644 --- a/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py +++ b/src/sqlbuild/cli/commands/_helpers/runtime/warehouses.py @@ -3,22 +3,20 @@ from __future__ import annotations import re -from collections.abc import Iterator -from contextlib import contextmanager -from contextvars import ContextVar, Token from pathlib import Path from sqlbuild.adapter.discovery.main.session_warehouse_support import ( adapter_supports_session_warehouse, ) +from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import ( + current_command_warehouse_scope, +) from sqlbuild.cli.commands.constants import ( CLI_WAREHOUSE_SOURCE, - COMMAND_WAREHOUSE_GROUPS, CONNECTION_DEFAULT_WAREHOUSE_SOURCE, CONNECTION_WAREHOUSE_SOURCE, ) from sqlbuild.cli.commands.exceptions import CliUserError -from sqlbuild.cli.commands.types import CliCommand from sqlbuild.cli.entry.models import ( AuthoredTargetWarehouse, CommandWarehouseScope, @@ -47,37 +45,6 @@ _CONNECTION_SOURCES: frozenset[str] = frozenset( {CONNECTION_WAREHOUSE_SOURCE, CONNECTION_DEFAULT_WAREHOUSE_SOURCE} ) -_CURRENT_SCOPE: ContextVar[CommandWarehouseScope | None] = ContextVar( - "sqlbuild_command_warehouse_scope", default=None -) - - -@contextmanager -def command_warehouse_scope( - *, - command: CliCommand | str | None, - cli_warehouse: str | None, - cli_vars: dict[str, object] | None = None, -) -> Iterator[CommandWarehouseScope]: - """Install the warehouse group, CLI override, and CLI vars for one command invocation.""" - - scope: CommandWarehouseScope = CommandWarehouseScope( - group=None if command is None else COMMAND_WAREHOUSE_GROUPS.get(CliCommand(command)), - cli_warehouse=cli_warehouse, - cli_vars={} if cli_vars is None else dict(cli_vars), - ) - token: Token[CommandWarehouseScope | None] = _CURRENT_SCOPE.set(scope) - try: - yield scope - finally: - _CURRENT_SCOPE.reset(token) - - -def current_command_warehouse_scope() -> CommandWarehouseScope: - """Return the active command's warehouse scope, or an empty one outside the CLI.""" - - scope: CommandWarehouseScope | None = _CURRENT_SCOPE.get() - return scope if scope is not None else CommandWarehouseScope(group=None) def validate_cli_warehouse(*, cli_warehouse: str | None, project_dir: Path) -> None: diff --git a/src/sqlbuild/cli/commands/main/entrypoint/entry.py b/src/sqlbuild/cli/commands/main/entrypoint/entry.py index fc82c37aef..f294f909a1 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/entry.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/entry.py @@ -5,15 +5,11 @@ import argparse from collections.abc import Mapping, Sequence -from sqlbuild.cli.commands._helpers.entry.dispatch_errors import dispatch_and_handle_errors from sqlbuild.cli.commands._helpers.entry.errors import cli_error_use_color from sqlbuild.cli.commands._helpers.entry.lazy_handlers import build_lazy_cli_handlers from sqlbuild.cli.commands._helpers.entry.parser import build_cli_parser from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace -from sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs import ( - dispatch_with_compute_logs, -) from sqlbuild.cli.entry.models import ( CliEntrypointHandlers, ParsedCliInvocation, @@ -58,6 +54,13 @@ def _main_with_dependencies( if invocation.args is None: return invocation.exit_code if invocation.exit_code is not None else 1 args: CliNamespace = invocation.args + from sqlbuild.cli.commands._helpers.entry.dispatch_errors import ( + dispatch_and_handle_errors, + ) + from sqlbuild.cli.commands.main.entrypoint._dispatch_with_compute_logs import ( + dispatch_with_compute_logs, + ) + return dispatch_with_compute_logs( args=args, identity=identity, diff --git a/src/sqlbuild/cli/commands/main/project/_compile.py b/src/sqlbuild/cli/commands/main/project/_compile.py index 39c537022e..635188cb63 100644 --- a/src/sqlbuild/cli/commands/main/project/_compile.py +++ b/src/sqlbuild/cli/commands/main/project/_compile.py @@ -25,9 +25,9 @@ from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.models import ( CompileAnalysis, - CompileCommandRequest, CompileWriteResult, ) +from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.models import CompileAnalysisSelection, CompilerDiagnostic from sqlbuild.compiler.compile.types import DiagnosticPhase from sqlbuild.compiler.profiling.main.collect import collect_compile_timings diff --git a/src/sqlbuild/cli/commands/models.py b/src/sqlbuild/cli/commands/models.py index 906928cc66..c42b6dbd76 100644 --- a/src/sqlbuild/cli/commands/models.py +++ b/src/sqlbuild/cli/commands/models.py @@ -20,14 +20,14 @@ ) from sqlbuild.cli.compile.models import ( # noqa: F401 CompileAnalysis, - CompileCommandRequest, - CompileProfileFlags, CompileWriteResult, SqlTestArtifactCacheRecord, SqlTestArtifactIdentityContext, ) from sqlbuild.cli.entry.models import ( # noqa: F401 CliEntrypointHandlers, + CompileCommandRequest, + CompileProfileFlags, ParsedCliInvocation, SelectorFileSummary, SelectorInputs, diff --git a/src/sqlbuild/cli/compile/main/run.py b/src/sqlbuild/cli/compile/main/run.py index 8aa083e950..52db97c08c 100644 --- a/src/sqlbuild/cli/compile/main/run.py +++ b/src/sqlbuild/cli/compile/main/run.py @@ -1,7 +1,7 @@ """Public compile command entrypoint.""" from sqlbuild.cli.commands.main.project._compile import run_compile as _run_compile -from sqlbuild.cli.compile.models import CompileCommandRequest +from sqlbuild.cli.entry.models import CompileCommandRequest def run_compile(request: CompileCommandRequest) -> int: diff --git a/src/sqlbuild/cli/compile/models.py b/src/sqlbuild/cli/compile/models.py index ffdf732c95..1bb0b44405 100644 --- a/src/sqlbuild/cli/compile/models.py +++ b/src/sqlbuild/cli/compile/models.py @@ -6,7 +6,6 @@ from pathlib import Path from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter -from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.output.models import WrittenTarget from sqlbuild.compiler.compile.models import ( CompiledObjectKey, @@ -19,36 +18,6 @@ from sqlbuild.compiler.planner.models import NativeSqlTestArtifact -@dataclass(frozen=True) -class CompileProfileFlags: - """Profiling toggles that skip compile phases for benchmarking.""" - - skip_discovery_sql_analysis: bool = False - skip_column_inference: bool = False - skip_contracts: bool = False - skip_write: bool = False - - -@dataclass(frozen=True) -class CompileCommandRequest: - """CLI inputs for one `sqb compile` invocation.""" - - project_dir: Path | None = None - no_sql_validation: bool = False - defer_to: str | None = None - selected_target: str | None = None - json_output: bool = False - manifest: bool = False - dag_path: str | None = None - no_color: bool = False - lineage_mode: CompileLineageMode = CompileLineageMode.FAST - select: tuple[str, ...] = () - exclude: tuple[str, ...] = () - cli_vars: dict[str, object] | None = None - profile_flags: CompileProfileFlags = CompileProfileFlags() - no_cache: bool = False - - @dataclass(frozen=True) class CompileAnalysis: """Compiled project analysis shared by compile output phases.""" diff --git a/src/sqlbuild/cli/entry/models.py b/src/sqlbuild/cli/entry/models.py index dbd83d921d..90ff1071a4 100644 --- a/src/sqlbuild/cli/entry/models.py +++ b/src/sqlbuild/cli/entry/models.py @@ -8,6 +8,7 @@ from sqlbuild.cli.commands.classes.cli_namespace import CliNamespace from sqlbuild.cli.commands.types import ( + CompileLineageMode, DagCommandHandler, DebugCommandHandler, FormatCommandHandler, @@ -16,10 +17,39 @@ ScopeCommandHandler, SkillsUpdateCommandHandler, ) -from sqlbuild.cli.compile.models import CompileCommandRequest from sqlbuild.spec.contracts.types import WarehouseGroup +@dataclass(frozen=True) +class CompileProfileFlags: + """Profiling toggles that skip compile phases for benchmarking.""" + + skip_discovery_sql_analysis: bool = False + skip_column_inference: bool = False + skip_contracts: bool = False + skip_write: bool = False + + +@dataclass(frozen=True) +class CompileCommandRequest: + """CLI inputs for one `sqb compile` invocation.""" + + project_dir: Path | None = None + no_sql_validation: bool = False + defer_to: str | None = None + selected_target: str | None = None + json_output: bool = False + manifest: bool = False + dag_path: str | None = None + no_color: bool = False + lineage_mode: CompileLineageMode = CompileLineageMode.FAST + select: tuple[str, ...] = () + exclude: tuple[str, ...] = () + cli_vars: dict[str, object] | None = None + profile_flags: CompileProfileFlags = CompileProfileFlags() + no_cache: bool = False + + @dataclass(frozen=True) class SelectorFileSummary: """Non-expanded provenance for one selector file.""" diff --git a/src/sqlbuild/cli/output/models.py b/src/sqlbuild/cli/output/models.py index 2c207f0727..c3f4f83752 100644 --- a/src/sqlbuild/cli/output/models.py +++ b/src/sqlbuild/cli/output/models.py @@ -51,14 +51,14 @@ CursorBoundsOwner, CursorResolutionStatus, IntegrationOutputKind, + PlannedCursorBounds, + WrittenTargetDiagnostic, ) from sqlbuild.compiler.auditing.types import ( AuditEvaluationMode, AuditSeverity, ThresholdOperator, ) -from sqlbuild.compiler.compile.models import CompilerDiagnostic -from sqlbuild.compiler.planner.models import CursorBounds from sqlbuild.runtime.observability.constants import ( RESOURCE_ATTEMPT_SKIPPED_EVENT, RESOURCE_SKIP_CODES, @@ -85,7 +85,7 @@ class CursorPlanDetails: requested_end: str | None bounds_owner: CursorBoundsOwner resolution_status: CursorResolutionStatus - resolved_bounds: CursorBounds | None + resolved_bounds: PlannedCursorBounds | None declared_grain: str | None effective_grain: str | None declared_batch_size: str | None @@ -577,7 +577,7 @@ class WrittenTarget: audit_count: int test_count: int target_dir: Path - diagnostics: tuple[CompilerDiagnostic, ...] = field(default_factory=tuple) + diagnostics: tuple[WrittenTargetDiagnostic, ...] = field(default_factory=tuple) @dataclass(frozen=True) diff --git a/src/sqlbuild/cli/output/types.py b/src/sqlbuild/cli/output/types.py index 5c84d3c6bd..3a8ca3333f 100644 --- a/src/sqlbuild/cli/output/types.py +++ b/src/sqlbuild/cli/output/types.py @@ -1,6 +1,15 @@ """Type-layer declarations for CLI output.""" from enum import StrEnum +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sqlbuild.compiler.compile.models import CompilerDiagnostic + from sqlbuild.compiler.planner.models import CursorBounds + +type WrittenTargetDiagnostic = CompilerDiagnostic + +type PlannedCursorBounds = CursorBounds class CursorBoundsOwner(StrEnum): diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 30f325738c..8197740900 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -11,9 +11,7 @@ from importlib.machinery import ModuleSpec from pathlib import Path from types import ModuleType -from typing import cast, get_type_hints - -from pydantic import ValidationError +from typing import TYPE_CHECKING, cast, get_type_hints from sqlbuild.compiler.discovery._helpers.filesystem.cached_files import ( parse_source_file_with_cache, @@ -109,7 +107,6 @@ SCOPED_DECLARATION_DIRECTORIES, ) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind -from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.exceptions import ProviderInputError from sqlbuild.python_nodes.main.read_asset_definition import read_asset_definition from sqlbuild.python_nodes.main.read_audit_factory_definition import ( @@ -138,6 +135,11 @@ from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry +if TYPE_CHECKING: + from pydantic import ValidationError + + from sqlbuild.provider.classes.provider import Provider + _PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" _FACTS_MEMO_KEY: str = "declaration_file_facts" @@ -1006,6 +1008,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, providers_root: Path = project_dir / "providers" if not providers_root.is_dir(): return () + from sqlbuild.provider.classes.provider import Provider discovered_providers: list[DiscoveredProvider] = [] seen_names: dict[str, Path] = {} @@ -1289,6 +1292,8 @@ def _provider_name(*, provider_class: type[Provider], file_path: Path, project_d def _provider_instance( *, provider_class: type[Provider], provider_name: str, file_path: Path, project_dir: Path ) -> Provider: + from pydantic import ValidationError + try: return provider_class() except ValidationError as error: @@ -1327,6 +1332,8 @@ def _provider_usages( discovered_provider: DiscoveredProvider | None = provider_by_name.get(parameter.name) if discovered_provider is None: continue + from sqlbuild.provider.classes.provider import Provider + annotation: object = type_hints.get(parameter.name, parameter.annotation) annotation_class_name: str | None = None annotation_module: str | None = None diff --git a/src/sqlbuild/compiler/discovery/models.py b/src/sqlbuild/compiler/discovery/models.py index 68453d746e..1bf81b98bf 100644 --- a/src/sqlbuild/compiler/discovery/models.py +++ b/src/sqlbuild/compiler/discovery/models.py @@ -14,9 +14,12 @@ SQL_SCENARIOS_OWNERSHIP_ROOT, SQL_TESTS_OWNERSHIP_ROOT, ) -from sqlbuild.compiler.discovery.types import LoaderConnectionMode, ScopedDeclarationFile +from sqlbuild.compiler.discovery.types import ( + LoaderConnectionMode, + ProjectProvider, + ScopedDeclarationFile, +) from sqlbuild.compiler.scopes.types import DeclarationKind, ScopeKind -from sqlbuild.provider.classes.provider import Provider from sqlbuild.python_nodes.models import AuditCase, ColumnLineageRef, RetryPolicy, SqlResourceRef from sqlbuild.python_nodes.types import PythonCheckSeverity from sqlbuild.runtime.event_exporting.constants import EVENT_EXPORT_KINDS @@ -600,8 +603,8 @@ class DiscoveredProvider: file_path: Path relative_path: Path name: str - provider_class: type[Provider] - settings: Provider + provider_class: type[ProjectProvider] + settings: ProjectProvider @dataclass(frozen=True) diff --git a/src/sqlbuild/compiler/discovery/types.py b/src/sqlbuild/compiler/discovery/types.py index fc1896e66c..916cf3e92a 100644 --- a/src/sqlbuild/compiler/discovery/types.py +++ b/src/sqlbuild/compiler/discovery/types.py @@ -5,10 +5,15 @@ from dataclasses import Field from enum import StrEnum from pathlib import Path -from typing import Any, ClassVar, NamedTuple, Protocol +from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, Protocol from sqlbuild.compiler.scopes.types import ScopeKind +if TYPE_CHECKING: + from sqlbuild.provider.classes.provider import Provider + +type ProjectProvider = Provider + class LoaderConnectionMode(StrEnum): """How a source loader owns the destination warehouse connection.""" diff --git a/src/sqlbuild/presentation/classes/transient_status_reporter.py b/src/sqlbuild/presentation/classes/transient_status_reporter.py index f89b1595fd..9a3aa9e5e9 100644 --- a/src/sqlbuild/presentation/classes/transient_status_reporter.py +++ b/src/sqlbuild/presentation/classes/transient_status_reporter.py @@ -4,10 +4,7 @@ import atexit from contextlib import AbstractContextManager -from typing import TextIO - -from rich.console import Console -from rich.status import Status +from typing import TYPE_CHECKING, TextIO from sqlbuild.errors.contracts.exceptions import SharedInputError from sqlbuild.presentation.classes.cli_style import CliStyle @@ -15,6 +12,10 @@ from sqlbuild.presentation.main._progress_spinners_disabled import progress_spinners_disabled from sqlbuild.presentation.main.transient_line_coordinator import shared_transient_line_coordinator +if TYPE_CHECKING: + from rich.console import Console + from rich.status import Status + class TransientStatusReporter: """Render one transient status line with optional spinner updates.""" @@ -35,9 +36,11 @@ def __init__( and hasattr(stream, "isatty") and stream.isatty() ) - self._console: Console | None = ( - Console(file=stream, no_color=(not use_color)) if self._enabled else None - ) + self._console: Console | None = None + if self._enabled: + from rich.console import Console + + self._console = Console(file=stream, no_color=(not use_color)) self._status_context: AbstractContextManager[Status] | None = None self._status: Status | None = None self._active: bool = False diff --git a/src/sqlbuild/provider/_helpers/injection.py b/src/sqlbuild/provider/_helpers/injection.py index 83aad8f654..e7680a16ef 100644 --- a/src/sqlbuild/provider/_helpers/injection.py +++ b/src/sqlbuild/provider/_helpers/injection.py @@ -5,12 +5,14 @@ import inspect from collections.abc import Callable from pathlib import Path -from typing import get_type_hints +from typing import TYPE_CHECKING, get_type_hints from sqlbuild.provider.classes.container import ProviderContainer -from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.exceptions import ProviderInjectionError, ProviderLookupError +if TYPE_CHECKING: + from sqlbuild.provider.classes.provider import Provider + _CONTEXT_PARAMETER_NAMES: frozenset[str] = frozenset({"ctx", "context", "_ctx", "hook_context"}) @@ -80,6 +82,8 @@ def _provider_for_parameter( annotation: object, providers: ProviderContainer | None, ) -> Provider | None: + from sqlbuild.provider.classes.provider import Provider + provider_annotation: type[Provider] | None = ( annotation if isinstance(annotation, type) and issubclass(annotation, Provider) else None ) diff --git a/src/sqlbuild/provider/_helpers/session.py b/src/sqlbuild/provider/_helpers/session.py index 5db65bfe2b..14285017d3 100644 --- a/src/sqlbuild/provider/_helpers/session.py +++ b/src/sqlbuild/provider/_helpers/session.py @@ -5,10 +5,10 @@ from typing import TYPE_CHECKING from sqlbuild.compiler.discovery.models import DiscoveredProvider -from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.classes.session import ProviderSession if TYPE_CHECKING: + from sqlbuild.provider.classes.provider import Provider from sqlbuild.runtime.event_exporting.classes.command_scope import EventExporterCommandScope diff --git a/src/sqlbuild/provider/classes/container.py b/src/sqlbuild/provider/classes/container.py index 1bff24e459..ee53dd8677 100644 --- a/src/sqlbuild/provider/classes/container.py +++ b/src/sqlbuild/provider/classes/container.py @@ -5,10 +5,10 @@ from collections.abc import Iterator from typing import TYPE_CHECKING -from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.exceptions import ProviderLookupError if TYPE_CHECKING: + from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.classes.session import ProviderSession diff --git a/src/sqlbuild/provider/classes/session.py b/src/sqlbuild/provider/classes/session.py index 675303fc4c..07e836710d 100644 --- a/src/sqlbuild/provider/classes/session.py +++ b/src/sqlbuild/provider/classes/session.py @@ -4,15 +4,18 @@ from collections.abc import Iterable, Mapping from threading import RLock +from typing import TYPE_CHECKING from sqlbuild.provider.classes.container import ProviderContainer -from sqlbuild.provider.classes.provider import Provider from sqlbuild.provider.exceptions import ( ProviderLookupError, ProviderSetupError, ProviderTeardownError, ) +if TYPE_CHECKING: + from sqlbuild.provider.classes.provider import Provider + class ProviderSession: """Command-scoped provider lifecycle manager.""" diff --git a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py index f10cda5377..d1d3de4b52 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/_test_types.py @@ -1008,3 +1008,13 @@ class CommandWarehouseExitTestCase: expected_connect_calls: int = 0 warehouses_section: str = '[targets.dev.warehouses]\nbuild = "BUILD_WH"\nquery = "ADHOC_WH"\n' local_config: str = "" + + +@dataclass(frozen=True) +class StartupImportFootprintTestCase: + """One fresh-process CLI invocation and the heavy modules it must not load.""" + + description: str + argv: tuple[str, ...] + expected_exit_code: int + forbidden_modules: tuple[str, ...] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py index a5fd9d6b29..249dd2d798 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_compile_startup_imports.py @@ -85,8 +85,8 @@ def test_given_legacy_model_imports_when_loading_focused_models_then_exports_kee expected_modules = { "CompileAnalysis": compile_models, - "CompileCommandRequest": compile_models, - "CompileProfileFlags": compile_models, + "CompileCommandRequest": entry_models, + "CompileProfileFlags": entry_models, "CompileWriteResult": compile_models, "SqlTestArtifactCacheRecord": compile_models, "SqlTestArtifactIdentityContext": compile_models, diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py new file mode 100644 index 0000000000..8f7093562a --- /dev/null +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_startup_import_footprint.py @@ -0,0 +1,142 @@ +"""Fresh-process guard that CLI startup loads only what the invoked command needs.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import cast + +import pytest + +from tests.integration.src.sqlbuild.cli.commands.main._test_types import ( + StartupImportFootprintTestCase, +) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import ( + write_compile_startup_project, +) + +WAREHOUSE_AND_INTEGRATION_MODULES: tuple[str, ...] = ( + "dagster", + "databricks", + "dlt", + "duckdb", + "google.cloud.bigquery", + "ingestr", + "jinja2", + "psycopg", + "pyarrow", + "pymssql", + "snowflake.connector", +) +COMPILE_ONLY_MODULES: tuple[str, ...] = ( + "filelock", + "polyglot_sql", + "pydantic", + "yaml", + "sqlbuild._native", + "sqlbuild.adapter.contract.classes.base_adapter", + "sqlbuild.cli.compile.models", + "sqlbuild.cli.output.models", + "sqlbuild.compiler.compile.models", + "sqlbuild.compiler.discovery.models", + "sqlbuild.compiler.planner.models", + "sqlbuild.lint", + "sqlbuild.rule_engine", + "sqlbuild.runtime.compute_logs", +) +PRESENTATION_AND_PROVIDER_MODULES: tuple[str, ...] = ( + "pydantic_settings", + "rich", + "sqlbuild.cli.commands.models", +) +PARSE_ONLY_FORBIDDEN: tuple[str, ...] = ( + *WAREHOUSE_AND_INTEGRATION_MODULES, + *COMPILE_ONLY_MODULES, + *PRESENTATION_AND_PROVIDER_MODULES, +) +COMPILE_FORBIDDEN: tuple[str, ...] = ( + *WAREHOUSE_AND_INTEGRATION_MODULES, + *PRESENTATION_AND_PROVIDER_MODULES, +) + +_FOOTPRINT_SCRIPT: str = """ +import contextlib, io, json, sys +from sqlbuild.cli.entry.main.entry import main + +argv = json.loads(sys.argv[1]) +forbidden = json.loads(sys.argv[2]) +with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + code = main(argv) +loaded = sorted( + name for name in forbidden + if any(module == name or module.startswith(name + ".") for module in sys.modules) +) +print(json.dumps({"code": code, "loaded": loaded})) +""" + + +@pytest.mark.parametrize( + "test_case", + [ + StartupImportFootprintTestCase( + description="version", + argv=("--version",), + expected_exit_code=0, + forbidden_modules=PARSE_ONLY_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="root help", + argv=("--help",), + expected_exit_code=0, + forbidden_modules=PARSE_ONLY_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="compile help", + argv=("compile", "--help"), + expected_exit_code=0, + forbidden_modules=PARSE_ONLY_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="unknown argument", + argv=("compile", "--unknown-option"), + expected_exit_code=2, + forbidden_modules=PARSE_ONLY_FORBIDDEN, + ), + StartupImportFootprintTestCase( + description="plain compile", + argv=("compile", "--json"), + expected_exit_code=0, + forbidden_modules=COMPILE_FORBIDDEN, + ), + ], + ids=lambda case: case.description, +) +def test_given_fresh_interpreter_when_running_cli_then_heavy_modules_stay_unloaded( + test_case: StartupImportFootprintTestCase, tmp_path: Path +) -> None: + write_compile_startup_project(tmp_path) + argv: list[str] = ["--project-dir", str(tmp_path), *test_case.argv] + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-c", + _FOOTPRINT_SCRIPT, + json.dumps(argv), + json.dumps(test_case.forbidden_modules), + ], + check=True, + capture_output=True, + text=True, + cwd=tmp_path, + ) + outcome: dict[str, object] = json.loads(result.stdout.strip().splitlines()[-1]) + + assert outcome["code"] == test_case.expected_exit_code + assert cast(list[str], outcome["loaded"]) == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py index 79321259e8..33ab46d16d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/debug/test_checks.py @@ -7,7 +7,7 @@ import pytest from sqlbuild.cli.commands._helpers.debug.checks import build_debug_result -from sqlbuild.cli.commands._helpers.runtime.warehouses import command_warehouse_scope +from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope from sqlbuild.cli.commands.models import DebugResult from tests.unit.src.sqlbuild.cli.commands._helpers.debug._test_types import ( DebugWarehouseVarsTestCase, diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py index 945ccd3bff..8acad574d4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/runtime/test_warehouses.py @@ -9,10 +9,8 @@ from sqlbuild.cli.commands._helpers.entry.parser import build_cli_parser from sqlbuild.cli.commands._helpers.runtime.connection import resolve_project_connection_config -from sqlbuild.cli.commands._helpers.runtime.warehouses import ( - command_warehouse_scope, - describe_connection_warehouse, -) +from sqlbuild.cli.commands._helpers.runtime.warehouse_scope import command_warehouse_scope +from sqlbuild.cli.commands._helpers.runtime.warehouses import describe_connection_warehouse from sqlbuild.cli.commands.constants import COMMAND_WAREHOUSE_GROUPS from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import CliCommand diff --git a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py index 9a643a83dc..394d401383 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/compile/test_compile_command.py @@ -15,9 +15,7 @@ from sqlbuild.cli.commands.main.project import _compile as compile_command from sqlbuild.cli.commands.types import CompileLineageMode from sqlbuild.cli.compile.main.run import run_compile -from sqlbuild.cli.compile.models import ( - CompileCommandRequest, -) +from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.models import CompilerDiagnostic from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.pipeline.models import ProjectGraph diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py index 9de30d0553..cfb5e25eb8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_compute_capture.py @@ -8,9 +8,7 @@ from sqlbuild.cli.commands.main.entrypoint import _dispatch_with_compute_logs as compute_dispatch from sqlbuild.cli.commands.main.entrypoint.entry import _main_with_dependencies -from sqlbuild.cli.compile.models import ( - CompileCommandRequest, -) +from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.diagnostics.classes.dynamic_stderr_handler import DynamicStderrHandler from sqlbuild.diagnostics.main.log_debug_event import log_debug_event from sqlbuild.diagnostics.main.log_sql import log_sql diff --git a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py index 98ea5b9414..9a41148b7b 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/entry/test_main.py @@ -30,9 +30,7 @@ ScenarioTestCommandRequest, ) from sqlbuild.cli.commands.types import CompileLineageMode -from sqlbuild.cli.compile.models import ( - CompileCommandRequest, -) +from sqlbuild.cli.entry.models import CompileCommandRequest from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import ProjectConfigError from sqlbuild.compiler.lineage.types import ColumnLineageMode From d47a1960561c68acbdcd20a2a2a32c6c8d08128d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:29:35 +0100 Subject: [PATCH 2/2] test(cli): expect --version not to load the native extension --- .../cli/commands/main/test_native_allocator_entry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py b/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py index 20c65d4a63..3a9f7b5e22 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_native_allocator_entry.py @@ -22,9 +22,11 @@ loaded_at_import = "sqlbuild._native" in sys.modules exit_code = main(["--version"]) +loaded_by_version = "sqlbuild._native" in sys.modules +import sqlbuild._native print(json.dumps({ "loaded_at_import": loaded_at_import, - "loaded_after_main": "sqlbuild._native" in sys.modules, + "loaded_by_version": loaded_by_version, "purge_delay": os.environ.get("MIMALLOC_PURGE_DELAY"), "exit_code": exit_code, })) @@ -65,7 +67,7 @@ def test_given_fresh_process_when_running_cli_entry_then_purge_delay_is_set_befo assert payload == { "loaded_at_import": False, - "loaded_after_main": True, + "loaded_by_version": False, "purge_delay": test_case.expected_purge_delay, "exit_code": 0, }