From e41244bb3e60c14029bdec9f1a138303c3dcaafc Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:08:08 +0100 Subject: [PATCH 1/9] feat!: merge Python node folders into python/ Discover loaders, tasks, assets, checks, factories, and audit factories anywhere under a single python/ root. The decorator defines each node kind, so folder-based kind validation is gone and undecorated helper modules under python/ are importable without becoming nodes. Remove the tasks/, assets/, checks/, loaders/, factories/, libs/, dagster/, and rivers_pipeline/ Python roots and the root definitions.py allowance. The Dagster and Rivers playgrounds now generate an orchestrator directory with definitions.py next to a nested waffle_shop/ SQLBuild project. Path selectors for Python nodes use the python/ root. BREAKING CHANGE: Python nodes and their helper modules must live under python/ (for example python/tasks/orders.py, imported as python.tasks.orders). Python under tasks/, assets/, checks/, loaders/, factories/, libs/, dagster/, rivers_pipeline/, or a root definitions.py now fails compilation with D016. Python path selectors use python/... instead of tasks/, assets/, checks/, or loaders/. --- .../cli/commands/_helpers/init/scaffold.py | 5 +- .../_helpers/playground/completion_output.py | 13 +- .../cli/commands/_helpers/playground/copy.py | 106 +++++---- .../loaders/waffle_loaders.py.txt | 0 .../loaders/waffle_sources.py.txt | 0 src/sqlbuild/cli/commands/constants.py | 1 + .../cli/commands/main/workspace/_init.py | 4 +- .../discovery/_helpers/filesystem/core.py | 161 +++---------- .../_helpers/filesystem/python_paths.py | 17 +- src/sqlbuild/compiler/discovery/constants.py | 4 +- src/sqlbuild/compiler/discovery/exceptions.py | 6 - src/sqlbuild/compiler/planner/constants.py | 3 +- .../compiler/python_nodes/constants.py | 2 +- .../{ => python}/loaders/strategy_sources.py | 0 .../{ => python}/loaders/waffle_sources.py | 0 .../commands/main/bigquery/test_bigquery.py | 8 +- .../test_build_no_tests_no_audits.py | 148 ++++++------ .../main/build/test_audit_factory_failures.py | 2 +- .../cli/commands/main/build/test_build.py | 100 ++++---- .../main/build/test_loader_watermark_build.py | 2 +- .../main/build/test_python_node_identity.py | 4 +- .../main/build/test_virtual_python_build.py | 66 +++--- .../test_virtual_source_freshness_build.py | 4 +- .../cli/commands/main/check/helpers.py | 36 +-- .../cli/commands/main/clone/test_clone.py | 6 +- .../main/compile/test_namespace_validation.py | 8 +- .../compile/test_python_project_layout.py | 224 +++++++++++++++++- .../main/databricks/test_databricks.py | 6 +- .../cli/commands/main/freshness/helpers.py | 2 +- .../cli/commands/main/init/_test_types.py | 10 + .../cli/commands/main/init/test_init.py | 81 ++++++- .../cli/commands/main/load/helpers.py | 8 +- .../cli/commands/main/load/test_load.py | 4 +- .../cli/commands/main/plan/helpers.py | 20 +- .../plan/test_virtual_plan_incremental.py | 2 +- .../commands/main/playground/_test_types.py | 8 + .../main/playground/test_playground.py | 60 +++++ .../commands/main/postgres/test_postgres.py | 8 +- .../commands/main/providers/test_providers.py | 28 +-- .../commands/main/snowflake/test_snowflake.py | 8 +- .../cli/commands/main/sqlserver/helpers.py | 2 +- .../commands/main/sqlserver/test_sqlserver.py | 12 +- .../sqlbuild/integrations/dagster/helpers.py | 10 +- .../dagster/test_sqlbuild_assets.py | 11 +- .../rivers/test_sqlbuild_assets.py | 7 +- .../cli/commands/main/test_load_command.py | 138 +++++------ .../compiler/pipeline/test_audit_factories.py | 2 +- .../sqlbuild/compiler/pipeline/test_main.py | 2 +- .../build/concurrent/test_concurrent_build.py | 2 +- .../_helpers/load_selection/helpers.py | 12 +- .../sqlbuild/cli/commands/main/dag/helpers.py | 30 +-- .../cli/commands/main/dag/test_dag_command.py | 4 +- .../cli/commands/main/init/test_run.py | 14 +- .../commands/main/playground/_test_types.py | 1 + .../main/playground/test_playground.py | 95 +++++--- .../cli/output/main/plan/test_plan_text.py | 20 +- .../output/main/plan_json/test_plan_json.py | 6 +- .../compile/_helpers/test_assembly.py | 2 +- .../compile/_helpers/test_audit_factories.py | 6 +- .../src/sqlbuild/compiler/dag/main/helpers.py | 8 +- .../discovery/_helpers/_test_types.py | 7 + .../test_filesystem_audit_factories.py | 29 ++- .../_helpers/test_filesystem_factories.py | 102 +++++--- .../_helpers/test_filesystem_loaders.py | 28 +-- .../_helpers/test_filesystem_python_nodes.py | 147 +++++++++++- .../_helpers/test_resource_identities.py | 2 +- .../sqlbuild/compiler/discovery/test_main.py | 196 +++++++++------ .../compiler/pipeline/_helpers/helpers.py | 2 +- .../compiler/planner/_helpers/helpers.py | 8 +- .../compiler/python_nodes/_helpers/helpers.py | 30 +-- .../python_nodes/_helpers/test_identity.py | 116 ++++----- .../python_nodes/_helpers/test_inventory.py | 18 +- .../_helpers/test_run_selectors.py | 4 +- .../python_nodes/_helpers/test_selectors.py | 31 ++- .../_helpers/test_unified_selectors.py | 8 +- .../executor/build/_helpers/helpers.py | 8 +- .../_helpers/test_external_source_loads.py | 4 +- .../sqlbuild/executor/load/test_execute.py | 44 ++-- .../src/sqlbuild/executor/load/test_run.py | 32 +-- .../executor/python_nodes/_helpers/helpers.py | 14 +- .../python_nodes/_helpers/test_execution.py | 54 ++--- .../_helpers/test_fingerprinting.py | 2 +- .../_helpers/test_ingress_execution.py | 2 +- .../sqlbuild/integrations/dagster/helpers.py | 12 +- .../state/main/test_python_node_identity.py | 2 +- .../{ => python}/loaders/waffle_sources.py | 0 86 files changed, 1499 insertions(+), 962 deletions(-) rename src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/{ => python}/loaders/waffle_loaders.py.txt (100%) rename src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/{ => python}/loaders/waffle_sources.py.txt (100%) rename tests/e2e/fixtures/source_loader_strategies/{ => python}/loaders/strategy_sources.py (100%) rename tests/e2e/fixtures/waffle_shop/{ => python}/loaders/waffle_sources.py (100%) rename website/examples/waffle-shop/{ => python}/loaders/waffle_sources.py (100%) diff --git a/src/sqlbuild/cli/commands/_helpers/init/scaffold.py b/src/sqlbuild/cli/commands/_helpers/init/scaffold.py index 012feb526e..f90180a96d 100644 --- a/src/sqlbuild/cli/commands/_helpers/init/scaffold.py +++ b/src/sqlbuild/cli/commands/_helpers/init/scaffold.py @@ -14,10 +14,7 @@ def scaffold_blank_project(*, base_dir: Path, project_name: str) -> None: "schemas", "sources", "seeds", - "loaders", - "tasks", - "assets", - "checks", + "python", "hooks/sql", "hooks/python", "tests/unit", diff --git a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py index 67b76d23fe..7d7a158fb3 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py @@ -2,6 +2,7 @@ from __future__ import annotations +from sqlbuild.cli.commands.constants import PLAYGROUND_ORCHESTRATED_PROJECT_DIR from sqlbuild.cli.commands.models import ( PlaygroundCommandRequest, PlaygroundTarget, @@ -37,7 +38,11 @@ def render_playground_completion_text( def _suggested_commands(*, template: PlaygroundTemplate, display_path: str) -> tuple[str, ...]: - commands: list[str] = [f"cd {display_path}"] + orchestrated: bool = template in (PlaygroundTemplate.DAGSTER, PlaygroundTemplate.RIVERS) + project_path: str = ( + f"{display_path}/{PLAYGROUND_ORCHESTRATED_PROJECT_DIR}" if orchestrated else display_path + ) + commands: list[str] = [f"cd {project_path}"] if template == PlaygroundTemplate.VIRTUAL: commands.extend( [ @@ -61,10 +66,12 @@ def _suggested_commands(*, template: PlaygroundTemplate, display_path: str) -> t ) else: commands.extend(["sqb compile", "sqb build", "sqb test", "sqb audit"]) + if orchestrated: + commands.append("cd ..") if template == PlaygroundTemplate.DAGSTER: - commands.append("DAGSTER_IS_DEV_CLI=1 dagster dev -f dagster/definitions.py") + commands.append("DAGSTER_IS_DEV_CLI=1 dagster dev -f definitions.py") if template == PlaygroundTemplate.RIVERS: - commands.append("rivers dev rivers_pipeline.definitions") + commands.append("rivers dev definitions") return tuple(commands) diff --git a/src/sqlbuild/cli/commands/_helpers/playground/copy.py b/src/sqlbuild/cli/commands/_helpers/playground/copy.py index 93d4d684d0..9951baf7a8 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/copy.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/copy.py @@ -6,13 +6,19 @@ from importlib.resources.abc import Traversable from pathlib import Path -from sqlbuild.cli.commands.constants import PLAYGROUND_TEMPLATE_VALUES +from sqlbuild.cli.commands.constants import ( + PLAYGROUND_ORCHESTRATED_PROJECT_DIR, + PLAYGROUND_TEMPLATE_VALUES, +) from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.types import PlaygroundTemplate +from sqlbuild.compiler.discovery.constants import PYTHON_NODE_ROOT _TEMPLATE_PACKAGE: str = "sqlbuild.cli.commands._helpers.playground" _WAFFLE_SHOP_TEMPLATE: str = "templates/waffle_shop" _LOADER_WAFFLE_SHOP_TEMPLATE: str = "templates/loader_waffle_shop" +_ORCHESTRATOR_DEFINITIONS_FILE: str = "definitions.py" +_ORCHESTRATOR_README_FILE: str = "README.md" _PYTHON_NODES_PROJECT_TOML: str = """name = "python_nodes_demo" adapter = "duckdb" @@ -112,7 +118,7 @@ def export_window(ctx): _PYTHON_NODES_LOADERS_PY: str = '''"""Loader examples for the Python nodes playground.""" from sqlbuild.loaders import loader -from tasks.orders import prepare_raw_orders +from python.tasks.orders import prepare_raw_orders @loader(depends_on=(prepare_raw_orders,)) @@ -132,7 +138,7 @@ def raw_orders(ctx): from sqlbuild.assets import asset from sqlbuild.refs import model -from tasks.orders import export_window, optional_partner_feed +from python.tasks.orders import export_window, optional_partner_feed @asset( @@ -159,7 +165,7 @@ def orders_export(ctx): from sqlbuild.checks import check -from assets.orders_export import orders_export +from python.assets.orders_export import orders_export @check(depends_on=orders_export, tags=["quality"], group="exports") @@ -423,7 +429,7 @@ def check_orders_export(ctx): sqlbuild_scenario_checks, ) -PROJECT_DIR = Path(__file__).resolve().parent.parent +PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop" SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) SQLBUILD_PROJECT.prepare_if_dev() @@ -455,21 +461,22 @@ def waffle_shop_scenarios(context: dg.AssetCheckExecutionContext): _DAGSTER_README: str = """# SQLBuild + Dagster playground -This directory contains a Dagster definitions file for the generated SQLBuild waffle shop project. +This directory is a Dagster code location. `definitions.py` orchestrates the SQLBuild waffle +shop project in `waffle_shop/`, which stays a plain SQLBuild project with no Dagster code inside. ## Run Dagster -Install SQLBuild with the Dagster extra, then start Dagster from the project root: +Install SQLBuild with the Dagster extra, then start Dagster from this directory: ```bash uv add "sqlbuild[dagster]" -DAGSTER_IS_DEV_CLI=1 uv run dagster dev -f dagster/definitions.py +DAGSTER_IS_DEV_CLI=1 uv run dagster dev -f definitions.py ``` Open the Dagster UI, materialize the `waffle_shop_assets` asset group, then run the scenario checks. -The definitions use `SqlBuildProject.prepare_if_dev()` to generate `target/sqlbuild_dag.json` -when Dagster starts in dev mode. +The definitions use `SqlBuildProject.prepare_if_dev()` to generate +`waffle_shop/target/sqlbuild_dag.json` when Dagster starts in dev mode. """ _RIVERS_DEFINITIONS: str = '''"""Rivers definitions for the SQLBuild waffle shop playground.""" @@ -485,7 +492,7 @@ def waffle_shop_scenarios(context: dg.AssetCheckExecutionContext): from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets -PROJECT_DIR = Path(__file__).resolve().parent.parent +PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop" SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) if __name__ == "__main__": @@ -529,28 +536,28 @@ def waffle_shop_assets(context: Any) -> Iterator[Any]: _RIVERS_README: str = """# SQLBuild + Rivers playground -This directory contains a Rivers repository definition for the generated SQLBuild -waffle shop project. +This directory is a Rivers code repository. `definitions.py` orchestrates the SQLBuild waffle +shop project in `waffle_shop/`, which stays a plain SQLBuild project with no Rivers code inside. ## Run Rivers -Install SQLBuild with the Rivers extra, then start Rivers from the project root: +Install SQLBuild with the Rivers extra, then start Rivers from this directory: ```bash uv add "sqlbuild[rivers]" -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions ``` You can also run a local materialization directly: ```bash -uv run python rivers_pipeline/definitions.py +uv run python definitions.py ``` The repository defines a `waffle_shop` job for UI-triggered materialization. -The definitions use `SqlBuildProject.prepare_if_dev()` to generate `target/sqlbuild_dag.json` -when `rivers dev` starts with `RIVERS_DEPLOYMENT=dev`. +The definitions use `SqlBuildProject.prepare_if_dev()` to generate +`waffle_shop/target/sqlbuild_dag.json` when `rivers dev` starts with `RIVERS_DEPLOYMENT=dev`. """ @@ -589,11 +596,23 @@ def create_playground_project( code="C702", help="reinstall SQLBuild or report a packaging issue", ) - _copy_tree(source=template_root, target=target_dir) if resolved_template == PlaygroundTemplate.DAGSTER: - _write_dagster_template_files(target_dir=target_dir) + _write_orchestrator_template_files( + template_root=template_root, + target_dir=target_dir, + definitions=_DAGSTER_DEFINITIONS, + readme=_DAGSTER_README, + ) + return if resolved_template == PlaygroundTemplate.RIVERS: - _write_rivers_template_files(target_dir=target_dir) + _write_orchestrator_template_files( + template_root=template_root, + target_dir=target_dir, + definitions=_RIVERS_DEFINITIONS, + readme=_RIVERS_README, + ) + return + _copy_tree(source=template_root, target=target_dir) if resolved_template == PlaygroundTemplate.VIRTUAL: _write_virtual_template_files(target_dir=target_dir) @@ -615,21 +634,12 @@ def _generated_name(template_name: str) -> str: return template_name -def _write_dagster_template_files(*, target_dir: Path) -> None: - dagster_dir: Path = target_dir / "dagster" - dagster_dir.mkdir() - (dagster_dir / "definitions.py").write_text(_DAGSTER_DEFINITIONS, encoding="utf-8") - (dagster_dir / "README.md").write_text(_DAGSTER_README, encoding="utf-8") - - -def _write_rivers_template_files(*, target_dir: Path) -> None: - rivers_dir: Path = target_dir / "rivers_pipeline" - rivers_dir.mkdir() - (rivers_dir / "__init__.py").write_text( - '"""Rivers playground definitions package."""\n', encoding="utf-8" - ) - (rivers_dir / "definitions.py").write_text(_RIVERS_DEFINITIONS, encoding="utf-8") - (rivers_dir / "README.md").write_text(_RIVERS_README, encoding="utf-8") +def _write_orchestrator_template_files( + *, template_root: Traversable, target_dir: Path, definitions: str, readme: str +) -> None: + _copy_tree(source=template_root, target=target_dir / PLAYGROUND_ORCHESTRATED_PROJECT_DIR) + (target_dir / _ORCHESTRATOR_DEFINITIONS_FILE).write_text(definitions, encoding="utf-8") + (target_dir / _ORCHESTRATOR_README_FILE).write_text(readme, encoding="utf-8") def _write_python_nodes_template_files(*, target_dir: Path) -> None: @@ -648,18 +658,18 @@ def _write_python_nodes_template_files(*, target_dir: Path) -> None: _PYTHON_NODES_FACT_ORDERS_SQL, encoding="utf-8", ) - tasks_dir: Path = target_dir / "tasks" - tasks_dir.mkdir() - (tasks_dir / "orders.py").write_text(_PYTHON_NODES_TASKS_PY, encoding="utf-8") - loaders_dir: Path = target_dir / "loaders" - loaders_dir.mkdir() - (loaders_dir / "orders.py").write_text(_PYTHON_NODES_LOADERS_PY, encoding="utf-8") - assets_dir: Path = target_dir / "assets" - assets_dir.mkdir() - (assets_dir / "orders_export.py").write_text(_PYTHON_NODES_ASSETS_PY, encoding="utf-8") - checks_dir: Path = target_dir / "checks" - checks_dir.mkdir() - (checks_dir / "orders_export.py").write_text(_PYTHON_NODES_CHECKS_PY, encoding="utf-8") + python_files: tuple[tuple[str, str], ...] = ( + ("tasks/orders.py", _PYTHON_NODES_TASKS_PY), + ("loaders/orders.py", _PYTHON_NODES_LOADERS_PY), + ("assets/orders_export.py", _PYTHON_NODES_ASSETS_PY), + ("checks/orders_export.py", _PYTHON_NODES_CHECKS_PY), + ) + relative_path: str + contents: str + for relative_path, contents in python_files: + python_file: Path = target_dir / PYTHON_NODE_ROOT / relative_path + python_file.parent.mkdir(parents=True, exist_ok=True) + python_file.write_text(contents, encoding="utf-8") for hook_language in ("sql", "python"): hook_dir: Path = target_dir / "hooks" / hook_language hook_dir.mkdir(parents=True) diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/loaders/waffle_loaders.py.txt b/src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/loaders/waffle_loaders.py.txt rename to src/sqlbuild/cli/commands/_helpers/playground/templates/loader_waffle_shop/python/loaders/waffle_loaders.py.txt diff --git a/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/loaders/waffle_sources.py.txt b/src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/python/loaders/waffle_sources.py.txt similarity index 100% rename from src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/loaders/waffle_sources.py.txt rename to src/sqlbuild/cli/commands/_helpers/playground/templates/waffle_shop/python/loaders/waffle_sources.py.txt diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 941a51f346..7a2148e822 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -47,6 +47,7 @@ SelectorKind.PATH, } ) +PLAYGROUND_ORCHESTRATED_PROJECT_DIR: str = "waffle_shop" PLAYGROUND_TEMPLATE_VALUES: tuple[str, ...] = tuple( template.value for template in PlaygroundTemplate ) diff --git a/src/sqlbuild/cli/commands/main/workspace/_init.py b/src/sqlbuild/cli/commands/main/workspace/_init.py index 44dd288c95..ae43d974da 100644 --- a/src/sqlbuild/cli/commands/main/workspace/_init.py +++ b/src/sqlbuild/cli/commands/main/workspace/_init.py @@ -41,8 +41,8 @@ def run_init(project_dir: Path | None) -> int: doc.blank() doc.section("Next steps") doc.line(" 1. Add sources to sources/") - doc.line(" 2. Add seeds to seeds/ or loaders to loaders/") - doc.line(" 3. Add tasks to tasks/, assets to assets/, or checks to checks/") + doc.line(" 2. Add seeds to seeds/") + doc.line(" 3. Add Python loaders, tasks, assets, checks, and factories to python/") doc.line( " 4. Add SQL hooks to hooks/sql/, Python hooks to hooks/python/, " "functions to functions/, or macros to macros/" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index d02b078ab4..981e3ebfa5 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -38,10 +38,8 @@ from sqlbuild.compiler.discovery.constants import ( CANONICAL_AUTHORED_ROOTS, MODEL_SCHEMAS_DIRECTORY_NAME, - PYTHON_FACTORY_FOLDER, PYTHON_INIT_MODULE_STEM, - PYTHON_LOADER_FOLDER, - PYTHON_NODE_KIND_VOWELS, + PYTHON_NODE_ROOT, SCHEMA_FILE_NAME, SEED_FILE_SUFFIX, SQL_TESTS_OWNERSHIP_ROOT, @@ -50,7 +48,6 @@ from sqlbuild.compiler.discovery.exceptions import ( DeclarationParseError, EventExporterDiscoveryError, - LoaderDiscoveryError, ModelSqlParseError, ProviderDiscoveryError, PythonNodeDiscoveryError, @@ -127,15 +124,6 @@ from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry -_PYTHON_NODE_KIND_FOLDERS: tuple[str, ...] = ("loaders", "tasks", "assets", "checks") -_PYTHON_NODE_FACTORY_FOLDERS: tuple[str, ...] = (*_PYTHON_NODE_KIND_FOLDERS, "factories") -_PYTHON_NODE_KIND_BY_FOLDER: dict[str, str] = { - "loaders": "loader", - "tasks": "task", - "assets": "asset", - "checks": "check", -} - @dataclass class _PythonNodeDiscoveryBucket: @@ -896,7 +884,7 @@ def discover_materialization_files( def discover_python_node_functions( *, project_dir: Path, providers: tuple[DiscoveredProvider, ...] = () ) -> DiscoveredPythonNodeFunctions: - """Discover decorated Python DAG node functions under node folders.""" + """Discover decorated Python DAG node functions under python/.""" bucket: _PythonNodeDiscoveryBucket = _discover_python_node_functions( project_dir=project_dir, @@ -930,7 +918,6 @@ def discover_hook_functions( module: ModuleType = _load_python_node_module( file_path=file_path, project_dir=project_dir, - node_folder="hooks/python", ) for _, value in inspect.getmembers(module, inspect.isfunction): if value.__module__ != module.__name__: @@ -1367,32 +1354,21 @@ def _discover_python_node_functions( ) -> _PythonNodeDiscoveryBucket: bucket: _PythonNodeDiscoveryBucket = _PythonNodeDiscoveryBucket() provider_by_name: dict[str, DiscoveredProvider] = _provider_by_name(providers) - node_folder: str - for node_folder in _PYTHON_NODE_FACTORY_FOLDERS: - node_root: Path = project_dir / node_folder - if not node_root.is_dir(): + node_root: Path = project_dir / PYTHON_NODE_ROOT + if not node_root.is_dir(): + return bucket + file_path: Path + for file_path in sorted(node_root.rglob("*.py")): + if file_path.stem == PYTHON_INIT_MODULE_STEM: continue - file_path: Path - for file_path in sorted(node_root.rglob("*.py")): - if file_path.stem == PYTHON_INIT_MODULE_STEM: - continue - module: ModuleType = ( - _load_loader_module(file_path=file_path, project_dir=project_dir) - if node_folder == PYTHON_LOADER_FOLDER - else _load_python_node_module( - file_path=file_path, - project_dir=project_dir, - node_folder=node_folder, - ) - ) - _append_module_python_nodes( - bucket=bucket, - module=module, - file_path=file_path, - project_dir=project_dir, - node_folder=node_folder, - provider_by_name=provider_by_name, - ) + module: ModuleType = _load_python_node_module(file_path=file_path, project_dir=project_dir) + _append_module_python_nodes( + bucket=bucket, + module=module, + file_path=file_path, + project_dir=project_dir, + provider_by_name=provider_by_name, + ) return bucket @@ -1402,7 +1378,6 @@ def _append_module_python_nodes( module: ModuleType, file_path: Path, project_dir: Path, - node_folder: str, provider_by_name: dict[str, DiscoveredProvider], ) -> None: _append_module_audit_factories( @@ -1410,20 +1385,17 @@ def _append_module_python_nodes( module=module, file_path=file_path, project_dir=project_dir, - node_folder=node_folder, ) - if node_folder != PYTHON_FACTORY_FOLDER: - for _, value in inspect.getmembers(module, inspect.isfunction): - if value.__module__ != module.__name__: - continue - _append_python_node_function( - bucket=bucket, - function=value, - file_path=file_path, - project_dir=project_dir, - expected_kind=_PYTHON_NODE_KIND_BY_FOLDER.get(node_folder), - provider_by_name=provider_by_name, - ) + for _, value in inspect.getmembers(module, inspect.isfunction): + if value.__module__ != module.__name__: + continue + _append_python_node_function( + bucket=bucket, + function=value, + file_path=file_path, + project_dir=project_dir, + provider_by_name=provider_by_name, + ) for _, value in inspect.getmembers(module, inspect.isfunction): if value.__module__ != module.__name__: continue @@ -1444,8 +1416,6 @@ def _append_module_python_nodes( function=generated_function, file_path=file_path, project_dir=project_dir, - expected_kind=_PYTHON_NODE_KIND_BY_FOLDER.get(node_folder), - factory_definition=factory_definition, provider_by_name=provider_by_name, ): raise PythonNodeDiscoveryError( @@ -1461,7 +1431,6 @@ def _append_module_audit_factories( module: ModuleType, file_path: Path, project_dir: Path, - node_folder: str, ) -> None: """Collect audit factories without routing them through Python-node discovery.""" @@ -1472,10 +1441,6 @@ def _append_module_audit_factories( if definition is None: continue relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) - if node_folder != PYTHON_FACTORY_FOLDER: - raise PythonNodeDiscoveryError( - f"Audit factory '{definition.name}' in {relative_path} must live under factories/" - ) if _python_node_definition_names(value): kinds: str = ", ".join(_python_node_definition_names(value)) raise PythonNodeDiscoveryError( @@ -1559,21 +1524,11 @@ def _append_python_node_function( function: Callable[..., object], file_path: Path, project_dir: Path, - expected_kind: str | None = None, - factory_definition: FactoryDefinition | None = None, provider_by_name: dict[str, DiscoveredProvider] | None = None, ) -> bool: resolved_provider_by_name: dict[str, DiscoveredProvider] = provider_by_name or {} loader_definition: LoaderDefinition | None = read_loader_definition(function) if loader_definition is not None: - _validate_python_node_kind( - actual_kind="loader", - expected_kind=expected_kind, - function_name=loader_definition.name, - factory_definition=factory_definition, - file_path=file_path, - project_dir=project_dir, - ) bucket.add_loader( DiscoveredLoaderFunction( file_path=file_path, @@ -1596,14 +1551,6 @@ def _append_python_node_function( return True task_definition: TaskDefinition | None = read_task_definition(function) if task_definition is not None: - _validate_python_node_kind( - actual_kind="task", - expected_kind=expected_kind, - function_name=task_definition.name, - factory_definition=factory_definition, - file_path=file_path, - project_dir=project_dir, - ) bucket.add_task( DiscoveredTaskFunction( file_path=file_path, @@ -1625,14 +1572,6 @@ def _append_python_node_function( return True asset_definition: AssetDefinition | None = read_asset_definition(function) if asset_definition is not None: - _validate_python_node_kind( - actual_kind="asset", - expected_kind=expected_kind, - function_name=asset_definition.name, - factory_definition=factory_definition, - file_path=file_path, - project_dir=project_dir, - ) bucket.add_asset( DiscoveredAssetFunction( file_path=file_path, @@ -1656,14 +1595,6 @@ def _append_python_node_function( return True check_definition: CheckDefinition | None = read_check_definition(function) if check_definition is not None: - _validate_python_node_kind( - actual_kind="check", - expected_kind=expected_kind, - function_name=check_definition.name, - factory_definition=factory_definition, - file_path=file_path, - project_dir=project_dir, - ) bucket.add_check( DiscoveredCheckFunction( file_path=file_path, @@ -1686,31 +1617,6 @@ def _append_python_node_function( return False -def _validate_python_node_kind( - *, - actual_kind: str, - expected_kind: str | None, - function_name: str, - factory_definition: FactoryDefinition | None, - file_path: Path, - project_dir: Path, -) -> None: - if expected_kind is None or actual_kind == expected_kind: - return - relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) - folder: str = relative_path.parts[0] - if factory_definition is None: - article: str = "an" if actual_kind[0] in PYTHON_NODE_KIND_VOWELS else "a" - raise PythonNodeDiscoveryError( - f"Python node '{function_name}' in {folder}/ is {article} {actual_kind}; " - f"{actual_kind}s must live in {actual_kind}s/ or be generated from factories/." - ) - raise PythonNodeDiscoveryError( - f"Factory {factory_definition.name} in {folder}/ returned a {actual_kind} " - f"'{function_name}'; mixed-kind factories must live in factories/." - ) - - def _call_factory( *, factory: Callable[..., object], @@ -1768,20 +1674,7 @@ def _normalize_factory_result( return tuple(functions) -def _load_loader_module(*, file_path: Path, project_dir: Path) -> ModuleType: - return _exec_project_module( - module_name="sqlbuild_project_loader_" - + "_".join( - _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts - ), - file_path=file_path, - project_dir=project_dir, - file_label="source loader", - error_type=LoaderDiscoveryError, - ) - - -def _load_python_node_module(*, file_path: Path, project_dir: Path, node_folder: str) -> ModuleType: +def _load_python_node_module(*, file_path: Path, project_dir: Path) -> ModuleType: return _exec_project_module( module_name="sqlbuild_project_python_node_" + "_".join( diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py index fe43286a0b..32c491a228 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/python_paths.py @@ -20,18 +20,11 @@ _DIRECT_PYTHON_ROOTS: frozenset[str] = frozenset( { "adapters", - "assets", - "checks", - "dagster", - "factories", - "loaders", - "libs", - "rules", "materializations", "providers", - "rivers_pipeline", + "python", + "rules", "sinks", - "tasks", } ) _LANGUAGE_PYTHON_ROOTS: frozenset[tuple[str, str]] = frozenset( @@ -39,7 +32,7 @@ ) _IGNORED_ROOTS: frozenset[str] = frozenset({"logs", "target", "tests", "venv"}) _LEGACY_DIAGNOSTIC_ROOTS: frozenset[str] = frozenset({"event_exporters"}) -_SUPPORTED_ROOT_FILES: frozenset[Path] = frozenset({Path("adapter.py"), Path("definitions.py")}) +_SUPPORTED_ROOT_FILES: frozenset[Path] = frozenset({Path("adapter.py")}) _SCOPED_DECLARATION_DIRECTORIES: frozenset[str] = ( INHERITED_DECLARATION_DIRECTORIES | LOCAL_DECLARATION_DIRECTORIES ) @@ -55,8 +48,8 @@ def validate_project_python_paths(*, project_dir: Path) -> None: raise ProjectPythonPathError( f"Unsupported project Python path(s): {rendered_paths}", help=( - "Move project Python under a documented resource, library, or integration path. " - "Factory support modules, including factories/**/_helpers.py, are allowed." + "Move pipeline Python nodes and the helper modules they import under python/, " + "or move SQLBuild extensions under their documented extension path." ), ) diff --git a/src/sqlbuild/compiler/discovery/constants.py b/src/sqlbuild/compiler/discovery/constants.py index 34d287fe32..81fe77aff0 100644 --- a/src/sqlbuild/compiler/discovery/constants.py +++ b/src/sqlbuild/compiler/discovery/constants.py @@ -15,9 +15,7 @@ PYTHON_INIT_MODULE_STEM: str = "__init__" PYTHON_CACHE_DIRECTORY_NAME: str = "__pycache__" LANGUAGE_PYTHON_ROOT_PART_COUNT: int = 2 -PYTHON_LOADER_FOLDER: str = "loaders" -PYTHON_FACTORY_FOLDER: str = "factories" -PYTHON_NODE_KIND_VOWELS: frozenset[str] = frozenset({"a", "e", "i", "o", "u"}) +PYTHON_NODE_ROOT: str = "python" PYTHON_UDF_DECORATOR_NAME: str = "udf" PYTHON_UDF_IMPORT_MODULES: frozenset[str] = frozenset({"sqlbuild", "sqlbuild.functions"}) diff --git a/src/sqlbuild/compiler/discovery/exceptions.py b/src/sqlbuild/compiler/discovery/exceptions.py index 6c203606dc..7779d906fb 100644 --- a/src/sqlbuild/compiler/discovery/exceptions.py +++ b/src/sqlbuild/compiler/discovery/exceptions.py @@ -75,12 +75,6 @@ class SqlScenarioParseError(DiscoveryError): code: str = "D009" -class LoaderDiscoveryError(DiscoveryError): - """Raised when project source loaders cannot be discovered.""" - - code: str = "D010" - - class PythonNodeDiscoveryError(DiscoveryError): """Raised when project Python nodes cannot be discovered.""" diff --git a/src/sqlbuild/compiler/planner/constants.py b/src/sqlbuild/compiler/planner/constants.py index 30678f1699..88de87825d 100644 --- a/src/sqlbuild/compiler/planner/constants.py +++ b/src/sqlbuild/compiler/planner/constants.py @@ -6,8 +6,7 @@ from sqlbuild.cursor_algebra.types import BoundSentinel PATH_SELECTOR_EXPLICIT_ROOT_ERROR: str = ( - "path selectors require an explicit root: use 'models/', 'tasks/', 'assets/', " - "'checks/', or 'loaders/'" + "path selectors require an explicit root: use 'models/' or 'python/'" ) EMPTY_FINGERPRINT_METADATA_JSON: str = "{}" RECORDED_RELATION_MISSING_WARNING_CODE: str = "RECORDED_RELATION_MISSING" diff --git a/src/sqlbuild/compiler/python_nodes/constants.py b/src/sqlbuild/compiler/python_nodes/constants.py index 008127a143..80b0c465e9 100644 --- a/src/sqlbuild/compiler/python_nodes/constants.py +++ b/src/sqlbuild/compiler/python_nodes/constants.py @@ -4,6 +4,6 @@ EMPTY_SELECTOR_FOLDER: str = "" SQL_MODEL_PATH_ROOT: str = "models" -PYTHON_NODE_PATH_ROOTS: frozenset[str] = frozenset({"tasks", "assets", "checks", "loaders"}) +PYTHON_NODE_PATH_ROOTS: frozenset[str] = frozenset({"python"}) UNIFIED_PATH_ROOTS: frozenset[str] = frozenset({*PYTHON_NODE_PATH_ROOTS, SQL_MODEL_PATH_ROOT}) TAG_NOT_FOUND_ERROR_CODE: str = "S008" diff --git a/tests/e2e/fixtures/source_loader_strategies/loaders/strategy_sources.py b/tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py similarity index 100% rename from tests/e2e/fixtures/source_loader_strategies/loaders/strategy_sources.py rename to tests/e2e/fixtures/source_loader_strategies/python/loaders/strategy_sources.py diff --git a/tests/e2e/fixtures/waffle_shop/loaders/waffle_sources.py b/tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py similarity index 100% rename from tests/e2e/fixtures/waffle_shop/loaders/waffle_sources.py rename to tests/e2e/fixtures/waffle_shop/python/loaders/waffle_sources.py diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py index 0fa89fe56a..83a8448523 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/bigquery/test_bigquery.py @@ -177,15 +177,15 @@ def test_given_python_result_when_running_check_on_bigquery_then_persists_node_r project_name="bigquery_node_results", dataset_name=dataset_name, ), - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" " return ctx.result(payload={'value': 42}, metadata={'source': 'bigquery'})\n" ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" @@ -991,7 +991,7 @@ def test_given_source_deferral_env_when_building_on_bigquery_then_reads_prod_and " - name: status\n" " type: STRING\n" ), - "loaders/raw_orders.py": ( + "python/loaders/raw_orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index cb225f0476..fdd763979d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -212,7 +212,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "factories/generated.py": """ + "python/factories/generated.py": """ from pathlib import Path from sqlbuild.assets import asset from sqlbuild.checks import check @@ -359,7 +359,7 @@ def test_given_existing_intermediate_target_when_running_source_only_then_reuses "[connection]\n" 'database = "run_existing_intermediate_project.duckdb"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" @@ -421,7 +421,7 @@ def test_given_task_selector_when_running_run_then_task_executes( "[connection]\n" 'database = "python_run_project.duckdb"\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -476,7 +476,7 @@ def test_given_asset_selector_when_running_run_then_asset_executes( "[connection]\n" 'database = "python_asset_run_project.duckdb"\n' ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from pathlib import Path\n" "from sqlbuild.assets import asset\n\n" "@asset\n" @@ -532,7 +532,7 @@ def test_given_task_selector_with_json_output_when_running_run_then_json_include "[connection]\n" 'database = "python_run_json_project.duckdb"\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" @@ -602,7 +602,7 @@ def test_given_failing_task_selector_when_running_run_then_command_fails( "[connection]\n" 'database = "python_failed_run_project.duckdb"\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def fail_orders(ctx):\n" @@ -670,7 +670,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "[connection]\n" 'database = "python_loader_run_project.duckdb"\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -678,10 +678,10 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ " Path(__file__).parents[1].joinpath('orders_ready.txt').write_text('ready')\n" " return ctx.result(metadata={'prepared': True})\n" ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n" - "from tasks.orders import prepare_orders\n\n" + "from python.tasks.orders import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" " marker = Path(__file__).parents[1].joinpath('orders_ready.txt')\n" @@ -762,7 +762,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built "MODEL (materialized table);\n\n" 'SELECT order_id, amount_cents FROM __source("raw_orders")\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" @@ -833,7 +833,7 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from pathlib import Path\n" "from sqlbuild.assets import asset\n" "from sqlbuild.refs import model\n\n" @@ -893,15 +893,15 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "[connection]\n" 'database = "python_task_asset_task_run_project.duckdb"\n' ), - "tasks/fetch_orders.py": ( + "python/tasks/fetch_orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def fetch_orders(ctx):\n" " return ctx.result(payload={'rows': 1})\n" ), - "tasks/notify_orders.py": ( + "python/tasks/notify_orders.py": ( "from pathlib import Path\n" - "from assets.orders import publish_orders\n" + "from python.assets.orders import publish_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=publish_orders)\n" "def notify_orders(ctx):\n" @@ -910,9 +910,9 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes " output.write_text(str(metadata['published']))\n" " return ctx.result()\n" ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from sqlbuild.assets import asset\n" - "from tasks.fetch_orders import fetch_orders\n\n" + "from python.tasks.fetch_orders import fetch_orders\n\n" "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" " payload = ctx.result_of(node_function=fetch_orders).payload\n" @@ -958,7 +958,7 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load "[connection]\n" 'database = "python_source_task_asset_run_project.duckdb"\n' ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" @@ -975,16 +975,16 @@ def test_given_source_task_asset_selection_when_running_run_then_task_reads_load " - name: amount_cents\n" " type: INTEGER\n" ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def summarize_loaded_orders(ctx):\n" " rows = ctx.query('SELECT COUNT(*) FROM raw_orders').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from sqlbuild.assets import asset\n" - "from tasks.orders import summarize_loaded_orders\n\n" + "from python.tasks.orders import summarize_loaded_orders\n\n" "@asset(depends_on=summarize_loaded_orders)\n" "def publish_loaded_orders(ctx):\n" " payload = ctx.result_of(node_function=summarize_loaded_orders).payload\n" @@ -1043,13 +1043,13 @@ def test_given_skip_and_asset_selection_with_json_when_running_run_then_json_rec "[connection]\n" 'database = "python_json_fields_run_project.duckdb"\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def optional_orders(ctx):\n" " return ctx.skip(reason='no files')\n\n" ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from sqlbuild.assets import asset\n\n" "@asset\n" "def observed_orders(ctx):\n" @@ -1131,7 +1131,7 @@ def test_given_independent_python_and_sql_selectors_when_running_run_then_all_br "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT order_id FROM __source("raw_orders")\n' ), - "tasks/branches.py": ( + "python/tasks/branches.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -1208,7 +1208,7 @@ def test_given_task_depends_on_model_when_running_run_then_task_runs_before_down "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" @@ -1266,7 +1266,7 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour "[connection]\n" 'database = "python_read_side_source_run_project.duckdb"\n' ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" @@ -1283,7 +1283,7 @@ def test_given_task_depends_on_source_when_running_run_then_task_runs_after_sour " - name: amount_cents\n" " type: INTEGER\n" ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import source\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" @@ -1355,7 +1355,7 @@ def test_given_sql_ready_task_fails_when_running_run_then_footer_json_and_exit_f "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __ref("stg_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" @@ -1421,7 +1421,7 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej "[connection]\n" 'database = "python_boundary_run_project.duckdb"\n' ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" @@ -1435,8 +1435,8 @@ def test_given_task_depends_on_terminal_loader_when_running_run_then_command_rej " - name: order_id\n" " type: INTEGER\n" ), - "tasks/orders.py": ( - "from loaders.orders import raw_orders\n" + "python/tasks/orders.py": ( + "from python.loaders.orders import raw_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=raw_orders)\n" "def summarize_orders(ctx):\n" @@ -1489,7 +1489,7 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo "[connection]\n" 'database = "python_intermediate_loader_run_project.duckdb"\n' ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='stage_orders',\n" @@ -1502,17 +1502,17 @@ def test_given_task_asset_depend_on_intermediate_loader_when_running_run_then_lo "def stage_orders(ctx):\n" " return [{'order_id': 1, 'amount_cents': 100}]\n" ), - "tasks/orders.py": ( - "from loaders.orders import stage_orders\n" + "python/tasks/orders.py": ( + "from python.loaders.orders import stage_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=stage_orders)\n" "def summarize_stage_orders(ctx):\n" " rows = ctx.query('SELECT COUNT(*) FROM stage_orders').fetchall()[0][0]\n" " return ctx.result(payload={'rows': rows}, metadata={'rows': rows})\n" ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from sqlbuild.assets import asset\n" - "from tasks.orders import summarize_stage_orders\n\n" + "from python.tasks.orders import summarize_stage_orders\n\n" "@asset(depends_on=summarize_stage_orders)\n" "def publish_stage_orders(ctx):\n" " payload = ctx.result_of(node_function=summarize_stage_orders).payload\n" @@ -1570,7 +1570,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "[connection]\n" 'database = "python_loader_task_loader_run_project.duckdb"\n' ), - "loaders/window.py": ( + "python/loaders/window.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='window_orders',\n" @@ -1580,9 +1580,9 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "def load_window_orders(ctx):\n" " return [{'order_id': 1}]\n" ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from pathlib import Path\n" - "from loaders.window import load_window_orders\n" + "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_raw_orders(ctx):\n" @@ -1590,9 +1590,9 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c " Path(__file__).parents[1].joinpath('prepared.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" - "from tasks.orders import prepare_raw_orders\n" + "from python.tasks.orders import prepare_raw_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_raw_orders,))\n" "def raw_orders(ctx):\n" @@ -1656,15 +1656,15 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "[connection]\n" 'database = "python_task_asset_loader_run_project.duckdb"\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def fetch_orders(ctx):\n" " return ctx.result(payload={'order_id': 1})\n" ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from pathlib import Path\n" - "from tasks.orders import fetch_orders\n" + "from python.tasks.orders import fetch_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" @@ -1673,9 +1673,9 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from pathlib import Path\n" - "from assets.orders import publish_orders\n" + "from python.assets.orders import publish_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_orders,))\n" "def raw_orders(ctx):\n" @@ -1739,7 +1739,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "[connection]\n" 'database = "python_loader_asset_loader_run_project.duckdb"\n' ), - "loaders/window.py": ( + "python/loaders/window.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='window_orders',\n" @@ -1749,9 +1749,9 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "def load_window_orders(ctx):\n" " return [{'order_id': 1}]\n" ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from pathlib import Path\n" - "from loaders.window import load_window_orders\n" + "from python.loaders.window import load_window_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=load_window_orders)\n" "def prepare_asset_orders(ctx):\n" @@ -1759,9 +1759,9 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ " Path(__file__).parents[1].joinpath('asset_ready.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" - "from assets.orders import prepare_asset_orders\n" + "from python.assets.orders import prepare_asset_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_asset_orders,))\n" "def raw_orders(ctx):\n" @@ -1825,7 +1825,7 @@ def test_given_loader_loader_chain_when_running_model_then_ingress_orders_chain( "[connection]\n" 'database = "python_loader_loader_run_project.duckdb"\n' ), - "loaders/window.py": ( + "python/loaders/window.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='window_orders',\n" @@ -1835,8 +1835,8 @@ def test_given_loader_loader_chain_when_running_model_then_ingress_orders_chain( "def load_window_orders(ctx):\n" " return [{'order_id': 1}]\n" ), - "loaders/raw.py": ( - "from loaders.window import load_window_orders\n" + "python/loaders/raw.py": ( + "from python.loaders.window import load_window_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(load_window_orders,))\n" "def raw_orders(ctx):\n" @@ -1898,7 +1898,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "[connection]\n" 'database = "python_sql_python_spine_run_project.duckdb"\n' ), - "loaders/window.py": ( + "python/loaders/window.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='window_orders',\n" @@ -1908,17 +1908,17 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "def load_window_orders(ctx):\n" " return [{'order_id': 7}]\n" ), - "tasks/prepare.py": ( - "from loaders.window import load_window_orders\n" + "python/tasks/prepare.py": ( + "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" " rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" ), - "assets/prepare.py": ( + "python/assets/prepare.py": ( "from pathlib import Path\n" - "from tasks.prepare import prepare_orders\n" + "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" @@ -1927,9 +1927,9 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" - "from assets.prepare import publish_prepared_orders\n" + "from python.assets.prepare import publish_prepared_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" @@ -1948,7 +1948,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" @@ -1956,17 +1956,17 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running " order_id = ctx.query('SELECT order_id FROM fact_orders').fetchall()[0][0]\n" " return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n" ), - "assets/export.py": ( - "from tasks.profile import profile_fact_orders\n" + "python/assets/export.py": ( + "from python.tasks.profile import profile_fact_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=profile_fact_orders)\n" "def export_fact_orders(ctx):\n" " payload = ctx.result_of(node_function=profile_fact_orders).payload\n" " return ctx.result(payload=payload, metadata={'exported': True})\n" ), - "tasks/notify.py": ( + "python/tasks/notify.py": ( "from pathlib import Path\n" - "from assets.export import export_fact_orders\n" + "from python.assets.export import export_fact_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" @@ -2049,15 +2049,15 @@ def test_given_source_task_loader_chain_when_running_run_then_command_rejects_bo " - name: order_id\n" " type: INTEGER\n" ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.refs import source\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=source('raw_orders'))\n" "def prepare_orders(ctx):\n" " return ctx.result()\n" ), - "loaders/orders.py": ( - "from tasks.orders import prepare_orders\n" + "python/loaders/orders.py": ( + "from python.tasks.orders import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='stage_orders',\n" @@ -2126,15 +2126,15 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou "models/stg_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('stg_orders'))\n" "def prepare_orders(ctx):\n" " return ctx.result()\n" ), - "loaders/orders.py": ( - "from tasks.orders import prepare_orders\n" + "python/loaders/orders.py": ( + "from python.tasks.orders import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='stage_orders',\n" @@ -2192,14 +2192,14 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "[connection]\n" 'database = "python_check_run_project.duckdb"\n' ), - "checks/orders.py": ( - "from tasks.orders import prepare_orders\n" + "python/checks/orders.py": ( + "from python.tasks.orders import prepare_orders\n" "from sqlbuild.checks import check\n\n" "@check(depends_on=prepare_orders)\n" "def check_orders_export(ctx):\n" " return ctx.pass_()\n" ), - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py index 920c010954..e3de912241 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_factory_failures.py @@ -48,7 +48,7 @@ def test_given_failing_factory_audit_when_building_then_existing_table_is_untouc AUDIT (); SELECT * FROM __ref("@model") WHERE NOT (@expression) """, - "factories/quality.py": """ + "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory @audit_factory diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index fa98936a5b..ce9a426481 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -657,14 +657,14 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "[connection]\n" 'database = "python_build_ingress_failure_project.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" " raise RuntimeError('prepare failed')\n" ), - "loaders/raw.py": ( - "from tasks.prepare import prepare_orders\n" + "python/loaders/raw.py": ( + "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" @@ -703,15 +703,15 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "[connection]\n" 'database = "python_build_ingress_skip_project.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), - "loaders/raw.py": ( - "from tasks.prepare import prepare_orders\n" + "python/loaders/raw.py": ( + "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" @@ -750,7 +750,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "[connection]\n" 'database = "python_build_non_json_result_project.duckdb"\n' ), - "tasks/bad.py": ( + "python/tasks/bad.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_bad_result(ctx):\n" @@ -774,7 +774,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "[connection]\n" 'database = "python_build_non_json_metadata_project.duckdb"\n' ), - "tasks/bad.py": ( + "python/tasks/bad.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_bad_metadata(ctx):\n" @@ -809,7 +809,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" @@ -848,7 +848,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" @@ -888,7 +888,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" @@ -917,7 +917,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "[connection]\n" 'database = "python_build_no_python_project.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -926,8 +926,8 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " marker.write_text('prepared')\n" " return ctx.result(payload={'order_id': 3})\n" ), - "loaders/raw.py": ( - "from tasks.prepare import prepare_orders\n" + "python/loaders/raw.py": ( + "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" @@ -945,7 +945,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" @@ -976,7 +976,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "[connection]\n" 'database = "python_build_source_relation_project.duckdb"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" @@ -991,7 +991,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap " - name: order_id\n" " type: INTEGER\n" ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import source\n" "from sqlbuild.tasks import task\n\n" @@ -1064,7 +1064,7 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "[connection]\n" 'database = "direct_build_missing_intermediate_project.duckdb"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" @@ -1093,7 +1093,7 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "[connection]\n" 'database = "direct_build_no_python_source_only_project.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -1102,9 +1102,9 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden " marker.write_text('prepared')\n" " return ctx.result()\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n" - "from tasks.prepare import prepare_events\n\n" + "from python.tasks.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events])\n" "def raw_events(ctx):\n" " return [{'event_id': 1}]\n" @@ -1170,7 +1170,7 @@ def test_given_direct_source_only_build_when_loader_has_ingress_then_enforces_so "[connection]\n" 'database = "direct_build_existing_intermediate_project.duckdb"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" @@ -1237,7 +1237,7 @@ def test_given_existing_intermediate_target_when_building_source_only_then_reuse "[connection]\n" 'database = "python_build_json_project.duckdb"\n' ), - "assets/orders.py": ( + "python/assets/orders.py": ( "from sqlbuild.assets import asset\n\n" "@asset\n" "def observed_orders(ctx):\n" @@ -1341,7 +1341,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "[connection]\n" 'database = "python_sql_python_spine_build_project.duckdb"\n' ), - "loaders/window.py": ( + "python/loaders/window.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='window_orders',\n" @@ -1351,17 +1351,17 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "def load_window_orders(ctx):\n" " return [{'order_id': 7}]\n" ), - "tasks/prepare.py": ( - "from loaders.window import load_window_orders\n" + "python/tasks/prepare.py": ( + "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" " rows = ctx.query('SELECT order_id FROM window_orders').fetchall()\n" " return ctx.result(payload={'order_id': rows[0][0]})\n" ), - "assets/prepare.py": ( + "python/assets/prepare.py": ( "from pathlib import Path\n" - "from tasks.prepare import prepare_orders\n" + "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" @@ -1370,9 +1370,9 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" - "from assets.prepare import publish_prepared_orders\n" + "from python.assets.prepare import publish_prepared_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" @@ -1391,7 +1391,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" @@ -1400,17 +1400,17 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s " order_id = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" " return ctx.result(payload={'order_id': order_id}, metadata={'rows': 1})\n" ), - "assets/export.py": ( - "from tasks.profile import profile_fact_orders\n" + "python/assets/export.py": ( + "from python.tasks.profile import profile_fact_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=profile_fact_orders)\n" "def export_fact_orders(ctx):\n" " payload = ctx.result_of(node_function=profile_fact_orders).payload\n" " return ctx.result(payload=payload, metadata={'exported': True})\n" ), - "tasks/notify.py": ( + "python/tasks/notify.py": ( "from pathlib import Path\n" - "from assets.export import export_fact_orders\n" + "from python.assets.export import export_fact_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" @@ -1497,7 +1497,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p "[connection]\n" 'database = "python_persisted_result_project.duckdb"\n' ), - "tasks/produce.py": ( + "python/tasks/produce.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" @@ -1515,7 +1515,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p sql="SELECT run_id FROM _sqlbuild_node_results WHERE node_name = 'produce_result'", ) first_run_id: str = str(first_run_rows[0][0]) - (project_dir / "tasks" / "produce.py").write_text( + (project_dir / "python" / "tasks" / "produce.py").write_text( ( "from sqlbuild.tasks import task\n\n" "@task\n" @@ -1528,7 +1528,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p command=("--no-color", "build", "--select", "produce_result"), project_dir=project_dir, ) - (project_dir / "tasks" / "produce.py").write_text( + (project_dir / "python" / "tasks" / "produce.py").write_text( ( "from sqlbuild.tasks import task\n\n" "@task\n" @@ -1549,10 +1549,10 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p ), ) failed_run_id: str = str(failed_run_rows[0][0]) - (project_dir / "tasks" / "consume.py").write_text( + (project_dir / "python" / "tasks" / "consume.py").write_text( ( "from pathlib import Path\n" - "from tasks.produce import produce_result\n" + "from python.tasks.produce import produce_result\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_result(ctx):\n" @@ -1640,7 +1640,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "[connection]\n" 'database = "python_persisted_loader_result_project.duckdb"\n' ), - "loaders/events.py": ( + "python/loaders/events.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" @@ -1658,9 +1658,9 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "models/fact_events.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_events")\n' ), - "tasks/consume.py": ( + "python/tasks/consume.py": ( "from pathlib import Path\n" - "from loaders.events import raw_events\n" + "from python.loaders.events import raw_events\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_loader_result(ctx):\n" @@ -1709,7 +1709,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "[connection]\n" 'database = "python_failed_loader_result_project.duckdb"\n' ), - "loaders/events.py": ( + "python/loaders/events.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_events(ctx):\n" @@ -1742,15 +1742,15 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "[connection]\n" 'database = "python_skipped_loader_result_project.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), - "loaders/events.py": ( - "from tasks.prepare import prepare_events\n" + "python/loaders/events.py": ( + "from python.tasks.prepare import prepare_events\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_events,))\n" "def raw_events(ctx):\n" @@ -1842,15 +1842,15 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti "[targets.prod]\n" 'schema = "prod"\n' ), - "tasks/produce.py": ( + "python/tasks/produce.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" " return ctx.result(payload={'target': ctx.target})\n" ), - "tasks/consume.py": ( + "python/tasks/consume.py": ( "from pathlib import Path\n" - "from tasks.produce import produce_result\n" + "from python.tasks.produce import produce_result\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def consume_result(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py index a06ca48837..c9388aaf81 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py @@ -52,7 +52,7 @@ def test_given_managed_loader_watermark_when_reloaded_then_incremental_consumes_ """ ).strip() + "\n", - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n\n" "@loader\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py index f67b193c43..7fb52be598 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -18,8 +18,8 @@ run_sqb, ) -_LOADERS_PATH: str = "loaders/orders.py" -_ASSETS_PATH: str = "assets/exports.py" +_LOADERS_PATH: str = "python/loaders/orders.py" +_ASSETS_PATH: str = "python/assets/exports.py" _ORIGINAL_COLUMNS: str = 'LoaderColumnSpec(name="order_id", type="INTEGER")' _ORIGINAL_RETRY: str = "RetryPolicy(max_attempts=2, retry_on=RuntimeError)" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py index 8629d350ac..5c63d0ef33 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py @@ -155,7 +155,7 @@ def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side "[targets.dev.state.connection]\n" 'database = "state.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -163,10 +163,10 @@ def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side " Path(__file__).parents[1].joinpath('prepared.txt').write_text('7')\n" " return ctx.result(payload={'order_id': 7})\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n" - "from tasks.prepare import prepare_orders\n\n" + "from python.tasks.prepare import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" @@ -184,7 +184,7 @@ def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model, source\n" "from sqlbuild.tasks import task\n\n" @@ -283,7 +283,7 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in "[targets.dev.state.connection]\n" 'database = "state.duckdb"\n' ), - "loaders/orders.py": ( + "python/loaders/orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" @@ -298,9 +298,9 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in " - name: value\n" " type: INTEGER\n" ), - "tasks/results.py": ( + "python/tasks/results.py": ( "from pathlib import Path\n" - "from loaders.orders import raw_orders\n" + "from python.loaders.orders import raw_orders\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('orders'))\n" @@ -322,9 +322,9 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in " )\n" " return ctx.result(metadata={'summarized': True})\n" ), - "assets/results.py": ( + "python/assets/results.py": ( "from sqlbuild.assets import asset\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@asset(depends_on=produce_result)\n" "def publish_result(ctx):\n" " payload = ctx.result_of(node_function=produce_result).payload\n" @@ -333,10 +333,10 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in "models/orders.sql": ( 'MODEL (materialized table);\n\nSELECT value FROM __source("raw_orders")\n' ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from assets.results import publish_result\n" - "from tasks.results import produce_result, summarize_loader\n\n" + "from python.assets.results import publish_result\n" + "from python.tasks.results import produce_result, summarize_loader\n\n" "@check(depends_on=(publish_result, summarize_loader))\n" "def check_produce_result(ctx):\n" " return (\n" @@ -431,7 +431,7 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in ).strip() + "\n", "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('orders'))\n" @@ -472,16 +472,16 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in ).strip() + "\n", "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('orders'))\n" "def produce_result(ctx):\n" " return ctx.result(payload={'value': 1})\n" ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" " return False\n" @@ -523,7 +523,7 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in ).strip() + "\n", "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" @@ -565,15 +565,15 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in """ ).strip() + "\n", - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from sqlbuild.compiler.python_nodes.types import SkipMode\n" "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" " return ctx.skip(reason='no input', mode=SkipMode.HARD)\n" ), - "loaders/events.py": ( - "from tasks.prepare import prepare_events\n" + "python/loaders/events.py": ( + "from python.tasks.prepare import prepare_events\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_events,))\n" "def raw_events(ctx):\n" @@ -628,7 +628,7 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in ).strip() + "\n", "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('orders'))\n" @@ -669,7 +669,7 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in ).strip() + "\n", "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS id\n", - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('orders'))\n" @@ -774,7 +774,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat "[targets.dev.state.connection]\n" 'database = "state.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -782,10 +782,10 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat " Path(__file__).parents[1].joinpath('prepared.txt').write_text('7')\n" " return ctx.result(payload={'order_id': 7})\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n" - "from tasks.prepare import prepare_orders\n\n" + "from python.tasks.prepare import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" @@ -803,7 +803,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" @@ -843,7 +843,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat ), ) == [(test_case.expected_warehouse_fingerprint_table_count,)] - (project_dir / "tasks" / "prepare.py").write_text( + (project_dir / "python" / "tasks" / "prepare.py").write_text( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -908,7 +908,7 @@ def test_given_virtual_read_side_python_failure_when_building_then_prints_python 'database = "state.duckdb"\n' ), "models/fact_orders.sql": "MODEL (materialized table);\n\nSELECT 7 AS order_id\n", - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" @@ -975,7 +975,7 @@ def test_given_virtual_read_side_python_skip_when_building_then_prints_python_sk 'database = "state.duckdb"\n' ), "models/fact_orders.sql": "MODEL (materialized table);\n\nSELECT 7 AS order_id\n", - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" @@ -1041,7 +1041,7 @@ def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_ "[targets.dev.state.connection]\n" 'database = "state.duckdb"\n' ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -1049,10 +1049,10 @@ def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_ " Path(__file__).parents[1].joinpath('prepared.txt').write_text('7')\n" " return ctx.result(payload={'order_id': 7})\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n" - "from tasks.prepare import prepare_orders\n\n" + "from python.tasks.prepare import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" @@ -1070,7 +1070,7 @@ def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_ "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from pathlib import Path\n" "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py index abbed11435..627c52cc97 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py @@ -480,7 +480,7 @@ def test_given_virtual_managed_source_freshness_when_unchanged_then_build_skips_ "sqlbuild_project.toml": build_virtual_plan_project_toml().replace( "[targets.dev]\n", '[targets.dev]\ndefer_sources_to = "dev"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n\n" "@loader\n" @@ -551,7 +551,7 @@ def test_given_virtual_managed_source_configured_freshness_when_building_then_re "sqlbuild_project.toml": build_virtual_plan_project_toml().replace( "[targets.dev]\n", '[targets.dev]\ndefer_sources_to = "dev"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from pathlib import Path\n" "from sqlbuild.loaders import loader\n\n" "@loader\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py index 3bccc00a63..c02364afa2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/helpers.py @@ -24,7 +24,7 @@ def prepare_python_check_project(*, tmp_path: Path) -> Path: [connection] database = "python_check_project.duckdb" """, - "tasks/orders.py": """ + "python/tasks/orders.py": """ from sqlbuild.tasks import task @@ -37,19 +37,19 @@ def export_orders(ctx): def export_customers(ctx): return ctx.result(payload={"rows": 2}, metadata={"rows": 2}) """, - "assets/orders.py": """ + "python/assets/orders.py": """ from sqlbuild.assets import asset -from tasks.orders import export_orders +from python.tasks.orders import export_orders @asset(depends_on=export_orders) def orders_asset(ctx): return ctx.result(payload={"asset_rows": 3}, metadata={"asset_rows": 3}) """, - "checks/orders.py": """ + "python/checks/orders.py": """ from sqlbuild.checks import check -from assets.orders import orders_asset -from tasks.orders import export_customers, export_orders +from python.assets.orders import orders_asset +from python.tasks.orders import export_customers, export_orders @check(depends_on=export_orders) @@ -106,7 +106,7 @@ def prepare_terminal_loader_check_project(*, tmp_path: Path) -> Path: [connection] database = "terminal_loader_check_project.duckdb" """, - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @@ -123,9 +123,9 @@ def raw_orders(ctx): - name: order_id type: INTEGER """, - "checks/raw.py": """ + "python/checks/raw.py": """ from sqlbuild.checks import check -from loaders.raw import raw_orders +from python.loaders.raw import raw_orders @check(depends_on=raw_orders) @@ -155,7 +155,7 @@ def prepare_read_side_python_check_project(*, tmp_path: Path) -> Path: SELECT 1 AS order_id """, - "assets/orders.py": """ + "python/assets/orders.py": """ from sqlbuild.assets import asset from sqlbuild.refs import model @@ -166,9 +166,9 @@ def orders_export(ctx): row = ctx.query(f"SELECT COUNT(*) AS order_count FROM {relation}").fetchone() return ctx.result(payload={"order_count": row[0]}, materialized=False) """, - "checks/orders.py": """ + "python/checks/orders.py": """ from sqlbuild.checks import check -from assets.orders import orders_export +from python.assets.orders import orders_export @check(depends_on=orders_export) @@ -208,7 +208,7 @@ def prepare_virtual_python_check_project(*, tmp_path: Path) -> Path: database = "state.duckdb" """, "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "tasks/export.py": """ + "python/tasks/export.py": """ from sqlbuild.refs import model from sqlbuild.tasks import task @@ -217,9 +217,9 @@ def prepare_virtual_python_check_project(*, tmp_path: Path) -> Path: def export_virtual_orders(ctx): return ctx.result(metadata={"rows": 1}) """, - "checks/export.py": """ + "python/checks/export.py": """ from sqlbuild.checks import check -from tasks.export import export_virtual_orders +from python.tasks.export import export_virtual_orders @check(depends_on=export_virtual_orders) @@ -259,7 +259,7 @@ def prepare_virtual_failing_python_check_project(*, tmp_path: Path) -> Path: database = "state.duckdb" """, "models/stg_orders.sql": "MODEL ();\n\nSELECT 1 AS id\n", - "tasks/export.py": """ + "python/tasks/export.py": """ from sqlbuild.refs import model from sqlbuild.tasks import task @@ -268,9 +268,9 @@ def prepare_virtual_failing_python_check_project(*, tmp_path: Path) -> Path: def export_virtual_orders(ctx): return ctx.result(metadata={"rows": 1}) """, - "checks/export.py": """ + "python/checks/export.py": """ from sqlbuild.checks import check -from tasks.export import export_virtual_orders +from python.tasks.export import export_virtual_orders @check(depends_on=export_virtual_orders) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py index 92c8bf9fc1..332c5d933d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clone/test_clone.py @@ -298,7 +298,7 @@ def test_given_invalid_origin_connection_when_exact_cloning_then_destination_ses """ ).strip() + "\n", - "loaders/raw.py": dedent( + "python/loaders/raw.py": dedent( """ from sqlbuild.loaders import loader @@ -378,7 +378,7 @@ def raw_customers(ctx): """ ).strip() + "\n", - "loaders/raw.py": dedent( + "python/loaders/raw.py": dedent( """ from sqlbuild.loaders import loader @@ -753,7 +753,7 @@ def test_given_destination_source_deferral_when_cloning_then_function_reads_defe """ ).strip() + "\n", - "loaders/raw.py": dedent( + "python/loaders/raw.py": dedent( """ from sqlbuild.loaders import loader diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py index f476734e74..35980a6788 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_namespace_validation.py @@ -145,7 +145,7 @@ def main(): repo_files={ "sqlbuild_project.toml": _PRESERVE_CONFIG, "sources/raw.yml": _MANAGED_SOURCE, - "loaders/raw_events.py": _MANAGED_SOURCE_LOADER, + "python/loaders/raw_events.py": _MANAGED_SOURCE_LOADER, }, expected_exit_code=1, expected_stderr_fragment="Managed source 'raw_events' has no logical schema", @@ -269,7 +269,7 @@ def main(): repo_files={ "sqlbuild_project.toml": _POSTGRES_CONFIG_WITHOUT_SCHEMA, "sources/raw.yml": _MANAGED_SOURCE, - "loaders/raw_events.py": _MANAGED_SOURCE_LOADER, + "python/loaders/raw_events.py": _MANAGED_SOURCE_LOADER, }, expected_exit_code=1, expected_stderr_fragment=( @@ -376,7 +376,7 @@ def main(): + '\n[targets.dev]\nloader_schema = "raw"\n' ), "sources/raw.yml": _MANAGED_SOURCE, - "loaders/raw_events.py": _MANAGED_SOURCE_LOADER, + "python/loaders/raw_events.py": _MANAGED_SOURCE_LOADER, }, expected_exit_code=1, expected_stderr_fragment="Named target 'dev' must explicitly set schema", @@ -394,7 +394,7 @@ def main(): "seeds/countries.csv": "code\nGB\n", "functions/sql/answer.sql": "FUNCTION (returns INTEGER);\n\n42\n", "sources/raw.yml": _MANAGED_SOURCE, - "loaders/raw_events.py": _MANAGED_SOURCE_LOADER, + "python/loaders/raw_events.py": _MANAGED_SOURCE_LOADER, }, expected_exit_code=0, expected_stderr_fragment="", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index 6d20890f98..8c27d9d1a5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -24,8 +24,117 @@ }, expected_exit_code=1, expected_stderr_fragments=( + "error[D016]", "Unsupported project Python path(s): audit_helpers/measurements.py", - "Factory support modules, including factories/**/_helpers.py, are allowed", + "Move pipeline Python nodes and the helper modules they import under python/", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed tasks Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "tasks/orders.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): tasks/orders.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed assets Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "assets/orders.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): assets/orders.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed checks Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "checks/orders.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): checks/orders.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed loaders Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "loaders/orders.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): loaders/orders.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed factories Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "factories/orders.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): factories/orders.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed libs Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "libs/cleaning.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): libs/cleaning.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed dagster Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "dagster/definitions.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): dagster/definitions.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed rivers_pipeline Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "rivers_pipeline/definitions.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): rivers_pipeline/definitions.py", + ), + ), + PythonProjectLayoutCompileTestCase( + description="removed root definitions.py Python root", + repo_files={ + "sqlbuild_project.toml": 'name = "python_layout"\nadapter = "duckdb"\n', + "definitions.py": "from sqlbuild.tasks import task\n", + }, + expected_exit_code=1, + expected_stderr_fragments=( + "error[D016]", + "Unsupported project Python path(s): definitions.py", ), ), ), @@ -85,5 +194,118 @@ def test_given_rule_harness_test_when_compiling_then_python_test_path_is_accepte assert "Project compiled 1 model" in result.stdout +@pytest.mark.parametrize( + "test_case", + ( + PythonProjectLayoutCompileTestCase( + description="python root with nested nodes and undecorated helpers", + repo_files={ + "sqlbuild_project.toml": ( + 'name = "python_root_layout"\n' + 'adapter = "duckdb"\n' + 'default_target = "dev"\n\n' + "[connections.developer]\n" + 'database = "python_root_layout.duckdb"\n\n' + "[targets.dev]\n" + 'connection = "developer"\n' + 'schema = "main"\n' + ), + "sources/raw.yml": ( + "sources:\n" + " - name: raw_orders\n" + " managed: true\n" + " write_strategy: table\n" + " columns:\n" + " - name: order_id\n" + " type: INTEGER\n" + " - name: status\n" + " type: VARCHAR\n" + ), + "models/fact_orders.sql": ( + "MODEL (materialized table);\n\n" + 'SELECT order_id, status FROM __source("raw_orders")\n' + ), + "python/_helpers.py": ( + "def normalize_status(value):\n return value.strip().lower()\n" + ), + "python/orders/__init__.py": "", + "python/orders/utils.py": ( + "from python._helpers import normalize_status\n\n\n" + "def order_rows():\n" + " return [\n" + " {'order_id': 1, 'status': normalize_status(' Shipped ')},\n" + " {'order_id': 2, 'status': normalize_status('PENDING')},\n" + " ]\n" + ), + "python/loaders/orders.py": ( + "from python.orders.utils import order_rows\n" + "from sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def raw_orders(ctx):\n" + " return order_rows()\n" + ), + "python/orders/exports/assets.py": ( + "from sqlbuild.assets import asset\n" + "from sqlbuild.refs import model\n\n\n" + "@asset(depends_on=model('fact_orders'))\n" + "def orders_export(ctx):\n" + " relation = ctx.relation(model('fact_orders'))\n" + " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()\n" + " return ctx.result(payload={'order_count': int(rows[0][0])})\n" + ), + "python/orders/exports/checks.py": ( + "from python.orders.exports.assets import orders_export\n" + "from sqlbuild.checks import check\n\n\n" + "@check(depends_on=orders_export)\n" + "def orders_export_not_empty(ctx):\n" + " payload = ctx.result_of(node_function=orders_export).payload\n" + " if payload['order_count'] <= 0:\n" + " return ctx.fail(message='orders export is empty')\n" + " return ctx.pass_(message='orders export is ready')\n" + ), + }, + expected_exit_code=0, + expected_stderr_fragments=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_python_root_project_when_compiling_and_building_then_nodes_run_with_helpers( + test_case: PythonProjectLayoutCompileTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_root_layout", + repo_files=test_case.repo_files, + ) + + compile_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + ) + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + ) + + compile_output: str = compile_result.stdout + compile_result.stderr + build_output: str = build_result.stdout + build_result.stderr + assert compile_result.returncode == test_case.expected_exit_code, compile_output + assert build_result.returncode == test_case.expected_exit_code, build_output + assert all( + fragment in build_output + for fragment in ( + "raw_orders", + "fact_orders", + "orders_export", + "orders_export_not_empty", + "orders export is ready", + ) + ) + assert "_helpers" not in build_output + assert "order_rows" not in build_output + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py index 4120d7eead..36438d4293 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/databricks/test_databricks.py @@ -168,15 +168,15 @@ def test_given_python_result_when_running_check_on_databricks_then_persists_node project_name="databricks_node_results", schema_name=schema_name, ), - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" " return ctx.result(payload={'value': 42}, metadata={'source': 'databricks'})\n" ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py index 528d54e0b1..b1f49e70ac 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/freshness/helpers.py @@ -37,7 +37,7 @@ def prepare_freshness_project( { False: {}, True: { - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_managed(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/init/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/init/_test_types.py index 88f05e7ecb..254eedc97b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/init/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/init/_test_types.py @@ -13,3 +13,13 @@ class InitE2ETestCase: expected_exit_code: int expected_paths: tuple[str, ...] expected_output_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class InitPythonRootE2ETestCase: + """Test case for building Python nodes added to an initialized project.""" + + description: str + project_files: dict[str, str] + unexpected_paths: tuple[str, ...] + expected_build_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py index a0c94f2770..a5b67872a8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/init/test_init.py @@ -7,7 +7,10 @@ import pytest -from tests.e2e.src.sqlbuild.cli.commands.main.init._test_types import InitE2ETestCase +from tests.e2e.src.sqlbuild.cli.commands.main.init._test_types import ( + InitE2ETestCase, + InitPythonRootE2ETestCase, +) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import run_sqb @@ -19,6 +22,8 @@ expected_exit_code=0, expected_paths=( "sqlbuild_project.toml", + "python", + "python/.gitkeep", "hooks/sql", "hooks/sql/.gitkeep", "hooks/python", @@ -30,6 +35,7 @@ ), expected_output_fragments=( "SQLBuild project created", + "Add Python loaders, tasks, assets, checks, and factories to python/", "Add SQL hooks to hooks/sql/, Python hooks to hooks/python/", "reusable audits to audits/generic/", "standalone audits to audits/singular/", @@ -55,3 +61,76 @@ def test_given_empty_project_directory_when_running_init_then_typed_resource_dir assert fragment in result.stdout for expected_path in test_case.expected_paths: assert (project_dir / expected_path).exists() + + +@pytest.mark.parametrize( + "test_case", + [ + InitPythonRootE2ETestCase( + description="initialized project builds Python nodes and helpers under python", + project_files={ + "models/marts/fact_orders.sql": ( + "MODEL (materialized table);\n\nSELECT 1 AS order_id\n" + ), + "python/_helpers.py": "def order_count(rows):\n return len(rows)\n", + "python/tasks/orders.py": ( + "from python._helpers import order_count\n" + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def count_orders(ctx):\n" + " return ctx.result(payload={'order_count': order_count([1, 2])})\n" + ), + "python/assets/orders_export.py": ( + "from python.tasks.orders import count_orders\n" + "from sqlbuild.assets import asset\n" + "from sqlbuild.refs import model\n\n\n" + "@asset(depends_on=(model('fact_orders'), count_orders))\n" + "def orders_export(ctx):\n" + " return ctx.result(payload={'ready': True})\n" + ), + }, + unexpected_paths=("tasks", "assets", "checks", "loaders", "factories", "libs"), + expected_build_fragments=( + "fact_orders", + "count_orders", + "orders_export", + "\u2713 Completed successfully", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_initialized_project_with_python_root_nodes_when_building_then_nodes_run( + test_case: InitPythonRootE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = tmp_path / "init_python_root_project" + project_dir.mkdir() + init_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "init"), + project_dir=project_dir, + ) + assert init_result.returncode == 0, init_result.stdout + init_result.stderr + relative_path: str + contents: str + for relative_path, contents in test_case.project_files.items(): + file_path: Path = project_dir / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(contents, encoding="utf-8") + + compile_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), + project_dir=project_dir, + ) + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + ) + + assert compile_result.returncode == 0, compile_result.stdout + compile_result.stderr + build_output: str = build_result.stdout + build_result.stderr + assert build_result.returncode == 0, build_output + for fragment in test_case.expected_build_fragments: + assert fragment in build_output + for unexpected_path in test_case.unexpected_paths: + assert not (project_dir / unexpected_path).exists() diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py index f755e66697..2fd1f06b52 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py @@ -33,7 +33,7 @@ def assert_source_only_ingress_dependency_case( source_yaml=("sources:\n - name: raw_events\n managed: true\n"), loader_py=( "from sqlbuild.loaders import loader\n" - "from tasks.prepare import prepare_events\n\n" + "from python.tasks.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events], write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" " {'name': 'load_seq', 'type': 'INTEGER'},\n" @@ -50,7 +50,7 @@ def assert_source_only_ingress_dependency_case( "SELECT event_id FROM {events.destination}')\n" ), ), - "tasks/prepare.py": ( + "python/tasks/prepare.py": ( "from pathlib import Path\n" "from sqlbuild.tasks import task\n\n" "@task\n" @@ -156,7 +156,7 @@ def build_schema_behavior_project_files(*, source_yaml: str, loader_py: str) -> 'database = "source_loader_schema_behavior.duckdb"\n' ), "sources/raw.yml": source_yaml, - "loaders/source_rows.py": loader_py, + "python/loaders/source_rows.py": loader_py, } @@ -260,7 +260,7 @@ def build_loader_waffle_shop_project_files(*, project_toml: str | None = None) - " - name: load_seq\n" " type: INTEGER\n" ), - "loaders/waffle_loaders.py": ( + "python/loaders/waffle_loaders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='append', cursor_column='load_seq', columns=[\n" " {'name': 'order_id', 'type': 'INTEGER'},\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 0ac0bbd6b6..255c678e4a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -623,7 +623,7 @@ def test_given_source_loader_reads_unselected_task_payload_when_loading_then_run test_case: SourceOnlyComplexIngressE2ETestCase, ) -> None: assert test_case.expected_error_fragment is not None - dependency_file: str = f"{test_case.dependency_dir}/prepare.py" + dependency_file: str = f"python/{test_case.dependency_dir}/prepare.py" project_dir: Path = prepare_inline_project( tmp_path=tmp_path, project_name="source_loader_schema_behavior", @@ -640,7 +640,7 @@ def test_given_source_loader_reads_unselected_task_payload_when_loading_then_run ), loader_py=( "from sqlbuild.loaders import loader\n" - f"from {test_case.dependency_dir}.prepare import prepare_events\n\n" + f"from python.{test_case.dependency_dir}.prepare import prepare_events\n\n" "@loader(depends_on=[prepare_events])\n" "def raw_events(ctx):\n" " payload = ctx.result_of(node_function=prepare_events).payload\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py index 2243457d74..4df324596c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/helpers.py @@ -147,7 +147,7 @@ def prepare_python_lifecycle_plan_project(*, tmp_path: Path) -> Path: "[connection]\n" 'database = "warehouse.duckdb"\n' ), - "loaders/window.py": ( + "python/loaders/window.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(\n" " destination='window_orders',\n" @@ -157,15 +157,15 @@ def prepare_python_lifecycle_plan_project(*, tmp_path: Path) -> Path: "def load_window_orders(ctx):\n" " return [{'order_id': 7}]\n" ), - "tasks/prepare.py": ( - "from loaders.window import load_window_orders\n" + "python/tasks/prepare.py": ( + "from python.loaders.window import load_window_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=load_window_orders)\n" "def prepare_orders(ctx):\n" " return ctx.result(payload={'order_id': 7})\n" ), - "assets/prepare.py": ( - "from tasks.prepare import prepare_orders\n" + "python/assets/prepare.py": ( + "from python.tasks.prepare import prepare_orders\n" "from sqlbuild.assets import asset\n\n" "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" @@ -174,8 +174,8 @@ def prepare_python_lifecycle_plan_project(*, tmp_path: Path) -> Path: " materialized=True\n" " )\n" ), - "loaders/raw.py": ( - "from assets.prepare import publish_prepared_orders\n" + "python/loaders/raw.py": ( + "from python.assets.prepare import publish_prepared_orders\n" "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" @@ -193,15 +193,15 @@ def prepare_python_lifecycle_plan_project(*, tmp_path: Path) -> Path: "models/fact_orders.sql": ( 'MODEL (materialized table);\n\nSELECT * FROM __source("raw_orders")\n' ), - "tasks/profile.py": ( + "python/tasks/profile.py": ( "from sqlbuild.refs import model\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" " return ctx.result(payload={'rows': 1})\n" ), - "tasks/notify.py": ( - "from tasks.profile import profile_fact_orders\n" + "python/tasks/notify.py": ( + "from python.tasks.profile import profile_fact_orders\n" "from sqlbuild.tasks import task\n\n" "@task(depends_on=profile_fact_orders)\n" "def notify_fact_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py index 4681156f78..e11a8ee0fa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/plan/test_virtual_plan_incremental.py @@ -209,7 +209,7 @@ def test_given_managed_source_without_observation_when_planning_then_matches_bui '[targets.dev]\nschema = "dev"\n\n', '[targets.dev]\nschema = "dev"\ndefer_sources_to = "dev"\n\n', ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py index 0bb1fa7a53..3925f0d5e7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -23,6 +23,14 @@ class PythonNodesPlaygroundLifecycleTestCase: expected_check_fragments: tuple[str, ...] +@dataclass(frozen=True) +class PlaygroundCompileBuildTestCase: + description: str + template: str + project_subdir: str + expected_build_fragments: tuple[str, ...] + + @dataclass(frozen=True) class UnicodeEmptyFixtureTestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py index 02b3fd424d..13fc14a02a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -8,6 +8,7 @@ from sqlbuild.cli.commands.main.workspace._playground import run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest from tests.e2e.src.sqlbuild.cli.commands.main.playground._test_types import ( + PlaygroundCompileBuildTestCase, PythonNodesPlaygroundLifecycleTestCase, VirtualPlaygroundLifecycleTestCase, ) @@ -222,3 +223,62 @@ def test_given_python_nodes_playground_when_running_lifecycle_then_it_succeeds( assert check_result.returncode == 0 for expected_fragment in test_case.expected_check_fragments: assert expected_fragment in check_output + + +@pytest.mark.parametrize( + "test_case", + [ + PlaygroundCompileBuildTestCase( + description="waffle shop playground loaders under python compile and build", + template="waffle_shop", + project_subdir="", + expected_build_fragments=("raw__customers", "raw__orders", "fact_orders"), + ), + PlaygroundCompileBuildTestCase( + description="loader waffle shop playground loaders under python compile and build", + template="loader_waffle_shop", + project_subdir="", + expected_build_fragments=("raw_orders", "raw_customers", "fact_waffle_orders"), + ), + PlaygroundCompileBuildTestCase( + description="dagster playground nested SQLBuild project compiles and builds", + template="dagster", + project_subdir="waffle_shop", + expected_build_fragments=("raw__customers", "raw__orders", "fact_orders"), + ), + PlaygroundCompileBuildTestCase( + description="rivers playground nested SQLBuild project compiles and builds", + template="rivers", + project_subdir="waffle_shop", + expected_build_fragments=("raw__customers", "raw__orders", "fact_orders"), + ), + ], + ids=lambda case: case.description, +) +def test_given_playground_template_when_compiling_and_building_then_it_succeeds( + test_case: PlaygroundCompileBuildTestCase, + tmp_path: Path, +) -> None: + assert ( + run_playground( + PlaygroundCommandRequest( + project_dir=tmp_path, target_path="playground", template=test_case.template + ) + ) + == 0 + ) + project_dir: Path = tmp_path / "playground" / test_case.project_subdir + + compile_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "compile"), project_dir=project_dir + ) + build_result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert compile_result.returncode == 0, compile_result.stdout + compile_result.stderr + build_output: str = build_result.stdout + build_result.stderr + assert build_result.returncode == 0, build_output + expected_fragment: str + for expected_fragment in (*test_case.expected_build_fragments, "\u2713 Completed successfully"): + assert expected_fragment in build_output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py index 0fdf5f26e5..1284767841 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/postgres/test_postgres.py @@ -258,15 +258,15 @@ def test_given_python_result_when_running_check_on_postgres_then_persists_node_r schema_name=schema_name, config=postgres_e2e_config, ), - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" " return ctx.result(payload={'value': 42}, metadata={'source': 'postgres'})\n" ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" @@ -399,7 +399,7 @@ def test_given_source_deferral_env_when_building_on_postgres_then_reads_prod_and " - name: status\n" " type: VARCHAR\n" ), - "loaders/raw_orders.py": ( + "python/loaders/raw_orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index 0a4e94d66e..aed52c64fb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -131,7 +131,7 @@ class MarkerProvider(Provider): """ ).strip() + "\n", - "loaders/raw_orders.py": dedent( + "python/loaders/raw_orders.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.loaders import loader @@ -143,7 +143,7 @@ def raw_orders(marker_provider: MarkerProvider): """ ).strip() + "\n", - "tasks/publish_orders.py": dedent( + "python/tasks/publish_orders.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.tasks import task @@ -319,7 +319,7 @@ def mark(self, value): """ ).strip() + "\n", - "tasks/provider_task.py": dedent( + "python/tasks/provider_task.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.tasks import task @@ -331,7 +331,7 @@ def provider_task(ctx, marker_provider: MarkerProvider): """ ).strip() + "\n", - "assets/provider_asset.py": dedent( + "python/assets/provider_asset.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.assets import asset @@ -343,7 +343,7 @@ def provider_asset(ctx, marker_provider: MarkerProvider): """ ).strip() + "\n", - "checks/provider_check.py": dedent( + "python/checks/provider_check.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.checks import check @@ -356,7 +356,7 @@ def provider_check(ctx, marker_provider: MarkerProvider): """ ).strip() + "\n", - "loaders/provider_loader.py": dedent( + "python/loaders/provider_loader.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.loaders import loader @@ -436,7 +436,7 @@ def raw_provider_events(ctx, marker_provider: MarkerProvider): repo_files={ "sqlbuild_project.toml": PROVIDER_FAILURE_PROJECT_TOML, "providers/marker.py": PROVIDER_MARKER_FILE, - "tasks/failing_task.py": dedent( + "python/tasks/failing_task.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.tasks import task @@ -458,7 +458,7 @@ def failing_task(ctx, marker_provider: MarkerProvider): repo_files={ "sqlbuild_project.toml": PROVIDER_FAILURE_PROJECT_TOML, "providers/marker.py": PROVIDER_MARKER_FILE, - "assets/failing_asset.py": dedent( + "python/assets/failing_asset.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.assets import asset @@ -480,7 +480,7 @@ def failing_asset(ctx, marker_provider: MarkerProvider): repo_files={ "sqlbuild_project.toml": PROVIDER_FAILURE_PROJECT_TOML, "providers/marker.py": PROVIDER_MARKER_FILE, - "checks/failing_check.py": dedent( + "python/checks/failing_check.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.checks import check @@ -502,7 +502,7 @@ def failing_check(ctx, marker_provider: MarkerProvider): repo_files={ "sqlbuild_project.toml": PROVIDER_FAILURE_PROJECT_TOML, "providers/marker.py": PROVIDER_MARKER_FILE, - "loaders/failing_loader.py": dedent( + "python/loaders/failing_loader.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.loaders import loader @@ -582,7 +582,7 @@ def test_given_provider_project_when_running_compile_or_plan_then_provider_setup repo_files={ "sqlbuild_project.toml": PROVIDER_FAILURE_PROJECT_TOML, "providers/marker.py": PROVIDER_MARKER_FILE, - "tasks/provider_task.py": dedent( + "python/tasks/provider_task.py": dedent( """ from providers.marker import MarkerProvider from sqlbuild.tasks import task @@ -632,7 +632,7 @@ def test_given_alias_imported_provider_annotation_when_running_command_then_cli_ repo_files={ "sqlbuild_project.toml": PROVIDER_FAILURE_PROJECT_TOML, "providers/marker.py": PROVIDER_MARKER_FILE, - "tasks/alias_task.py": dedent( + "python/tasks/alias_task.py": dedent( """ import importlib.util import sys @@ -677,7 +677,7 @@ def alias_task(ctx, marker_provider: AliasMarkerProvider): repo_files={ "sqlbuild_project.toml": build_virtual_plan_project_toml(), "providers/marker.py": PROVIDER_MARKER_FILE, - "tasks/failing_task.py": dedent( + "python/tasks/failing_task.py": dedent( """ from sqlbuild.refs import model from providers.marker import MarkerProvider @@ -839,7 +839,7 @@ def mark(self, label): """ ).strip() + "\n", - "loaders/events.py": dedent( + "python/loaders/events.py": dedent( """ from providers.concurrent_marker import ConcurrentMarkerProvider from sqlbuild.loaders import loader diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py index bf0e530265..13fbb876be 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/snowflake/test_snowflake.py @@ -186,15 +186,15 @@ def test_given_python_result_when_running_check_on_snowflake_then_persists_node_ project_name="snowflake_node_results", schema_name=schema_name, ), - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" " return ctx.result(payload={'value': 42}, metadata={'source': 'snowflake'})\n" ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" @@ -868,7 +868,7 @@ def test_given_source_deferral_env_when_building_on_snowflake_then_reads_prod_an " - name: status\n" " type: VARCHAR\n" ), - "loaders/raw_orders.py": ( + "python/loaders/raw_orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py index 008705a424..26421ea52c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/helpers.py @@ -400,7 +400,7 @@ def prepare_sqlserver_source_loader_strategies( source_path.read_text(encoding="utf-8").replace("type: TIMESTAMP", "type: DATETIME2"), encoding="utf-8", ) - loader_path: Path = project_dir / "loaders" / "strategy_sources.py" + loader_path: Path = project_dir / "python" / "loaders" / "strategy_sources.py" old_sql: str = ( 'f"CREATE TABLE {ctx.destination} AS "\n' " \"SELECT 1 AS status_id, 'loaded' AS status_name, " diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py index e544ba5552..f9780db95d 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/sqlserver/test_sqlserver.py @@ -120,7 +120,7 @@ def test_given_managed_source_when_cloning_sqlserver_then_source_precedes_depend "[targets.prod.clone]\n" "allow_as_clone_destination = true\n" ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_customers(ctx):\n" @@ -438,15 +438,15 @@ def test_given_python_result_when_running_check_on_sqlserver_then_persists_node_ schema_name=schema_name, config=config, ), - "tasks/results.py": ( + "python/tasks/results.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def produce_result(ctx):\n" " return ctx.result(payload={'value': 42}, metadata={'source': 'sqlserver'})\n" ), - "checks/results.py": ( + "python/checks/results.py": ( "from sqlbuild.checks import check\n" - "from tasks.results import produce_result\n\n" + "from python.tasks.results import produce_result\n\n" "@check(depends_on=produce_result)\n" "def check_produce_result(ctx):\n" " return ctx.result_of(node_function=produce_result).payload['value'] == 42\n" @@ -1190,7 +1190,7 @@ def test_given_source_loader_project_when_building_on_sqlserver_then_model_reads " - name: status\n" " type: NVARCHAR(100)\n" ), - "loaders/raw_orders.py": ( + "python/loaders/raw_orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" @@ -1262,7 +1262,7 @@ def test_given_source_deferral_env_when_building_on_sqlserver_then_reads_prod_an " - name: status\n" " type: NVARCHAR(100)\n" ), - "loaders/raw_orders.py": ( + "python/loaders/raw_orders.py": ( "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py index c5b92da8b3..4b3bfccf4e 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/helpers.py @@ -26,11 +26,11 @@ def prepare_python_nodes_integration_project(root: Path) -> Path: project_dir: Path = root / "python_nodes_project" models_dir: Path = project_dir / "models" - tasks_dir: Path = project_dir / "tasks" - factories_dir: Path = project_dir / "factories" - assets_dir: Path = project_dir / "assets" - loaders_dir: Path = project_dir / "loaders" - checks_dir: Path = project_dir / "checks" + tasks_dir: Path = project_dir / "python" / "tasks" + factories_dir: Path = project_dir / "python" / "factories" + assets_dir: Path = project_dir / "python" / "assets" + loaders_dir: Path = project_dir / "python" / "loaders" + checks_dir: Path = project_dir / "python" / "checks" for directory in (models_dir, tasks_dir, factories_dir, assets_dir, loaders_dir, checks_dir): directory.mkdir(parents=True) (project_dir / "sqlbuild_project.toml").write_text( diff --git a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py index 0fe9b377a5..d6b60417fb 100644 --- a/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/dagster/test_sqlbuild_assets.py @@ -537,7 +537,7 @@ def test_given_chained_source_loader_when_dagster_selects_source_then_reuses_int "[connection]\n" 'database = "dagster_chained_source_loader.duckdb"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" @@ -628,7 +628,7 @@ def test_given_chained_source_loader_when_dagster_selects_source_without_interme "[connection]\n" 'database = "dagster_missing_intermediate_source_loader.duckdb"\n' ), - "loaders/raw.py": ( + "python/loaders/raw.py": ( "from sqlbuild.loaders import loader\n\n" "@loader(write_strategy='table', columns=[\n" " {'name': 'event_id', 'type': 'INTEGER'},\n" @@ -702,14 +702,13 @@ def test_given_generated_dagster_playground_when_materializing_assets_then_build ) == 0 ) - project_dir: Path = tmp_path / playground_name + orchestrator_dir: Path = tmp_path / playground_name + project_dir: Path = orchestrator_dir / "waffle_shop" sqb_bin_dir: Path = REPO_ROOT / ".venv" / "bin" monkeypatch.setenv("DAGSTER_IS_DEV_CLI", "1") monkeypatch.setenv("PATH", f"{sqb_bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") - generated_defs: dict[str, object] = runpy.run_path( - str(project_dir / "dagster" / "definitions.py") - ) + generated_defs: dict[str, object] = runpy.run_path(str(orchestrator_dir / "definitions.py")) waffle_shop_assets: object = generated_defs["waffle_shop_assets"] sqlbuild_project: SqlBuildProject = generated_defs["SQLBUILD_PROJECT"] # type: ignore[assignment] diff --git a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py index c4a5e2d0b7..ae77782cf8 100644 --- a/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py +++ b/tests/e2e/src/sqlbuild/integrations/rivers/test_sqlbuild_assets.py @@ -180,14 +180,13 @@ def test_given_generated_rivers_playground_when_materializing_assets_then_build_ ) == 0 ) - project_dir: Path = tmp_path / playground_name + orchestrator_dir: Path = tmp_path / playground_name + project_dir: Path = orchestrator_dir / "waffle_shop" sqb_bin_dir: Path = REPO_ROOT / ".venv" / "bin" monkeypatch.setenv("RIVERS_DEPLOYMENT", "dev") monkeypatch.setenv("PATH", f"{sqb_bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") - generated_defs: dict[str, object] = runpy.run_path( - str(project_dir / "rivers_pipeline" / "definitions.py") - ) + generated_defs: dict[str, object] = runpy.run_path(str(orchestrator_dir / "definitions.py")) repo: Any = generated_defs["repo"] sqlbuild_project: SqlBuildProject = generated_defs["SQLBUILD_PROJECT"] # type: ignore[assignment] diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py index 351611341e..79e0b1a17a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_load_command.py @@ -100,7 +100,7 @@ def raw_orders(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -138,7 +138,9 @@ def raw_orders(ctx): """.strip() + "\n", "sources/raw.yml": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["sources/raw.yml"], - "loaders/raw_orders.py": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["loaders/raw_orders.py"], + "python/loaders/raw_orders.py": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES[ + "python/loaders/raw_orders.py" + ], "models/stg_orders.sql": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["models/stg_orders.sql"], } @@ -277,7 +279,7 @@ def raw_orders(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader( @@ -328,7 +330,9 @@ def raw_orders(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["loaders/raw_orders.py"], + "python/loaders/raw_orders.py": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES[ + "python/loaders/raw_orders.py" + ], "models/stg_orders.sql": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["models/stg_orders.sql"], } @@ -369,7 +373,9 @@ def raw_orders(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["loaders/raw_orders.py"], + "python/loaders/raw_orders.py": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES[ + "python/loaders/raw_orders.py" + ], "models/stg_orders.sql": _BUILD_RUN_AUTO_LOAD_PROJECT_FILES["models/stg_orders.sql"], } @@ -479,7 +485,7 @@ def raw_orders(ctx): _BUILD_RUN_AUTO_LOAD_FAILURE_PROJECT_FILES: dict[str, str] = { **_BUILD_RUN_AUTO_LOAD_PROJECT_FILES, - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -490,7 +496,7 @@ def raw_orders(ctx): _BUILD_RUN_AUTO_LOAD_RELOAD_PROJECT_FILES: dict[str, str] = { **_BUILD_RUN_AUTO_LOAD_PROJECT_FILES, - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -522,7 +528,7 @@ def raw_orders(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_customers.py": """ + "python/loaders/raw_customers.py": """ from sqlbuild.loaders import loader @loader @@ -547,7 +553,7 @@ def raw_customers(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -581,7 +587,7 @@ def raw_orders(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -634,14 +640,14 @@ def raw_orders(ctx): write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader def raw_orders(ctx): return [{"order_id": 3, "status": "selected"}] """, - "loaders/raw_events.py": """ + "python/loaders/raw_events.py": """ from sqlbuild.loaders import loader @loader @@ -689,7 +695,7 @@ def raw_events(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -1849,7 +1855,7 @@ def test_given_build_skips_loader_when_manifest_is_written_then_marks_source_aut type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -1902,14 +1908,14 @@ def raw_orders(ctx): write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader def raw_orders(ctx): return [{"order_id": 3, "status": "selected"}] """, - "loaders/raw_events.py": """ + "python/loaders/raw_events.py": """ from sqlbuild.loaders import loader @loader @@ -1957,7 +1963,7 @@ def raw_events(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -2061,7 +2067,7 @@ def test_given_source_loader_when_running_load_then_writes_source_table( type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -2114,14 +2120,14 @@ def raw_orders(ctx): write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader def raw_orders(ctx): return [{"order_id": 3, "status": "selected"}] """, - "loaders/raw_events.py": """ + "python/loaders/raw_events.py": """ from sqlbuild.loaders import loader @loader @@ -2169,7 +2175,7 @@ def raw_events(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -2311,7 +2317,7 @@ def test_given_source_loader_when_running_pipeline_then_drops_stale_staging_firs type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2345,7 +2351,7 @@ def raw_append_events(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2388,7 +2394,7 @@ def raw_merge_customers(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2423,7 +2429,7 @@ def raw_merge_composite(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2468,7 +2474,7 @@ def raw_delete_insert_events(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from datetime import datetime from sqlbuild.loaders import loader @@ -2516,7 +2522,7 @@ def raw_delete_insert_ts(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2551,7 +2557,7 @@ def raw_delete_insert_empty(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader run_count = 0 @@ -2583,7 +2589,7 @@ def raw_no_cursor(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2613,7 +2619,7 @@ def raw_empty_cursor(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2641,7 +2647,7 @@ def raw_missing_cursor_target(ctx): type: BOOLEAN """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2668,7 +2674,7 @@ def raw_reload_context(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2703,7 +2709,7 @@ def raw_cursor_overrides(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2737,7 +2743,7 @@ def raw_append_events(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2780,7 +2786,7 @@ def raw_merge_customers(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2815,7 +2821,7 @@ def raw_merge_composite(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -2860,7 +2866,7 @@ def raw_delete_insert_events(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from datetime import datetime from sqlbuild.loaders import loader @@ -2908,7 +2914,7 @@ def raw_delete_insert_ts(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -3210,7 +3216,7 @@ def delete_insert_cursor_spy( type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader run_count = 0 @@ -3242,7 +3248,7 @@ def raw_no_cursor(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -3272,7 +3278,7 @@ def raw_empty_cursor(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -3504,7 +3510,7 @@ def test_given_cursor_override_flags_when_running_load_then_passes_typed_context type: BIGINT """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ import threading import time @@ -3605,7 +3611,7 @@ def test_given_multiple_source_loaders_when_running_pipeline_then_uses_concurren type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ import threading import time @@ -3728,7 +3734,7 @@ def test_given_loader_dag_when_running_pipeline_then_independent_branches_overla type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from datetime import date, datetime from sqlbuild.loaders import loader @@ -3822,7 +3828,7 @@ def test_given_generator_loader_with_inferred_columns_when_running_load_then_wri type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -3876,7 +3882,7 @@ def test_given_generator_loader_yields_multiple_rows_when_running_load_then_writ type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -3970,7 +3976,7 @@ def test_given_generator_loader_uses_batch_size_when_running_pipeline_then_appen type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4002,7 +4008,7 @@ def raw_missing_known(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4032,7 +4038,7 @@ def raw_empty_generator(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4062,7 +4068,7 @@ def raw_late_null(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4093,7 +4099,7 @@ def raw_batch_size_two(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4182,7 +4188,7 @@ def test_given_batched_loader_variants_when_running_pipeline_then_writes_expecte type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4245,7 +4251,7 @@ def test_given_loader_writes_many_rows_when_running_load_then_formats_human_row_ type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4301,7 +4307,7 @@ def test_given_loader_returns_empty_rows_when_running_load_then_writes_empty_dec type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4351,7 +4357,7 @@ def test_given_self_managed_loader_when_running_load_then_uses_loader_written_ta write_strategy: table """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4375,7 +4381,7 @@ def raw_conflict(ctx): managed: true """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4397,7 +4403,7 @@ def raw_missing_strategy(ctx): write_strategy: table """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4443,7 +4449,7 @@ def test_given_loader_returns_conflicting_types_when_running_load_then_fails_cle load_batch_size: 1 """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4473,7 +4479,7 @@ def raw_batched_conflict(ctx): load_batch_size: 1 """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4501,7 +4507,7 @@ def raw_batched_non_dict(ctx): type: INTEGER """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4537,7 +4543,7 @@ def raw_append_failure(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4573,7 +4579,7 @@ def raw_merge_failure(ctx): type: VARCHAR """.strip() + "\n", - "loaders/raw.py": """ + "python/loaders/raw.py": """ from sqlbuild.loaders import loader @loader @@ -4656,7 +4662,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": _RAW_ORDERS_LOADER, + "python/loaders/raw_orders.py": _RAW_ORDERS_LOADER, }, select=("missing_source",), exclude=(), @@ -4692,7 +4698,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": _RAW_ORDERS_LOADER, + "python/loaders/raw_orders.py": _RAW_ORDERS_LOADER, }, select=(), exclude=("missing_source",), @@ -4713,7 +4719,7 @@ def test_given_later_loader_batch_fails_when_running_load_then_drops_staging( expression: SELECT 1 AS customer_id """.strip() + "\n", - "loaders/raw_orders.py": _RAW_ORDERS_LOADER, + "python/loaders/raw_orders.py": _RAW_ORDERS_LOADER, }, select=(), exclude=("raw_customers",), @@ -4779,7 +4785,7 @@ def test_given_invalid_load_selectors_when_running_load_then_it_raises_clear_err write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": _RAW_ORDERS_LOADER, + "python/loaders/raw_orders.py": _RAW_ORDERS_LOADER, }, select=(), exclude=("raw_orders",), diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py index 7841791409..15d13ced77 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_audit_factories.py @@ -35,7 +35,7 @@ def test_given_generated_and_direct_attachments_when_compiled_then_audit_inputs_ AUDIT (); SELECT * FROM __ref("@model") WHERE NOT (@expression) """, - "factories/quality.py": """ + "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory @audit_factory diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py index 344afba1af..f2a9dd39a0 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_main.py @@ -416,7 +416,7 @@ def test_given_project_files_when_running_compile_pipeline_then_produces_valid_o project_files={ "sqlbuild_project.toml": _PROJECT_TOML, "models/orders.sql": "MODEL (materialized table);\n\nSELECT 1 AS order_id", - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" diff --git a/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py b/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py index 94595c6f72..c07fce1696 100644 --- a/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py +++ b/tests/integration/src/sqlbuild/executor/build/concurrent/test_concurrent_build.py @@ -189,7 +189,7 @@ def mark(self, label): """ ).strip() + "\n", - "loaders/events.py": dedent( + "python/loaders/events.py": dedent( """ from providers.concurrent_marker import ConcurrentMarkerProvider from sqlbuild.loaders import loader diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py index 1cbd25afaa..f16717b056 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/load_selection/helpers.py @@ -49,20 +49,20 @@ def build_load_selection_inputs() -> DiscoveredProjectInputs: ), loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/loaders.py"), - relative_path=Path("loaders/loaders.py"), + file_path=Path("python/loaders/loaders.py"), + relative_path=Path("python/loaders/loaders.py"), name="fetch_orders", function=fetch_orders, ), DiscoveredLoaderFunction( - file_path=Path("loaders/loaders.py"), - relative_path=Path("loaders/loaders.py"), + file_path=Path("python/loaders/loaders.py"), + relative_path=Path("python/loaders/loaders.py"), name="refresh_orders", function=refresh_orders, ), DiscoveredLoaderFunction( - file_path=Path("loaders/loaders.py"), - relative_path=Path("loaders/loaders.py"), + file_path=Path("python/loaders/loaders.py"), + relative_path=Path("python/loaders/loaders.py"), name="load_raw_orders", function=load_raw_orders, depends_on=(fetch_orders,), diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py index 9beb227898..a8aa26f04d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/helpers.py @@ -77,14 +77,14 @@ def prepare_python_dag_project(root: Path) -> Path: """Create a local project with Python tasks, assets, and checks.""" project_dir: Path = prepare_static_dag_project(root) - tasks_dir: Path = project_dir / "tasks" - assets_dir: Path = project_dir / "assets" - loaders_dir: Path = project_dir / "loaders" - checks_dir: Path = project_dir / "checks" - tasks_dir.mkdir() - assets_dir.mkdir() - loaders_dir.mkdir() - checks_dir.mkdir() + tasks_dir: Path = project_dir / "python" / "tasks" + assets_dir: Path = project_dir / "python" / "assets" + loaders_dir: Path = project_dir / "python" / "loaders" + checks_dir: Path = project_dir / "python" / "checks" + tasks_dir.mkdir(parents=True) + assets_dir.mkdir(parents=True) + loaders_dir.mkdir(parents=True) + checks_dir.mkdir(parents=True) (tasks_dir / "prepare_orders.py").write_text( "\n".join( ( @@ -108,7 +108,7 @@ def prepare_python_dag_project(root: Path) -> Path: "\n".join( ( "from sqlbuild.loaders import loader", - "from tasks.prepare_orders import prepare_orders", + "from python.tasks.prepare_orders import prepare_orders", "", "@loader(", " depends_on=(prepare_orders,),", @@ -125,8 +125,8 @@ def prepare_python_dag_project(root: Path) -> Path: "\n".join( ( "from sqlbuild.assets import asset", - "from loaders.warehouse_export import warehouse_export", - "from tasks.prepare_orders import prepare_orders", + "from python.loaders.warehouse_export import warehouse_export", + "from python.tasks.prepare_orders import prepare_orders", "", "@asset(", " depends_on=(prepare_orders, warehouse_export),", @@ -150,9 +150,9 @@ def prepare_python_dag_project(root: Path) -> Path: "\n".join( ( "from sqlbuild.checks import check", - "from assets.orders_export import orders_export", - "from loaders.warehouse_export import warehouse_export", - "from tasks.prepare_orders import prepare_orders", + "from python.assets.orders_export import orders_export", + "from python.loaders.warehouse_export import warehouse_export", + "from python.tasks.prepare_orders import prepare_orders", "", "@check(", " depends_on=(orders_export, prepare_orders, warehouse_export),", @@ -172,7 +172,7 @@ def prepare_python_dag_project(root: Path) -> Path: "\n".join( ( "from sqlbuild.checks import check", - "from loaders.warehouse_export import warehouse_export", + "from python.loaders.warehouse_export import warehouse_export", "", "@check(depends_on=warehouse_export, tags=['loader'])", "def check_loader_export(ctx):", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py index 534e98ec16..7d34289ba8 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/dag/test_dag_command.py @@ -141,7 +141,7 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit "task:prepare_orders", "loader:warehouse_export", ], - "path": "checks/check_orders_export.py", + "path": "python/checks/check_orders_export.py", "description": "Orders export is present", "severity": "error", "tags": ["daily"], @@ -154,7 +154,7 @@ def test_given_local_project_when_running_dag_json_then_outputs_static_graph_wit "kind": "python_check", "name": "check_loader_export", "checked_asset_ids": ["loader:warehouse_export"], - "path": "checks/check_loader_export.py", + "path": "python/checks/check_loader_export.py", "severity": "error", "tags": ["loader"], }, diff --git a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py index 4aebeb67b9..43e3916b90 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/init/test_run.py @@ -21,10 +21,7 @@ "schemas", "sources", "seeds", - "loaders", - "tasks", - "assets", - "checks", + "python", "hooks/sql", "hooks/python", "tests/unit", @@ -41,10 +38,7 @@ "schemas/.gitkeep", "sources/.gitkeep", "seeds/.gitkeep", - "loaders/.gitkeep", - "tasks/.gitkeep", - "assets/.gitkeep", - "checks/.gitkeep", + "python/.gitkeep", "hooks/sql/.gitkeep", "hooks/python/.gitkeep", "tests/unit/.gitkeep", @@ -64,8 +58,8 @@ "Config: sqlbuild_project.toml", "Next steps:", "1. Add sources to sources/", - "2. Add seeds to seeds/ or loaders to loaders/", - "3. Add tasks to tasks/, assets to assets/, or checks to checks/", + "2. Add seeds to seeds/", + "3. Add Python loaders, tasks, assets, checks, and factories to python/", "4. Add SQL hooks to hooks/sql/, Python hooks to hooks/python/", "reusable audits to audits/generic/", "standalone audits to audits/singular/", diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py index 1957e2ad77..9cc621ba1c 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -21,3 +21,4 @@ class RunPlaygroundTestCase: expected_stdout_fragments: tuple[str, ...] template: str = "waffle_shop" expected_color_fragments: tuple[str, ...] = () + project_subdir: str = "" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index f93ced3ba9..6c9f083558 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -35,7 +35,7 @@ Path("audits/generic/expression_is_true.sql"), Path("functions/sql/udf__is_completed_order.sql"), Path("functions/sql/table_fn__customer_orders.sql"), - Path("loaders/waffle_sources.py"), + Path("python/loaders/waffle_sources.py"), Path("macros/currency.py"), Path("models/marts/_sqlbuild/_macros/currency.py"), Path("models/marts/_sqlbuild/_macros/datetime.py"), @@ -65,24 +65,35 @@ template="dagster", expected_files=( Path("README.md"), + Path("definitions.py"), + Path("waffle_shop/README.md"), + Path("waffle_shop/sqlbuild_project.toml"), + Path("waffle_shop/models/marts/fact_orders.sql"), + Path("waffle_shop/sources/raw.yml"), + Path("waffle_shop/python/loaders/waffle_sources.py"), + ), + unexpected_paths=( + Path("target"), Path("sqlbuild_project.toml"), - Path("models/marts/fact_orders.sql"), - Path("sources/raw.yml"), - Path("loaders/waffle_sources.py"), - Path("dagster/definitions.py"), - Path("dagster/README.md"), + Path("dagster"), + Path("waffle_shop/dagster"), + Path("waffle_shop/definitions.py"), ), - unexpected_paths=(Path("target"),), expected_file_fragments=( - (Path("sqlbuild_project.toml"), ('default_target = "dev"',)), + (Path("waffle_shop/sqlbuild_project.toml"), ('default_target = "dev"',)), ( - Path("sources/raw.yml"), + Path("waffle_shop/sources/raw.yml"), ("managed: true",), ), ( - Path("loaders/waffle_sources.py"), + Path("waffle_shop/python/loaders/waffle_sources.py"), ("def raw__customers(ctx", "def raw__orders(ctx"), ), + ( + Path("definitions.py"), + ('PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop"',), + ), + (Path("README.md"), ("dagster dev -f definitions.py",)), ), ), CreatePlaygroundProjectTestCase( @@ -91,19 +102,24 @@ template="rivers", expected_files=( Path("README.md"), + Path("definitions.py"), + Path("waffle_shop/sqlbuild_project.toml"), + Path("waffle_shop/models/marts/fact_orders.sql"), + Path("waffle_shop/sources/raw.yml"), + Path("waffle_shop/python/loaders/waffle_sources.py"), + ), + unexpected_paths=( + Path("target"), Path("sqlbuild_project.toml"), - Path("models/marts/fact_orders.sql"), - Path("sources/raw.yml"), - Path("loaders/waffle_sources.py"), - Path("rivers_pipeline/__init__.py"), - Path("rivers_pipeline/definitions.py"), - Path("rivers_pipeline/README.md"), + Path("rivers_pipeline"), + Path("waffle_shop/rivers_pipeline"), + Path("waffle_shop/definitions.py"), ), - unexpected_paths=(Path("target"),), expected_file_fragments=( ( - Path("rivers_pipeline/definitions.py"), + Path("definitions.py"), ( + 'PROJECT_DIR = Path(__file__).resolve().parent / "waffle_shop"', "import rivers as rs", "from sqlbuild.integrations.rivers import", "@sqlbuild_assets(project=SQLBUILD_PROJECT)", @@ -112,8 +128,8 @@ ), ), ( - Path("rivers_pipeline/README.md"), - ("rivers dev rivers_pipeline.definitions", "`waffle_shop` job"), + Path("README.md"), + ("rivers dev definitions", "`waffle_shop` job"), ), ), ), @@ -127,7 +143,7 @@ Path("models/fact_waffle_orders.sql"), Path("models/customer_revenue.sql"), Path("sources/raw.yml"), - Path("loaders/waffle_loaders.py"), + Path("python/loaders/waffle_loaders.py"), ), unexpected_paths=(Path("target"), Path("sqlbuild_local.toml")), expected_file_fragments=( @@ -137,14 +153,14 @@ ), (Path("sources/raw.yml"), ("managed: true",)), ( - Path("loaders/waffle_loaders.py"), + Path("python/loaders/waffle_loaders.py"), ("def raw_orders(ctx):", "def raw_customers(ctx):"), ), ), unexpected_file_fragments=( (Path("sources/raw.yml"), ("loader:",)), ( - Path("loaders/waffle_loaders.py"), + Path("python/loaders/waffle_loaders.py"), ("def load_raw_orders(ctx):", "def load_raw_customers(ctx):"), ), ), @@ -161,7 +177,7 @@ Path("seeds/lookups.yml"), Path("seeds/waffle_price_tiers.csv"), Path("sources/raw.yml"), - Path("loaders/waffle_loaders.py"), + Path("python/loaders/waffle_loaders.py"), Path("tests/unit/test_fact_waffle_orders.sql"), Path("tests/scenarios/customer_revenue_minimal.sql"), ), @@ -215,10 +231,10 @@ Path("sqlbuild_project.toml"), Path("sources/raw.yml"), Path("models/fact_orders.sql"), - Path("tasks/orders.py"), - Path("loaders/orders.py"), - Path("assets/orders_export.py"), - Path("checks/orders_export.py"), + Path("python/tasks/orders.py"), + Path("python/loaders/orders.py"), + Path("python/assets/orders_export.py"), + Path("python/checks/orders_export.py"), ), unexpected_paths=(Path("target"), Path("sqlbuild_local.toml")), expected_file_fragments=( @@ -227,7 +243,7 @@ ("[connections.developer]", 'connection = "developer"'), ), ( - Path("tasks/orders.py"), + Path("python/tasks/orders.py"), ( "from sqlbuild.tasks import SkipMode, task", "def prepare_raw_orders", @@ -236,11 +252,11 @@ ), ), ( - Path("loaders/orders.py"), + Path("python/loaders/orders.py"), ("@loader(depends_on=(prepare_raw_orders,))", "def raw_orders"), ), ( - Path("assets/orders_export.py"), + Path("python/assets/orders_export.py"), ( 'ctx.relation(model("fact_orders"))', "materialized=False", @@ -248,14 +264,14 @@ ), ), ( - Path("checks/orders_export.py"), + Path("python/checks/orders_export.py"), ( "@check(depends_on=orders_export", "ctx.result_of(node_function=orders_export)", ), ), ), - unexpected_file_fragments=((Path("tasks/orders.py"), ("sqlbuild.compiler",)),), + unexpected_file_fragments=((Path("python/tasks/orders.py"), ("sqlbuild.compiler",)),), ), ], ids=lambda case: case.description, @@ -338,8 +354,11 @@ def test_given_existing_target_when_creating_playground_then_it_raises_user_erro "SQLBuild playground created", "Project: demo_dagster_shop", "Example: waffle shop + Dagster", - "dagster dev -f dagster/definitions.py", + "cd demo_dagster_shop/waffle_shop", + "cd ..", + "dagster dev -f definitions.py", ), + project_subdir="waffle_shop", ), RunPlaygroundTestCase( description="prints Rivers next steps after creating Rivers playground", @@ -349,8 +368,11 @@ def test_given_existing_target_when_creating_playground_then_it_raises_user_erro "SQLBuild playground created", "Project: demo_rivers_shop", "Example: waffle shop + Rivers", - "rivers dev rivers_pipeline.definitions", + "cd demo_rivers_shop/waffle_shop", + "cd ..", + "rivers dev definitions", ), + project_subdir="waffle_shop", ), RunPlaygroundTestCase( description="prints loader-focused waffle shop next steps", @@ -414,7 +436,8 @@ def test_given_playground_command_when_running_then_it_prints_next_steps( expected_fragment: str for expected_fragment in test_case.expected_stdout_fragments: assert expected_fragment in captured.out - assert (tmp_path / test_case.target_path / "sqlbuild_project.toml").is_file() + project_dir: Path = tmp_path / test_case.target_path / test_case.project_subdir + assert (project_dir / "sqlbuild_project.toml").is_file() assert (tmp_path / test_case.target_path / ".agents/skills/sqlbuild/SKILL.md").is_file() diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py index 070f29c2e0..ef0b9ac251 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan/test_plan_text.py @@ -726,9 +726,9 @@ { "dependencies": [ { - "module": "tasks.helpers", + "module": "python.tasks.helpers", "qualname": "order_label", - "source_path": "tasks/helpers.py", + "source_path": "python/tasks/helpers.py", "source_text": "def order_label():\n return 'old'\n", } ] @@ -739,9 +739,9 @@ { "dependencies": [ { - "module": "tasks.helpers", + "module": "python.tasks.helpers", "qualname": "order_label", - "source_path": "tasks/helpers.py", + "source_path": "python/tasks/helpers.py", "source_text": "def order_label():\n return 'new'\n", } ] @@ -759,7 +759,7 @@ "dependency diff:", "- return 1", "+ return 2", - "tasks/helpers.py :: tasks.helpers :: order_label", + "python/tasks/helpers.py :: python.tasks.helpers :: order_label", "- return 'old'", "+ return 'new'", ), @@ -1592,9 +1592,9 @@ def test_given_plan_output_when_formatting_then_contains_expected_fragments( { "dependencies": [ { - "module": "tasks.helpers", + "module": "python.tasks.helpers", "qualname": "order_label", - "source_path": "tasks/helpers.py", + "source_path": "python/tasks/helpers.py", "source_text": "def order_label():\n return 'old'\n", } ] @@ -1605,9 +1605,9 @@ def test_given_plan_output_when_formatting_then_contains_expected_fragments( { "dependencies": [ { - "module": "tasks.helpers", + "module": "python.tasks.helpers", "qualname": "order_label", - "source_path": "tasks/helpers.py", + "source_path": "python/tasks/helpers.py", "source_text": "def order_label():\n return 'new'\n", } ] @@ -1617,7 +1617,7 @@ def test_given_plan_output_when_formatting_then_contains_expected_fragments( ), ), expected_fragments=( - "\033[2m # tasks/helpers.py :: tasks.helpers :: order_label\033[0m", + "\033[2m # python/tasks/helpers.py :: python.tasks.helpers :: order_label\033[0m", "\033[38;5;167m - return 'old'\033[0m", "\033[32m + return 'new'\033[0m", ), diff --git a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py index e10780b8b5..e6fe42c491 100644 --- a/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py +++ b/tests/unit/src/sqlbuild/cli/output/main/plan_json/test_plan_json.py @@ -576,7 +576,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence { "module": "tasks.helpers", "qualname": "order_label", - "source_path": "tasks/helpers.py", + "source_path": "python/tasks/helpers.py", "source_text": "def order_label():\n return 'old'\n", } ] @@ -589,7 +589,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence { "module": "tasks.helpers", "qualname": "order_label", - "source_path": "tasks/helpers.py", + "source_path": "python/tasks/helpers.py", "source_text": "def order_label():\n return 'new'\n", } ] @@ -605,7 +605,7 @@ def test_given_direct_diagnostics_policy_when_formatting_plan_json_then_evidence '"dependency_diff"', '"- return 1"', '"+ return 2"', - "tasks/helpers.py :: tasks.helpers :: order_label", + "python/tasks/helpers.py :: tasks.helpers :: order_label", "\"- return 'old'\"", "\"+ return 'new'\"", ), diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py index c8565b8942..c62f187349 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_assembly.py @@ -340,7 +340,7 @@ write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py index a621fcfeaa..88a38806c0 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_audit_factories.py @@ -47,7 +47,7 @@ def test_given_factory_attachment_when_building_inputs_then_cases_append_with_pr "sqlbuild_project.toml": _PROJECT_FILE, "models/orders.sql": f"MODEL ({test_case.model_header}); SELECT 1 AS amount", "audits/generic/expression_is_true.sql": _AUDIT_FILE, - "factories/quality.py": """ + "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, AuditSeverity, audit_factory @audit_factory @@ -72,7 +72,7 @@ def unused_quality(): assert tuple(audit.name for audit in schema_entry.audits) == test_case.expected_audit_names location: SourceLocation | None = schema_entry.audits[0].location assert location is not None - assert location.path == Path("factories/quality.py") + assert location.path == Path("python/factories/quality.py") assert location.line > 0 assert "audit_factories" not in inputs.model_inputs[0].config.values assert test_case.expected_warning_code in tuple( @@ -114,7 +114,7 @@ def test_given_invalid_factory_attachment_when_building_inputs_then_compile_erro { "sqlbuild_project.toml": _PROJECT_FILE, "models/orders.sql": f"MODEL ({test_case.model_header}); SELECT 1 AS amount", - "factories/quality.py": f""" + "python/factories/quality.py": f""" from sqlbuild.audits import AuditCase, audit_factory @audit_factory diff --git a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py index 1a82a2dab2..5e7458b8de 100644 --- a/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/dag/main/helpers.py @@ -106,14 +106,14 @@ def raw_orders_loader() -> list[dict[str, object]]: effective_target_schema="raw", loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/orders.py"), - relative_path=Path("loaders/orders.py"), + file_path=Path("python/loaders/orders.py"), + relative_path=Path("python/loaders/orders.py"), name="shared_order_feed", function=shared_order_feed, ), DiscoveredLoaderFunction( - file_path=Path("loaders/orders.py"), - relative_path=Path("loaders/orders.py"), + file_path=Path("python/loaders/orders.py"), + relative_path=Path("python/loaders/orders.py"), name="raw_orders_loader", function=raw_orders_loader, depends_on=(shared_order_feed,), diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index b57da39078..18fe822771 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -796,6 +796,13 @@ class DiscoverCheckFunctionsTestCase: expected_check_tags: tuple[tuple[str, ...], ...] +@dataclass(frozen=True) +class DiscoverPythonRootNodesTestCase: + description: str + files: dict[str, str] + expected_nodes: tuple[tuple[str, str, str], ...] + + @dataclass(frozen=True) class ExpectedBooleanTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py index 5b6bd61560..5de55e3831 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_audit_factories.py @@ -21,7 +21,7 @@ AuditFactoryDiscoveryTestCase( description="audit and node factories coexist", files={ - "factories/quality.py": """ + "python/factories/quality.py": """ from sqlbuild.audits import AuditCase, audit_factory from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -40,7 +40,26 @@ def refresh(ctx): }, expected_factory_names=("quality_checks",), expected_task_names=("refresh_quality",), - ) + ), + AuditFactoryDiscoveryTestCase( + description="audit factory beside a direct task outside a factories folder", + files={ + "python/quality.py": """ +from sqlbuild.audits import AuditCase, audit_factory +from sqlbuild.tasks import task + +@audit_factory +def quality_checks(): + return [AuditCase(name="positive_amount", definition="expression_is_true", arguments={"expression": "amount > 0"})] + +@task(name="refresh_quality") +def refresh(ctx): + return None +""", + }, + expected_factory_names=("quality_checks",), + expected_task_names=("refresh_quality",), + ), ], ids=lambda case: case.description, ) @@ -57,7 +76,7 @@ def test_given_audit_and_node_factories_when_discovering_then_both_are_collected test_case.expected_factory_names ) assert discovered.audit_factories[0].cases[0].name == "positive_amount" - assert discovered.audit_factories[0].relative_path == Path("factories/quality.py") + assert discovered.audit_factories[0].relative_path == Path(next(iter(test_case.files))) assert discovered.audit_factories[0].line > 0 assert tuple(task.name for task in discovered.tasks) == test_case.expected_task_names @@ -90,7 +109,7 @@ def test_given_invalid_audit_factory_when_discovering_then_clear_error_is_raised write_repo_files( tmp_path, { - "factories/quality.py": f""" + "python/factories/quality.py": f""" from sqlbuild.audits import audit_factory @audit_factory @@ -127,7 +146,7 @@ def duplicate_factory(): """ write_repo_files( tmp_path, - {"factories/a.py": source, "factories/nested/b.py": source}, + {"python/factories/a.py": source, "python/factories/nested/b.py": source}, ) with pytest.raises(PythonNodeDiscoveryError, match=test_case.expected_error_fragment): diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py index af3b8131c6..e23e33b0d1 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_factories.py @@ -21,7 +21,7 @@ DiscoverPythonNodeFactoriesTestCase( description="discovers mixed nodes returned by a factory", files={ - "factories/generated.py": """ + "python/factories/generated.py": """ from sqlbuild.assets import asset from sqlbuild.checks import check from sqlbuild.factories import factory @@ -78,7 +78,7 @@ def regional_pipeline(): DiscoverPythonNodeFactoriesTestCase( description="discovers tuple nodes returned by a factory", files={ - "loaders/generated.py": """ + "python/loaders/generated.py": """ from sqlbuild.factories import factory from sqlbuild.loaders import loader @@ -104,7 +104,7 @@ def generated_loaders(): DiscoverPythonNodeFactoriesTestCase( description="discovers set node returned by a factory", files={ - "checks/generated.py": """ + "python/checks/generated.py": """ from sqlbuild.checks import check from sqlbuild.factories import factory @@ -126,7 +126,7 @@ def generated_check(ctx): DiscoverPythonNodeFactoriesTestCase( description="discovers single node returned by a factory in assets folder", files={ - "assets/generated.py": """ + "python/assets/generated.py": """ from sqlbuild.assets import asset from sqlbuild.factories import factory @@ -148,7 +148,7 @@ def export(ctx): DiscoverPythonNodeFactoriesTestCase( description="discovers factory in loaders folder", files={ - "loaders/generated.py": """ + "python/loaders/generated.py": """ from sqlbuild.factories import factory from sqlbuild.loaders import loader @@ -167,6 +167,57 @@ def load(ctx): expected_check_names=(), expected_loader_dependency_counts=(0,), ), + DiscoverPythonNodeFactoriesTestCase( + description="discovers node kind from its decorator regardless of python subfolder", + files={ + "python/checks/generated.py": """ +from sqlbuild.assets import asset + + +@asset(name="orders_export") +def export(ctx): + return {} +""", + }, + expected_loader_names=(), + expected_task_names=(), + expected_asset_names=("orders_export",), + expected_check_names=(), + expected_asset_dependency_counts=(0,), + ), + DiscoverPythonNodeFactoriesTestCase( + description="discovers mixed-kind factory next to direct nodes using a shared helper", + files={ + "python/_helpers.py": """ +REGION = "emea" +""", + "python/orders/pipeline.py": """ +from python._helpers import REGION +from sqlbuild.checks import check +from sqlbuild.factories import factory +from sqlbuild.tasks import task + + +@task(name="prepare_orders") +def prepare_orders(ctx): + return {"region": REGION} + + +@factory +def regional_checks(): + @check(name=f"orders_{REGION}_exists", depends_on=prepare_orders) + def generated_check(ctx): + return True + return generated_check +""", + }, + expected_loader_names=(), + expected_task_names=("prepare_orders",), + expected_asset_names=(), + expected_check_names=("orders_emea_exists",), + expected_task_dependency_counts=(0,), + expected_check_dependency_counts=(1,), + ), ), ids=lambda case: case.description, ) @@ -203,7 +254,7 @@ def test_given_factory_nodes_when_discovering_python_nodes_then_returns_generate DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns invalid shape", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -221,7 +272,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns none", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -239,7 +290,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns dict", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -257,7 +308,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns bytes", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -275,7 +326,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns object", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -293,7 +344,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns class", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -315,7 +366,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returns nested structures", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -339,7 +390,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory returned item is not decorated", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -361,7 +412,7 @@ def broken_factory(): DiscoverPythonNodeFactoriesTestCase( description="raises when factory requires arguments", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -379,7 +430,7 @@ def broken_factory(region): DiscoverPythonNodeFactoriesTestCase( description="raises when factory body fails", files={ - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory @@ -394,27 +445,6 @@ def broken_factory(): expected_check_names=(), expected_error_fragment="failed during discovery: boom", ), - DiscoverPythonNodeFactoriesTestCase( - description="raises when direct node kind does not match folder", - files={ - "checks/generated.py": """ -from sqlbuild.assets import asset - - -@asset(name="orders_export") -def export(ctx): - return {} -""", - }, - expected_loader_names=(), - expected_task_names=(), - expected_asset_names=(), - expected_check_names=(), - expected_error_fragment=( - "Python node 'orders_export' in checks/ is an asset; " - "assets must live in assets/ or be generated from factories/." - ), - ), ), ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py index d700029f9c..8acb9febbe 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_loaders.py @@ -7,7 +7,7 @@ import pytest from sqlbuild.compiler.discovery._helpers.filesystem.core import discover_python_node_functions -from sqlbuild.compiler.discovery.exceptions import LoaderDiscoveryError +from sqlbuild.compiler.discovery.exceptions import PythonNodeDiscoveryError from sqlbuild.compiler.discovery.models import DiscoveredLoaderFunction from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverLoaderFunctionsTestCase, @@ -18,16 +18,16 @@ "test_case", [ DiscoverLoaderFunctionsTestCase( - description="discovers decorated source loaders from loaders directory", + description="discovers decorated source loaders from the python directory", files={ - "loaders/github.py": """ + "python/loaders/github.py": """ from sqlbuild.loaders import loader @loader def github_events(ctx): return [] """, - "loaders/stripe.py": """ + "python/loaders/stripe.py": """ from sqlbuild.loaders import loader @loader(destination="raw.customers") @@ -47,7 +47,7 @@ def stripe_customers(ctx): DiscoverLoaderFunctionsTestCase( description="discovers intermediate loader write and schema metadata", files={ - "loaders/events.py": """ + "python/loaders/events.py": """ from sqlbuild.loaders import loader @loader( @@ -77,7 +77,7 @@ def events(ctx): DiscoverLoaderFunctionsTestCase( description="discovers loader dependencies from decorator metadata", files={ - "loaders/events.py": """ + "python/loaders/events.py": """ from sqlbuild.loaders import loader @loader @@ -101,7 +101,7 @@ def enriched_events(ctx): DiscoverLoaderFunctionsTestCase( description="discovers explicit loader names and dependencies", files={ - "loaders/events.py": """ + "python/loaders/events.py": """ from sqlbuild.loaders import loader @loader(name="fetch_events") @@ -123,7 +123,7 @@ def make_enriched(ctx): expected_contracts=(None, None), ), DiscoverLoaderFunctionsTestCase( - description="returns empty tuple when loaders directory does not exist", + description="returns empty tuple when python directory does not exist", files={}, expected_names=(), expected_targets=(), @@ -137,9 +137,9 @@ def make_enriched(ctx): DiscoverLoaderFunctionsTestCase( description="ignores undecorated functions and init files", files={ - "loaders/__init__.py": "", - "loaders/helpers.py": "def helper(): return None\n", - "loaders/orders.py": """ + "python/loaders/__init__.py": "", + "python/loaders/helpers.py": "def helper(): return None\n", + "python/loaders/orders.py": """ from sqlbuild.loaders import loader @loader @@ -200,7 +200,7 @@ def test_given_project_dir_when_discovering_loaders_then_returns_expected( [ DiscoverLoaderFunctionsTestCase( description="raises clear error when loader file import fails", - files={"loaders/broken.py": "import missing_loader_dependency\n"}, + files={"python/loaders/broken.py": "import missing_loader_dependency\n"}, expected_names=(), expected_targets=(), expected_dependency_counts=(), @@ -209,7 +209,7 @@ def test_given_project_dir_when_discovering_loaders_then_returns_expected( expected_unique_keys=(), expected_column_names=(), expected_contracts=(), - expected_error_fragment="Failed to import source loader file", + expected_error_fragment="Failed to import Python node file python/loaders/broken.py", ) ], ids=lambda case: case.description, @@ -225,5 +225,5 @@ def test_given_loader_import_error_when_discovering_loaders_then_raises_clear_er file_path.parent.mkdir(parents=True, exist_ok=True) file_path.write_text(contents, encoding="utf-8") - with pytest.raises(LoaderDiscoveryError, match=test_case.expected_error_fragment): + with pytest.raises(PythonNodeDiscoveryError, match=test_case.expected_error_fragment): discover_python_node_functions(project_dir=tmp_path) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index c84d689d24..2d349349ac 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -13,10 +13,12 @@ from sqlbuild.compiler.discovery.models import ( DiscoveredAssetFunction, DiscoveredCheckFunction, + DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, ) from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverCheckFunctionsTestCase, + DiscoverPythonRootNodesTestCase, DiscoverTaskAssetFunctionsTestCase, ) @@ -27,7 +29,7 @@ DiscoverTaskAssetFunctionsTestCase( description="discovers decorated tasks and assets from explicit folders", files={ - "tasks/windows.py": """ + "python/tasks/windows.py": """ from sqlbuild.tasks import task @task(tags=("api",), group="ingestion") @@ -38,9 +40,9 @@ def fetch_window(ctx): def helper(): return None """, - "assets/exports.py": """ + "python/assets/exports.py": """ from sqlbuild.assets import asset -from tasks.windows import fetch_window +from python.tasks.windows import fetch_window @asset( depends_on=fetch_window, @@ -50,7 +52,7 @@ def helper(): def export_customers(ctx): return {"uri": "s3://exports/customers.parquet"} """, - "assets/__init__.py": """ + "python/assets/__init__.py": """ from sqlbuild.assets import asset @asset @@ -125,7 +127,7 @@ def test_given_project_dir_when_discovering_python_nodes_then_returns_expected( [ DiscoverTaskAssetFunctionsTestCase( description="raises clear error when task import fails", - files={"tasks/broken.py": "import missing_task_dependency\n"}, + files={"python/tasks/broken.py": "import missing_task_dependency\n"}, expected_task_names=(), expected_task_dependency_counts=(), expected_task_tags=(), @@ -159,16 +161,16 @@ def test_given_python_node_import_error_when_discovering_then_raises_clear_error DiscoverCheckFunctionsTestCase( description="discovers decorated checks from explicit folder", files={ - "assets/exports.py": """ + "python/assets/exports.py": """ from sqlbuild.assets import asset @asset def export_customers(ctx): return {"uri": "s3://exports/customers.parquet"} """, - "checks/exports.py": """ + "python/checks/exports.py": """ from sqlbuild.checks import check -from assets.exports import export_customers +from python.assets.exports import export_customers @check(depends_on=export_customers, severity="warn", tags=("exports",)) def export_customers_exists(ctx): @@ -177,9 +179,9 @@ def export_customers_exists(ctx): def helper(): return None """, - "checks/__init__.py": """ + "python/checks/__init__.py": """ from sqlbuild.checks import check -from assets.exports import export_customers +from python.assets.exports import export_customers @check(depends_on=export_customers) def ignored_init_check(ctx): @@ -223,3 +225,128 @@ def test_given_project_dir_when_discovering_checks_then_returns_expected( ) assert tuple(check.severity.value for check in checks) == test_case.expected_check_severities assert tuple(check.tags for check in checks) == test_case.expected_check_tags + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoverPythonRootNodesTestCase( + description="discovers every node kind in one module directly under python", + files={ + "python/orders.py": """ +from sqlbuild.assets import asset +from sqlbuild.checks import check +from sqlbuild.loaders import loader +from sqlbuild.tasks import task + +@task +def prepare_orders(ctx): + return None + +@loader(depends_on=[prepare_orders]) +def raw_orders(ctx): + return [] + +@asset(depends_on=prepare_orders) +def orders_export(ctx): + return None + +@check(depends_on=orders_export) +def orders_export_exists(ctx): + return True +""", + }, + expected_nodes=( + ("loader", "raw_orders", "python/orders.py"), + ("task", "prepare_orders", "python/orders.py"), + ("asset", "orders_export", "python/orders.py"), + ("check", "orders_export_exists", "python/orders.py"), + ), + ), + DiscoverPythonRootNodesTestCase( + description="discovers nodes in nested folders that import undecorated helpers", + files={ + "python/_helpers.py": """ +def export_uri(name): + return f"s3://exports/{name}.parquet" +""", + "python/orders/__init__.py": "", + "python/orders/utils.py": """ +from sqlbuild.tasks import task + +def order_label(value): + return str(value) + +LABEL = task +""", + "python/orders/ingest/loaders.py": """ +from sqlbuild.loaders import loader + +@loader +def raw_orders(ctx): + return [] +""", + "python/orders/pipeline/tasks.py": """ +from python.orders.utils import order_label +from sqlbuild.tasks import task + +@task +def label_orders(ctx): + return order_label(1) +""", + "python/exports/assets.py": """ +from python._helpers import export_uri +from python.orders.pipeline.tasks import label_orders +from sqlbuild.assets import asset + +@asset(depends_on=label_orders) +def orders_export(ctx): + return {"uri": export_uri("orders")} +""", + "python/exports/deep/nested/checks.py": """ +from python.exports.assets import orders_export +from sqlbuild.checks import check + +@check(depends_on=orders_export) +def orders_export_exists(ctx): + return True +""", + }, + expected_nodes=( + ("loader", "raw_orders", "python/orders/ingest/loaders.py"), + ("task", "label_orders", "python/orders/pipeline/tasks.py"), + ("asset", "orders_export", "python/exports/assets.py"), + ("check", "orders_export_exists", "python/exports/deep/nested/checks.py"), + ), + ), + DiscoverPythonRootNodesTestCase( + description="treats helper-only python root as having no nodes", + files={ + "python/_helpers.py": "def order_label(value):\n return str(value)\n", + "python/orders/utils.py": "REGION = 'emea'\n", + }, + expected_nodes=(), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_root_when_discovering_python_nodes_then_decorators_define_node_kinds( + test_case: DiscoverPythonRootNodesTestCase, + tmp_path: Path, +) -> None: + relative_path: str + contents: str + for relative_path, contents in test_case.files.items(): + file_path: Path = tmp_path / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(contents, encoding="utf-8") + + result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=tmp_path) + + discovered: tuple[tuple[str, str, str], ...] = ( + *(("loader", node.name, node.relative_path.as_posix()) for node in result.loaders), + *(("task", node.name, node.relative_path.as_posix()) for node in result.tasks), + *(("asset", node.name, node.relative_path.as_posix()) for node in result.assets), + *(("check", node.name, node.relative_path.as_posix()) for node in result.checks), + ) + assert discovered == test_case.expected_nodes diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py index 82603c2ad9..fcf38552a4 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_resource_identities.py @@ -194,7 +194,7 @@ def test_given_noncanonical_name_when_validating_resource_identity_then_coded_er DiscoveredResourceIdentityErrorTestCase( description="Python task name", project_files={ - "tasks/orders.py": ( + "python/tasks/orders.py": ( "from sqlbuild.tasks import task\n\n" "@task\n" "def DailyTask(ctx):\n" diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 5beac5442b..057ebc199c 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -6,6 +6,7 @@ import pytest +from sqlbuild.compiler.discovery.exceptions import ProjectPythonPathError from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.observability import EventDispatcher, LifecycleEvent, dispatcher_scope @@ -63,30 +64,30 @@ """, "audits/generic/not_null.sql": "AUDIT ();\nSELECT 1\n", "macros/name_helpers.py": "def slug() -> str:\n return 'slug'\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader(name="raw_orders") def fetch_orders(ctx): return [] """, - "tasks/windows.py": """ + "python/tasks/windows.py": """ from sqlbuild.tasks import task @task(tags=("api",), group="ingestion") def fetch_window(ctx): return {"window": "today"} """, - "assets/exports.py": """ + "python/assets/exports.py": """ from sqlbuild.assets import asset @asset(columns=[{"name": "customer_id", "type": "string"}]) def export_customers(ctx): return {"uri": "s3://exports/customers.parquet"} """, - "checks/exports.py": """ + "python/checks/exports.py": """ from sqlbuild.checks import check -from assets.exports import export_customers +from python.assets.exports import export_customers @check(depends_on=export_customers, severity="warn") def export_customers_exists(ctx): @@ -341,7 +342,7 @@ def test_given_valid_sql_and_failing_python_import_when_discovering_then_phases_ base_repo_files() | { "models/orders.sql": "MODEL ();\n\nSELECT 1 AS order_id\n", - "tasks/broken.py": "import dependency_that_does_not_exist\n", + "python/tasks/broken.py": "import dependency_that_does_not_exist\n", }, ) events: list[LifecycleEvent] = [] @@ -404,7 +405,7 @@ def test_given_ignored_siblings_when_discovering_then_count_uses_relevant_collec repo_files=base_repo_files() | { "sources/raw.yml": "sources:\n - name: raw_orders\n managed: true\n", - "loaders/generated.py": """ + "python/loaders/generated.py": """ from sqlbuild.factories import factory from sqlbuild.loaders import loader @@ -423,7 +424,7 @@ def load(ctx): description="generated task and asset dependency validates", repo_files=base_repo_files() | { - "factories/generated.py": """ + "python/factories/generated.py": """ from sqlbuild.assets import asset from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -449,7 +450,7 @@ def export(ctx): description="single-kind factory validates in factories folder", repo_files=base_repo_files() | { - "factories/generated_tasks.py": """ + "python/factories/generated_tasks.py": """ from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -469,9 +470,9 @@ def prepare(ctx): description="factory can import ordinary private helper module", repo_files=base_repo_files() | { - "factories/orders/_helpers.py": "TASK_NAME = 'prepare_orders'\n", - "factories/orders/generated.py": """ -from factories.orders._helpers import TASK_NAME + "python/factories/orders/_helpers.py": "TASK_NAME = 'prepare_orders'\n", + "python/factories/orders/generated.py": """ +from python.factories.orders._helpers import TASK_NAME from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -511,18 +512,19 @@ def test_given_generated_factory_nodes_when_discovering_inputs_then_validates_ex "test_case", ( ProjectPythonPathAcceptanceTestCase( - description="documented integration Python paths", + description="undecorated helper modules under python root", repo_files={ - "definitions.py": "ROOT_DEFINITIONS = True\n", - "dagster/definitions.py": "DAGSTER_DEFINITIONS = True\n", - "rivers_pipeline/definitions.py": "RIVERS_DEFINITIONS = True\n", + "python/_helpers.py": "REGION = 'emea'\n", + "python/orders/__init__.py": "", + "python/orders/utils.py": "def order_label(value):\n return str(value)\n", + "adapter.py": "ADAPTER = None\n", }, expected_project_name="demo", ), ), ids=lambda case: case.description, ) -def test_given_documented_integration_python_when_discovering_inputs_then_paths_are_accepted( +def test_given_supported_project_python_when_discovering_inputs_then_paths_are_accepted( test_case: ProjectPythonPathAcceptanceTestCase, tmp_path: Path, write_repo_files: Callable[[Path, dict[str, str]], None], @@ -534,6 +536,73 @@ def test_given_documented_integration_python_when_discovering_inputs_then_paths_ assert result.project_config.name == test_case.expected_project_name +@pytest.mark.parametrize( + "test_case", + ( + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed tasks root", + repo_files=base_repo_files() | {"tasks/orders.py": "VALUE = 1\n"}, + expected_error_fragment="tasks/orders.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed assets root", + repo_files=base_repo_files() | {"assets/orders.py": "VALUE = 1\n"}, + expected_error_fragment="assets/orders.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed checks root", + repo_files=base_repo_files() | {"checks/orders.py": "VALUE = 1\n"}, + expected_error_fragment="checks/orders.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed loaders root", + repo_files=base_repo_files() | {"loaders/orders.py": "VALUE = 1\n"}, + expected_error_fragment="loaders/orders.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed factories root", + repo_files=base_repo_files() | {"factories/orders.py": "VALUE = 1\n"}, + expected_error_fragment="factories/orders.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed libs root", + repo_files=base_repo_files() | {"libs/cleaning.py": "VALUE = 1\n"}, + expected_error_fragment="libs/cleaning.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed dagster root", + repo_files=base_repo_files() | {"dagster/definitions.py": "VALUE = 1\n"}, + expected_error_fragment="dagster/definitions.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed rivers_pipeline root", + repo_files=base_repo_files() | {"rivers_pipeline/definitions.py": "VALUE = 1\n"}, + expected_error_fragment="rivers_pipeline/definitions.py", + ), + DiscoverProjectInputsErrorTestCase( + description="raises when Python lives in removed root definitions root", + repo_files=base_repo_files() | {"definitions.py": "VALUE = 1\n"}, + expected_error_fragment="definitions.py", + ), + ), + ids=lambda case: case.description, +) +def test_given_python_in_removed_root_when_discovering_inputs_then_d016_is_raised( + test_case: DiscoverProjectInputsErrorTestCase, + tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + write_repo_files(tmp_path, test_case.repo_files) + + with pytest.raises(ProjectPythonPathError) as error_info: + discover_project_inputs(project_dir=tmp_path) + + assert error_info.value.code == "D016" + assert f"Unsupported project Python path(s): {test_case.expected_error_fragment}" in str( + error_info.value + ) + + @pytest.mark.parametrize( "test_case", ( @@ -563,7 +632,7 @@ def test_given_non_project_with_python_when_discovering_then_missing_config_erro description="generated duplicate names fail validation", repo_files=base_repo_files() | { - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -586,7 +655,7 @@ def generated_tasks(): description="generated name colliding with top-level node fails validation", repo_files=base_repo_files() | { - "tasks/top_level.py": """ + "python/tasks/top_level.py": """ from sqlbuild.tasks import task @@ -594,7 +663,7 @@ def generated_tasks(): def top_level_profile(ctx): return None """, - "assets/generated.py": """ + "python/assets/generated.py": """ from sqlbuild.assets import asset from sqlbuild.factories import factory @@ -613,7 +682,7 @@ def generated_profile(ctx): description="generated dependency cycle fails validation", repo_files=base_repo_files() | { - "tasks/generated.py": """ + "python/tasks/generated.py": """ from sqlbuild.factories import factory from sqlbuild.tasks import task @@ -636,7 +705,7 @@ def second_body(ctx): description="generated loader SQL dependency fails validation", repo_files=base_repo_files() | { - "loaders/generated.py": """ + "python/loaders/generated.py": """ from sqlbuild.factories import factory from sqlbuild.loaders import loader from sqlbuild.refs import model @@ -656,7 +725,7 @@ def raw_orders(ctx): description="generated check SQL dependency fails validation", repo_files=base_repo_files() | { - "checks/generated.py": """ + "python/checks/generated.py": """ from sqlbuild.checks import check from sqlbuild.factories import factory from sqlbuild.refs import model @@ -677,7 +746,7 @@ def check_orders(ctx): repo_files=base_repo_files() | { "sources/raw.yml": "sources:\n - name: raw_orders\n managed: true\n", - "loaders/generated.py": """ + "python/loaders/generated.py": """ from sqlbuild.factories import factory from sqlbuild.loaders import loader @@ -692,29 +761,6 @@ def load(ctx): }, expected_error_fragment="Managed source 'raw_orders' in sources/raw.yml requires loader", ), - DiscoverFactoryValidationTestCase( - description="kind-folder factory returning foreign kind fails validation", - repo_files=base_repo_files() - | { - "assets/generated.py": """ -from sqlbuild.factories import factory -from sqlbuild.loaders import loader - - -@factory -def gen(): - @loader(name="raw_orders") - def load(ctx): - return [] - - return load -""", - }, - expected_error_fragment=( - "Factory gen in assets/ returned a loader 'raw_orders'; " - "mixed-kind factories must live in factories/." - ), - ), ), ids=lambda case: case.description, ) @@ -866,14 +912,14 @@ def test_given_invalid_generated_factory_nodes_when_discovering_inputs_then_rais description="raises when loader names are duplicated", repo_files=base_repo_files() | { - "loaders/a.py": """ + "python/loaders/a.py": """ from sqlbuild.loaders import loader @loader def raw_orders(ctx): return [] """, - "loaders/b.py": """ + "python/loaders/b.py": """ from sqlbuild.loaders import loader @loader @@ -887,14 +933,14 @@ def raw_orders(ctx): description="raises when explicit loader names are duplicated", repo_files=base_repo_files() | { - "loaders/a.py": """ + "python/loaders/a.py": """ from sqlbuild.loaders import loader @loader(name="raw_orders") def load_a(ctx): return [] """, - "loaders/b.py": """ + "python/loaders/b.py": """ from sqlbuild.loaders import loader @loader(name="raw_orders") @@ -914,7 +960,7 @@ def load_b(ctx): table: fetch_orders """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -934,7 +980,7 @@ def fetch_orders(ctx): table: raw_orders """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader @@ -948,7 +994,7 @@ def raw_orders(ctx): description="raises when loader dependency is not decorated", repo_files=base_repo_files() | { - "loaders/events.py": """ + "python/loaders/events.py": """ from sqlbuild.loaders import loader def fetch_events(ctx): @@ -965,7 +1011,7 @@ def enriched_events(ctx): description="raises when loader dependencies contain a cycle", repo_files=base_repo_files() | { - "loaders/events.py": """ + "python/loaders/events.py": """ from sqlbuild.loaders import loader def fetch_events(ctx): @@ -991,7 +1037,7 @@ def enriched_events(ctx): write_strategy: table """.strip() + "\n", - "loaders/raw_orders.py": """ + "python/loaders/raw_orders.py": """ from sqlbuild.loaders import loader @loader(write_strategy="table", columns=[{"name": "id", "type": "INTEGER"}]) @@ -1005,14 +1051,14 @@ def raw_orders(ctx): description="raises when task and asset names are duplicated", repo_files=base_repo_files() | { - "tasks/export_customers.py": """ + "python/tasks/export_customers.py": """ from sqlbuild.tasks import task @task def export_customers(ctx): return None """, - "assets/export_customers.py": """ + "python/assets/export_customers.py": """ from sqlbuild.assets import asset @asset @@ -1026,16 +1072,16 @@ def export_customers(ctx): description="raises when check name collides with task name", repo_files=base_repo_files() | { - "tasks/export_customers.py": """ + "python/tasks/export_customers.py": """ from sqlbuild.tasks import task @task def export_customers(ctx): return None """, - "checks/export_customers.py": """ + "python/checks/export_customers.py": """ from sqlbuild.checks import check -from tasks.export_customers import export_customers +from python.tasks.export_customers import export_customers @check(depends_on=export_customers, name="export_customers") def export_customers_check(ctx): @@ -1049,7 +1095,7 @@ def export_customers_check(ctx): repo_files=base_repo_files() | { "models/marts/export_customers.sql": "MODEL ();\n\nselect 1\n", - "tasks/export_customers.py": """ + "python/tasks/export_customers.py": """ from sqlbuild.tasks import task @task @@ -1074,7 +1120,7 @@ def export_customers(ctx): """.strip() + "\n", "seeds/export_customers.csv": "customer_id\n1\n", - "assets/export_customers.py": """ + "python/assets/export_customers.py": """ from sqlbuild.assets import asset @asset @@ -1096,16 +1142,16 @@ def export_customers(ctx): table: export_customers_exists """.strip() + "\n", - "tasks/export_customers.py": """ + "python/tasks/export_customers.py": """ from sqlbuild.tasks import task @task def export_customers(ctx): return None """, - "checks/export_customers.py": """ + "python/checks/export_customers.py": """ from sqlbuild.checks import check -from tasks.export_customers import export_customers +from python.tasks.export_customers import export_customers @check(depends_on=export_customers) def export_customers_exists(ctx): @@ -1155,7 +1201,7 @@ def main(amount): amount > 100 """.strip() + "\n", - "loaders/fetch_orders.py": """ + "python/loaders/fetch_orders.py": """ from sqlbuild.loaders import loader @loader @@ -1223,7 +1269,7 @@ def notify(ctx): description="raises when task dependency is not decorated", repo_files=base_repo_files() | { - "tasks/windows.py": """ + "python/tasks/windows.py": """ from sqlbuild.tasks import task def fetch_window(ctx): @@ -1240,7 +1286,7 @@ def export_window(ctx): description="raises when task and asset dependencies contain a cycle", repo_files=base_repo_files() | { - "tasks/windows.py": """ + "python/tasks/windows.py": """ from sqlbuild.tasks import task def fetch_window(ctx): @@ -1259,7 +1305,7 @@ def enrich_window(ctx): description="raises when check dependency is not decorated", repo_files=base_repo_files() | { - "checks/exports.py": """ + "python/checks/exports.py": """ from sqlbuild.checks import check def export_customers(ctx): @@ -1276,7 +1322,7 @@ def export_customers_exists(ctx): description="raises when loader declares SQL model dependency", repo_files=base_repo_files() | { - "loaders/orders.py": """ + "python/loaders/orders.py": """ from sqlbuild.loaders import loader from sqlbuild.refs import model @@ -1291,7 +1337,7 @@ def load_orders(ctx): description="raises when check declares SQL model dependency", repo_files=base_repo_files() | { - "checks/orders.py": """ + "python/checks/orders.py": """ from sqlbuild.checks import check from sqlbuild.refs import model @@ -1306,16 +1352,16 @@ def check_orders(ctx): description="raises when check depends on check", repo_files=base_repo_files() | { - "assets/exports.py": """ + "python/assets/exports.py": """ from sqlbuild.assets import asset @asset def export_customers(ctx): return None """, - "checks/exports.py": """ + "python/checks/exports.py": """ from sqlbuild.checks import check -from assets.exports import export_customers +from python.assets.exports import export_customers @check(depends_on=export_customers) def export_customers_exists(ctx): diff --git a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py index 7dd7c537b6..e508eda85f 100644 --- a/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/pipeline/_helpers/helpers.py @@ -72,7 +72,7 @@ def build_previous_python_identity_map( definition='{"source_text": "def prepare_orders(ctx):\\n return old_helper()\\n"}', metadata_json=( '{"dependencies": [{"module": "tasks.helpers", ' - '"qualname": "old_helper", "source_path": "tasks/helpers.py", ' + '"qualname": "old_helper", "source_path": "python/tasks/helpers.py", ' '"source_text": "def old_helper():\\n return 1\\n"}]}' ), ts=datetime(2026, 1, 15, 12, 0, 0), diff --git a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py index f12e04bba9..1ab2d85e0a 100644 --- a/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/planner/_helpers/helpers.py @@ -369,16 +369,16 @@ def build_source_load_nodes_project() -> CompiledProject: ), loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name="fetch_orders", function=fetch_orders, destination="staging_fetch_orders", write_strategy=SourceWriteStrategy.TABLE, ), DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name="load_orders", function=load_orders, depends_on=(fetch_orders,), diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py index 05d7c93ebf..778546978a 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/helpers.py @@ -101,7 +101,7 @@ def build_external_loader_python_node_graph() -> PythonNodeGraph: loader_functions=( DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="load_events", function=load_events, connection_mode=LoaderConnectionMode.EXTERNAL, @@ -120,7 +120,7 @@ def build_terminal_loader_python_node_graph() -> PythonNodeGraph: loader_functions=( DiscoveredLoaderFunction( file_path=Path("/project/loaders/orders.py"), - relative_path=Path("loaders/orders.py"), + relative_path=Path("python/loaders/orders.py"), name="raw_orders", function=raw_orders, ), @@ -137,7 +137,7 @@ def build_orders_python_node_graph() -> PythonNodeGraph: loader_functions=( DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="load_events", function=load_events, depends_on=(prepare_orders,), @@ -146,7 +146,7 @@ def build_orders_python_node_graph() -> PythonNodeGraph: task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="prepare_orders", function=prepare_orders, tags=("orders", "daily"), @@ -155,7 +155,7 @@ def build_orders_python_node_graph() -> PythonNodeGraph: asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_orders", function=export_orders, depends_on=(imported_prepare_orders,), @@ -169,7 +169,7 @@ def build_orders_python_node_graph() -> PythonNodeGraph: check_functions=( DiscoveredCheckFunction( file_path=Path("/project/checks/orders.py"), - relative_path=Path("checks/orders.py"), + relative_path=Path("python/checks/orders.py"), name="check_orders_export", function=check_orders_export, depends_on=(export_orders,), @@ -260,7 +260,7 @@ def build_sql_ref_python_node_graph(*, dependency: SqlResourceRef) -> PythonNode task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="profile_orders", function=prepare_orders, depends_on=(dependency,), @@ -278,7 +278,7 @@ def build_sql_downstream_task_to_loader_python_node_graph() -> PythonNodeGraph: loader_functions=( DiscoveredLoaderFunction( file_path=Path("/project/loaders/orders.py"), - relative_path=Path("loaders/orders.py"), + relative_path=Path("python/loaders/orders.py"), name="load_events", function=load_events, depends_on=(prepare_orders,), @@ -287,7 +287,7 @@ def build_sql_downstream_task_to_loader_python_node_graph() -> PythonNodeGraph: task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="prepare_orders", function=prepare_orders, depends_on=(model_ref("orders"),), @@ -362,13 +362,13 @@ def _build_loader_dependency_python_node_graph( intermediate_loaders: tuple[DiscoveredLoaderFunction, ...] = ( DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="fetch_pages", function=fetch_pages, ), DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="load_events", function=load_events, depends_on=(fetch_pages,), @@ -377,7 +377,7 @@ def _build_loader_dependency_python_node_graph( terminal_loaders: tuple[DiscoveredLoaderFunction, ...] = ( DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="raw_orders", function=raw_orders, ), @@ -390,7 +390,7 @@ def _build_loader_dependency_python_node_graph( task_functions: tuple[DiscoveredTaskFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="summarize_orders", function=summarize_orders, depends_on=(dependency_function,), @@ -399,7 +399,7 @@ def _build_loader_dependency_python_node_graph( asset_functions: tuple[DiscoveredAssetFunction, ...] = ( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_orders", function=export_orders, depends_on=(dependency_function,), @@ -408,7 +408,7 @@ def _build_loader_dependency_python_node_graph( check_functions: tuple[DiscoveredCheckFunction, ...] = ( DiscoveredCheckFunction( file_path=Path("/project/checks/orders.py"), - relative_path=Path("checks/orders.py"), + relative_path=Path("python/checks/orders.py"), name="check_loaded_orders", function=check_loaded_orders, depends_on=(dependency_function,), diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py index c709395bf7..719197ed47 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_identity.py @@ -26,7 +26,7 @@ PythonNodeIdentityTestCase( description="includes same-file helper dependency", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ def normalize_order(value): return value.strip().lower() @@ -36,10 +36,10 @@ def build_orders(ctx): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("normalize_order",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("def normalize_order(value):", "source_text"), @@ -47,32 +47,32 @@ def build_orders(ctx): PythonNodeIdentityTestCase( description="includes transitive imported first-party helper dependencies", repo_files={ - "tasks/orders.py": """ -from libs.cleaning import normalize_orders + "python/tasks/orders.py": """ +from python.helpers.cleaning import normalize_orders def build_orders(ctx): return normalize_orders(ctx) """.strip() + "\n", - "libs/cleaning.py": """ -from libs.status import clean_status + "python/helpers/cleaning.py": """ +from python.helpers.status import clean_status def normalize_orders(ctx): return clean_status(" Pending ") """.strip() + "\n", - "libs/status.py": """ + "python/helpers/status.py": """ def clean_status(value): return value.strip().lower() """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("normalize_orders", "clean_status"), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=( @@ -84,7 +84,7 @@ def clean_status(value): PythonNodeIdentityTestCase( description="excludes helpers loaded from venv under git root", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ import vendor_helpers @@ -98,10 +98,10 @@ def normalize_order(value): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("dependencies",), @@ -111,7 +111,7 @@ def normalize_order(value): PythonNodeIdentityTestCase( description="excludes helpers loaded from external third-party path", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ import external_vendor_helpers @@ -125,10 +125,10 @@ def normalize_order(value): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("dependencies",), @@ -138,7 +138,7 @@ def normalize_order(value): PythonNodeIdentityTestCase( description="ignores stdlib builtins and dynamic references without crashing", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ import json @@ -153,10 +153,10 @@ def build_orders(ctx): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("dependencies",), @@ -165,15 +165,15 @@ def build_orders(ctx): PythonNodeIdentityTestCase( description="terminates cyclic helper traversal", repo_files={ - "tasks/orders.py": """ -from libs.cycle import helper_a + "python/tasks/orders.py": """ +from python.helpers.cycle import helper_a def build_orders(ctx): return helper_a(" Pending ") """.strip() + "\n", - "libs/cycle.py": """ + "python/helpers/cycle.py": """ def helper_a(value): return helper_b(value) @@ -183,10 +183,10 @@ def helper_b(value): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("helper_a", "helper_b"), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("def helper_a(value):", "def helper_b(value):"), @@ -194,24 +194,24 @@ def helper_b(value): PythonNodeIdentityTestCase( description="includes first-party module attribute helper dependency", repo_files={ - "tasks/orders.py": """ -from libs import cleaning + "python/tasks/orders.py": """ +from python.helpers import cleaning def build_orders(ctx): return cleaning.normalize_order(" Pending ") """.strip() + "\n", - "libs/cleaning.py": """ + "python/helpers/cleaning.py": """ def normalize_order(value): return value.strip().lower() """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("normalize_order",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("def normalize_order(value):", "source_text"), @@ -219,7 +219,7 @@ def normalize_order(value): PythonNodeIdentityTestCase( description="includes first-party class constructor dependency", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ class Cleaner: def clean(self, value): return value.strip().lower() @@ -230,10 +230,10 @@ def build_orders(ctx): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("Cleaner",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("class Cleaner:", "source_text"), @@ -241,7 +241,7 @@ def build_orders(ctx): PythonNodeIdentityTestCase( description="includes first-party class method dependency", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ class Cleaner: @staticmethod def clean(value): @@ -253,10 +253,10 @@ def build_orders(ctx): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("Cleaner.clean",), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("def clean(value):", "source_text"), @@ -264,30 +264,30 @@ def build_orders(ctx): PythonNodeIdentityTestCase( description="sorts dependencies deterministically by source path and qualname", repo_files={ - "tasks/orders.py": """ -from libs.z_helpers import z_clean -from libs.a_helpers import a_clean + "python/tasks/orders.py": """ +from python.helpers.z_helpers import z_clean +from python.helpers.a_helpers import a_clean def build_orders(ctx): return z_clean(a_clean(" Pending ")) """.strip() + "\n", - "libs/z_helpers.py": """ + "python/helpers/z_helpers.py": """ def z_clean(value): return value.strip() """.strip() + "\n", - "libs/a_helpers.py": """ + "python/helpers/a_helpers.py": """ def a_clean(value): return value.lower() """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=("a_clean", "z_clean"), expected_definition_fragments=("def build_orders(ctx):", "source_text"), expected_metadata_fragments=("def a_clean(value):", "def z_clean(value):"), @@ -295,16 +295,16 @@ def a_clean(value): PythonNodeIdentityTestCase( description="stores decorator config in definition payload", repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ def build_orders(ctx): return None """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", - expected_source_path="tasks/orders.py", + expected_source_path="python/tasks/orders.py", expected_dependency_qualnames=(), expected_definition_fragments=("orders", "daily", "source_text"), expected_metadata_fragments=("dependencies",), @@ -379,7 +379,7 @@ def test_given_python_node_when_building_identity_then_includes_expected_first_p PythonNodeIdentityChangeTestCase( description="nested function source changes update body and version identity", before_repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ def build_orders(ctx): def normalize_order(value): return value.strip().lower() @@ -389,7 +389,7 @@ def normalize_order(value): + "\n", }, after_repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ def build_orders(ctx): def normalize_order(value): return value.strip().upper() @@ -398,7 +398,7 @@ def normalize_order(value): """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", expected_definition_hash_changed=True, @@ -407,36 +407,36 @@ def normalize_order(value): PythonNodeIdentityChangeTestCase( description="first-party helper source changes update version identity only", before_repo_files={ - "tasks/orders.py": """ -from libs.cleaning import normalize_order + "python/tasks/orders.py": """ +from python.helpers.cleaning import normalize_order def build_orders(ctx): return normalize_order(" Pending ") """.strip() + "\n", - "libs/cleaning.py": """ + "python/helpers/cleaning.py": """ def normalize_order(value): return value.strip().lower() """.strip() + "\n", }, after_repo_files={ - "tasks/orders.py": """ -from libs.cleaning import normalize_order + "python/tasks/orders.py": """ +from python.helpers.cleaning import normalize_order def build_orders(ctx): return normalize_order(" Pending ") """.strip() + "\n", - "libs/cleaning.py": """ + "python/helpers/cleaning.py": """ def normalize_order(value): return value.strip().upper() """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", expected_definition_hash_changed=False, @@ -445,20 +445,20 @@ def normalize_order(value): PythonNodeIdentityChangeTestCase( description="decorator config changes update body and version identity", before_repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ def build_orders(ctx): return None """.strip() + "\n", }, after_repo_files={ - "tasks/orders.py": """ + "python/tasks/orders.py": """ def build_orders(ctx): return None """.strip() + "\n", }, - entry_module_path="tasks/orders.py", + entry_module_path="python/tasks/orders.py", function_name="build_orders", node_type="task", expected_definition_hash_changed=True, diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py index 5970d9c0ec..0d7d26d11f 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_inventory.py @@ -98,7 +98,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node loader_functions=( DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="load_events", function=load_events, depends_on=(prepare_orders,), @@ -107,7 +107,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="prepare_orders", function=prepare_orders, tags=test_case.expected_task_tags, @@ -120,7 +120,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_orders", function=export_orders, depends_on=(imported_prepare_orders,), @@ -132,7 +132,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node ), DiscoveredAssetFunction( file_path=Path("/project/assets/notifications.py"), - relative_path=Path("assets/notifications.py"), + relative_path=Path("python/assets/notifications.py"), name="notify_orders", function=notify_orders, meta={"provider_usages": "user-owned"}, @@ -141,7 +141,7 @@ def test_given_discovered_python_functions_when_building_graph_then_indexes_node check_functions=( DiscoveredCheckFunction( file_path=Path("/project/checks/orders.py"), - relative_path=Path("checks/orders.py"), + relative_path=Path("python/checks/orders.py"), name="check_orders_export", function=check_orders_export, depends_on=(export_orders,), @@ -275,7 +275,7 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ loader_functions=( DiscoveredLoaderFunction( file_path=Path("/project/loaders/events.py"), - relative_path=Path("loaders/events.py"), + relative_path=Path("python/loaders/events.py"), name="load_events", function=load_events, destination="raw.events", @@ -289,7 +289,7 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="prepare_orders", function=prepare_orders, tags=("orders",), @@ -302,7 +302,7 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_orders", function=export_orders, tags=("orders",), @@ -316,7 +316,7 @@ def test_given_fully_configured_python_nodes_when_building_then_identity_config_ check_functions=( DiscoveredCheckFunction( file_path=Path("/project/checks/orders.py"), - relative_path=Path("checks/orders.py"), + relative_path=Path("python/checks/orders.py"), name="check_orders_export", function=check_orders_export, depends_on=(export_orders,), diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index 46418de951..f393b6c4b2 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -31,7 +31,7 @@ ), PythonSqlSelectorTestCase( description="selects expanded Python asset path for run", - select=("+path:assets",), + select=("+path:python/assets",), exclude=(), expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), @@ -82,7 +82,7 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( ), PythonSqlSelectorErrorTestCase( description="rejects direct asset path with unselected Python dependency for run", - select=("path:assets",), + select=("path:python/assets",), exclude=(), expected_error_type=ValueError, expected_error_fragment=( diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py index d50c63b7fe..3a9d8b4aa0 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_selectors.py @@ -103,39 +103,47 @@ def test_given_python_node_typed_selector_when_parsing_then_returns_expected_kin exclude=(), expected_names=frozenset({"prepare_orders", "export_orders"}), ), + PythonNodeSelectorTestCase( + description="selects every Python node by the python root path", + select=("path:python",), + exclude=(), + expected_names=frozenset( + {"prepare_orders", "load_events", "export_orders", "check_orders_export"} + ), + ), PythonNodeSelectorTestCase( description="selects Python nodes by explicit tasks path", - select=("path:tasks",), + select=("path:python/tasks",), exclude=(), expected_names=frozenset({"prepare_orders"}), ), PythonNodeSelectorTestCase( description="selects Python nodes by explicit loaders path", - select=("path:loaders",), + select=("path:python/loaders",), exclude=(), expected_names=frozenset({"load_events"}), ), PythonNodeSelectorTestCase( description="selects Python nodes by explicit assets path", - select=("path:assets",), + select=("path:python/assets",), exclude=(), expected_names=frozenset({"export_orders"}), ), PythonNodeSelectorTestCase( description="selects Python nodes by explicit assets slash path", - select=("assets/",), + select=("python/assets/",), exclude=(), expected_names=frozenset({"export_orders"}), ), PythonNodeSelectorTestCase( description="selects Python nodes by leading slash assets path", - select=("/assets",), + select=("/python/assets",), exclude=(), expected_names=frozenset({"export_orders"}), ), PythonNodeSelectorTestCase( description="selects Python nodes by explicit checks path with upstream expansion", - select=("+path:checks",), + select=("+path:python/checks",), exclude=(), expected_names=frozenset({"prepare_orders", "export_orders", "check_orders_export"}), ), @@ -206,12 +214,19 @@ def test_given_python_node_selectors_when_resolving_then_returns_expected_names( expected_error_type=ValueError, expected_error_fragment="path selectors require an explicit root", ), + PythonNodeSelectorErrorTestCase( + description="raises when path selector uses a kind folder outside the python root", + select=("path:tasks",), + exclude=(), + expected_error_type=ValueError, + expected_error_fragment="use 'models/' or 'python/'", + ), PythonNodeSelectorErrorTestCase( description="raises when explicit path selector matches no Python nodes", - select=("path:tasks/missing",), + select=("path:python/tasks/missing",), exclude=(), expected_error_type=ValueError, - expected_error_fragment="no Python nodes found under path 'tasks/missing'", + expected_error_fragment="no Python nodes found under path 'python/tasks/missing'", ), PythonNodeSelectorErrorTestCase( description="raises when SQL source selector is used for Python nodes", diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py index 90d8bc3ef6..2430ea467c 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_unified_selectors.py @@ -74,28 +74,28 @@ ), PythonSqlSelectorTestCase( description="selects explicit Python task path", - select=("path:tasks",), + select=("path:python/tasks",), exclude=(), expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders"}), ), PythonSqlSelectorTestCase( description="selects expanded explicit Python asset path", - select=("+path:assets",), + select=("+path:python/assets",), exclude=(), expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), ), PythonSqlSelectorTestCase( description="selects expanded explicit Python asset slash path", - select=("+assets/",), + select=("+python/assets/",), exclude=(), expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), ), PythonSqlSelectorTestCase( description="selects expanded explicit Python asset leading slash path", - select=("+/assets",), + select=("+/python/assets",), exclude=(), expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py index fb4e0f42ae..d453bfd9a6 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/helpers.py @@ -144,8 +144,8 @@ def failing_source_node_test_loader(_ctx: object) -> list[dict[str, object]]: def build_discovered_source_loader(*, loader_name: str) -> DiscoveredLoaderFunction: return DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=loader_name, function=source_node_test_loader, ) @@ -153,8 +153,8 @@ def build_discovered_source_loader(*, loader_name: str) -> DiscoveredLoaderFunct def build_failing_discovered_source_loader(*, loader_name: str) -> DiscoveredLoaderFunction: return DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=loader_name, function=failing_source_node_test_loader, ) diff --git a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py index cfa446b1f9..b194c12040 100644 --- a/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py +++ b/tests/unit/src/sqlbuild/executor/build/_helpers/test_external_source_loads.py @@ -103,8 +103,8 @@ def external_loader(ctx: LoaderContext) -> None: ), loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=external_loader, connection_mode=LoaderConnectionMode.EXTERNAL, diff --git a/tests/unit/src/sqlbuild/executor/load/test_execute.py b/tests/unit/src/sqlbuild/executor/load/test_execute.py index 93e4e17df4..56e7a4cd4c 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_execute.py +++ b/tests/unit/src/sqlbuild/executor/load/test_execute.py @@ -92,8 +92,8 @@ def raw_orders_loader(ctx: LoaderContext) -> Iterator[dict[str, object]]: write_strategy=SourceWriteStrategy.TABLE, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name="raw_orders_loader", function=raw_orders_loader, ), @@ -155,8 +155,8 @@ def failing_loader(ctx: LoaderContext) -> Iterator[dict[str, object]]: write_strategy=SourceWriteStrategy.TABLE, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name="raw_orders_loader", function=failing_loader, ), @@ -229,8 +229,8 @@ def raw_orders_loader(ctx: LoaderContext) -> None: cursor_column="updated_at", ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=raw_orders_loader, ), @@ -324,8 +324,8 @@ def raw_orders_loader( meta={"sqlbuild_loader_node": True}, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=raw_orders_loader, destination="staging_raw_orders", @@ -401,8 +401,8 @@ def raw_orders_loader(ctx: LoaderContext) -> None: meta={"sqlbuild_loader_node": True}, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=raw_orders_loader, destination="staging_raw_orders", @@ -469,8 +469,8 @@ def raw_orders_loader( meta={"sqlbuild_loader_node": True}, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=raw_orders_loader, destination="staging_raw_orders", @@ -550,8 +550,8 @@ def raw_orders_loader( loader=test_case.loader_name, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=raw_orders_loader, connection_mode=LoaderConnectionMode.EXTERNAL, @@ -607,8 +607,8 @@ def external_generator(ctx: LoaderContext) -> Iterator[dict[str, object]]: result: LoadExecutionResult = execute_source_load( source_entry=SourceEntry(name="raw_orders", table="orders", loader="external"), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/external.py"), - relative_path=Path("loaders/external.py"), + file_path=Path("python/loaders/external.py"), + relative_path=Path("python/loaders/external.py"), name="external", function=external_generator, connection_mode=LoaderConnectionMode.EXTERNAL, @@ -667,8 +667,8 @@ def external_envelope(ctx: LoaderContext) -> NodeResultEnvelope: result: LoadExecutionResult = execute_source_load( source_entry=SourceEntry(name="raw_orders", table="orders", loader="external"), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/external.py"), - relative_path=Path("loaders/external.py"), + file_path=Path("python/loaders/external.py"), + relative_path=Path("python/loaders/external.py"), name="external", function=external_envelope, connection_mode=LoaderConnectionMode.EXTERNAL, @@ -747,8 +747,8 @@ def raw_orders_loader(ctx: LoaderContext) -> None: loader=test_case.loader_name, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=raw_orders_loader, connection_mode=LoaderConnectionMode.EXTERNAL, @@ -809,8 +809,8 @@ def fetch_orders(ctx: LoaderContext) -> None: meta={"sqlbuild_loader_node": True}, ), loader_function=DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=fetch_orders, destination=test_case.loader_target, diff --git a/tests/unit/src/sqlbuild/executor/load/test_run.py b/tests/unit/src/sqlbuild/executor/load/test_run.py index b62f1df0b3..57f4c2b1cd 100644 --- a/tests/unit/src/sqlbuild/executor/load/test_run.py +++ b/tests/unit/src/sqlbuild/executor/load/test_run.py @@ -73,8 +73,8 @@ def external_loader(ctx: LoaderContext) -> None: ), loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/raw.py"), - relative_path=Path("loaders/raw.py"), + file_path=Path("python/loaders/raw.py"), + relative_path=Path("python/loaders/raw.py"), name=test_case.loader_name, function=external_loader, connection_mode=LoaderConnectionMode.EXTERNAL, @@ -136,8 +136,8 @@ def loader(_ctx: LoaderContext) -> None: sources=(SourceEntry(name="ordered", loader="ordered_loader"),), loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/ordered.py"), - relative_path=Path("loaders/ordered.py"), + file_path=Path("python/loaders/ordered.py"), + relative_path=Path("python/loaders/ordered.py"), name="ordered_loader", function=loader, connection_mode=test_case.connection_mode, @@ -231,27 +231,27 @@ def c_loader(_ctx: LoaderContext) -> None: ), loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/a.py"), - relative_path=Path("loaders/a.py"), + file_path=Path("python/loaders/a.py"), + relative_path=Path("python/loaders/a.py"), name="a_loader", function=a_loader, ), DiscoveredLoaderFunction( - file_path=Path("loaders/x.py"), - relative_path=Path("loaders/x.py"), + file_path=Path("python/loaders/x.py"), + relative_path=Path("python/loaders/x.py"), name="x_loader", function=x_loader, ), DiscoveredLoaderFunction( - file_path=Path("loaders/b.py"), - relative_path=Path("loaders/b.py"), + file_path=Path("python/loaders/b.py"), + relative_path=Path("python/loaders/b.py"), name="b_loader", function=b_loader, depends_on=(a_loader,), ), DiscoveredLoaderFunction( - file_path=Path("loaders/c.py"), - relative_path=Path("loaders/c.py"), + file_path=Path("python/loaders/c.py"), + relative_path=Path("python/loaders/c.py"), name="c_loader", function=c_loader, depends_on=(b_loader, x_loader), @@ -324,14 +324,14 @@ def second_loader(_ctx: LoaderContext) -> None: sources=sources, loader_functions=( DiscoveredLoaderFunction( - file_path=Path("loaders/first.py"), - relative_path=Path("loaders/first.py"), + file_path=Path("python/loaders/first.py"), + relative_path=Path("python/loaders/first.py"), name="first_loader", function=first_loader, ), DiscoveredLoaderFunction( - file_path=Path("loaders/second.py"), - relative_path=Path("loaders/second.py"), + file_path=Path("python/loaders/second.py"), + relative_path=Path("python/loaders/second.py"), name="second_loader", function=second_loader, ), diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py index 9d1cc43dcd..1b7ba331ab 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/helpers.py @@ -496,7 +496,7 @@ def build_ingress_task_asset_graph() -> PythonNodeGraph: asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="publish_ingress_orders", function=publish_ingress_orders, depends_on=(prepare_ingress_orders,), @@ -509,7 +509,7 @@ def build_ingress_task_asset_graph() -> PythonNodeGraph: def ingress_task_function() -> DiscoveredTaskFunction: return DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="prepare_ingress_orders", function=prepare_ingress_orders, ) @@ -518,7 +518,7 @@ def ingress_task_function() -> DiscoveredTaskFunction: def ingress_loader_function() -> DiscoveredLoaderFunction: return DiscoveredLoaderFunction( file_path=Path("/project/loaders/orders.py"), - relative_path=Path("loaders/orders.py"), + relative_path=Path("python/loaders/orders.py"), name="load_ingress_orders", function=load_ingress_orders, depends_on=(prepare_ingress_orders,), @@ -568,7 +568,7 @@ def build_read_side_sql_task_asset_graph() -> PythonNodeGraph: task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="profile_stg_orders", function=profile_stg_orders, depends_on=(model("stg_orders"),), @@ -577,7 +577,7 @@ def build_read_side_sql_task_asset_graph() -> PythonNodeGraph: asset_functions=( DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_stg_profile", function=export_stg_profile, depends_on=(profile_stg_orders,), @@ -619,7 +619,7 @@ def python_check_function_for_case(description: str) -> DiscoveredCheckFunction: }.get(description, passing_python_check) return DiscoveredCheckFunction( file_path=Path("/project/checks/orders.py"), - relative_path=Path("checks/orders.py"), + relative_path=Path("python/checks/orders.py"), name="check_upstream_task", function=function, depends_on=(check_upstream_task,), @@ -635,7 +635,7 @@ def build_python_check_graph(*, check_function: DiscoveredCheckFunction) -> Pyth task_functions=( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="upstream_task", function=check_upstream_task, ), diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py index 8ff6291176..012182863a 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_execution.py @@ -93,7 +93,7 @@ def malformed_task() -> PythonNodeResult: nodes=( DiscoveredTaskFunction( file_path=Path("/project/tasks/bad.py"), - relative_path=Path("tasks/bad.py"), + relative_path=Path("python/tasks/bad.py"), name="malformed_task", function=malformed_task, retry=RetryPolicy(max_attempts=3, retry_on=Exception, jitter=False), @@ -153,7 +153,7 @@ def test_given_successful_callable_when_result_persistence_fails_then_resource_o ) node: DiscoveredTaskFunction = DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="persist_orders", function=lambda: "ok", ) @@ -206,7 +206,7 @@ def test_given_successful_callable_when_owned_finalizer_fails_then_resource_only dispatcher.subscribe_lifecycle(subscriber=events.append, accepts_opaque=False) node: DiscoveredTaskFunction = DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="finalize_orders", function=lambda: "ok", ) @@ -259,13 +259,13 @@ def test_given_task_asset_chain_when_executing_python_nodes_then_records_results nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="fetch_orders", function=fetch_orders, ), DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_orders", function=export_orders, depends_on=(fetch_orders,), @@ -321,13 +321,13 @@ def test_given_provider_parameters_when_executing_python_nodes_then_providers_ar nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/provider.py"), - relative_path=Path("tasks/provider.py"), + relative_path=Path("python/tasks/provider.py"), name="provider_task", function=provider_task, ), DiscoveredAssetFunction( file_path=Path("/project/assets/provider.py"), - relative_path=Path("assets/provider.py"), + relative_path=Path("python/assets/provider.py"), name="provider_asset", function=provider_asset, ), @@ -379,13 +379,13 @@ def test_given_provider_container_when_executing_python_nodes_then_context_expos nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/provider.py"), - relative_path=Path("tasks/provider.py"), + relative_path=Path("python/tasks/provider.py"), name="context_provider_task", function=context_provider_task, ), DiscoveredAssetFunction( file_path=Path("/project/assets/provider.py"), - relative_path=Path("assets/provider.py"), + relative_path=Path("python/assets/provider.py"), name="context_provider_asset", function=context_provider_asset, ), @@ -439,13 +439,13 @@ def test_given_missing_provider_container_when_executing_python_nodes_then_failu nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/provider.py"), - relative_path=Path("tasks/provider.py"), + relative_path=Path("python/tasks/provider.py"), name="provider_task", function=provider_task, ), DiscoveredAssetFunction( file_path=Path("/project/assets/provider.py"), - relative_path=Path("assets/provider.py"), + relative_path=Path("python/assets/provider.py"), name="provider_asset", function=provider_asset, ), @@ -497,7 +497,7 @@ def test_given_missing_context_provider_when_executing_python_node_then_failure_ nodes=( DiscoveredTaskFunction( file_path=Path("/project/tasks/provider.py"), - relative_path=Path("tasks/provider.py"), + relative_path=Path("python/tasks/provider.py"), name="missing_context_provider_task", function=missing_context_provider_task, ), @@ -545,13 +545,13 @@ def test_given_ready_python_node_when_executing_then_uses_existing_run_state( run_state: PythonNodeRunState = PythonNodeRunState() task_node: DiscoveredTaskFunction = DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="fetch_orders", function=fetch_orders, ) asset_node: DiscoveredAssetFunction = DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_orders", function=export_orders, depends_on=(fetch_orders,), @@ -636,7 +636,7 @@ def test_given_cursor_overrides_when_executing_python_nodes_then_context_receive nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="cursor_window", function=cursor_window, ), @@ -688,13 +688,13 @@ def test_given_hard_skipped_upstream_when_executing_python_nodes_then_skips_down nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="skip_empty_orders", function=hard_skip_empty_orders, ), DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_after_skip", function=export_after_skip, depends_on=(hard_skip_empty_orders,), @@ -780,26 +780,26 @@ def test_given_mixed_python_skips_when_executing_nodes_then_fan_in_matches_mode( nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="skip_empty_orders", function=skip_function, ), DiscoveredTaskFunction( file_path=Path("/project/tasks/sibling.py"), - relative_path=Path("tasks/sibling.py"), + relative_path=Path("python/tasks/sibling.py"), name="successful_sibling", function=successful_sibling, ), DiscoveredAssetFunction( file_path=Path("/project/assets/intermediate.py"), - relative_path=Path("assets/intermediate.py"), + relative_path=Path("python/assets/intermediate.py"), name="export_after_skip", function=export_after_skip, depends_on=(skip_function,), ), DiscoveredAssetFunction( file_path=Path("/project/assets/final.py"), - relative_path=Path("assets/final.py"), + relative_path=Path("python/assets/final.py"), name="export_after_mixed_skip", function=export_after_mixed_skip, depends_on=(export_after_skip, successful_sibling), @@ -848,13 +848,13 @@ def test_given_failed_upstream_when_executing_python_nodes_then_blocks_downstrea nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="fail_orders", function=fail_orders, ), DiscoveredAssetFunction( file_path=Path("/project/assets/orders.py"), - relative_path=Path("assets/orders.py"), + relative_path=Path("python/assets/orders.py"), name="export_after_failure", function=export_after_failure, depends_on=(fail_orders,), @@ -910,7 +910,7 @@ def test_given_retry_policy_when_transient_failures_then_retries_and_succeeds( nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="flaky_task", function=flaky_task, retry=RetryPolicy( @@ -1035,7 +1035,7 @@ def test_given_retry_policy_when_attempts_exhausted_then_records_final_exception nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="flaky_task", function=flaky_task, retry=RetryPolicy( @@ -1094,7 +1094,7 @@ def test_given_retry_policy_when_exception_is_not_selected_then_does_not_retry( nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="flaky_task", function=flaky_task, retry=RetryPolicy(max_attempts=3, retry_on=TimeoutError, jitter=False), @@ -1148,7 +1148,7 @@ def test_given_retry_policy_when_backoff_exceeds_cap_then_sleep_is_capped( nodes: tuple[DiscoveredTaskFunction | DiscoveredAssetFunction, ...] = ( DiscoveredTaskFunction( file_path=Path("/project/tasks/orders.py"), - relative_path=Path("tasks/orders.py"), + relative_path=Path("python/tasks/orders.py"), name="flaky_task", function=flaky_task, retry=RetryPolicy( diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py index 1a1be44d0f..24230a7762 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_fingerprinting.py @@ -52,7 +52,7 @@ def test_given_python_identity_when_writing_fingerprint_then_uses_fingerprint_st identity=PythonNodeIdentity( node_type="task", node_name="build_orders", - source_path="tasks/orders.py", + source_path="python/tasks/orders.py", source_hash="source-hash", definition_hash="definition-hash", version_hash="version-hash", diff --git a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py index 5ff9074a7e..14e3f59a08 100644 --- a/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py +++ b/tests/unit/src/sqlbuild/executor/python_nodes/_helpers/test_ingress_execution.py @@ -228,7 +228,7 @@ def test_given_pre_recorded_hard_skip_when_scheduler_bypasses_downstream_then_on bypassed_node: DiscoveredPythonNode = DiscoveredPythonNode( kind=PythonNodeKind.TASK, file_path=Path("/project/tasks/bypassed.py"), - relative_path=Path("tasks/bypassed.py"), + relative_path=Path("python/tasks/bypassed.py"), name="bypassed_task", function=scheduler_bypassed_task, ) diff --git a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py index c89331807f..ade0b9641a 100644 --- a/tests/unit/src/sqlbuild/integrations/dagster/helpers.py +++ b/tests/unit/src/sqlbuild/integrations/dagster/helpers.py @@ -121,14 +121,14 @@ def build_dagster_test_dag() -> Mapping[str, Any]: "kind": "loader", "name": "shared_order_feed", "asset_key": ["shared_order_feed"], - "path": "loaders/orders.py", + "path": "python/loaders/orders.py", }, { "id": "loader:raw_orders_loader", "kind": "loader", "name": "raw_orders_loader", "asset_key": ["raw_orders_loader"], - "path": "loaders/orders.py", + "path": "python/loaders/orders.py", }, { "id": "seed:waffle_types", @@ -215,7 +215,7 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: "kind": "task", "name": "prepare_orders", "asset_key": ["task", "prepare_orders"], - "path": "tasks/prepare_orders.py", + "path": "python/tasks/prepare_orders.py", "tags": ["daily"], "group": "python", "meta": {"owner": "data"}, @@ -225,7 +225,7 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: "kind": "asset", "name": "orders_export", "asset_key": ["asset", "orders_export"], - "path": "assets/orders_export.py", + "path": "python/assets/orders_export.py", "tags": ["daily"], "group": "exports", "columns": [{"name": "order_id", "type": "integer"}], @@ -237,7 +237,7 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: "kind": "check", "name": "check_orders_export", "asset_key": ["check", "check_orders_export"], - "path": "checks/check_orders_export.py", + "path": "python/checks/check_orders_export.py", "tags": ["daily"], }, ] @@ -254,7 +254,7 @@ def build_python_augmented_dagster_test_dag() -> Mapping[str, Any]: "kind": "python_check", "name": "check_orders_export", "checked_asset_ids": ["asset:orders_export"], - "path": "checks/check_orders_export.py", + "path": "python/checks/check_orders_export.py", "severity": "error", "tags": ["daily"], }, diff --git a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py index f70a8f48a3..163e6f83f1 100644 --- a/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py +++ b/tests/unit/src/sqlbuild/virtual/state/main/test_python_node_identity.py @@ -40,7 +40,7 @@ def test_given_virtual_python_identity_when_recording_then_reads_previous_finger identity: PythonNodeIdentity = PythonNodeIdentity( node_type=test_case.expected_node_type, node_name=test_case.expected_node_name, - source_path="tasks/orders.py", + source_path="python/tasks/orders.py", source_hash="source-hash", definition_hash="definition-hash", version_hash=test_case.expected_version_hash, diff --git a/website/examples/waffle-shop/loaders/waffle_sources.py b/website/examples/waffle-shop/python/loaders/waffle_sources.py similarity index 100% rename from website/examples/waffle-shop/loaders/waffle_sources.py rename to website/examples/waffle-shop/python/loaders/waffle_sources.py From 73b1e2c12adcf1db691d26a365cbdf45ffc7d032 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:10:15 +0100 Subject: [PATCH 2/9] docs: document the python/ node root and external orchestrator layout Update the Python nodes, project Python, selector, init, playground, quickstart, dlt, Dagster, and Rivers pages for the single python/ root and for orchestrator definitions that live outside the SQLBuild project, then regenerate the bundled skill references. --- .../sqlbuild/references/docs/cli/init.md | 5 +-- .../references/docs/cli/playground.md | 23 +++++++----- .../docs/concepts/project-configuration.md | 32 ++++++++++++----- .../docs/concepts/python-nodes/assets.md | 6 ++-- .../docs/concepts/python-nodes/checks.md | 8 ++--- .../docs/concepts/python-nodes/factories.md | 32 ++++------------- .../docs/concepts/python-nodes/loaders.md | 15 ++++---- .../docs/concepts/python-nodes/overview.md | 35 +++++++++++++++---- .../docs/concepts/python-nodes/providers.md | 9 ++--- .../docs/concepts/python-nodes/tasks.md | 4 +-- .../references/docs/concepts/selectors.md | 4 +-- .../docs/integrations/dagster-reference.md | 8 ++--- .../references/docs/integrations/dagster.md | 10 +++--- .../references/docs/integrations/dlt.md | 2 +- .../references/docs/integrations/rivers.md | 14 ++++---- .../sqlbuild/references/docs/quickstart.md | 5 +-- website/src/content/docs/docs/cli/init.mdx | 5 +-- .../src/content/docs/docs/cli/playground.mdx | 23 +++++++----- .../docs/concepts/project-configuration.mdx | 32 ++++++++++++----- .../docs/concepts/python-nodes/assets.mdx | 6 ++-- .../docs/concepts/python-nodes/checks.mdx | 8 ++--- .../docs/concepts/python-nodes/factories.mdx | 32 ++++------------- .../docs/concepts/python-nodes/loaders.mdx | 15 ++++---- .../docs/concepts/python-nodes/overview.mdx | 34 ++++++++++++++---- .../docs/concepts/python-nodes/providers.mdx | 9 ++--- .../docs/docs/concepts/python-nodes/tasks.mdx | 4 +-- .../content/docs/docs/concepts/selectors.mdx | 4 +-- .../docs/integrations/dagster-reference.mdx | 8 ++--- .../docs/docs/integrations/dagster.mdx | 10 +++--- .../content/docs/docs/integrations/dlt.mdx | 2 +- .../content/docs/docs/integrations/rivers.mdx | 14 ++++---- website/src/content/docs/docs/quickstart.mdx | 5 +-- 32 files changed, 239 insertions(+), 184 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md index 51fe4c115c..35088402fd 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/init.md @@ -30,10 +30,7 @@ my-project/ schemas/ sources/ seeds/ - loaders/ - tasks/ - assets/ - checks/ + python/ hooks/ sql/ python/ diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md index 326d8cc75c..f66dc9aea5 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md @@ -21,8 +21,8 @@ The positional argument is the directory to create (default `sqlbuild-playground | Template | Description | |----------|-------------| | `waffle_shop` | Default. DuckDB-backed project with models, tests, scenarios, and macros. | -| `dagster` | Waffle shop project plus a `dagster/` directory with a ready-to-run `definitions.py`. | -| `rivers` | Waffle shop project plus a `rivers_pipeline/` directory with a Rivers repository definition. | +| `dagster` | A Dagster code location: a ready-to-run `definitions.py` next to the waffle shop SQLBuild project in `waffle_shop/`. | +| `rivers` | A Rivers code repository: a `definitions.py` repository definition next to the waffle shop SQLBuild project in `waffle_shop/`. | | `python_nodes` | A small DuckDB project demonstrating [Python nodes](../concepts/python-nodes/overview.md): a task feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | ## What it creates @@ -31,6 +31,7 @@ A complete DuckDB-backed project with: - Staging views, fact/dimension tables, and incremental models - Sources with inline expression data (no external setup) +- Python source loaders in `python/loaders/` - Seeds, SQL functions, and a custom materialization - Built-in and custom audits - SQL unit tests and multi-model tests @@ -38,14 +39,20 @@ A complete DuckDB-backed project with: - Python macros - AI agent skill files (auto-installed for OpenCode, Claude Code, and other agents) -The `dagster` template adds: +The `dagster` and `rivers` templates keep orchestrator code outside the SQLBuild project: -- `dagster/definitions.py` - Dagster definitions with `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource` -- `dagster/README.md` - Setup instructions +```text +sqlbuild-playground/ + definitions.py # Dagster or Rivers definitions pointing at waffle_shop/ + README.md # setup instructions + waffle_shop/ # the waffle shop SQLBuild project +``` + +The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource`; the Rivers `definitions.py` defines a `waffle_shop` job. Run `sqb` commands from `waffle_shop/` and the orchestrator from the top-level directory. The `python_nodes` template instead creates a focused Python-nodes project: -- `tasks/orders.py`, `loaders/orders.py`, `assets/orders_export.py`, `checks/orders_export.py` +- `python/tasks/orders.py`, `python/loaders/orders.py`, `python/assets/orders_export.py`, `python/checks/orders_export.py` - A `fact_orders` SQL model over a managed `raw_orders` source - Examples of the SQL boundary, `ctx.relation(model(...))`, soft-skip fan-in, and a Python check @@ -61,13 +68,13 @@ sqb build sqb playground --template dagster cd sqlbuild-playground uv pip install 'sqlbuild[dagster]' -dagster dev -f dagster/definitions.py +dagster dev -f definitions.py # With Rivers integration sqb playground --template rivers cd sqlbuild-playground uv pip install 'sqlbuild[rivers]' -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions # With Python nodes sqb playground --template python_nodes diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index ef42122937..9322d66e68 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -531,16 +531,32 @@ See [skills CLI reference](../cli/skills.md) for usage details. ## Project Python -Project-owned Python must live in a supported extension location such as `factories/`, `libs/`, -`macros/`, `providers/`, or another documented Python resource root. Factory locations contain -normal Python: constants, classes, undecorated helper functions, and modules such as `_helpers.py` -are allowed, while decorators determine which functions become SQLBuild resources. Compilation -rejects Python under unsupported project roots, so indirectly importable modules cannot create an -unofficial project structure. +Project-owned Python lives in a small set of roots. Anything that runs as part of the pipeline graph +goes in `python/`: loaders, tasks, assets, checks, factories, and the helper modules they import. +Modules under `python/` are normal Python: constants, classes, undecorated helper functions, and +helper-only modules such as `python/_helpers.py` are allowed, while decorators determine which +functions become graph nodes. See [Python nodes](python-nodes/overview.md#the-python-folder). + +Code that extends SQLBuild itself keeps its own root: + +| Location | Holds | +|----------|-------| +| `rules/` | Custom project rules | +| `materializations/` | Custom materializations | +| `adapters/`, `adapter.py` | Custom adapters | +| `providers/` | Runtime providers | +| `sinks/` | Lifecycle event and command output sinks | +| `hooks/python/` | Python model lifecycle hooks | +| `functions/python/` | Python UDFs | +| `macros/`, `enums/`, `constants/` and scoped `_macros/`, `_enums/`, `_constants/` | Python macros and declarations | + +Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create +an unofficial project structure. Keep repository pytest tests outside the SQLBuild project's `tests/` directory, which is reserved for -SQLBuild SQL tests and scenarios. Documented integration paths such as `dagster/`, -`rivers_pipeline/`, and their `definitions.py` modules are also supported. +SQLBuild SQL tests and scenarios. Orchestrator code such as Dagster or Rivers definitions lives outside +the SQLBuild project directory and points at it; see [Dagster](../integrations/dagster.md) and +[Rivers](../integrations/rivers.md). ## sqlbuild_local.toml diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md index d2fe11ed27..a78487b7b4 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/assets.md @@ -20,10 +20,10 @@ Assets are Python nodes that produce or observe an external artifact - an export ## Defining an asset -Place Python files under `assets/` and decorate functions with `@asset`: +Place Python files under `python/` (the playgrounds use `python/assets/`) and decorate functions with `@asset`: ```python -# assets/exports.py +# python/assets/exports.py from sqlbuild.assets import asset, AssetContext @asset @@ -57,7 +57,7 @@ Assets declare dependencies with `depends_on` (a single function, tuple, or list ```python from sqlbuild.assets import asset -from tasks.orders import export_orders +from python.tasks.orders import export_orders @asset(depends_on=export_orders) def orders_dashboard(ctx): diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index fc6946658f..b81c8232ad 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -22,12 +22,12 @@ Checks are separate graph nodes, not callbacks embedded in a task or asset. A ch ## Defining a check -Place Python files under `checks/` and decorate functions with `@check`. `depends_on` is required: +Place Python files under `python/` (the playgrounds use `python/checks/`) and decorate functions with `@check`. `depends_on` is required: ```python -# checks/orders.py +# python/checks/orders.py from sqlbuild.checks import check, CheckContext -from tasks.orders import export_orders +from python.tasks.orders import export_orders @check(depends_on=export_orders) def check_orders_exported(ctx: CheckContext): @@ -121,7 +121,7 @@ sqb check --select tag:exports | | Checks | Audits | |---|--------|--------| | Validates | Python tasks, assets, loaders | SQL relations | -| Authored in | `checks/` (Python) | `MODEL()` headers / `audits/` (SQL) | +| Authored in | `python/` (Python) | `MODEL()` headers / `audits/` (SQL) | | Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` | | Severity | `error`, `warn` | `error`, `warn` | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md index 76c1df8031..f5c8a2e866 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/factories.md @@ -15,7 +15,7 @@ Factories are an advanced feature. Most projects author nodes directly; reach fo A `@factory` function takes **no arguments** and returns one or more decorated node functions: ```python -# factories/exports.py +# python/factories/exports.py from sqlbuild.factories import factory from sqlbuild.assets import asset @@ -36,38 +36,18 @@ The returned nodes are discovered and added to the graph exactly as if you had w A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@task`, `@asset`, `@loader`, or `@check` function. -## Folder rules +## Where factories live -Where a factory lives determines what it is allowed to emit: +Factories live anywhere under `python/`, like every other Python node. The decorators on the returned functions decide each node's kind, so a factory may emit a single kind or a mix, and it can sit next to hand-written nodes in the same file. The playgrounds keep factories in `python/factories/`. -| Location | May emit | -|----------|----------| -| `loaders/` | loaders only | -| `tasks/` | tasks only | -| `assets/` | assets only | -| `checks/` | checks only | -| `factories/` | any kind, including a mix | - -- A **single-kind** factory can live in that kind's folder (e.g. an asset-only factory in `assets/`), keeping it next to the nodes it generates. It can also live in `factories/`. -- A factory that emits **more than one kind** must live in `factories/`. - -This keeps each kind folder honest: everything in `assets/`, whether hand-written or factory-generated, is an asset. - -SQLBuild enforces this at discovery time. A factory in a kind folder that returns a foreign kind raises an error pointing you to `factories/`: - -``` -Factory export_pipeline in assets/ returned a loader 'raw_orders'; -mixed-kind factories must live in factories/. -``` - -The `factories/` folder is not created by `sqb init` - add it when you need it. +Factory modules are normal Python: constants, classes, and undecorated helpers are allowed, and shared helpers can live in their own modules, for example `python/factories/_helpers.py` imported as `from python.factories._helpers import TABLES`. ## Mixed-kind factories -A factory in `factories/` can generate a whole related pipeline at once - a loader, the asset that reads it, and a check on the result: +A single factory can generate a whole related pipeline at once - a loader, the asset that reads it, and a check on the result: ```python -# factories/orders.py +# python/factories/orders.py from sqlbuild.factories import factory from sqlbuild.loaders import loader from sqlbuild.assets import asset diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index d97906b3ab..eed742a5e3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -26,7 +26,7 @@ Loaders are Python functions that load data into source tables. They replace exp ## How it works -1. Write a Python function under `loaders/` decorated with `@loader` +1. Write a Python function under `python/` decorated with `@loader` 2. Declare a managed source in `sources/*.yml` with `managed: true` and **the same name as the loader function** 3. SQLBuild calls the function, writes returned rows to a staging table, then applies the configured write strategy to the target @@ -34,10 +34,10 @@ Loaders participate in the build lifecycle. When `sqb build` runs, managed sourc ## Defining a loader -Place Python files under `loaders/` in your project directory. Each file can contain one or more loader functions: +Place Python files under `python/` in your project directory (the playgrounds use `python/loaders/`). Each file can contain one or more loader functions: ```python -# loaders/raw_sources.py +# python/loaders/raw_sources.py from sqlbuild.loaders import loader from sqlbuild.executor.load.models import LoaderContext @@ -357,9 +357,10 @@ When a loader returns rows with columns not present in the existing target table ``` my-project/ - loaders/ - raw_sources.py # loader functions - api_sources.py # more loader functions + python/ + loaders/ + raw_sources.py # loader functions + api_sources.py # more loader functions sources/ raw.yml # managed source declarations (managed: true) models/ @@ -367,7 +368,7 @@ my-project/ stg_customers.sql # __source("raw_customers") ``` -SQLBuild discovers all `.py` files under `loaders/` recursively (excluding `__init__.py` and files starting with `_`). Each file is scanned for functions decorated with `@loader`. +SQLBuild imports all `.py` files under `python/` recursively (excluding `__init__.py`). Functions decorated with `@loader` become loaders; undecorated helpers do not become nodes. ## Config reference diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index dcfc26c99f..7433ce2673 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -8,6 +8,7 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/overview/ ## Contents +- The `python/` folder - The SQL boundary - Decorators - Identity tracking @@ -20,14 +21,14 @@ Online: https://sqlbuild.com/docs/concepts/python-nodes/overview/ Python nodes let your project grow beyond warehouse-only SQL while keeping the SQL graph clean. They are ordinary Python functions, decorated to become nodes in the same DAG as your SQL models, and they run as part of `sqb build`. -There are four kinds, authored in dedicated top-level folders: +There are four kinds. The decorator says what each function is, so all of them live under one `python/` folder: -| Kind | Decorator | Folder | Purpose | -|------|-----------|--------|---------| -| Loader | `@loader` | `loaders/` | Load external data into a managed source | -| Task | `@task` | `tasks/` | Run Python computation or side effects | -| Asset | `@asset` | `assets/` | Produce or observe an external artifact | -| Check | `@check` | `checks/` | Validate tasks, assets, and loaders | +| Kind | Decorator | Purpose | +|------|-----------|---------| +| Loader | `@loader` | Load external data into a managed source | +| Task | `@task` | Run Python computation or side effects | +| Asset | `@asset` | Produce or observe an external artifact | +| Check | `@check` | Validate tasks, assets, and loaders | All four share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: @@ -40,6 +41,26 @@ All four share the same decorator conventions, dependency model, selection synta Nodes can also be generated programmatically with [`@factory`](factories.md) instead of authored one at a time. +## The `python/` folder + +Anything that runs as part of the pipeline graph goes in `python/`: loaders, tasks, assets, checks, [factories](factories.md), and the helper modules they import. SQLBuild imports every `.py` file under `python/` recursively (except `__init__.py`) and registers each decorated function by its decorator. Undecorated functions and helper-only modules are allowed and never become nodes. + +How you organize files inside `python/` is up to you. The playgrounds group nodes by kind: + +``` +python/ + _helpers.py # shared helpers, not nodes + loaders/orders.py + tasks/orders.py + assets/orders_export.py + checks/orders_export.py + factories/regional.py +``` + +Grouping by domain (`python/orders/`, `python/customers/`) works just as well, and a single file may mix kinds. Import other project modules through the `python` package, for example `from python.tasks.orders import export_orders` or `from python._helpers import normalize_status`. + +Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `materializations/`, `adapters/`, `providers/`, `sinks/`, `hooks/python/`, `functions/python/`, and `macros/`. Python outside the supported locations fails compilation with `D016`; see [Project Python](../project-configuration.md#project-python) for the full list. + ## The SQL boundary The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md index b42ac14b84..1ae87b4d78 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/providers.md @@ -202,8 +202,9 @@ my-project/ record_notification.sql python/ notify.py - loaders/ - load_orders.py - tasks/ - export_orders.py + python/ + loaders/ + load_orders.py + tasks/ + export_orders.py ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md index 6929c54594..4af571d7de 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/tasks.md @@ -20,10 +20,10 @@ Tasks are Python functions that run as part of the DAG. Use them for computation ## Defining a task -Place Python files under `tasks/` and decorate functions with `@task`: +Place Python files under `python/` (the playgrounds use `python/tasks/`) and decorate functions with `@task`: ```python -# tasks/orders.py +# python/tasks/orders.py from sqlbuild.tasks import task, TaskContext @task diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index 4664f9edc4..b625635644 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -70,7 +70,7 @@ sqb build --select models/marts sqb build --select path:models/intermediate ``` -Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/`, `tasks/`, `assets/`, `checks/`, or `loaders/`. Nested paths work too: `models/staging/orders`. +Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/` for SQL models or `python/` for Python nodes. Nested paths work too: `models/staging/orders` or `python/tasks`. ### Seed and source @@ -167,5 +167,5 @@ path selector 'fact_orders~' requires names on both sides of '~' If a path selector omits the root directory, SQLBuild asks for the explicit form: ``` -path selectors require an explicit root: use 'models/', 'tasks/', 'assets/', 'checks/', or 'loaders/' +path selectors require an explicit root: use 'models/' or 'python/' ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md index c7a4339a9d..ca5204b7bd 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster-reference.md @@ -23,7 +23,7 @@ Project metadata and DAG artifact preparation. from sqlbuild.integrations.dagster import SqlBuildProject project = SqlBuildProject( - project_dir=".", + project_dir="my_sqlbuild_project", target_path="target", dag_filename="sqlbuild_dag.json", sqb_command=("sqb",), @@ -55,13 +55,13 @@ Dagster `ConfigurableResource` that shells out to the SQLBuild CLI. from sqlbuild.integrations.dagster import SqlBuildCliResource, SqlBuildProject # From a project -resource = SqlBuildCliResource(SqlBuildProject(project_dir=".")) +resource = SqlBuildCliResource(SqlBuildProject(project_dir="my_sqlbuild_project")) # Or with explicit paths resource = SqlBuildCliResource( - project_dir=".", + project_dir="my_sqlbuild_project", sqb_command=["sqb"], - dag_path="target/sqlbuild_dag.json", + dag_path="my_sqlbuild_project/target/sqlbuild_dag.json", ) ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md index 3d224501da..361ffc5e31 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dagster.md @@ -36,10 +36,10 @@ This installs `dagster` and `dagster-webserver` alongside SQLBuild. ```bash sqb playground --template dagster cd sqlbuild-playground -dagster dev -f dagster/definitions.py +dagster dev -f definitions.py ``` -This creates the waffle shop project with a `dagster/definitions.py` that includes asset definitions, scenario checks, and a configured resource. Open the Dagster UI, materialize the assets, then run the scenario checks. +This creates a Dagster code location with a `definitions.py` that includes asset definitions, scenario checks, and a configured resource, next to the waffle shop SQLBuild project in `waffle_shop/`. Dagster code stays outside the SQLBuild project directory. Open the Dagster UI, materialize the assets, then run the scenario checks. ## How it works @@ -61,7 +61,7 @@ from sqlbuild.integrations.dagster import ( ) import dagster as dg -project = SqlBuildProject(project_dir=".") +project = SqlBuildProject(project_dir="my_sqlbuild_project") project.prepare_if_dev() # auto-generates DAG artifact in dagster dev @sqlbuild_assets(project=project) @@ -78,6 +78,8 @@ defs = dg.Definitions( dagster dev -f definitions.py ``` +`definitions.py` belongs to your Dagster code location and sits next to the SQLBuild project directory (`my_sqlbuild_project/` here), not inside it. SQLBuild projects contain no orchestrator code. + Dagster discovers every SQLBuild model, loader, task, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. For production deployments, use `project.prepare()` or `sqb compile --dag` to generate the DAG artifact explicitly in your CI pipeline. @@ -144,7 +146,7 @@ def my_scenario_checks(context, sqb): `SqlBuildProject.prepare()` regenerates the DAG artifact by running `sqb compile --dag`. Use `prepare_if_dev()` to only regenerate during local development: ```python -project = SqlBuildProject(project_dir=".") +project = SqlBuildProject(project_dir="my_sqlbuild_project") project.prepare_if_dev() # only runs when DAGSTER_IS_DEV_CLI is set ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md index 518cd79879..3bdc163c2a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/dlt.md @@ -136,7 +136,7 @@ SELECT id, title, state FROM __source("raw_github_issues") When you need dlt capabilities beyond the declarative surface (custom transforms, `delete_insert`, sources not covered above), wrap a dlt pipeline in a [source loader](../concepts/python-nodes/loaders.md). The loader calls `dlt.pipeline(...).run(...)` and returns `None`; dlt handles the writes and SQLBuild treats the source as loaded. ```python -# loaders/github_sources.py +# python/loaders/github_sources.py import dlt from dlt.sources.rest_api import rest_api_source from sqlbuild.loaders import loader diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md index 6608230229..9760133072 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/integrations/rivers.md @@ -34,10 +34,10 @@ This installs `rivers` alongside SQLBuild. ```bash sqb playground --template rivers cd sqlbuild-playground -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions ``` -This creates the waffle shop project with a `rivers_pipeline/definitions.py` that includes asset definitions and a configured job. Open the Rivers UI to inspect assets and trigger materializations. +This creates a Rivers code repository with a `definitions.py` that includes asset definitions and a configured job, next to the waffle shop SQLBuild project in `waffle_shop/`. Rivers code stays outside the SQLBuild project directory. Open the Rivers UI to inspect assets and trigger materializations. ## How it works @@ -60,7 +60,7 @@ import rivers as rs from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets -PROJECT_DIR = Path(__file__).resolve().parent.parent +PROJECT_DIR = Path(__file__).resolve().parent / "my_sqlbuild_project" SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) SQLBUILD_PROJECT.prepare_if_dev() @@ -90,12 +90,14 @@ repo = rs.CodeRepository( ) ``` +`definitions.py` belongs to your Rivers code repository and sits next to the SQLBuild project directory (`my_sqlbuild_project/` here), not inside it. SQLBuild projects contain no orchestrator code. + ## SqlBuildProject `SqlBuildProject` manages paths and DAG artifact generation: ```python -project = SqlBuildProject(project_dir=Path(".")) +project = SqlBuildProject(project_dir=Path("my_sqlbuild_project")) ``` | Field | Default | Description | @@ -171,11 +173,11 @@ def my_assets(context): Run the pipeline directly without the Rivers UI: ```bash -uv run python rivers_pipeline/definitions.py +uv run python definitions.py ``` Or start the Rivers development server: ```bash -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions ``` diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md index 761f0c7509..7453c17f8a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/quickstart.md @@ -222,8 +222,9 @@ waffle-shop/ sqlbuild_project.toml # project config sources/ raw.yml # source declarations (loader-managed and expression-backed) - loaders/ - waffle_sources.py # Python loaders for raw__customers and raw__orders + python/ + loaders/ + waffle_sources.py # Python loaders for raw__customers and raw__orders seeds/ waffle_types.csv # seed data lookups.yml # seed declarations diff --git a/website/src/content/docs/docs/cli/init.mdx b/website/src/content/docs/docs/cli/init.mdx index ae2787f5ba..0035524475 100644 --- a/website/src/content/docs/docs/cli/init.mdx +++ b/website/src/content/docs/docs/cli/init.mdx @@ -29,10 +29,7 @@ my-project/ schemas/ sources/ seeds/ - loaders/ - tasks/ - assets/ - checks/ + python/ hooks/ sql/ python/ diff --git a/website/src/content/docs/docs/cli/playground.mdx b/website/src/content/docs/docs/cli/playground.mdx index 4b94e7d775..0fe8fcd757 100644 --- a/website/src/content/docs/docs/cli/playground.mdx +++ b/website/src/content/docs/docs/cli/playground.mdx @@ -20,8 +20,8 @@ The positional argument is the directory to create (default `sqlbuild-playground | Template | Description | |----------|-------------| | `waffle_shop` | Default. DuckDB-backed project with models, tests, scenarios, and macros. | -| `dagster` | Waffle shop project plus a `dagster/` directory with a ready-to-run `definitions.py`. | -| `rivers` | Waffle shop project plus a `rivers_pipeline/` directory with a Rivers repository definition. | +| `dagster` | A Dagster code location: a ready-to-run `definitions.py` next to the waffle shop SQLBuild project in `waffle_shop/`. | +| `rivers` | A Rivers code repository: a `definitions.py` repository definition next to the waffle shop SQLBuild project in `waffle_shop/`. | | `python_nodes` | A small DuckDB project demonstrating [Python nodes](/docs/concepts/python-nodes/overview): a task feeding a loader, a model read by a Python asset through `ctx.relation(model(...))`, a soft-skip fan-in, `materialized=False`, and a Python check. | ## What it creates @@ -30,6 +30,7 @@ A complete DuckDB-backed project with: - Staging views, fact/dimension tables, and incremental models - Sources with inline expression data (no external setup) +- Python source loaders in `python/loaders/` - Seeds, SQL functions, and a custom materialization - Built-in and custom audits - SQL unit tests and multi-model tests @@ -37,14 +38,20 @@ A complete DuckDB-backed project with: - Python macros - AI agent skill files (auto-installed for OpenCode, Claude Code, and other agents) -The `dagster` template adds: +The `dagster` and `rivers` templates keep orchestrator code outside the SQLBuild project: -- `dagster/definitions.py` - Dagster definitions with `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource` -- `dagster/README.md` - Setup instructions +```text +sqlbuild-playground/ + definitions.py # Dagster or Rivers definitions pointing at waffle_shop/ + README.md # setup instructions + waffle_shop/ # the waffle shop SQLBuild project +``` + +The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource`; the Rivers `definitions.py` defines a `waffle_shop` job. Run `sqb` commands from `waffle_shop/` and the orchestrator from the top-level directory. The `python_nodes` template instead creates a focused Python-nodes project: -- `tasks/orders.py`, `loaders/orders.py`, `assets/orders_export.py`, `checks/orders_export.py` +- `python/tasks/orders.py`, `python/loaders/orders.py`, `python/assets/orders_export.py`, `python/checks/orders_export.py` - A `fact_orders` SQL model over a managed `raw_orders` source - Examples of the SQL boundary, `ctx.relation(model(...))`, soft-skip fan-in, and a Python check @@ -60,13 +67,13 @@ sqb build sqb playground --template dagster cd sqlbuild-playground uv pip install 'sqlbuild[dagster]' -dagster dev -f dagster/definitions.py +dagster dev -f definitions.py # With Rivers integration sqb playground --template rivers cd sqlbuild-playground uv pip install 'sqlbuild[rivers]' -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions # With Python nodes sqb playground --template python_nodes diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index d60f9978b5..545e3478f2 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -511,16 +511,32 @@ See [skills CLI reference](/docs/cli/skills) for usage details. ## Project Python -Project-owned Python must live in a supported extension location such as `factories/`, `libs/`, -`macros/`, `providers/`, or another documented Python resource root. Factory locations contain -normal Python: constants, classes, undecorated helper functions, and modules such as `_helpers.py` -are allowed, while decorators determine which functions become SQLBuild resources. Compilation -rejects Python under unsupported project roots, so indirectly importable modules cannot create an -unofficial project structure. +Project-owned Python lives in a small set of roots. Anything that runs as part of the pipeline graph +goes in `python/`: loaders, tasks, assets, checks, factories, and the helper modules they import. +Modules under `python/` are normal Python: constants, classes, undecorated helper functions, and +helper-only modules such as `python/_helpers.py` are allowed, while decorators determine which +functions become graph nodes. See [Python nodes](/docs/concepts/python-nodes/overview#the-python-folder). + +Code that extends SQLBuild itself keeps its own root: + +| Location | Holds | +|----------|-------| +| `rules/` | Custom project rules | +| `materializations/` | Custom materializations | +| `adapters/`, `adapter.py` | Custom adapters | +| `providers/` | Runtime providers | +| `sinks/` | Lifecycle event and command output sinks | +| `hooks/python/` | Python model lifecycle hooks | +| `functions/python/` | Python UDFs | +| `macros/`, `enums/`, `constants/` and scoped `_macros/`, `_enums/`, `_constants/` | Python macros and declarations | + +Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create +an unofficial project structure. Keep repository pytest tests outside the SQLBuild project's `tests/` directory, which is reserved for -SQLBuild SQL tests and scenarios. Documented integration paths such as `dagster/`, -`rivers_pipeline/`, and their `definitions.py` modules are also supported. +SQLBuild SQL tests and scenarios. Orchestrator code such as Dagster or Rivers definitions lives outside +the SQLBuild project directory and points at it; see [Dagster](/docs/integrations/dagster) and +[Rivers](/docs/integrations/rivers). ## sqlbuild_local.toml diff --git a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx index 286c431533..19a06bc589 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/assets.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/assets.mdx @@ -7,10 +7,10 @@ Assets are Python nodes that produce or observe an external artifact - an export ## Defining an asset -Place Python files under `assets/` and decorate functions with `@asset`: +Place Python files under `python/` (the playgrounds use `python/assets/`) and decorate functions with `@asset`: ```python -# assets/exports.py +# python/assets/exports.py from sqlbuild.assets import asset, AssetContext @@ -45,7 +45,7 @@ Assets declare dependencies with `depends_on` (a single function, tuple, or list ```python from sqlbuild.assets import asset -from tasks.orders import export_orders +from python.tasks.orders import export_orders @asset(depends_on=export_orders) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index 051290c618..6b8112dff7 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -9,12 +9,12 @@ Checks are separate graph nodes, not callbacks embedded in a task or asset. A ch ## Defining a check -Place Python files under `checks/` and decorate functions with `@check`. `depends_on` is required: +Place Python files under `python/` (the playgrounds use `python/checks/`) and decorate functions with `@check`. `depends_on` is required: ```python -# checks/orders.py +# python/checks/orders.py from sqlbuild.checks import check, CheckContext -from tasks.orders import export_orders +from python.tasks.orders import export_orders @check(depends_on=export_orders) @@ -109,7 +109,7 @@ sqb check --select tag:exports | | Checks | Audits | |---|--------|--------| | Validates | Python tasks, assets, loaders | SQL relations | -| Authored in | `checks/` (Python) | `MODEL()` headers / `audits/` (SQL) | +| Authored in | `python/` (Python) | `MODEL()` headers / `audits/` (SQL) | | Run by | `sqb build`, `sqb check` | `sqb build`, `sqb audit` | | Severity | `error`, `warn` | `error`, `warn` | diff --git a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx index 22972af2ca..c2af78862c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/factories.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/factories.mdx @@ -12,7 +12,7 @@ Factories are an advanced feature. Most projects author nodes directly; reach fo A `@factory` function takes **no arguments** and returns one or more decorated node functions: ```python -# factories/exports.py +# python/factories/exports.py from sqlbuild.factories import factory from sqlbuild.assets import asset @@ -34,38 +34,18 @@ The returned nodes are discovered and added to the graph exactly as if you had w A factory may return a single node or a list, tuple, or set of nodes. Every returned item must be a decorated `@task`, `@asset`, `@loader`, or `@check` function. -## Folder rules +## Where factories live -Where a factory lives determines what it is allowed to emit: +Factories live anywhere under `python/`, like every other Python node. The decorators on the returned functions decide each node's kind, so a factory may emit a single kind or a mix, and it can sit next to hand-written nodes in the same file. The playgrounds keep factories in `python/factories/`. -| Location | May emit | -|----------|----------| -| `loaders/` | loaders only | -| `tasks/` | tasks only | -| `assets/` | assets only | -| `checks/` | checks only | -| `factories/` | any kind, including a mix | - -- A **single-kind** factory can live in that kind's folder (e.g. an asset-only factory in `assets/`), keeping it next to the nodes it generates. It can also live in `factories/`. -- A factory that emits **more than one kind** must live in `factories/`. - -This keeps each kind folder honest: everything in `assets/`, whether hand-written or factory-generated, is an asset. - -SQLBuild enforces this at discovery time. A factory in a kind folder that returns a foreign kind raises an error pointing you to `factories/`: - -``` -Factory export_pipeline in assets/ returned a loader 'raw_orders'; -mixed-kind factories must live in factories/. -``` - -The `factories/` folder is not created by `sqb init` - add it when you need it. +Factory modules are normal Python: constants, classes, and undecorated helpers are allowed, and shared helpers can live in their own modules, for example `python/factories/_helpers.py` imported as `from python.factories._helpers import TABLES`. ## Mixed-kind factories -A factory in `factories/` can generate a whole related pipeline at once - a loader, the asset that reads it, and a check on the result: +A single factory can generate a whole related pipeline at once - a loader, the asset that reads it, and a check on the result: ```python -# factories/orders.py +# python/factories/orders.py from sqlbuild.factories import factory from sqlbuild.loaders import loader from sqlbuild.assets import asset diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index f0d928638b..cd8cdc46e4 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -7,7 +7,7 @@ Loaders are Python functions that load data into source tables. They replace exp ## How it works -1. Write a Python function under `loaders/` decorated with `@loader` +1. Write a Python function under `python/` decorated with `@loader` 2. Declare a managed source in `sources/*.yml` with `managed: true` and **the same name as the loader function** 3. SQLBuild calls the function, writes returned rows to a staging table, then applies the configured write strategy to the target @@ -15,10 +15,10 @@ Loaders participate in the build lifecycle. When `sqb build` runs, managed sourc ## Defining a loader -Place Python files under `loaders/` in your project directory. Each file can contain one or more loader functions: +Place Python files under `python/` in your project directory (the playgrounds use `python/loaders/`). Each file can contain one or more loader functions: ```python -# loaders/raw_sources.py +# python/loaders/raw_sources.py from sqlbuild.loaders import loader from sqlbuild.executor.load.models import LoaderContext @@ -338,9 +338,10 @@ When a loader returns rows with columns not present in the existing target table ``` my-project/ - loaders/ - raw_sources.py # loader functions - api_sources.py # more loader functions + python/ + loaders/ + raw_sources.py # loader functions + api_sources.py # more loader functions sources/ raw.yml # managed source declarations (managed: true) models/ @@ -348,7 +349,7 @@ my-project/ stg_customers.sql # __source("raw_customers") ``` -SQLBuild discovers all `.py` files under `loaders/` recursively (excluding `__init__.py` and files starting with `_`). Each file is scanned for functions decorated with `@loader`. +SQLBuild imports all `.py` files under `python/` recursively (excluding `__init__.py`). Functions decorated with `@loader` become loaders; undecorated helpers do not become nodes. ## Config reference diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index d051238541..0fdb61d35d 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -5,14 +5,14 @@ description: "Tasks, assets, loaders, and checks as first-class nodes in the SQL Python nodes let your project grow beyond warehouse-only SQL while keeping the SQL graph clean. They are ordinary Python functions, decorated to become nodes in the same DAG as your SQL models, and they run as part of `sqb build`. -There are four kinds, authored in dedicated top-level folders: +There are four kinds. The decorator says what each function is, so all of them live under one `python/` folder: -| Kind | Decorator | Folder | Purpose | -|------|-----------|--------|---------| -| Loader | `@loader` | `loaders/` | Load external data into a managed source | -| Task | `@task` | `tasks/` | Run Python computation or side effects | -| Asset | `@asset` | `assets/` | Produce or observe an external artifact | -| Check | `@check` | `checks/` | Validate tasks, assets, and loaders | +| Kind | Decorator | Purpose | +|------|-----------|---------| +| Loader | `@loader` | Load external data into a managed source | +| Task | `@task` | Run Python computation or side effects | +| Asset | `@asset` | Produce or observe an external artifact | +| Check | `@check` | Validate tasks, assets, and loaders | All four share the same decorator conventions, dependency model, selection syntax, and runtime context helpers. The pages for each kind cover their specifics: @@ -25,6 +25,26 @@ All four share the same decorator conventions, dependency model, selection synta Nodes can also be generated programmatically with [`@factory`](/docs/concepts/python-nodes/factories) instead of authored one at a time. +## The `python/` folder + +Anything that runs as part of the pipeline graph goes in `python/`: loaders, tasks, assets, checks, [factories](/docs/concepts/python-nodes/factories), and the helper modules they import. SQLBuild imports every `.py` file under `python/` recursively (except `__init__.py`) and registers each decorated function by its decorator. Undecorated functions and helper-only modules are allowed and never become nodes. + +How you organize files inside `python/` is up to you. The playgrounds group nodes by kind: + +``` +python/ + _helpers.py # shared helpers, not nodes + loaders/orders.py + tasks/orders.py + assets/orders_export.py + checks/orders_export.py + factories/regional.py +``` + +Grouping by domain (`python/orders/`, `python/customers/`) works just as well, and a single file may mix kinds. Import other project modules through the `python` package, for example `from python.tasks.orders import export_orders` or `from python._helpers import normalize_status`. + +Code that extends SQLBuild itself keeps its own folder, such as `rules/`, `materializations/`, `adapters/`, `providers/`, `sinks/`, `hooks/python/`, `functions/python/`, and `macros/`. Python outside the supported locations fails compilation with `D016`; see [Project Python](/docs/concepts/project-configuration#project-python) for the full list. + ## The SQL boundary The most important rule: **SQL models never depend on Python nodes.** The dependency direction is strictly one way. diff --git a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx index 3d049b2715..1019f76824 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/providers.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/providers.mdx @@ -186,8 +186,9 @@ my-project/ record_notification.sql python/ notify.py - loaders/ - load_orders.py - tasks/ - export_orders.py + python/ + loaders/ + load_orders.py + tasks/ + export_orders.py ``` diff --git a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx index 5531605fad..a33e5eeca4 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/tasks.mdx @@ -7,10 +7,10 @@ Tasks are Python functions that run as part of the DAG. Use them for computation ## Defining a task -Place Python files under `tasks/` and decorate functions with `@task`: +Place Python files under `python/` (the playgrounds use `python/tasks/`) and decorate functions with `@task`: ```python -# tasks/orders.py +# python/tasks/orders.py from sqlbuild.tasks import task, TaskContext diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index f9b0ee178e..6b8db58d05 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -57,7 +57,7 @@ sqb build --select models/marts sqb build --select path:models/intermediate ``` -Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/`, `tasks/`, `assets/`, `checks/`, or `loaders/`. Nested paths work too: `models/staging/orders`. +Any name containing `/` is treated as a path selector, so `path:models/marts` and a bare `models/marts` work the same way. Path selectors require an explicit root directory: `models/` for SQL models or `python/` for Python nodes. Nested paths work too: `models/staging/orders` or `python/tasks`. ### Seed and source @@ -154,5 +154,5 @@ path selector 'fact_orders~' requires names on both sides of '~' If a path selector omits the root directory, SQLBuild asks for the explicit form: ``` -path selectors require an explicit root: use 'models/', 'tasks/', 'assets/', 'checks/', or 'loaders/' +path selectors require an explicit root: use 'models/' or 'python/' ``` diff --git a/website/src/content/docs/docs/integrations/dagster-reference.mdx b/website/src/content/docs/docs/integrations/dagster-reference.mdx index 94310410bc..7e88ef5a43 100644 --- a/website/src/content/docs/docs/integrations/dagster-reference.mdx +++ b/website/src/content/docs/docs/integrations/dagster-reference.mdx @@ -11,7 +11,7 @@ Project metadata and DAG artifact preparation. from sqlbuild.integrations.dagster import SqlBuildProject project = SqlBuildProject( - project_dir=".", + project_dir="my_sqlbuild_project", target_path="target", dag_filename="sqlbuild_dag.json", sqb_command=("sqb",), @@ -43,13 +43,13 @@ Dagster `ConfigurableResource` that shells out to the SQLBuild CLI. from sqlbuild.integrations.dagster import SqlBuildCliResource, SqlBuildProject # From a project -resource = SqlBuildCliResource(SqlBuildProject(project_dir=".")) +resource = SqlBuildCliResource(SqlBuildProject(project_dir="my_sqlbuild_project")) # Or with explicit paths resource = SqlBuildCliResource( - project_dir=".", + project_dir="my_sqlbuild_project", sqb_command=["sqb"], - dag_path="target/sqlbuild_dag.json", + dag_path="my_sqlbuild_project/target/sqlbuild_dag.json", ) ``` diff --git a/website/src/content/docs/docs/integrations/dagster.mdx b/website/src/content/docs/docs/integrations/dagster.mdx index 7e3b61b4e4..c8b0939558 100644 --- a/website/src/content/docs/docs/integrations/dagster.mdx +++ b/website/src/content/docs/docs/integrations/dagster.mdx @@ -20,10 +20,10 @@ This installs `dagster` and `dagster-webserver` alongside SQLBuild. ```bash sqb playground --template dagster cd sqlbuild-playground -dagster dev -f dagster/definitions.py +dagster dev -f definitions.py ``` -This creates the waffle shop project with a `dagster/definitions.py` that includes asset definitions, scenario checks, and a configured resource. Open the Dagster UI, materialize the assets, then run the scenario checks. +This creates a Dagster code location with a `definitions.py` that includes asset definitions, scenario checks, and a configured resource, next to the waffle shop SQLBuild project in `waffle_shop/`. Dagster code stays outside the SQLBuild project directory. Open the Dagster UI, materialize the assets, then run the scenario checks. ## How it works @@ -45,7 +45,7 @@ from sqlbuild.integrations.dagster import ( ) import dagster as dg -project = SqlBuildProject(project_dir=".") +project = SqlBuildProject(project_dir="my_sqlbuild_project") project.prepare_if_dev() # auto-generates DAG artifact in dagster dev @sqlbuild_assets(project=project) @@ -62,6 +62,8 @@ defs = dg.Definitions( dagster dev -f definitions.py ``` +`definitions.py` belongs to your Dagster code location and sits next to the SQLBuild project directory (`my_sqlbuild_project/` here), not inside it. SQLBuild projects contain no orchestrator code. + Dagster discovers every SQLBuild model, loader, task, and asset as a Dagster asset. Selecting a subset in the Dagster UI automatically scopes the `sqb build` invocation to those nodes via `--select`. For production deployments, use `project.prepare()` or `sqb compile --dag` to generate the DAG artifact explicitly in your CI pipeline. @@ -128,7 +130,7 @@ def my_scenario_checks(context, sqb): `SqlBuildProject.prepare()` regenerates the DAG artifact by running `sqb compile --dag`. Use `prepare_if_dev()` to only regenerate during local development: ```python -project = SqlBuildProject(project_dir=".") +project = SqlBuildProject(project_dir="my_sqlbuild_project") project.prepare_if_dev() # only runs when DAGSTER_IS_DEV_CLI is set ``` diff --git a/website/src/content/docs/docs/integrations/dlt.mdx b/website/src/content/docs/docs/integrations/dlt.mdx index 6ed0539b12..7341376c11 100644 --- a/website/src/content/docs/docs/integrations/dlt.mdx +++ b/website/src/content/docs/docs/integrations/dlt.mdx @@ -132,7 +132,7 @@ SELECT id, title, state FROM __source("raw_github_issues") When you need dlt capabilities beyond the declarative surface (custom transforms, `delete_insert`, sources not covered above), wrap a dlt pipeline in a [source loader](/docs/concepts/python-nodes/loaders). The loader calls `dlt.pipeline(...).run(...)` and returns `None`; dlt handles the writes and SQLBuild treats the source as loaded. ```python -# loaders/github_sources.py +# python/loaders/github_sources.py import dlt from dlt.sources.rest_api import rest_api_source from sqlbuild.loaders import loader diff --git a/website/src/content/docs/docs/integrations/rivers.mdx b/website/src/content/docs/docs/integrations/rivers.mdx index 27943b33e1..c8a17a11a0 100644 --- a/website/src/content/docs/docs/integrations/rivers.mdx +++ b/website/src/content/docs/docs/integrations/rivers.mdx @@ -20,10 +20,10 @@ This installs `rivers` alongside SQLBuild. ```bash sqb playground --template rivers cd sqlbuild-playground -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions ``` -This creates the waffle shop project with a `rivers_pipeline/definitions.py` that includes asset definitions and a configured job. Open the Rivers UI to inspect assets and trigger materializations. +This creates a Rivers code repository with a `definitions.py` that includes asset definitions and a configured job, next to the waffle shop SQLBuild project in `waffle_shop/`. Rivers code stays outside the SQLBuild project directory. Open the Rivers UI to inspect assets and trigger materializations. ## How it works @@ -46,7 +46,7 @@ import rivers as rs from sqlbuild.integrations.rivers import SqlBuildProject, sqlbuild_assets -PROJECT_DIR = Path(__file__).resolve().parent.parent +PROJECT_DIR = Path(__file__).resolve().parent / "my_sqlbuild_project" SQLBUILD_PROJECT = SqlBuildProject(project_dir=PROJECT_DIR) SQLBUILD_PROJECT.prepare_if_dev() @@ -78,12 +78,14 @@ repo = rs.CodeRepository( ) ``` +`definitions.py` belongs to your Rivers code repository and sits next to the SQLBuild project directory (`my_sqlbuild_project/` here), not inside it. SQLBuild projects contain no orchestrator code. + ## SqlBuildProject `SqlBuildProject` manages paths and DAG artifact generation: ```python -project = SqlBuildProject(project_dir=Path(".")) +project = SqlBuildProject(project_dir=Path("my_sqlbuild_project")) ``` | Field | Default | Description | @@ -159,11 +161,11 @@ def my_assets(context): Run the pipeline directly without the Rivers UI: ```bash -uv run python rivers_pipeline/definitions.py +uv run python definitions.py ``` Or start the Rivers development server: ```bash -uv run rivers dev rivers_pipeline.definitions +uv run rivers dev definitions ``` diff --git a/website/src/content/docs/docs/quickstart.mdx b/website/src/content/docs/docs/quickstart.mdx index 2d9ca32718..530954868c 100644 --- a/website/src/content/docs/docs/quickstart.mdx +++ b/website/src/content/docs/docs/quickstart.mdx @@ -208,8 +208,9 @@ waffle-shop/ sqlbuild_project.toml # project config sources/ raw.yml # source declarations (loader-managed and expression-backed) - loaders/ - waffle_sources.py # Python loaders for raw__customers and raw__orders + python/ + loaders/ + waffle_sources.py # Python loaders for raw__customers and raw__orders seeds/ waffle_types.csv # seed data lookups.yml # seed declarations From 4c5fb90e8ea42168bafb512f953a8fa00a9e09fe Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:18:36 +0100 Subject: [PATCH 3/9] docs: regenerate bundled skill references --- .../.agents/skills/sqlbuild/references/docs/cli/compile.md | 4 +++- .../.agents/skills/sqlbuild/references/docs/cli/query.md | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md index 43b86abd84..03525419f1 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/compile.md @@ -46,7 +46,9 @@ sqb --project-dir compile [flags] 3. **SQL validation** - validates SQL syntax (when SQL analysis is enabled) 4. **Column lineage** - analyzes column-level dependencies across models (fast mode by default) 5. **Contract validation** - checks declared column contracts against inferred query output -6. **Rules** - evaluates selected native built-ins, then selected custom Python Rules +6. **Rules** - evaluates selected native built-ins, then selected custom Python Rules. Rules that + need inferred type proofs, such as `SQBRCONTRACT105`, are skipped when SQL analysis is disabled, + and compile prints one note naming them 7. **Artifact write** - writes compiled SQL to `target/compiled/` when Rules pass ## Focused compilation diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md index 617f80f46e..1d9110dc78 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/query.md @@ -83,6 +83,13 @@ order_id,customer_id,status 2,200,completed ``` +## Errors + +When the warehouse rejects the SQL, `sqb query` prints the warehouse message as +`error[C108]: ...` on stderr and exits with code `2`. Nothing is written to stdout, so +redirected `json` or `csv` output is never partial. Invalid command input, such as a missing +query file, exits with code `1`. + ## Examples ```bash From c252a9e984b3575c5e3530bb51ffce5927e73d4c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:26:14 +0100 Subject: [PATCH 4/9] fix: import python/ modules under their package names Discovery imports every python/ module as python. through the normal import system, so helper packages can use relative imports and each module initializes once per discovery pass. python.* entries are evicted at the start of each pass so sequential projects and edited files are imported afresh. --- .../discovery/_helpers/filesystem/core.py | 32 ++- .../cli/commands/main/build/_test_types.py | 12 ++ .../main/build/test_python_node_identity.py | 92 ++++++++ .../discovery/_helpers/_test_types.py | 27 +++ .../_helpers/test_filesystem_python_nodes.py | 200 ++++++++++++++++++ 5 files changed, 362 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 981e3ebfa5..b2db7a8725 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -1357,11 +1357,15 @@ def _discover_python_node_functions( node_root: Path = project_dir / PYTHON_NODE_ROOT if not node_root.is_dir(): return bucket + _evict_python_root_modules() + importlib.invalidate_caches() file_path: Path for file_path in sorted(node_root.rglob("*.py")): if file_path.stem == PYTHON_INIT_MODULE_STEM: continue - module: ModuleType = _load_python_node_module(file_path=file_path, project_dir=project_dir) + module: ModuleType = _import_python_root_module( + file_path=file_path, project_dir=project_dir + ) _append_module_python_nodes( bucket=bucket, module=module, @@ -1372,6 +1376,32 @@ def _discover_python_node_functions( return bucket +def _evict_python_root_modules() -> None: + """Drop python.* modules so each discovery pass imports the current project afresh.""" + + module_name: str + for module_name in tuple(sys.modules): + if module_name == PYTHON_NODE_ROOT or module_name.startswith(f"{PYTHON_NODE_ROOT}."): + sys.modules.pop(module_name, None) + + +def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleType: + """Import a python/ module under its package name, reusing it within a discovery pass.""" + + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) + module_name: str = ".".join(relative_path.with_suffix("").parts) + old_path: list[str] = list(sys.path) + sys.path.insert(0, str(project_dir)) + try: + return importlib.import_module(module_name) + except Exception as error: + raise PythonNodeDiscoveryError( + f"Failed to import Python node file {relative_path}: {error}" + ) from error + finally: + sys.path = old_path + + def _append_module_python_nodes( *, bucket: _PythonNodeDiscoveryBucket, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 7bce2e6d25..0ed2a4cb58 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1120,3 +1120,15 @@ class DroppedIncrementalRelationE2ETestCase: description: str incremental_strategy: str expected_rows: tuple[tuple[int, str], ...] + + +@dataclass(frozen=True) +class PythonHelperPackageIdentityE2ETestCase: + """Test case for node identity after editing a relative-import helper package.""" + + description: str + edited_path: str + original_text: str + edited_text: str + expected_identity_status: str + expected_version_count: int diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py index 7fb52be598..a2cc997b96 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_identity.py @@ -10,6 +10,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + PythonHelperPackageIdentityE2ETestCase, PythonNodeIdentityBuildE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( @@ -145,5 +146,96 @@ def export_orders(ctx): ) == [(test_case.expected_loader_version_count,)] +@pytest.mark.parametrize( + "test_case", + [ + PythonHelperPackageIdentityE2ETestCase( + description="relative-import helper edit changes the dependent task identity", + edited_path="python/helpers/clean.py", + original_text="value.strip().lower()", + edited_text="value.strip().upper()", + expected_identity_status="changed", + expected_version_count=2, + ), + ], + ids=lambda case: case.description, +) +def test_given_helper_package_with_relative_imports_when_editing_helper_then_task_identity_changes( + test_case: PythonHelperPackageIdentityE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_helper_package_project", + repo_files={ + "sqlbuild_project.toml": dedent( + """ + name = "python_helper_package_project" + adapter = "duckdb" + + [connection] + database = "warehouse.duckdb" + """ + ).strip() + + "\n", + "python/helpers/__init__.py": "", + "python/helpers/values.py": 'STATUS = "shipped"\n', + "python/helpers/clean.py": dedent( + """ + from .values import STATUS + + + def normalize_status(value): + return f"{value.strip().lower()}:{STATUS}" + """ + ).lstrip(), + "python/orders.py": dedent( + """ + import python.helpers.clean + from sqlbuild.tasks import task + + + @task + def orders(ctx): + return ctx.result( + payload={"status": python.helpers.clean.normalize_status(" Shipped ")} + ) + """ + ).lstrip(), + }, + ) + first_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + assert first_build.returncode == 0, first_build.stdout + first_build.stderr + + edited: Path = project_dir / test_case.edited_path + edited.write_text( + edited.read_text(encoding="utf-8").replace(test_case.original_text, test_case.edited_text), + encoding="utf-8", + ) + plan: subprocess.CompletedProcess[str] = run_sqb( + command=("plan", "--json"), project_dir=project_dir + ) + second_build: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), project_dir=project_dir + ) + + assert plan.returncode == 0, plan.stdout + plan.stderr + assert second_build.returncode == 0, second_build.stdout + second_build.stderr + payload: dict[str, object] = json.loads(plan.stdout) + statuses_by_name: dict[str, str] = { + str(entry["name"]): str(entry["identity_status"]) for entry in payload["python_nodes"] + } + assert statuses_by_name["orders"] == test_case.expected_identity_status + assert query_duckdb( + db_path=project_dir / "warehouse.duckdb", + sql=( + "SELECT COUNT(DISTINCT version_hash) FROM main._sqlbuild_fingerprints " + "WHERE node_type = 'task' AND node_name = 'orders'" + ), + ) == [(test_case.expected_version_count,)] + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index 18fe822771..cd63e5c802 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -803,6 +803,33 @@ class DiscoverPythonRootNodesTestCase: expected_nodes: tuple[tuple[str, str, str], ...] +@dataclass(frozen=True) +class DiscoverPythonRootImportTestCase: + description: str + files: dict[str, str] + expected_task_names: tuple[str, ...] + expected_init_count: int = 0 + + +@dataclass(frozen=True) +class PythonRootProjectIsolationTestCase: + description: str + first_files: dict[str, str] + second_files: dict[str, str] + expected_first_result: object + expected_second_result: object + + +@dataclass(frozen=True) +class PythonRootHelperIdentityTestCase: + description: str + files: dict[str, str] + edited_path: str + original_text: str + edited_text: str + expected_dependency_path: str + + @dataclass(frozen=True) class ExpectedBooleanTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 2d349349ac..ff7a319c4b 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -16,10 +16,15 @@ DiscoveredPythonNodeFunctions, DiscoveredTaskFunction, ) +from sqlbuild.compiler.python_nodes.main.identity import build_python_node_identity +from sqlbuild.compiler.python_nodes.models import PythonNodeIdentity from tests.unit.src.sqlbuild.compiler.discovery._helpers._test_types import ( DiscoverCheckFunctionsTestCase, + DiscoverPythonRootImportTestCase, DiscoverPythonRootNodesTestCase, DiscoverTaskAssetFunctionsTestCase, + PythonRootHelperIdentityTestCase, + PythonRootProjectIsolationTestCase, ) @@ -350,3 +355,198 @@ def test_given_python_root_when_discovering_python_nodes_then_decorators_define_ *(("check", node.name, node.relative_path.as_posix()) for node in result.checks), ) assert discovered == test_case.expected_nodes + + +_HELPER_PACKAGE_FILES: dict[str, str] = { + "python/helpers/__init__.py": "", + "python/helpers/values.py": """ +from pathlib import Path + +Path(__file__).resolve().parents[2].joinpath("init_count.txt").open("a").write("values\\n") +STATUS = "shipped" +""", + "python/helpers/clean.py": """ +from .values import STATUS + + +def normalize_status(value): + return f"{value.strip().lower()}:{STATUS}" +""", + "python/orders.py": """ +import python.helpers.clean +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return python.helpers.clean.normalize_status(" Shipped ") +""", +} + + +def _write_files(*, root: Path, files: dict[str, str]) -> None: + relative_path: str + contents: str + for relative_path, contents in files.items(): + file_path: Path = root / relative_path + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(contents, encoding="utf-8") + + +@pytest.mark.parametrize( + "test_case", + [ + DiscoverPythonRootImportTestCase( + description="imports helper packages with relative imports once per discovery pass", + files=_HELPER_PACKAGE_FILES, + expected_task_names=("orders",), + expected_init_count=1, + ), + DiscoverPythonRootImportTestCase( + description="shares one helper module between nodes that import it absolutely", + files={ + "python/_counter.py": """ +from pathlib import Path + +Path(__file__).resolve().parents[1].joinpath("init_count.txt").open("a").write("counter\\n") +""", + "python/tasks/first.py": """ +import python._counter +from sqlbuild.tasks import task + + +@task +def first_task(ctx): + return None +""", + "python/tasks/second.py": """ +from python import _counter +from sqlbuild.tasks import task + + +@task +def second_task(ctx): + return None +""", + }, + expected_task_names=("first_task", "second_task"), + expected_init_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_root_helpers_when_discovering_then_modules_import_under_package_names( + test_case: DiscoverPythonRootImportTestCase, + tmp_path: Path, +) -> None: + _write_files(root=tmp_path, files=test_case.files) + + result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=tmp_path) + + assert tuple(node.name for node in result.tasks) == test_case.expected_task_names + assert all(node.function.__module__.startswith("python.") for node in result.tasks) + init_lines: list[str] = (tmp_path / "init_count.txt").read_text().splitlines() + assert len(init_lines) == test_case.expected_init_count + + +@pytest.mark.parametrize( + "test_case", + [ + PythonRootProjectIsolationTestCase( + description="second project imports its own python helpers", + first_files={ + "python/_values.py": "STATUS = 'first'\n", + "python/orders.py": """ +from python._values import STATUS +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return STATUS +""", + }, + second_files={ + "python/_values.py": "STATUS = 'second'\n", + "python/orders.py": """ +from python._values import STATUS +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return STATUS +""", + }, + expected_first_result="first", + expected_second_result="second", + ), + ], + ids=lambda case: case.description, +) +def test_given_two_projects_when_discovering_sequentially_then_python_modules_are_isolated( + test_case: PythonRootProjectIsolationTestCase, + tmp_path: Path, +) -> None: + first_dir: Path = tmp_path / "first" + second_dir: Path = tmp_path / "second" + _write_files(root=first_dir, files=test_case.first_files) + _write_files(root=second_dir, files=test_case.second_files) + + first: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=first_dir) + second: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=second_dir) + + assert first.tasks[0].function(None) == test_case.expected_first_result + assert second.tasks[0].function(None) == test_case.expected_second_result + + +@pytest.mark.parametrize( + "test_case", + [ + PythonRootHelperIdentityTestCase( + description="editing a relative-import helper changes the dependent node identity", + files=_HELPER_PACKAGE_FILES, + edited_path="python/helpers/clean.py", + original_text="value.strip().lower()", + edited_text="value.strip().upper()", + expected_dependency_path="python/helpers/clean.py", + ), + ], + ids=lambda case: case.description, +) +def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_changes( + test_case: PythonRootHelperIdentityTestCase, + tmp_path: Path, +) -> None: + _write_files(root=tmp_path, files=test_case.files) + before_task: DiscoveredTaskFunction = discover_python_node_functions( + project_dir=tmp_path + ).tasks[0] + before: PythonNodeIdentity = build_python_node_identity( + node_type="task", + node_name=before_task.name, + function=before_task.function, + project_dir=tmp_path, + ) + edited: Path = tmp_path / test_case.edited_path + edited.write_text( + edited.read_text(encoding="utf-8").replace(test_case.original_text, test_case.edited_text), + encoding="utf-8", + ) + + after_task: DiscoveredTaskFunction = discover_python_node_functions(project_dir=tmp_path).tasks[ + 0 + ] + after: PythonNodeIdentity = build_python_node_identity( + node_type="task", + node_name=after_task.name, + function=after_task.function, + project_dir=tmp_path, + ) + + assert before.source_hash == after.source_hash + assert before.version_hash != after.version_hash + assert test_case.expected_dependency_path in { + dependency.source_path for dependency in after.dependencies + } + assert test_case.edited_text in after.metadata_json From 109795fd6b92ac7ff91ca8ba284a2469242051e6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:26:51 +0100 Subject: [PATCH 5/9] fix: register factory-returned module nodes once A factory may return a decorated node that direct discovery already registers from its own python/ module. Skip that identical callable in factory output while keeping the duplicate-name error for distinct nodes. --- .../discovery/_helpers/filesystem/core.py | 18 +++++ .../compile/test_python_project_layout.py | 68 +++++++++++++++++ .../sqlbuild/compiler/discovery/test_main.py | 73 +++++++++++++++++++ 3 files changed, 159 insertions(+) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index b2db7a8725..d37c569efb 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -1371,6 +1371,7 @@ def _discover_python_node_functions( module=module, file_path=file_path, project_dir=project_dir, + node_root=node_root, provider_by_name=provider_by_name, ) return bucket @@ -1402,12 +1403,27 @@ def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleT sys.path = old_path +def _is_python_root_module_function(*, function: Callable[..., object], node_root: Path) -> bool: + """Return whether direct discovery registers this callable from its own python/ module.""" + + if not any(kind != "factory" for kind in _python_node_definition_names(function)): + return False + module: ModuleType | None = sys.modules.get(getattr(function, "__module__", "")) + if module is None or vars(module).get(getattr(function, "__name__", "")) is not function: + return False + module_file: object = getattr(module, "__file__", None) + if not isinstance(module_file, str) or Path(module_file).stem == PYTHON_INIT_MODULE_STEM: + return False + return _is_relative_to(path=Path(module_file).resolve(), parent=node_root.resolve()) + + def _append_module_python_nodes( *, bucket: _PythonNodeDiscoveryBucket, module: ModuleType, file_path: Path, project_dir: Path, + node_root: Path, provider_by_name: dict[str, DiscoveredProvider], ) -> None: _append_module_audit_factories( @@ -1441,6 +1457,8 @@ def _append_module_python_nodes( index: int generated_function: Callable[..., object] for index, generated_function in enumerate(generated_functions): + if _is_python_root_module_function(function=generated_function, node_root=node_root): + continue if not _append_python_node_function( bucket=bucket, function=generated_function, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index 8c27d9d1a5..2efd260982 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import subprocess from pathlib import Path @@ -307,5 +308,72 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi assert "order_rows" not in build_output +@pytest.mark.parametrize( + "test_case", + ( + PythonProjectLayoutCompileTestCase( + description="factory returning its module-level task registers one node", + repo_files={ + "sqlbuild_project.toml": 'name = "factory_layout"\nadapter = "duckdb"\n', + "python/factories/orders.py": ( + "from sqlbuild.factories import factory\n" + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def orders(ctx):\n" + " return None\n\n\n" + "@factory\n" + "def order_nodes():\n" + " return orders\n" + ), + }, + expected_exit_code=0, + expected_stderr_fragments=(), + ), + PythonProjectLayoutCompileTestCase( + description="factory returning a distinct same-name task still conflicts", + repo_files={ + "sqlbuild_project.toml": 'name = "factory_layout"\nadapter = "duckdb"\n', + "python/factories/orders.py": ( + "from sqlbuild.factories import factory\n" + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def orders(ctx):\n" + " return None\n\n\n" + "@factory\n" + "def order_nodes():\n" + " @task(name='orders')\n" + " def generated_orders(ctx):\n" + " return None\n" + " return generated_orders\n" + ), + }, + expected_exit_code=1, + expected_stderr_fragments=("Duplicate Python node found for 'orders'",), + ), + ), + ids=lambda case: case.description, +) +def test_given_factory_returning_module_node_when_planning_then_node_registers_once( + test_case: PythonProjectLayoutCompileTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="factory_layout", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("plan", "--json"), + project_dir=project_dir, + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments) + if result.returncode == 0: + payload: dict[str, list[dict[str, object]]] = json.loads(result.stdout) + assert [entry["name"] for entry in payload["python_nodes"]] == ["orders"] + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 057ebc199c..094a395e13 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -466,6 +466,55 @@ def prepare(ctx): }, expected_task_names=("prepare_orders",), ), + DiscoverFactoryValidationTestCase( + description="factory returning its own module-level node registers it once", + repo_files=base_repo_files() + | { + "python/factories/orders.py": """ +from sqlbuild.factories import factory +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return None + + +@factory +def order_nodes(): + return orders +""", + }, + expected_task_names=("orders",), + ), + DiscoverFactoryValidationTestCase( + description="factory returning an imported python module node registers it once", + repo_files=base_repo_files() + | { + "python/a_factories/orders.py": """ +from python.tasks.orders import prepare_orders +from sqlbuild.factories import factory +from sqlbuild.tasks import task + + +@factory +def order_nodes(): + @task(name="publish_orders", depends_on=prepare_orders) + def publish(ctx): + return None + return [prepare_orders, publish] +""", + "python/tasks/orders.py": """ +from sqlbuild.tasks import task + + +@task +def prepare_orders(ctx): + return None +""", + }, + expected_task_names=("publish_orders", "prepare_orders"), + ), DiscoverFactoryValidationTestCase( description="factory can import ordinary private helper module", repo_files=base_repo_files() @@ -678,6 +727,30 @@ def generated_profile(ctx): }, expected_error_fragment="Duplicate Python node found for 'profile'", ), + DiscoverFactoryValidationTestCase( + description="factory returning a distinct node named like a module node fails validation", + repo_files=base_repo_files() + | { + "python/factories/orders.py": """ +from sqlbuild.factories import factory +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return None + + +@factory +def order_nodes(): + @task(name="orders") + def generated_orders(ctx): + return None + return generated_orders +""", + }, + expected_error_fragment="Duplicate Python node found for 'orders'", + ), DiscoverFactoryValidationTestCase( description="generated dependency cycle fails validation", repo_files=base_repo_files() From 0424c0743eea28838011edaf4da5b1ef81e2709b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:27:45 +0100 Subject: [PATCH 6/9] fix: install wrapper playground skills in the SQLBuild project Dagster and Rivers playgrounds nest the SQLBuild project under waffle_shop/. Install the scaffolded agent skills there, where sqb skills and stale-skill maintenance resolve them, and document the location. --- .../references/docs/cli/playground.md | 4 +- .../_helpers/playground/completion_output.py | 7 ++- .../commands/_helpers/playground/scaffold.py | 14 +++++- src/sqlbuild/cli/commands/constants.py | 3 ++ .../commands/main/playground/_test_types.py | 8 ++++ .../main/playground/test_playground.py | 46 ++++++++++++++++++- .../src/content/docs/docs/cli/playground.mdx | 4 +- 7 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md index f66dc9aea5..b288863a33 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/playground.md @@ -45,10 +45,10 @@ The `dagster` and `rivers` templates keep orchestrator code outside the SQLBuild sqlbuild-playground/ definitions.py # Dagster or Rivers definitions pointing at waffle_shop/ README.md # setup instructions - waffle_shop/ # the waffle shop SQLBuild project + waffle_shop/ # the waffle shop SQLBuild project, including agent skill files ``` -The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource`; the Rivers `definitions.py` defines a `waffle_shop` job. Run `sqb` commands from `waffle_shop/` and the orchestrator from the top-level directory. +The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource`; the Rivers `definitions.py` defines a `waffle_shop` job. Run `sqb` commands from `waffle_shop/` and the orchestrator from the top-level directory. Agent skill files are installed in `waffle_shop/`, where `sqb skills` and stale-skill checks maintain them. The `python_nodes` template instead creates a focused Python-nodes project: diff --git a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py index 7d7a158fb3..e206cab35c 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/completion_output.py @@ -2,7 +2,10 @@ from __future__ import annotations -from sqlbuild.cli.commands.constants import PLAYGROUND_ORCHESTRATED_PROJECT_DIR +from sqlbuild.cli.commands.constants import ( + PLAYGROUND_ORCHESTRATED_PROJECT_DIR, + PLAYGROUND_ORCHESTRATED_TEMPLATES, +) from sqlbuild.cli.commands.models import ( PlaygroundCommandRequest, PlaygroundTarget, @@ -38,7 +41,7 @@ def render_playground_completion_text( def _suggested_commands(*, template: PlaygroundTemplate, display_path: str) -> tuple[str, ...]: - orchestrated: bool = template in (PlaygroundTemplate.DAGSTER, PlaygroundTemplate.RIVERS) + orchestrated: bool = template in PLAYGROUND_ORCHESTRATED_TEMPLATES project_path: str = ( f"{display_path}/{PLAYGROUND_ORCHESTRATED_PROJECT_DIR}" if orchestrated else display_path ) diff --git a/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py b/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py index 687ed5adc2..96954cb9ba 100644 --- a/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py +++ b/src/sqlbuild/cli/commands/_helpers/playground/scaffold.py @@ -6,6 +6,10 @@ from sqlbuild.cli.commands._helpers.playground.copy import create_playground_project from sqlbuild.cli.commands._helpers.skills.update import update_sqlbuild_skills +from sqlbuild.cli.commands.constants import ( + PLAYGROUND_ORCHESTRATED_PROJECT_DIR, + PLAYGROUND_ORCHESTRATED_TEMPLATES, +) from sqlbuild.cli.commands.models import ( PlaygroundCommandRequest, PlaygroundTarget, @@ -30,4 +34,12 @@ def write_playground_project(*, target: PlaygroundTarget) -> None: """Create the playground project files and refresh bundled skills.""" create_playground_project(target_dir=target.target_dir, template=target.template.value) - _ = update_sqlbuild_skills(project_dir=target.target_dir) + _ = update_sqlbuild_skills(project_dir=_playground_project_dir(target=target)) + + +def _playground_project_dir(*, target: PlaygroundTarget) -> Path: + """Return the SQLBuild project directory inside a generated playground.""" + + if target.template in PLAYGROUND_ORCHESTRATED_TEMPLATES: + return target.target_dir / PLAYGROUND_ORCHESTRATED_PROJECT_DIR + return target.target_dir diff --git a/src/sqlbuild/cli/commands/constants.py b/src/sqlbuild/cli/commands/constants.py index 7a2148e822..c4c976c281 100644 --- a/src/sqlbuild/cli/commands/constants.py +++ b/src/sqlbuild/cli/commands/constants.py @@ -48,6 +48,9 @@ } ) PLAYGROUND_ORCHESTRATED_PROJECT_DIR: str = "waffle_shop" +PLAYGROUND_ORCHESTRATED_TEMPLATES: frozenset[PlaygroundTemplate] = frozenset( + {PlaygroundTemplate.DAGSTER, PlaygroundTemplate.RIVERS} +) PLAYGROUND_TEMPLATE_VALUES: tuple[str, ...] = tuple( template.value for template in PlaygroundTemplate ) diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py index 9cc621ba1c..6308a38a41 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -22,3 +22,11 @@ class RunPlaygroundTestCase: template: str = "waffle_shop" expected_color_fragments: tuple[str, ...] = () project_subdir: str = "" + + +@dataclass(frozen=True) +class WrapperPlaygroundSkillsTestCase: + description: str + template: str + project_subdir: str + skill_file: str diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 6c9f083558..35426ce512 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -9,12 +9,15 @@ completion_output as playground_completion_output, ) from sqlbuild.cli.commands._helpers.playground.copy import create_playground_project +from sqlbuild.cli.commands._helpers.skills.update import maintain_sqlbuild_skills from sqlbuild.cli.commands.exceptions import CliUserError from sqlbuild.cli.commands.main.workspace._playground import run_playground from sqlbuild.cli.commands.models import PlaygroundCommandRequest +from sqlbuild.cli.output.models import SkillMaintenanceResult from tests.unit.src.sqlbuild.cli.commands.main.playground._test_types import ( CreatePlaygroundProjectTestCase, RunPlaygroundTestCase, + WrapperPlaygroundSkillsTestCase, ) @@ -438,7 +441,7 @@ def test_given_playground_command_when_running_then_it_prints_next_steps( assert expected_fragment in captured.out project_dir: Path = tmp_path / test_case.target_path / test_case.project_subdir assert (project_dir / "sqlbuild_project.toml").is_file() - assert (tmp_path / test_case.target_path / ".agents/skills/sqlbuild/SKILL.md").is_file() + assert (project_dir / ".agents/skills/sqlbuild/SKILL.md").is_file() @pytest.mark.parametrize( @@ -479,3 +482,44 @@ def test_given_color_terminal_when_running_playground_command_then_it_styles_key expected_fragment: str for expected_fragment in test_case.expected_color_fragments: assert expected_fragment in captured.out + + +@pytest.mark.parametrize( + "test_case", + [ + WrapperPlaygroundSkillsTestCase( + description="dagster playground installs skills maintained from the nested project", + template="dagster", + project_subdir="waffle_shop", + skill_file=".agents/skills/sqlbuild/SKILL.md", + ), + WrapperPlaygroundSkillsTestCase( + description="rivers playground installs skills maintained from the nested project", + template="rivers", + project_subdir="waffle_shop", + skill_file=".claude/skills/sqlbuild/SKILL.md", + ), + ], + ids=lambda case: case.description, +) +def test_given_wrapper_playground_when_maintaining_skills_then_nested_project_sees_installed_skills( + test_case: WrapperPlaygroundSkillsTestCase, + tmp_path: Path, +) -> None: + exit_code: int = run_playground( + PlaygroundCommandRequest( + project_dir=tmp_path, target_path="orchestrator", template=test_case.template + ) + ) + wrapper_dir: Path = tmp_path / "orchestrator" + project_dir: Path = wrapper_dir / test_case.project_subdir + + fresh: SkillMaintenanceResult = maintain_sqlbuild_skills(project_dir=project_dir) + installed_skill: Path = project_dir / test_case.skill_file + installed_skill.write_text("edited\n", encoding="utf-8") + stale: SkillMaintenanceResult = maintain_sqlbuild_skills(project_dir=project_dir) + + assert exit_code == 0 + assert not (wrapper_dir / test_case.skill_file).exists() + assert fresh.message == "" + assert "SQLBuild skill files are out of date" in stale.message diff --git a/website/src/content/docs/docs/cli/playground.mdx b/website/src/content/docs/docs/cli/playground.mdx index 0fe8fcd757..719b76d243 100644 --- a/website/src/content/docs/docs/cli/playground.mdx +++ b/website/src/content/docs/docs/cli/playground.mdx @@ -44,10 +44,10 @@ The `dagster` and `rivers` templates keep orchestrator code outside the SQLBuild sqlbuild-playground/ definitions.py # Dagster or Rivers definitions pointing at waffle_shop/ README.md # setup instructions - waffle_shop/ # the waffle shop SQLBuild project + waffle_shop/ # the waffle shop SQLBuild project, including agent skill files ``` -The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource`; the Rivers `definitions.py` defines a `waffle_shop` job. Run `sqb` commands from `waffle_shop/` and the orchestrator from the top-level directory. +The Dagster `definitions.py` uses `sqlbuild_assets`, `sqlbuild_scenario_checks`, and `SqlBuildCliResource`; the Rivers `definitions.py` defines a `waffle_shop` job. Run `sqb` commands from `waffle_shop/` and the orchestrator from the top-level directory. Agent skill files are installed in `waffle_shop/`, where `sqb skills` and stale-skill checks maintain them. The `python_nodes` template instead creates a focused Python-nodes project: From e7dbd68038d8f378c9ddc514ef71fe10afc18969 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:32:07 +0100 Subject: [PATCH 7/9] test: align python/ discovery regressions with fensu rules Check node decorators directly when de-duplicating factory output, split the factory E2E into success and conflict tests, and use expected_ fields and the shared repo-file fixture in the new regressions. --- .../discovery/_helpers/filesystem/core.py | 8 +- .../cli/commands/main/compile/_test_types.py | 7 ++ .../compile/test_python_project_layout.py | 40 ++++++++-- .../commands/main/playground/_test_types.py | 3 +- .../main/playground/test_playground.py | 10 +-- .../_helpers/test_filesystem_python_nodes.py | 74 +++++++++---------- 6 files changed, 88 insertions(+), 54 deletions(-) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index d37c569efb..5c888f449c 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -1406,7 +1406,13 @@ def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleT def _is_python_root_module_function(*, function: Callable[..., object], node_root: Path) -> bool: """Return whether direct discovery registers this callable from its own python/ module.""" - if not any(kind != "factory" for kind in _python_node_definition_names(function)): + node_definitions: tuple[object | None, ...] = ( + read_loader_definition(function), + read_task_definition(function), + read_asset_definition(function), + read_check_definition(function), + ) + if all(definition is None for definition in node_definitions): return False module: ModuleType | None = sys.modules.get(getattr(function, "__module__", "")) if module is None or vars(module).get(getattr(function, "__name__", "")) is not function: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index b2220b0da7..5dbf791413 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -237,3 +237,10 @@ class TypeProofRuleCompileTestCase: expected_returncode: int expected_rule_findings: int expected_note_count: int + + +@dataclass(frozen=True) +class FactoryModuleNodePlanTestCase: + description: str + repo_files: dict[str, str] + expected_python_node_names: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index 2efd260982..37d90ab2f1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -9,6 +9,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + FactoryModuleNodePlanTestCase, PythonProjectLayoutCompileTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -311,7 +312,7 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi @pytest.mark.parametrize( "test_case", ( - PythonProjectLayoutCompileTestCase( + FactoryModuleNodePlanTestCase( description="factory returning its module-level task registers one node", repo_files={ "sqlbuild_project.toml": 'name = "factory_layout"\nadapter = "duckdb"\n', @@ -326,9 +327,37 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi " return orders\n" ), }, - expected_exit_code=0, - expected_stderr_fragments=(), + expected_python_node_names=("orders",), ), + ), + ids=lambda case: case.description, +) +def test_given_factory_returning_module_node_when_planning_then_node_registers_once( + test_case: FactoryModuleNodePlanTestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="factory_layout", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("plan", "--json"), + project_dir=project_dir, + ) + + assert result.returncode == 0, result.stdout + result.stderr + payload: dict[str, list[dict[str, object]]] = json.loads(result.stdout) + assert ( + tuple(entry["name"] for entry in payload["python_nodes"]) + == test_case.expected_python_node_names + ) + + +@pytest.mark.parametrize( + "test_case", + ( PythonProjectLayoutCompileTestCase( description="factory returning a distinct same-name task still conflicts", repo_files={ @@ -353,7 +382,7 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi ), ids=lambda case: case.description, ) -def test_given_factory_returning_module_node_when_planning_then_node_registers_once( +def test_given_factory_returning_distinct_same_name_node_when_planning_then_conflict_is_reported( test_case: PythonProjectLayoutCompileTestCase, tmp_path: Path, ) -> None: @@ -370,9 +399,6 @@ def test_given_factory_returning_module_node_when_planning_then_node_registers_o assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments) - if result.returncode == 0: - payload: dict[str, list[dict[str, object]]] = json.loads(result.stdout) - assert [entry["name"] for entry in payload["python_nodes"]] == ["orders"] if __name__ == "__main__": diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py index 6308a38a41..09386b3e33 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/_test_types.py @@ -29,4 +29,5 @@ class WrapperPlaygroundSkillsTestCase: description: str template: str project_subdir: str - skill_file: str + expected_skill_file: str + expected_stale_message: str = "SQLBuild skill files are out of date" diff --git a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py index 35426ce512..31e7652458 100644 --- a/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py +++ b/tests/unit/src/sqlbuild/cli/commands/main/playground/test_playground.py @@ -491,13 +491,13 @@ def test_given_color_terminal_when_running_playground_command_then_it_styles_key description="dagster playground installs skills maintained from the nested project", template="dagster", project_subdir="waffle_shop", - skill_file=".agents/skills/sqlbuild/SKILL.md", + expected_skill_file=".agents/skills/sqlbuild/SKILL.md", ), WrapperPlaygroundSkillsTestCase( description="rivers playground installs skills maintained from the nested project", template="rivers", project_subdir="waffle_shop", - skill_file=".claude/skills/sqlbuild/SKILL.md", + expected_skill_file=".claude/skills/sqlbuild/SKILL.md", ), ], ids=lambda case: case.description, @@ -515,11 +515,11 @@ def test_given_wrapper_playground_when_maintaining_skills_then_nested_project_se project_dir: Path = wrapper_dir / test_case.project_subdir fresh: SkillMaintenanceResult = maintain_sqlbuild_skills(project_dir=project_dir) - installed_skill: Path = project_dir / test_case.skill_file + installed_skill: Path = project_dir / test_case.expected_skill_file installed_skill.write_text("edited\n", encoding="utf-8") stale: SkillMaintenanceResult = maintain_sqlbuild_skills(project_dir=project_dir) assert exit_code == 0 - assert not (wrapper_dir / test_case.skill_file).exists() + assert not (wrapper_dir / test_case.expected_skill_file).exists() assert fresh.message == "" - assert "SQLBuild skill files are out of date" in stale.message + assert test_case.expected_stale_message in stale.message diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index ff7a319c4b..58bdf14144 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path import pytest @@ -27,6 +28,32 @@ PythonRootProjectIsolationTestCase, ) +_HELPER_PACKAGE_FILES: dict[str, str] = { + "python/helpers/__init__.py": "", + "python/helpers/values.py": """ +from pathlib import Path + +Path(__file__).resolve().parents[2].joinpath("init_count.txt").open("a").write("values\\n") +STATUS = "shipped" +""", + "python/helpers/clean.py": """ +from .values import STATUS + + +def normalize_status(value): + return f"{value.strip().lower()}:{STATUS}" +""", + "python/orders.py": """ +import python.helpers.clean +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return python.helpers.clean.normalize_status(" Shipped ") +""", +} + @pytest.mark.parametrize( "test_case", @@ -357,42 +384,6 @@ def test_given_python_root_when_discovering_python_nodes_then_decorators_define_ assert discovered == test_case.expected_nodes -_HELPER_PACKAGE_FILES: dict[str, str] = { - "python/helpers/__init__.py": "", - "python/helpers/values.py": """ -from pathlib import Path - -Path(__file__).resolve().parents[2].joinpath("init_count.txt").open("a").write("values\\n") -STATUS = "shipped" -""", - "python/helpers/clean.py": """ -from .values import STATUS - - -def normalize_status(value): - return f"{value.strip().lower()}:{STATUS}" -""", - "python/orders.py": """ -import python.helpers.clean -from sqlbuild.tasks import task - - -@task -def orders(ctx): - return python.helpers.clean.normalize_status(" Shipped ") -""", -} - - -def _write_files(*, root: Path, files: dict[str, str]) -> None: - relative_path: str - contents: str - for relative_path, contents in files.items(): - file_path: Path = root / relative_path - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.write_text(contents, encoding="utf-8") - - @pytest.mark.parametrize( "test_case", [ @@ -438,8 +429,9 @@ def second_task(ctx): def test_given_python_root_helpers_when_discovering_then_modules_import_under_package_names( test_case: DiscoverPythonRootImportTestCase, tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: - _write_files(root=tmp_path, files=test_case.files) + write_repo_files(tmp_path, test_case.files) result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=tmp_path) @@ -487,11 +479,12 @@ def orders(ctx): def test_given_two_projects_when_discovering_sequentially_then_python_modules_are_isolated( test_case: PythonRootProjectIsolationTestCase, tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: first_dir: Path = tmp_path / "first" second_dir: Path = tmp_path / "second" - _write_files(root=first_dir, files=test_case.first_files) - _write_files(root=second_dir, files=test_case.second_files) + write_repo_files(first_dir, test_case.first_files) + write_repo_files(second_dir, test_case.second_files) first: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=first_dir) second: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=second_dir) @@ -517,8 +510,9 @@ def test_given_two_projects_when_discovering_sequentially_then_python_modules_ar def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_changes( test_case: PythonRootHelperIdentityTestCase, tmp_path: Path, + write_repo_files: Callable[[Path, dict[str, str]], None], ) -> None: - _write_files(root=tmp_path, files=test_case.files) + write_repo_files(tmp_path, test_case.files) before_task: DiscoveredTaskFunction = discover_python_node_functions( project_dir=tmp_path ).tasks[0] From e387c22f234f34e0f09ac2919623ceae8c911ced Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 05:44:13 +0100 Subject: [PATCH 8/9] fix: bind python/ to the project and de-duplicate factory nodes by identity Discovery installs an explicit python package whose __path__ is the project's python/ folder, so project imports never resolve to an unrelated python package elsewhere on sys.path. Unrelated python.* modules displaced during discovery are restored afterwards. Discovery now imports every python/ module and registers direct nodes before running factories, then skips factory results that are the same callable direct discovery registered. A factory that binds a generated node to a module global registers it once instead of dropping it. --- .../discovery/_helpers/filesystem/core.py | 158 ++++++++++++------ .../cli/commands/main/compile/_test_types.py | 8 + .../compile/test_python_project_layout.py | 74 ++++++++ .../discovery/_helpers/_test_types.py | 9 + .../_helpers/test_filesystem_python_nodes.py | 45 +++++ .../sqlbuild/compiler/discovery/test_main.py | 24 ++- 6 files changed, 270 insertions(+), 48 deletions(-) diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index 5c888f449c..7c85fa536f 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -124,6 +124,8 @@ from sqlbuild.runtime.observability.models import LifecycleEvent from sqlbuild.spec.contracts.models import SchemaModelEntry, SchemaSeedEntry +_PROJECT_PYTHON_PACKAGE_MARKER: str = "__sqlbuild_project_python__" + @dataclass class _PythonNodeDiscoveryBucket: @@ -1357,33 +1359,102 @@ def _discover_python_node_functions( node_root: Path = project_dir / PYTHON_NODE_ROOT if not node_root.is_dir(): return bucket - _evict_python_root_modules() - importlib.invalidate_caches() - file_path: Path - for file_path in sorted(node_root.rglob("*.py")): - if file_path.stem == PYTHON_INIT_MODULE_STEM: - continue - module: ModuleType = _import_python_root_module( - file_path=file_path, project_dir=project_dir - ) - _append_module_python_nodes( - bucket=bucket, - module=module, - file_path=file_path, - project_dir=project_dir, - node_root=node_root, - provider_by_name=provider_by_name, + saved_modules: dict[str, ModuleType] = _install_python_root_package( + node_root=node_root, project_dir=project_dir + ) + try: + modules: tuple[tuple[Path, ModuleType], ...] = tuple( + (file_path, _import_python_root_module(file_path=file_path, project_dir=project_dir)) + for file_path in sorted(node_root.rglob("*.py")) + if file_path.stem != PYTHON_INIT_MODULE_STEM ) + direct_functions: dict[int, Callable[..., object]] = {} + file_path: Path + module: ModuleType + for file_path, module in modules: + direct_functions.update( + (id(function), function) + for function in _append_module_direct_nodes( + bucket=bucket, + module=module, + file_path=file_path, + project_dir=project_dir, + provider_by_name=provider_by_name, + ) + ) + for file_path, module in modules: + _append_module_factory_nodes( + bucket=bucket, + module=module, + file_path=file_path, + project_dir=project_dir, + provider_by_name=provider_by_name, + direct_functions=direct_functions, + ) + finally: + _restore_python_root_modules(saved_modules=saved_modules) return bucket -def _evict_python_root_modules() -> None: - """Drop python.* modules so each discovery pass imports the current project afresh.""" +def _python_root_module_names() -> tuple[str, ...]: + return tuple( + module_name + for module_name in sys.modules + if module_name == PYTHON_NODE_ROOT or module_name.startswith(f"{PYTHON_NODE_ROOT}.") + ) + + +def _install_python_root_package(*, node_root: Path, project_dir: Path) -> dict[str, ModuleType]: + """Bind python to this project's python/ folder and return displaced unrelated modules.""" + existing: ModuleType | None = sys.modules.get(PYTHON_NODE_ROOT) + owned: bool = existing is None or bool(getattr(existing, _PROJECT_PYTHON_PACKAGE_MARKER, False)) + saved_modules: dict[str, ModuleType] = {} module_name: str - for module_name in tuple(sys.modules): - if module_name == PYTHON_NODE_ROOT or module_name.startswith(f"{PYTHON_NODE_ROOT}."): - sys.modules.pop(module_name, None) + for module_name in _python_root_module_names(): + module: ModuleType | None = sys.modules.pop(module_name, None) + if not owned and module is not None: + saved_modules[module_name] = module + importlib.invalidate_caches() + init_file: Path = node_root / f"{PYTHON_INIT_MODULE_STEM}.py" + spec: ModuleSpec | None = ( + importlib.util.spec_from_file_location( + PYTHON_NODE_ROOT, init_file, submodule_search_locations=[str(node_root)] + ) + if init_file.is_file() + else ModuleSpec(PYTHON_NODE_ROOT, None, is_package=True) + ) + if spec is None: + raise PythonNodeDiscoveryError(f"Could not load Python node package {node_root}") + spec.submodule_search_locations = [str(node_root)] + package: ModuleType = importlib.util.module_from_spec(spec) + setattr(package, _PROJECT_PYTHON_PACKAGE_MARKER, True) + sys.modules[PYTHON_NODE_ROOT] = package + if spec.loader is None: + return saved_modules + old_path: list[str] = list(sys.path) + sys.path.insert(0, str(project_dir)) + try: + spec.loader.exec_module(package) + except Exception as error: + _restore_python_root_modules(saved_modules=saved_modules) + raise PythonNodeDiscoveryError( + f"Failed to import Python node file {init_file.relative_to(project_dir)}: {error}" + ) from error + finally: + sys.path = old_path + return saved_modules + + +def _restore_python_root_modules(*, saved_modules: dict[str, ModuleType]) -> None: + """Put back an unrelated python package that discovery displaced.""" + + if not saved_modules: + return + module_name: str + for module_name in _python_root_module_names(): + sys.modules.pop(module_name, None) + sys.modules.update(saved_modules) def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleType: @@ -1403,51 +1474,44 @@ def _import_python_root_module(*, file_path: Path, project_dir: Path) -> ModuleT sys.path = old_path -def _is_python_root_module_function(*, function: Callable[..., object], node_root: Path) -> bool: - """Return whether direct discovery registers this callable from its own python/ module.""" - - node_definitions: tuple[object | None, ...] = ( - read_loader_definition(function), - read_task_definition(function), - read_asset_definition(function), - read_check_definition(function), - ) - if all(definition is None for definition in node_definitions): - return False - module: ModuleType | None = sys.modules.get(getattr(function, "__module__", "")) - if module is None or vars(module).get(getattr(function, "__name__", "")) is not function: - return False - module_file: object = getattr(module, "__file__", None) - if not isinstance(module_file, str) or Path(module_file).stem == PYTHON_INIT_MODULE_STEM: - return False - return _is_relative_to(path=Path(module_file).resolve(), parent=node_root.resolve()) - - -def _append_module_python_nodes( +def _append_module_direct_nodes( *, bucket: _PythonNodeDiscoveryBucket, module: ModuleType, file_path: Path, project_dir: Path, - node_root: Path, provider_by_name: dict[str, DiscoveredProvider], -) -> None: +) -> tuple[Callable[..., object], ...]: _append_module_audit_factories( bucket=bucket, module=module, file_path=file_path, project_dir=project_dir, ) + registered: list[Callable[..., object]] = [] for _, value in inspect.getmembers(module, inspect.isfunction): if value.__module__ != module.__name__: continue - _append_python_node_function( + if _append_python_node_function( bucket=bucket, function=value, file_path=file_path, project_dir=project_dir, provider_by_name=provider_by_name, - ) + ): + registered.append(value) + return tuple(registered) + + +def _append_module_factory_nodes( + *, + bucket: _PythonNodeDiscoveryBucket, + module: ModuleType, + file_path: Path, + project_dir: Path, + provider_by_name: dict[str, DiscoveredProvider], + direct_functions: dict[int, Callable[..., object]], +) -> None: for _, value in inspect.getmembers(module, inspect.isfunction): if value.__module__ != module.__name__: continue @@ -1463,7 +1527,7 @@ def _append_module_python_nodes( index: int generated_function: Callable[..., object] for index, generated_function in enumerate(generated_functions): - if _is_python_root_module_function(function=generated_function, node_root=node_root): + if direct_functions.get(id(generated_function)) is generated_function: continue if not _append_python_node_function( bucket=bucket, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 5dbf791413..ab7d11f2b6 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -244,3 +244,11 @@ class FactoryModuleNodePlanTestCase: description: str repo_files: dict[str, str] expected_python_node_names: tuple[str, ...] + + +@dataclass(frozen=True) +class UnrelatedPythonPackageBuildTestCase: + description: str + unrelated_files: dict[str, str] + repo_files: dict[str, str] + expected_build_fragments: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py index 37d90ab2f1..b01925c069 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_project_layout.py @@ -11,6 +11,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( FactoryModuleNodePlanTestCase, PythonProjectLayoutCompileTestCase, + UnrelatedPythonPackageBuildTestCase, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb @@ -329,6 +330,24 @@ def test_given_python_root_project_when_compiling_and_building_then_nodes_run_wi }, expected_python_node_names=("orders",), ), + FactoryModuleNodePlanTestCase( + description="factory binding its generated task to a module global registers it", + repo_files={ + "sqlbuild_project.toml": 'name = "factory_layout"\nadapter = "duckdb"\n', + "python/factories/orders.py": ( + "from sqlbuild.factories import factory\n" + "from sqlbuild.tasks import task\n\n\n" + "@factory\n" + "def order_nodes():\n" + " global orders\n\n" + " @task\n" + " def orders(ctx):\n" + " return None\n\n" + " return orders\n" + ), + }, + expected_python_node_names=("orders",), + ), ), ids=lambda case: case.description, ) @@ -401,5 +420,60 @@ def test_given_factory_returning_distinct_same_name_node_when_planning_then_conf assert all(fragment in result.stderr for fragment in test_case.expected_stderr_fragments) +@pytest.mark.parametrize( + "test_case", + ( + UnrelatedPythonPackageBuildTestCase( + description="project python helpers win over an unrelated python package on PYTHONPATH", + unrelated_files={ + "python/__init__.py": "UNRELATED = True\n", + "python/helpers.py": "raise RuntimeError('unrelated python package imported')\n", + }, + repo_files={ + "sqlbuild_project.toml": 'name = "python_collision"\nadapter = "duckdb"\n', + "python/helpers/__init__.py": "", + "python/helpers/values.py": "STATUS = 'shipped'\n", + "python/helpers/clean.py": ( + "from .values import STATUS\n\n\ndef label():\n return STATUS\n" + ), + "python/orders.py": ( + "import python.helpers.clean\n" + "from sqlbuild.tasks import task\n\n\n" + "@task\n" + "def orders(ctx):\n" + " return ctx.result(payload={'status': python.helpers.clean.label()})\n" + ), + }, + expected_build_fragments=("orders", "\u2713 Completed successfully"), + ), + ), + ids=lambda case: case.description, +) +def test_given_unrelated_python_package_on_path_when_building_then_project_python_is_used( + test_case: UnrelatedPythonPackageBuildTestCase, + tmp_path: Path, +) -> None: + site_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="site", + repo_files=test_case.unrelated_files, + ) + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="python_collision", + repo_files=test_case.repo_files, + ) + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + env={"PYTHONPATH": str(site_dir)}, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == 0, output + assert all(fragment in output for fragment in test_case.expected_build_fragments) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py index cd63e5c802..d37e523799 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/_test_types.py @@ -830,6 +830,15 @@ class PythonRootHelperIdentityTestCase: expected_dependency_path: str +@dataclass(frozen=True) +class UnrelatedPythonPackageTestCase: + description: str + unrelated_files: dict[str, str] + project_files: dict[str, str] + expected_task_names: tuple[str, ...] + expected_task_result: object + + @dataclass(frozen=True) class ExpectedBooleanTestCase: description: str diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 58bdf14144..8b032893aa 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -2,8 +2,11 @@ from __future__ import annotations +import importlib +import sys from collections.abc import Callable from pathlib import Path +from types import ModuleType import pytest @@ -26,6 +29,7 @@ DiscoverTaskAssetFunctionsTestCase, PythonRootHelperIdentityTestCase, PythonRootProjectIsolationTestCase, + UnrelatedPythonPackageTestCase, ) _HELPER_PACKAGE_FILES: dict[str, str] = { @@ -544,3 +548,44 @@ def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_cha dependency.source_path for dependency in after.dependencies } assert test_case.edited_text in after.metadata_json + + +@pytest.mark.parametrize( + "test_case", + [ + UnrelatedPythonPackageTestCase( + description="project python package wins over an imported unrelated python package", + unrelated_files={ + "site/python/__init__.py": "UNRELATED = True\n", + "site/python/other.py": "VALUE = 'unrelated'\n", + }, + project_files=_HELPER_PACKAGE_FILES, + expected_task_names=("orders",), + expected_task_result="shipped:shipped", + ), + ], + ids=lambda case: case.description, +) +def test_given_unrelated_python_package_when_discovering_then_project_resolves_and_package_returns( + test_case: UnrelatedPythonPackageTestCase, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + write_repo_files: Callable[[Path, dict[str, str]], None], +) -> None: + project_dir: Path = tmp_path / "project" + write_repo_files(tmp_path, test_case.unrelated_files) + write_repo_files(project_dir, test_case.project_files) + monkeypatch.delitem(sys.modules, "python", raising=False) + monkeypatch.delitem(sys.modules, "python.other", raising=False) + monkeypatch.syspath_prepend(str(tmp_path / "site")) + unrelated: ModuleType = importlib.import_module("python") + unrelated_other: ModuleType = importlib.import_module("python.other") + monkeypatch.setitem(sys.modules, "python", unrelated) + monkeypatch.setitem(sys.modules, "python.other", unrelated_other) + + result: DiscoveredPythonNodeFunctions = discover_python_node_functions(project_dir=project_dir) + + assert tuple(node.name for node in result.tasks) == test_case.expected_task_names + assert result.tasks[0].function(None) == test_case.expected_task_result + assert sys.modules["python"] is unrelated + assert sys.modules["python.other"] is unrelated_other diff --git a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py index 094a395e13..4cac20c490 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/test_main.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/test_main.py @@ -482,6 +482,28 @@ def orders(ctx): @factory def order_nodes(): + return orders +""", + }, + expected_task_names=("orders",), + ), + DiscoverFactoryValidationTestCase( + description="factory binding its generated node to a module global registers it once", + repo_files=base_repo_files() + | { + "python/factories/orders.py": """ +from sqlbuild.factories import factory +from sqlbuild.tasks import task + + +@factory +def order_nodes(): + global orders + + @task + def orders(ctx): + return None + return orders """, }, @@ -513,7 +535,7 @@ def prepare_orders(ctx): return None """, }, - expected_task_names=("publish_orders", "prepare_orders"), + expected_task_names=("prepare_orders", "publish_orders"), ), DiscoverFactoryValidationTestCase( description="factory can import ordinary private helper module", From 1df059688d24f4bcfdfcd11a4a67b36ea590b8e0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sun, 27 Sep 2026 06:07:51 +0100 Subject: [PATCH 9/9] test: resolve the project root from python/ node fixtures --- .../test_build_no_tests_no_audits.py | 38 +++++++++---------- .../cli/commands/main/build/test_build.py | 20 +++++----- .../main/build/test_loader_watermark_build.py | 2 +- .../main/build/test_virtual_python_build.py | 26 ++++++------- .../test_virtual_source_freshness_build.py | 4 +- .../cli/commands/main/load/helpers.py | 2 +- .../commands/main/providers/test_providers.py | 2 +- 7 files changed, 47 insertions(+), 47 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index fdd763979d..e63afac281 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -221,7 +221,7 @@ def test_given_factory_generated_nodes_when_running_commands_then_lifecycle_succ from sqlbuild.tasks import task -PROJECT_DIR = Path(__file__).parents[1] +PROJECT_DIR = Path(__file__).parents[2] @factory @@ -426,7 +426,7 @@ def test_given_task_selector_when_running_run_then_task_executes( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('task_marker.txt').write_text('ran')\n" + " Path(__file__).parents[2].joinpath('task_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'})\n" ), }, @@ -481,7 +481,7 @@ def test_given_asset_selector_when_running_run_then_asset_executes( "from sqlbuild.assets import asset\n\n" "@asset\n" "def prepared_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('asset_marker.txt').write_text('ran')\n" + " Path(__file__).parents[2].joinpath('asset_marker.txt').write_text('ran')\n" " return ctx.result(payload={'status': 'ok'}, materialized=True)\n" ), }, @@ -675,7 +675,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('orders_ready.txt').write_text('ready')\n" + " Path(__file__).parents[2].joinpath('orders_ready.txt').write_text('ready')\n" " return ctx.result(metadata={'prepared': True})\n" ), "python/loaders/orders.py": ( @@ -684,7 +684,7 @@ def test_given_task_loader_source_model_chain_when_running_model_then_task_runs_ "from python.tasks.orders import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('orders_ready.txt')\n" + " marker = Path(__file__).parents[2].joinpath('orders_ready.txt')\n" " if not marker.exists():\n" " raise RuntimeError('orders were not prepared')\n" " return [{'order_id': 1, 'amount_cents': 100}]\n" @@ -769,7 +769,7 @@ def test_given_model_and_task_selector_when_running_run_then_task_can_read_built "@task(depends_on=model('fact_orders'))\n" "def summarize_orders(ctx):\n" " rows = ctx.query('SELECT COUNT(*) FROM fact_orders').fetchall()[0][0]\n" - " Path(__file__).parents[1].joinpath('summary.txt').write_text(str(rows))\n" + " Path(__file__).parents[2].joinpath('summary.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" ), }, @@ -840,7 +840,7 @@ def test_given_asset_depends_on_terminal_model_when_running_run_then_asset_reads "@asset(depends_on=model('fact_orders'))\n" "def export_fact_orders(ctx):\n" " rows = ctx.query('SELECT COUNT(*) FROM fact_orders').fetchall()[0][0]\n" - " Path(__file__).parents[1].joinpath('export.txt').write_text(str(rows))\n" + " Path(__file__).parents[2].joinpath('export.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" ), }, @@ -906,7 +906,7 @@ def test_given_task_asset_task_chain_when_running_final_task_then_chain_executes "@task(depends_on=publish_orders)\n" "def notify_orders(ctx):\n" " metadata = ctx.result_of(node_function=publish_orders).metadata\n" - " output = Path(__file__).parents[1].joinpath('notify.txt')\n" + " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(metadata['published']))\n" " return ctx.result()\n" ), @@ -1136,11 +1136,11 @@ def test_given_independent_python_and_sql_selectors_when_running_run_then_all_br "from sqlbuild.tasks import task\n\n" "@task\n" "def branch_a(ctx):\n" - " Path(__file__).parents[1].joinpath('branch_a.txt').write_text('a')\n" + " Path(__file__).parents[2].joinpath('branch_a.txt').write_text('a')\n" " return ctx.result()\n\n" "@task\n" "def branch_b(ctx):\n" - " Path(__file__).parents[1].joinpath('branch_b.txt').write_text('b')\n" + " Path(__file__).parents[2].joinpath('branch_b.txt').write_text('b')\n" " return ctx.result()\n" ), }, @@ -1587,7 +1587,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "@task(depends_on=load_window_orders)\n" "def prepare_raw_orders(ctx):\n" " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" - " Path(__file__).parents[1].joinpath('prepared.txt').write_text(str(rows))\n" + " Path(__file__).parents[2].joinpath('prepared.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows})\n" ), "python/loaders/raw.py": ( @@ -1596,7 +1596,7 @@ def test_given_loader_task_loader_chain_when_running_model_then_ingress_orders_c "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_raw_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('window orders were not prepared')\n" " return [{'order_id': 1}]\n" @@ -1669,7 +1669,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "@asset(depends_on=fetch_orders)\n" "def publish_orders(ctx):\n" " payload = ctx.result_of(node_function=fetch_orders).payload\n" - " marker = Path(__file__).parents[1].joinpath('asset_ready.txt')\n" + " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" ), @@ -1679,7 +1679,7 @@ def test_given_task_asset_loader_chain_when_running_model_then_ingress_orders_ch "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('asset_ready.txt')\n" + " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('asset was not ready')\n" " return [{'order_id': 1}]\n" @@ -1756,7 +1756,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "@asset(depends_on=load_window_orders)\n" "def prepare_asset_orders(ctx):\n" " rows = ctx.query('SELECT COUNT(*) FROM window_orders').fetchall()[0][0]\n" - " Path(__file__).parents[1].joinpath('asset_ready.txt').write_text(str(rows))\n" + " Path(__file__).parents[2].joinpath('asset_ready.txt').write_text(str(rows))\n" " return ctx.result(metadata={'rows': rows}, materialized=True)\n" ), "python/loaders/raw.py": ( @@ -1765,7 +1765,7 @@ def test_given_loader_asset_loader_chain_when_running_model_then_ingress_orders_ "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(prepare_asset_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('asset_ready.txt')\n" + " marker = Path(__file__).parents[2].joinpath('asset_ready.txt')\n" " if marker.read_text() != '1':\n" " raise RuntimeError('asset orders were not prepared')\n" " return [{'order_id': 1}]\n" @@ -1923,7 +1923,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" " payload = ctx.result_of(node_function=prepare_orders).payload\n" - " marker = Path(__file__).parents[1].joinpath('prepared_order_id.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" ), @@ -1933,7 +1933,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared_order_id.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( @@ -1971,7 +1971,7 @@ def test_given_loader_task_asset_loader_model_task_asset_task_spine_when_running "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" " payload = ctx.result_of(node_function=export_fact_orders).payload\n" - " output = Path(__file__).parents[1].joinpath('notify.txt')\n" + " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(payload['order_id']))\n" " return ctx.result(metadata={'notified': True})\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py index ce9a426481..859ec396d7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_build.py @@ -922,7 +922,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" " return ctx.result(payload={'order_id': 3})\n" ), @@ -951,7 +951,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('profile.txt')\n" + " marker = Path(__file__).parents[2].joinpath('profile.txt')\n" " marker.write_text('profiled')\n" " return ctx.result()\n" ), @@ -999,7 +999,7 @@ def test_given_shared_downstream_failure_when_building_then_blocks_all_source_ap "def profile_raw_orders(ctx):\n" " relation = ctx.relation(source('raw_orders'))\n" " value = ctx.query(f'SELECT order_id FROM {relation}').fetchall()[0][0]\n" - " marker = Path(__file__).parents[1].joinpath('source_profile.txt')\n" + " marker = Path(__file__).parents[2].joinpath('source_profile.txt')\n" " marker.write_text(str(value))\n" " return ctx.result(metadata={'source_order_id': value})\n" ), @@ -1098,7 +1098,7 @@ def test_given_python_lifecycle_edge_case_when_building_then_direct_build_harden "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " marker.write_text('prepared')\n" " return ctx.result()\n" ), @@ -1366,7 +1366,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "@asset(depends_on=prepare_orders)\n" "def publish_prepared_orders(ctx):\n" " payload = ctx.result_of(node_function=prepare_orders).payload\n" - " marker = Path(__file__).parents[1].joinpath('prepared_order_id.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " marker.write_text(str(payload['order_id']))\n" " return ctx.result(payload=payload, materialized=True)\n" ), @@ -1376,7 +1376,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "from sqlbuild.loaders import loader\n\n" "@loader(depends_on=(publish_prepared_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared_order_id.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared_order_id.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( @@ -1415,7 +1415,7 @@ def test_given_python_sql_python_spine_when_building_then_orders_python_around_s "@task(depends_on=export_fact_orders)\n" "def notify_fact_orders(ctx):\n" " payload = ctx.result_of(node_function=export_fact_orders).payload\n" - " output = Path(__file__).parents[1].joinpath('notify.txt')\n" + " output = Path(__file__).parents[2].joinpath('notify.txt')\n" " output.write_text(str(payload['order_id']))\n" " return ctx.result(metadata={'notified': True})\n" ), @@ -1561,7 +1561,7 @@ def test_given_prior_python_task_result_when_later_task_reads_result_then_uses_p f" failed = ctx.result_of(node_function=produce_result, run_id='{failed_run_id}')\n" " history = ctx.results_of(node_function=produce_result, limit=2)\n" " values = ','.join(str(item.payload['value']) for item in history)\n" - " output = Path(__file__).parents[1].joinpath('consumed.txt')\n" + " output = Path(__file__).parents[2].joinpath('consumed.txt')\n" " output.write_text(\n" " f\"{latest.payload['value']}:{first.payload['value']}:\"\n" ' f"{failed.status}:{failed.payload is None}:{values}:"\n' @@ -1665,7 +1665,7 @@ def test_given_prior_loader_result_when_later_task_reads_result_then_uses_loader "@task\n" "def consume_loader_result(ctx):\n" " result = ctx.result_of(node_function=raw_events)\n" - " output = Path(__file__).parents[1].joinpath('loader_result.txt')\n" + " output = Path(__file__).parents[2].joinpath('loader_result.txt')\n" " output.write_text(\n" " f\"{result.metadata['loader_name']}:{result.metadata['source_name']}:\"\n" " f\"{result.metadata['rows_loaded']}\"\n" @@ -1855,7 +1855,7 @@ def test_given_same_node_results_in_multiple_targets_when_reading_then_uses_acti "@task\n" "def consume_result(ctx):\n" " result = ctx.result_of(node_function=produce_result)\n" - " output = Path(__file__).parents[1].joinpath('target_result.txt')\n" + " output = Path(__file__).parents[2].joinpath('target_result.txt')\n" " output.write_text(str(result.payload['target']))\n" " return ctx.result()\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py index c9388aaf81..d0b28ade69 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_loader_watermark_build.py @@ -57,7 +57,7 @@ def test_given_managed_loader_watermark_when_reloaded_then_incremental_consumes_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " maximum = int(Path(__file__).parents[1].joinpath('maximum.txt').read_text())\n" + " maximum = int(Path(__file__).parents[2].joinpath('maximum.txt').read_text())\n" " return [{'id': value, 'amount': value * 100} " "for value in range(1, maximum + 1)]\n" ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py index 5c63d0ef33..ebdeed1565 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py @@ -160,7 +160,7 @@ def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('prepared.txt').write_text('7')\n" + " Path(__file__).parents[2].joinpath('prepared.txt').write_text('7')\n" " return ctx.result(payload={'order_id': 7})\n" ), "python/loaders/raw.py": ( @@ -169,7 +169,7 @@ def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side "from python.tasks.prepare import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( @@ -192,14 +192,14 @@ def test_given_virtual_python_nodes_when_building_then_runs_loader_and_read_side "def profile_fact_orders(ctx):\n" " relation = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()[0][0]\n" - " Path(__file__).parents[1].joinpath('profile.txt').write_text(str(rows))\n" + " Path(__file__).parents[2].joinpath('profile.txt').write_text(str(rows))\n" " return ctx.result(payload={'rows': rows})\n" "\n" "@task(depends_on=source('raw_orders'))\n" "def profile_raw_orders(ctx):\n" " relation = ctx.relation(source('raw_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()[0][0]\n" - " output = Path(__file__).parents[1].joinpath('source_profile.txt')\n" + " output = Path(__file__).parents[2].joinpath('source_profile.txt')\n" " output.write_text(str(rows))\n" " return ctx.result(payload={'rows': rows})\n" ), @@ -311,12 +311,12 @@ def test_given_virtual_python_result_when_building_then_persists_node_results_in "def summarize_loader(ctx):\n" " result = ctx.result_of(node_function=raw_orders)\n" " history = ctx.results_of(node_function=raw_orders, limit=1)\n" - " output = Path(__file__).parents[1].joinpath('loader_result.txt')\n" + " output = Path(__file__).parents[2].joinpath('loader_result.txt')\n" " output.write_text(\n" " f\"{result.metadata['loader_name']}:{result.metadata['source_name']}:\"\n" " f\"{result.metadata['rows_loaded']}\"\n" " )\n" - " history_output = Path(__file__).parents[1].joinpath('history_result.txt')\n" + " history_output = Path(__file__).parents[2].joinpath('history_result.txt')\n" " history_output.write_text(\n" " f\"{ctx.result_of(node_function=produce_result).payload['value']}:{len(history)}\"\n" " )\n" @@ -779,7 +779,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('prepared.txt').write_text('7')\n" + " Path(__file__).parents[2].joinpath('prepared.txt').write_text('7')\n" " return ctx.result(payload={'order_id': 7})\n" ), "python/loaders/raw.py": ( @@ -788,7 +788,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat "from python.tasks.prepare import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( @@ -811,7 +811,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat "def profile_fact_orders(ctx):\n" " relation = ctx.relation(model('fact_orders'))\n" " rows = ctx.query(f'SELECT COUNT(*) FROM {relation}').fetchall()[0][0]\n" - " Path(__file__).parents[1].joinpath('profile.txt').write_text(str(rows))\n" + " Path(__file__).parents[2].joinpath('profile.txt').write_text(str(rows))\n" " return ctx.result(payload={'rows': rows})\n" ), }, @@ -848,7 +848,7 @@ def test_given_virtual_python_identities_when_replanning_then_reads_virtual_stat "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('prepared.txt').write_text('8')\n" + " Path(__file__).parents[2].joinpath('prepared.txt').write_text('8')\n" " return ctx.result(payload={'order_id': 8})\n", encoding="utf-8", ) @@ -1046,7 +1046,7 @@ def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_ "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('prepared.txt').write_text('7')\n" + " Path(__file__).parents[2].joinpath('prepared.txt').write_text('7')\n" " return ctx.result(payload={'order_id': 7})\n" ), "python/loaders/raw.py": ( @@ -1055,7 +1055,7 @@ def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_ "from python.tasks.prepare import prepare_orders\n\n" "@loader(depends_on=(prepare_orders,))\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('prepared.txt')\n" + " marker = Path(__file__).parents[2].joinpath('prepared.txt')\n" " return [{'order_id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( @@ -1076,7 +1076,7 @@ def test_given_virtual_python_nodes_when_no_python_then_only_loader_side_python_ "from sqlbuild.tasks import task\n\n" "@task(depends_on=model('fact_orders'))\n" "def profile_fact_orders(ctx):\n" - " Path(__file__).parents[1].joinpath('profile.txt').write_text('ran')\n" + " Path(__file__).parents[2].joinpath('profile.txt').write_text('ran')\n" " return ctx.result()\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py index 627c52cc97..cb7cfb1ca4 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py @@ -485,7 +485,7 @@ def test_given_virtual_managed_source_freshness_when_unchanged_then_build_skips_ "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " marker = Path(__file__).parents[1].joinpath('raw_order_id.txt')\n" + " marker = Path(__file__).parents[2].joinpath('raw_order_id.txt')\n" " return [{'id': int(marker.read_text())}]\n" ), "sources/raw.yml": ( @@ -556,7 +556,7 @@ def test_given_virtual_managed_source_configured_freshness_when_building_then_re "from sqlbuild.loaders import loader\n\n" "@loader\n" "def raw_orders(ctx):\n" - " root = Path(__file__).parents[1]\n" + " root = Path(__file__).parents[2]\n" " return [{\n" " 'id': int(root.joinpath('raw_order_id.txt').read_text()),\n" " 'data_version': int(root.joinpath('raw_data_version.txt').read_text()),\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py index 2fd1f06b52..37704c6616 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/helpers.py @@ -55,7 +55,7 @@ def assert_source_only_ingress_dependency_case( "from sqlbuild.tasks import task\n\n" "@task\n" "def prepare_events(ctx):\n" - " Path(__file__).parents[1].joinpath('prepared.txt').write_text('prepared')\n" + " Path(__file__).parents[2].joinpath('prepared.txt').write_text('prepared')\n" " return ctx.result()\n" ), }, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py index aed52c64fb..7b5fb5d0aa 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/providers/test_providers.py @@ -640,7 +640,7 @@ def test_given_alias_imported_provider_annotation_when_running_command_then_cli_ from sqlbuild.tasks import task - provider_path = Path(__file__).parents[1] / "providers" / "marker.py" + provider_path = Path(__file__).parents[2] / "providers" / "marker.py" spec = importlib.util.spec_from_file_location("alias_marker", provider_path) alias_marker = importlib.util.module_from_spec(spec) sys.modules["alias_marker"] = alias_marker