From 6e24715ebdefd227ace7203ab931910157aa2ae6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:31:28 +0100 Subject: [PATCH 1/6] feat(compiler): add compiler engine switch and engine-namespaced caches Select the compiler engine with SQLBUILD_COMPILER_ENGINE (python by default, or native) or the hidden --compiler-engine global option. All frontier stages run through one compile_frontier seam; the native engine currently falls back to the Python stages. Compile JSON and sqb debug report the engine, compile reuse keys include it, and the native engine keeps its compiler, Rules and lineage stores in -native-v1 directories so the Python stores stay untouched. --- .../src/engine/_helpers/cache.rs | 21 +- .../src/engine/_helpers/evaluation.rs | 2 +- .../engine/tests/test_incremental_rules.rs | 41 ++++ .../src/engine/tests/test_types.rs | 7 + crates/sqlbuild-rules/src/models.rs | 2 + .../cli/commands/_helpers/compile/output.py | 2 + .../cli/commands/_helpers/debug/checks.py | 2 + .../cli/commands/_helpers/entry/parser.py | 8 + .../cli/commands/_helpers/entry/parsing.py | 7 + .../cli/commands/_helpers/lineage/cache.py | 13 +- .../cli/commands/classes/cli_namespace.py | 2 + .../cli/commands/main/entrypoint/entry.py | 23 +- .../cli/compile_reuse/_helpers/attempt.py | 6 +- .../_helpers/runtime_identity.py | 9 +- src/sqlbuild/cli/compile_reuse/constants.py | 15 +- .../compiler/discovery/main/discover.py | 19 +- .../fact_cache/main/_compile_cache_root.py | 3 +- src/sqlbuild/compiler/frontier/__init__.py | 1 + .../compiler/frontier/_helpers/__init__.py | 0 .../compiler/frontier/_helpers/engine.py | 26 +++ .../frontier/_helpers/native_frontier.py | 12 + .../frontier/_helpers/stage_capture.py | 37 ++++ .../compiler/frontier/classes/__init__.py | 1 + .../frontier/classes/stage_capture_encoder.py | 160 ++++++++++++++ .../classes/stage_capture_sequence.py | 19 ++ src/sqlbuild/compiler/frontier/constants.py | 20 ++ src/sqlbuild/compiler/frontier/exceptions.py | 5 + .../compiler/frontier/main/__init__.py | 1 + .../frontier/main/_compile_frontier.py | 20 ++ .../frontier/main/compiler_cache_directory.py | 21 ++ .../frontier/main/compiler_engine_override.py | 26 +++ .../frontier/main/engine_cache_name.py | 13 ++ .../frontier/main/resolve_compiler_engine.py | 10 + src/sqlbuild/compiler/frontier/types.py | 18 ++ .../pipeline/_helpers/analysis_selection.py | 27 +++ .../pipeline/main/compiled_project.py | 112 ++++++---- .../compiler/scopes/_helpers/cache.py | 9 +- src/sqlbuild/compiler/scopes/constants.py | 3 +- .../scopes/main/load_or_build_scope_index.py | 37 ++-- .../_helpers/engine/custom_rules.py | 5 +- .../rule_engine/_helpers/engine/native.py | 7 + .../rule_engine/_helpers/run/cache_paths.py | 22 ++ .../rule_engine/_helpers/run/native_memo.py | 9 +- .../rule_engine/_helpers/run/rules.py | 5 +- src/sqlbuild/rule_engine/constants.py | 8 +- .../cli/commands/main/compile/_test_types.py | 34 +++ .../cli/commands/main/compile/helpers.py | 54 +++++ .../compile/test_compiler_engine_switch.py | 151 +++++++++++++ .../commands/_helpers/entry/_test_types.py | 30 +++ .../entry/test_compiler_engine_option.py | 113 ++++++++++ .../cli/compile_reuse/_helpers/_test_types.py | 19 ++ .../cli/compile_reuse/_helpers/helpers.py | 46 ++++ .../test_compiler_engine_reuse_key.py | 105 +++++++++ .../sqlbuild/compiler/frontier/__init__.py | 0 .../sqlbuild/compiler/frontier/_test_types.py | 98 +++++++++ .../src/sqlbuild/compiler/frontier/helpers.py | 87 ++++++++ .../compiler/frontier/test_cache_namespace.py | 86 ++++++++ .../frontier/test_resolve_compiler_engine.py | 155 +++++++++++++ .../compiler/frontier/test_stage_capture.py | 205 ++++++++++++++++++ .../_helpers/host/test_custom_host_pool.py | 7 +- 60 files changed, 1903 insertions(+), 103 deletions(-) create mode 100644 src/sqlbuild/compiler/frontier/__init__.py create mode 100644 src/sqlbuild/compiler/frontier/_helpers/__init__.py create mode 100644 src/sqlbuild/compiler/frontier/_helpers/engine.py create mode 100644 src/sqlbuild/compiler/frontier/_helpers/native_frontier.py create mode 100644 src/sqlbuild/compiler/frontier/_helpers/stage_capture.py create mode 100644 src/sqlbuild/compiler/frontier/classes/__init__.py create mode 100644 src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py create mode 100644 src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py create mode 100644 src/sqlbuild/compiler/frontier/constants.py create mode 100644 src/sqlbuild/compiler/frontier/exceptions.py create mode 100644 src/sqlbuild/compiler/frontier/main/__init__.py create mode 100644 src/sqlbuild/compiler/frontier/main/_compile_frontier.py create mode 100644 src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py create mode 100644 src/sqlbuild/compiler/frontier/main/compiler_engine_override.py create mode 100644 src/sqlbuild/compiler/frontier/main/engine_cache_name.py create mode 100644 src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py create mode 100644 src/sqlbuild/compiler/frontier/types.py create mode 100644 src/sqlbuild/rule_engine/_helpers/run/cache_paths.py create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py create mode 100644 tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py create mode 100644 tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/__init__.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/_test_types.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/helpers.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py create mode 100644 tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index ed1749bf96..80efa9b7dc 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -16,21 +16,24 @@ pub(crate) struct RuleCacheEntry { } pub(crate) struct Cache { - root: PathBuf, + native_rules_path: PathBuf, } impl Cache { - pub(crate) fn open(project_dir: &Path) -> Result { - Ok(Self { - root: project_dir.join("target/rules-cache"), - }) + pub(crate) fn open(project_dir: &Path, rules_cache_path: &str) -> Result { + let native_rules_path = if rules_cache_path.is_empty() { + project_dir.join("target/rules-cache/bulk/native.json") + } else { + PathBuf::from(rules_cache_path) + }; + Ok(Self { native_rules_path }) } pub(crate) fn native_rules_bucket( &self, ruleset_identity: &str, ) -> Result { - read_bucket(&self.root.join("bulk/native.json"), ruleset_identity) + read_bucket(&self.native_rules_path, ruleset_identity) } pub(crate) fn put_native_rules_bucket( @@ -38,11 +41,7 @@ impl Cache { ruleset_identity: &str, bucket: &RuleCacheBucket, ) -> Result<(), String> { - write_bucket( - &self.root.join("bulk/native.json"), - ruleset_identity, - bucket, - ) + write_bucket(&self.native_rules_path, ruleset_identity, bucket) } } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 6652778977..ad6787b3bd 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -320,7 +320,7 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result Result<(), String> { + let test_cases = [test_types::ConfiguredCachePathTestCase { + description: "an engine-owned cache file replaces the default bulk cache", + cache_file: "target/rules-cache-native-v1/bulk/native.json", + expected_default_cache_exists: false, + expected_warm_hits: 3, + }]; + for test_case in &test_cases { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut rest = split_request( + &project_dir, + &json!({"select": ["SQBR"], "cache": {"enabled": true}}), + ); + let cache_path = project_dir.path().join(test_case.cache_file); + rest["rules_cache_path"] = json!(cache_path); + let models = orders_models(); + + let cold = evaluate_split(&rest, &models)?; + let warm = evaluate_split(&rest, &models)?; + + assert!(cache_path.is_file(), "{}", test_case.description); + assert_eq!( + project_dir + .path() + .join("target/rules-cache/bulk/native.json") + .exists(), + test_case.expected_default_cache_exists, + "{}", + test_case.description + ); + assert_eq!( + warm["cache_hits"], test_case.expected_warm_hits, + "{}", + test_case.description + ); + assert_eq!(warm["faults"], cold["faults"], "{}", test_case.description); + } + Ok(()) +} + #[test] fn given_corrupt_model_cache_when_evaluating_then_every_model_reevaluates() -> Result<(), String> { let test_cases = [test_types::ModelCacheReuseTestCase { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 8dbb4535ad..8f825fa48a 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -242,6 +242,13 @@ pub(crate) struct ModelCacheReuseTestCase { pub(crate) expected_misses: u64, } +pub(crate) struct ConfiguredCachePathTestCase { + pub(crate) description: &'static str, + pub(crate) cache_file: &'static str, + pub(crate) expected_default_cache_exists: bool, + pub(crate) expected_warm_hits: u64, +} + pub(crate) struct CachePruningTestCase { pub(crate) description: &'static str, pub(crate) expected_paths: &'static [&'static str], diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index b9bfbd3af2..e7576c976c 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -729,6 +729,7 @@ pub struct EvaluateRequest { pub initial_findings: Vec, pub defer_suppressions: bool, pub custom_rules: Vec, + pub rules_cache_path: String, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -765,6 +766,7 @@ impl Default for EvaluateRequest { initial_findings: vec![], defer_suppressions: false, custom_rules: vec![], + rules_cache_path: String::new(), } } } diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 7a873ed3de..232054aa0c 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -38,6 +38,7 @@ ) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities @@ -208,6 +209,7 @@ def format_compile_json( sql_validation_enabled=sql_validation_enabled, ), "compile_timings": timings_ms, + "compiler_engine": resolve_compiler_engine().value, "lineage_mode": lineage_mode.value, "resources": _resources(graph=graph, lineage=lineage), "artifacts": _artifacts(written=written, manifest=manifest), diff --git a/src/sqlbuild/cli/commands/_helpers/debug/checks.py b/src/sqlbuild/cli/commands/_helpers/debug/checks.py index 85cade7924..aa01e5c13c 100644 --- a/src/sqlbuild/cli/commands/_helpers/debug/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/debug/checks.py @@ -41,6 +41,7 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -272,6 +273,7 @@ def _build_runtime_lines() -> list[DebugLine]: sqlbuild_version = "unknown" return [ DebugLine(label="sqlbuild version", message=sqlbuild_version), + DebugLine(label="compiler engine", message=resolve_compiler_engine().value), DebugLine(label="python version", message=platform.python_version()), DebugLine(label="python path", message=sys.executable), DebugLine(label="os info", message=platform.platform()), diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index ba08ac6710..17e9c79555 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -29,6 +29,7 @@ ) from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode from sqlbuild.compiler.contract_adoption.types import ContractAction +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_OPTION, COMPILER_ENGINE_VALUES from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.scopes.types import DeclarationKind @@ -98,6 +99,13 @@ def _add_global_args(*, parser: argparse.ArgumentParser, suppress_defaults: bool default=argparse.SUPPRESS if suppress_defaults else False, help="enable verbose output and mirror internal diagnostics to stderr", ) + parser.add_argument( + COMPILER_ENGINE_OPTION, + dest="compiler_engine", + choices=COMPILER_ENGINE_VALUES, + default=argparse.SUPPRESS if suppress_defaults else None, + help=argparse.SUPPRESS, + ) def _add_global_args_to_subcommands( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py index 56bc7f1221..70575c1dce 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py @@ -25,6 +25,8 @@ SelectorFileSummary, SelectorInputs, ) +from sqlbuild.compiler.frontier.exceptions import CompilerEngineError +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.integrations.dbt.types import DbtInteropCommand @@ -115,6 +117,11 @@ def parse_cli_invocation( if unknown_args: parser.error(f"unrecognized arguments: {' '.join(unknown_args)}") args.verbose = args.verbose or args.debug + if args.compiler_engine is None: + try: + _ = resolve_compiler_engine() + except CompilerEngineError as error: + parser.error(str(error)) if ( args.debug and args.command == CliCommand.DBT diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py index 1454a47f27..eef67adaca 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py @@ -16,11 +16,14 @@ from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME from sqlbuild.cli.commands.models import LineageNode, RelationLineageIndex from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.frontier.constants import CACHE_DIRECTORY_NAME +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name from sqlbuild.compiler.pipeline.models import ProjectGraph _CACHE_SCHEMA_VERSION: int = 1 _CACHE_ALGORITHM_VERSION: str = "relation-lineage-graph-v4" -_CACHE_RELATIVE_PATH: Path = Path("cache/lineage/v1/structural-graph.sqlite3") +_LINEAGE_CACHE_DIRECTORY_NAME: str = "lineage" +_CACHE_FILE_RELATIVE_PATH: Path = Path("v1/structural-graph.sqlite3") _CACHE_MAX_BYTES: int = 50_000_000 _CACHE_MAX_ROWS: int = 1_000_000 _SQLITE_TIMEOUT_SECONDS: float = 5.0 @@ -176,7 +179,13 @@ def _is_excluded(*, path: Path, project_dir: Path) -> bool: def _cache_path(*, project_dir: Path) -> Path: - return project_dir / TARGET_DIRECTORY_NAME / _CACHE_RELATIVE_PATH + return ( + project_dir + / TARGET_DIRECTORY_NAME + / CACHE_DIRECTORY_NAME + / engine_cache_name(_LINEAGE_CACHE_DIRECTORY_NAME) + / _CACHE_FILE_RELATIVE_PATH + ) def _relation_index(*, graph: ProjectGraph) -> RelationLineageIndex: diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index b07cb18c1c..474c482885 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -48,6 +48,7 @@ "start_cursor_int": None, "end_cursor_int": None, "no_color": False, + "compiler_engine": None, "fail_fast": False, "full_refresh": False, "as_target": None, @@ -217,6 +218,7 @@ class CliNamespace: start_cursor_int: str | None end_cursor_int: str | None no_color: bool + compiler_engine: str | None fail_fast: bool full_refresh: bool as_target: str | None diff --git a/src/sqlbuild/cli/commands/main/entrypoint/entry.py b/src/sqlbuild/cli/commands/main/entrypoint/entry.py index f294f909a1..da0ef22774 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/entry.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/entry.py @@ -14,6 +14,8 @@ CliEntrypointHandlers, ParsedCliInvocation, ) +from sqlbuild.compiler.frontier.main.compiler_engine_override import compiler_engine_override +from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.observability.main.invocation_external_context_scope import ( @@ -61,13 +63,16 @@ def _main_with_dependencies( dispatch_with_compute_logs, ) - return dispatch_with_compute_logs( - args=args, - identity=identity, - operation=lambda: dispatch_and_handle_errors( + with compiler_engine_override( + None if args.compiler_engine is None else CompilerEngine(args.compiler_engine) + ): + return dispatch_with_compute_logs( args=args, - invocation=invocation, - handlers=handlers, - use_color=use_color, - ), - ) + identity=identity, + operation=lambda: dispatch_and_handle_errors( + args=args, + invocation=invocation, + handlers=handlers, + use_color=use_color, + ), + ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index d7a0e729b3..dac7203b29 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -45,7 +45,7 @@ PROJECT_CONFIG_FILENAMES, REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE, - REUSE_ENTRY_DIRECTORY_PARTS, + REUSE_ENTRY_DIRECTORY_NAME, REUSE_ENTRY_SUFFIX, REUSE_HIT_MESSAGE, TOTAL_TIMING, @@ -62,6 +62,7 @@ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.presentation.main.supports_color import supports_color @@ -81,7 +82,8 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: attempt: CompileReuseAttempt = replace( bypass, outcome=CompileReuseOutcome.MISS, - entry_path=Path(project_dir, *REUSE_ENTRY_DIRECTORY_PARTS) + entry_path=compiler_cache_directory(Path(project_dir)) + / REUSE_ENTRY_DIRECTORY_NAME / f"{entry_slot_name(selected_target=request.selected_target)}{REUSE_ENTRY_SUFFIX}", runtime=runtime_identity(), search_path=search_path_stamps(project_dir=project_dir), diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index b3eddc3024..3c792b9ba5 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -17,8 +17,10 @@ PROJECT_ROOT_PATH_MTIME_NS, REUSE_FORMAT_VERSION, TRACKED_ENVIRONMENT_PREFIXES, + UNTRACKED_ENVIRONMENT_NAMES, ) from sqlbuild.cli.compile_reuse.models import CompileReuseRequest, SettingsEnvironmentInputs +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine def runtime_identity() -> dict[str, str]: @@ -96,10 +98,12 @@ def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool def tracked_environment_names(*, template_names: tuple[str, ...]) -> tuple[str, ...]: - """Return SQLBuild-prefixed variables plus the variables templates read.""" + """Return tracked SQLBuild variables (not engine or capture settings) plus template reads.""" prefixed: set[str] = { - name for name in os.environ if name.startswith(TRACKED_ENVIRONMENT_PREFIXES) + name + for name in os.environ + if name.startswith(TRACKED_ENVIRONMENT_PREFIXES) and name not in UNTRACKED_ENVIRONMENT_NAMES } return tuple(sorted(prefixed.union(template_names))) @@ -155,6 +159,7 @@ def invocation_digest(*, request: CompileReuseRequest, project_dir: str, use_col "exclude": list(request.exclude), "cli_vars": request.cli_vars, "sys_path": list(sys.path), + "compiler_engine": resolve_compiler_engine().value, } encoded: bytes = json.dumps( payload, sort_keys=True, separators=(",", ":"), default=repr diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 95f701e062..e8896df728 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -7,9 +7,19 @@ from sqlbuild.cli.compile_reuse.classes.compile_artifact_write_registry import ( CompileArtifactWriteRegistry, ) +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, +) REUSE_FORMAT_VERSION: int = 1 -REUSE_ENTRY_DIRECTORY_PARTS: tuple[str, ...] = ("target", "cache", "compiler", "project-reuse-v1") +REUSE_ENTRY_DIRECTORY_NAME: str = "project-reuse-v1" +REUSE_ENTRY_DIRECTORY_PARTS: tuple[str, ...] = ( + "target", + "cache", + "compiler", + REUSE_ENTRY_DIRECTORY_NAME, +) REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" @@ -25,6 +35,9 @@ REUSE_DISABLE_VALUE: str = "1" REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") +UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( + {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} +) MISSING_ENVIRONMENT_VALUE: str = "\0missing" MISSING_FILE_DIGEST: str = "\0missing" PROJECT_CONFIG_FILENAMES: tuple[str, ...] = ("sqlbuild_project.toml", "sqlbuild_project.yml") diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index ccf9207c03..6154330a7f 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -2,6 +2,7 @@ from __future__ import annotations +from functools import partial from pathlib import Path from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( @@ -26,6 +27,8 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryCacheRequest from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -41,12 +44,16 @@ def discover_project_inputs( """Load all raw project inputs from disk before semantic resolution.""" with OperationLifecycle(operation_kind="project", operation_name="project_discovery"): - return _discover_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - cache_request=cache_request, - declaration_reuse=declaration_reuse, + return compile_frontier( + until=CompilerStage.DISCOVERED_PROJECT_INPUTS, + python_stage=partial( + _discover_project_inputs, + project_dir=project_dir, + sql_analysis_enabled_override=sql_analysis_enabled_override, + extract_output_column_locations=extract_output_column_locations, + cache_request=cache_request, + declaration_reuse=declaration_reuse, + ), ) diff --git a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py b/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py index 39067cf45e..27ffbcebd1 100644 --- a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py +++ b/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py @@ -9,6 +9,7 @@ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.spec.contracts.models import TargetConfig @@ -24,4 +25,4 @@ def compile_cache_root( or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE ): return None - return project_dir / "target" / "cache" / "compiler" + return compiler_cache_directory(project_dir) diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py new file mode 100644 index 0000000000..86ae110a66 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/__init__.py @@ -0,0 +1 @@ +"""Compiler engine selection and the native frontier seam.""" diff --git a/src/sqlbuild/compiler/frontier/_helpers/__init__.py b/src/sqlbuild/compiler/frontier/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/frontier/_helpers/engine.py b/src/sqlbuild/compiler/frontier/_helpers/engine.py new file mode 100644 index 0000000000..875cbbd141 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/_helpers/engine.py @@ -0,0 +1,26 @@ +"""Engine parsing and environment lookup shared by the frontier entry points.""" + +import os + +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + COMPILER_ENGINE_VALUES, + DEFAULT_COMPILER_ENGINE, +) +from sqlbuild.compiler.frontier.exceptions import CompilerEngineError +from sqlbuild.compiler.frontier.types import CompilerEngine + + +def active_compiler_engine() -> CompilerEngine: + """Return the engine selected by SQLBUILD_COMPILER_ENGINE, defaulting to Python.""" + + raw_value: str | None = os.environ.get(COMPILER_ENGINE_ENV_VAR) + if not raw_value: + return DEFAULT_COMPILER_ENGINE + try: + return CompilerEngine(raw_value) + except ValueError as error: + raise CompilerEngineError( + f"{COMPILER_ENGINE_ENV_VAR} must be one of {', '.join(COMPILER_ENGINE_VALUES)} " + f"(got {raw_value!r})" + ) from error diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py new file mode 100644 index 0000000000..4c8aebc06a --- /dev/null +++ b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py @@ -0,0 +1,12 @@ +"""The native side of the compiler frontier.""" + +from collections.abc import Callable + +from sqlbuild.compiler.frontier.types import CompilerStage + + +def native_frontier[T](*, until: CompilerStage, python_stage: Callable[[], T]) -> T: + """Produce the frontier object for `until` natively; no stage is native yet.""" + + _ = until + return python_stage() diff --git a/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py b/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py new file mode 100644 index 0000000000..da8a53de2a --- /dev/null +++ b/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py @@ -0,0 +1,37 @@ +"""Dump frontier objects as canonical JSON so two engines can be compared stage by stage.""" + +import json +import os +from pathlib import Path + +from sqlbuild.compiler.frontier.classes.stage_capture_encoder import StageCaptureEncoder +from sqlbuild.compiler.frontier.classes.stage_capture_sequence import StageCaptureSequence +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR, STAGE_CAPTURE_SUFFIX +from sqlbuild.compiler.frontier.types import CompilerStage + + +def write_stage_capture(*, stage: CompilerStage, value: object) -> None: + """Write one frontier object when SQLBUILD_COMPILER_STAGE_CAPTURE_DIR names a directory.""" + + directory: str | None = os.environ.get(STAGE_CAPTURE_DIR_ENV_VAR) + if not directory: + return + path: Path = Path(directory) / ( + f"{StageCaptureSequence.next_value():03d}-{stage.value}{STAGE_CAPTURE_SUFFIX}" + ) + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(render_stage_capture(value), encoding="utf-8") + + +def render_stage_capture(value: object) -> str: + """Render a value as indented JSON that keeps insertion order and sorts only sets.""" + + return ( + json.dumps( + StageCaptureEncoder().encode(value), + indent=1, + ensure_ascii=False, + allow_nan=False, + ) + + "\n" + ) diff --git a/src/sqlbuild/compiler/frontier/classes/__init__.py b/src/sqlbuild/compiler/frontier/classes/__init__.py new file mode 100644 index 0000000000..27eb23ee2d --- /dev/null +++ b/src/sqlbuild/compiler/frontier/classes/__init__.py @@ -0,0 +1 @@ +"""Compiler frontier runtime classes.""" diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py new file mode 100644 index 0000000000..b7d31c891a --- /dev/null +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py @@ -0,0 +1,160 @@ +"""Convert frontier objects into canonical JSON-compatible values.""" + +from __future__ import annotations + +import dataclasses +import datetime +import decimal +import enum +import functools +import hashlib +import json +import math +import re +import types +from collections.abc import Mapping +from collections.abc import Set as AbstractSet +from pathlib import PurePath +from typing import Any + +from sqlbuild.compiler.frontier.constants import ( + STAGE_CAPTURE_SKIPPED_SLOTS, + STAGE_CAPTURE_UNORDERED_ATTRIBUTES, +) + +_CALLABLE_TYPES: tuple[type, ...] = ( + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + types.BuiltinMethodType, + type, +) + + +class StageCaptureEncoder: + """Encode one object graph deterministically, replacing callables with qualified names.""" + + def __init__(self) -> None: + self._active: set[int] = set() + + def encode(self, value: object) -> object: + """Return a JSON-compatible value that keeps insertion order and sorts only sets.""" + + if isinstance(value, enum.Enum): + return {"__enum__": f"{qualified_name(type(value))}.{value.name}"} + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else {"__float__": repr(value)} + scalar: object | None = _encode_scalar(value) + if scalar is not None: + return scalar + identity: int = id(value) + if identity in self._active: + return {"__cycle__": qualified_name(type(value))} + self._active.add(identity) + try: + return self._encode_container(value) + finally: + self._active.discard(identity) + + def _encode_container(self, value: object) -> object: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + "__type__": qualified_name(type(value)), + **{ + field.name: self.encode(getattr(value, field.name, None)) + for field in dataclasses.fields(value) + }, + } + if isinstance(value, Mapping): + return self._encode_mapping(value) + if isinstance(value, AbstractSet): + return {"__set__": sorted((self.encode(item) for item in value), key=_sort_key)} + if isinstance(value, (list, tuple)): + return [self.encode(item) for item in value] + if isinstance(value, functools.partial): + return { + "__partial__": self.encode(value.func), + "args": self.encode(value.args), + "keywords": self.encode(value.keywords), + } + attributes: dict[str, object] | None = _instance_attributes(value) + type_name: str = qualified_name(type(value)) + if attributes is None: + return {"__opaque__": type_name} + unordered: frozenset[str] = STAGE_CAPTURE_UNORDERED_ATTRIBUTES.get(type_name, frozenset()) + return { + "__type__": type_name, + **{ + name: self._encode_attribute(value=item, unordered=name in unordered) + for name, item in attributes.items() + }, + } + + def _encode_attribute(self, *, value: object, unordered: bool) -> object: + if not unordered or not isinstance(value, Mapping): + return self.encode(value) + return {"__unordered_mapping__": sorted(self._mapping_pairs(value), key=_sort_key)} + + def _encode_mapping(self, value: Mapping[Any, object]) -> object: + if all(isinstance(key, str) for key in value): + return {str(key): self.encode(item) for key, item in value.items()} + return {"__mapping__": self._mapping_pairs(value)} + + def _mapping_pairs(self, value: Mapping[Any, object]) -> list[list[object]]: + return [[self.encode(key), self.encode(item)] for key, item in value.items()] + + +def qualified_name(value: object) -> str: + """Return `module:qualname` for a callable or class.""" + + module: object = getattr(value, "__module__", None) + name: object = getattr(value, "__qualname__", None) or getattr(value, "__name__", None) + if name is None: + return qualified_name(type(value)) + return f"{module}:{name}" if isinstance(module, str) else str(name) + + +def _encode_scalar(value: object) -> object | None: + if isinstance(value, PurePath): + return {"__path__": value.as_posix()} + if isinstance(value, decimal.Decimal): + return {"__decimal__": str(value)} + if isinstance(value, (datetime.date, datetime.time)): + return {"__datetime__": value.isoformat()} + if isinstance(value, (bytes, bytearray, memoryview)): + data: bytes = bytes(value) + return {"__bytes__": len(data), "sha256": hashlib.sha256(data).hexdigest()} + if isinstance(value, re.Pattern): + return {"__pattern__": str(value.pattern)} + if isinstance(value, types.ModuleType): + return {"__module__": value.__name__} + if isinstance(value, _CALLABLE_TYPES): + return {"__callable__": qualified_name(value)} + return None + + +def _sort_key(item: object) -> str: + return json.dumps(item, sort_keys=True, ensure_ascii=False) + + +def _instance_attributes(value: object) -> dict[str, object] | None: + instance_dict: object = getattr(value, "__dict__", None) + attributes: dict[str, object] = ( + {str(name): item for name, item in instance_dict.items()} + if isinstance(instance_dict, dict) + else {} + ) + for cls in type(value).__mro__: + slots: object = cls.__dict__.get("__slots__", ()) + for name in (slots,) if isinstance(slots, str) else slots: + if ( + isinstance(name, str) + and name not in STAGE_CAPTURE_SKIPPED_SLOTS + and hasattr(value, name) + ): + attributes[name] = getattr(value, name) + if not attributes and not isinstance(instance_dict, dict): + return None + return attributes diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py new file mode 100644 index 0000000000..0ec43b4b13 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py @@ -0,0 +1,19 @@ +"""Process-wide ordering of stage captures.""" + +import itertools +import threading +from typing import ClassVar + + +class StageCaptureSequence: + """Number stage captures in the order this process produced them.""" + + _counter: ClassVar[itertools.count[int]] = itertools.count(1) + _lock: ClassVar[threading.Lock] = threading.Lock() + + @classmethod + def next_value(cls) -> int: + """Return the next capture number.""" + + with cls._lock: + return next(cls._counter) diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py new file mode 100644 index 0000000000..b74f1abdf5 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -0,0 +1,20 @@ +"""Compiler engine switch, cache namespace, and stage capture constants.""" + +from sqlbuild.compiler.frontier.types import CompilerEngine + +COMPILER_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" +COMPILER_ENGINE_OPTION: str = "--compiler-engine" +DEFAULT_COMPILER_ENGINE: CompilerEngine = CompilerEngine.PYTHON +COMPILER_ENGINE_VALUES: tuple[str, ...] = tuple(engine.value for engine in CompilerEngine) +NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" +TARGET_DIRECTORY_NAME: str = "target" +CACHE_DIRECTORY_NAME: str = "cache" +COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" +STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +STAGE_CAPTURE_SUFFIX: str = ".json" +STAGE_CAPTURE_SKIPPED_SLOTS: frozenset[str] = frozenset({"__dict__", "__weakref__"}) +STAGE_CAPTURE_UNORDERED_ATTRIBUTES: dict[str, frozenset[str]] = { + "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( + {"schemas", "analysis_shapes", "expression_shapes", "shared_analyses"} + ) +} diff --git a/src/sqlbuild/compiler/frontier/exceptions.py b/src/sqlbuild/compiler/frontier/exceptions.py new file mode 100644 index 0000000000..81f0c64dc3 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/exceptions.py @@ -0,0 +1,5 @@ +"""Compiler engine selection errors.""" + + +class CompilerEngineError(ValueError): + """Raised when the requested compiler engine is not a supported value.""" diff --git a/src/sqlbuild/compiler/frontier/main/__init__.py b/src/sqlbuild/compiler/frontier/main/__init__.py new file mode 100644 index 0000000000..c834fc5400 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/__init__.py @@ -0,0 +1 @@ +"""Public compiler engine and frontier entry points.""" diff --git a/src/sqlbuild/compiler/frontier/main/_compile_frontier.py b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py new file mode 100644 index 0000000000..a872314978 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py @@ -0,0 +1,20 @@ +"""The single seam between the Python compiler stages and the native frontier.""" + +from collections.abc import Callable + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier._helpers.native_frontier import native_frontier +from sqlbuild.compiler.frontier._helpers.stage_capture import write_stage_capture +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage + + +def compile_frontier[T](*, until: CompilerStage, python_stage: Callable[[], T]) -> T: + """Produce the frontier object for `until` on the active engine, capturing it on request.""" + + result: T = ( + python_stage() + if active_compiler_engine() is CompilerEngine.PYTHON + else native_frontier(until=until, python_stage=python_stage) + ) + write_stage_capture(stage=until, value=result) + return result diff --git a/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py b/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py new file mode 100644 index 0000000000..aafb64585e --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py @@ -0,0 +1,21 @@ +"""Project-local compiler store root for the active engine.""" + +from pathlib import Path + +from sqlbuild.compiler.frontier.constants import ( + CACHE_DIRECTORY_NAME, + COMPILER_CACHE_DIRECTORY_NAME, + TARGET_DIRECTORY_NAME, +) +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name + + +def compiler_cache_directory(project_dir: Path) -> Path: + """Return the compiler store root owned by the active engine.""" + + return ( + project_dir + / TARGET_DIRECTORY_NAME + / CACHE_DIRECTORY_NAME + / engine_cache_name(COMPILER_CACHE_DIRECTORY_NAME) + ) diff --git a/src/sqlbuild/compiler/frontier/main/compiler_engine_override.py b/src/sqlbuild/compiler/frontier/main/compiler_engine_override.py new file mode 100644 index 0000000000..fbfc87cd3d --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/compiler_engine_override.py @@ -0,0 +1,26 @@ +"""Select the compiler engine for one CLI invocation.""" + +import os +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine + + +@contextmanager +def compiler_engine_override(engine: CompilerEngine | None) -> Iterator[None]: + """Select an engine for one invocation so worker threads and subprocesses see it too.""" + + if engine is None: + yield + return + previous: str | None = os.environ.get(COMPILER_ENGINE_ENV_VAR) + os.environ[COMPILER_ENGINE_ENV_VAR] = engine.value + try: + yield + finally: + if previous is None: + _ = os.environ.pop(COMPILER_ENGINE_ENV_VAR, None) + else: + os.environ[COMPILER_ENGINE_ENV_VAR] = previous diff --git a/src/sqlbuild/compiler/frontier/main/engine_cache_name.py b/src/sqlbuild/compiler/frontier/main/engine_cache_name.py new file mode 100644 index 0000000000..da1eca742c --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/engine_cache_name.py @@ -0,0 +1,13 @@ +"""Engine namespacing for on-disk compiler stores.""" + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier.constants import NATIVE_CACHE_NAMESPACE_SUFFIX +from sqlbuild.compiler.frontier.types import CompilerEngine + + +def engine_cache_name(base: str) -> str: + """Name an on-disk store so engines never read each other's entries.""" + + if active_compiler_engine() is CompilerEngine.PYTHON: + return base + return f"{base}{NATIVE_CACHE_NAMESPACE_SUFFIX}" diff --git a/src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py b/src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py new file mode 100644 index 0000000000..bddadba080 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py @@ -0,0 +1,10 @@ +"""Public compiler engine resolution entry point.""" + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier.types import CompilerEngine + + +def resolve_compiler_engine() -> CompilerEngine: + """Return the active compiler engine, raising CompilerEngineError for an unknown value.""" + + return active_compiler_engine() diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py new file mode 100644 index 0000000000..f7d4daa79d --- /dev/null +++ b/src/sqlbuild/compiler/frontier/types.py @@ -0,0 +1,18 @@ +"""Compiler engine and frontier stage identities.""" + +from enum import StrEnum + + +class CompilerEngine(StrEnum): + """Which compiler implementation produces the frontier objects.""" + + PYTHON = "python" + NATIVE = "native" + + +class CompilerStage(StrEnum): + """Frontier objects handed from one compiler stage to the next, in pipeline order.""" + + DISCOVERED_PROJECT_INPUTS = "discovered_project_inputs" + COMPILE_PROJECT_INPUTS = "compile_project_inputs" + COMPILED_PROJECT = "compiled_project" diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py index 850c2996c4..1a5c79e100 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py @@ -4,10 +4,13 @@ from dataclasses import replace +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, CompiledObjectKey, CompiledProject, + CompileProjectInputs, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineOptions @@ -99,3 +102,27 @@ def resolve_configured_rule_selection( policies=PlannerPolicies(auto_load_sources=analysis_selection.auto_load_sources), ) return frozenset(model.key for model in scopes.stale_warning_scope.models_by_name.values()) + + +def resolve_analysis_model_names( + *, + compile_inputs: CompileProjectInputs, + inference_profile: ExpressionInferenceProfile, + selection: CompileAnalysisSelection | None, +) -> frozenset[str] | None: + """Return the models selected for deep analysis, or None to analyse every model.""" + + if selection is None or (not selection.select and not selection.exclude): + return None + structural_project: CompiledProject = assemble_project( + inputs=compile_inputs, + inference_profile=inference_profile, + skip_column_inference=True, + analysis_model_names=frozenset(), + ) + scopes: PlannerScopeResolution = resolve_planner_scopes( + project=structural_project, + selection=PlannerSelection(select=selection.select, exclude=selection.exclude), + policies=PlannerPolicies(auto_load_sources=selection.auto_load_sources), + ) + return frozenset(scopes.stale_warning_scope.models_by_name) diff --git a/src/sqlbuild/compiler/pipeline/main/compiled_project.py b/src/sqlbuild/compiler/pipeline/main/compiled_project.py index 62d0b230ac..748d8db834 100644 --- a/src/sqlbuild/compiler/pipeline/main/compiled_project.py +++ b/src/sqlbuild/compiler/pipeline/main/compiled_project.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import replace +from functools import partial from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -21,7 +22,10 @@ TABLE_PROMOTION_MODE_SETTING_KEY, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild.compiler.pipeline._helpers.analysis_selection import resolve_analysis_model_names from sqlbuild.compiler.pipeline._helpers.target_defaults import apply_target_defaults from sqlbuild.compiler.pipeline._helpers.target_validation import ( validate_managed_loader_target_isolation, @@ -29,14 +33,6 @@ validate_named_target_schema_strategy, validate_project_targets, ) -from sqlbuild.compiler.planner.main.selection._resolve_planner_scopes import ( - resolve_planner_scopes, -) -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScopeResolution, - PlannerSelection, -) from sqlbuild.compiler.profiling.main.record import record_compile_timing from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( @@ -62,6 +58,37 @@ def build_compiled_project( ) -> CompiledProject: """Build one compiled project with adapter defaults and target validation applied.""" + return compile_frontier( + until=CompilerStage.COMPILED_PROJECT, + python_stage=partial( + _build_python_compiled_project, + discovered_inputs=discovered_inputs, + adapter=adapter, + selected_target=selected_target, + no_sql_validation=no_sql_validation, + skip_column_inference=skip_column_inference, + column_lineage_mode=column_lineage_mode, + cli_vars=cli_vars, + external_sql_reference_resolver=external_sql_reference_resolver, + resolved_connection=resolved_connection, + analysis_selection=analysis_selection, + ), + ) + + +def _build_python_compiled_project( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None, + no_sql_validation: bool, + skip_column_inference: bool, + column_lineage_mode: ColumnLineageMode, + cli_vars: dict[str, object] | None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, + resolved_connection: dict[str, object] | None, + analysis_selection: CompileAnalysisSelection | None, +) -> CompiledProject: validate_named_target_schema_strategy( discovered_inputs=discovered_inputs, adapter=adapter, @@ -74,22 +101,11 @@ def build_compiled_project( ) no_cache: bool = analysis_selection is not None and analysis_selection.no_cache with record_compile_timing("attachment_ms"): - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileProjectInputs = _frontier_compile_inputs( discovered_inputs=discovered_inputs, - adapter_context=CompileAdapterContext( - value_renderer=adapter, - collection_rendering=resolve_effective_collection_rendering( - project_config=discovered_inputs.project_config, - declaration_override=None, - ), - python_functions_inherit_default_namespace=( - adapter.python_functions_inherit_default_namespace() - ), - sql_lexical_syntax=adapter.sql_lexical_syntax, - ), + adapter=adapter, selected_target=selected_target, no_sql_validation=no_sql_validation, - defer_model_sql_validation=True, cli_vars=cli_vars, resolved_connection=resolved_connection, external_sql_reference_resolver=external_sql_reference_resolver, @@ -98,7 +114,7 @@ def build_compiled_project( if analysis_selection is not None and analysis_selection.on_inputs_ready is not None: analysis_selection.on_inputs_ready(inputs=compile_inputs) inference_profile: ExpressionInferenceProfile = adapter.expression_inference_profile() - analysis_model_names: frozenset[str] | None = _resolve_analysis_model_names( + analysis_model_names: frozenset[str] | None = resolve_analysis_model_names( compile_inputs=compile_inputs, inference_profile=inference_profile, selection=analysis_selection, @@ -146,23 +162,39 @@ def build_compiled_project( ) -def _resolve_analysis_model_names( +def _frontier_compile_inputs( *, - compile_inputs: CompileProjectInputs, - inference_profile: ExpressionInferenceProfile, - selection: CompileAnalysisSelection | None, -) -> frozenset[str] | None: - if selection is None or (not selection.select and not selection.exclude): - return None - structural_project: CompiledProject = assemble_project( - inputs=compile_inputs, - inference_profile=inference_profile, - skip_column_inference=True, - analysis_model_names=frozenset(), - ) - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=structural_project, - selection=PlannerSelection(select=selection.select, exclude=selection.exclude), - policies=PlannerPolicies(auto_load_sources=selection.auto_load_sources), + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None, + no_sql_validation: bool, + cli_vars: dict[str, object] | None, + resolved_connection: dict[str, object] | None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, + no_cache: bool, +) -> CompileProjectInputs: + return compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial( + build_compile_inputs, + discovered_inputs=discovered_inputs, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered_inputs.project_config, + declaration_override=None, + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + selected_target=selected_target, + no_sql_validation=no_sql_validation, + defer_model_sql_validation=True, + cli_vars=cli_vars, + resolved_connection=resolved_connection, + external_sql_reference_resolver=external_sql_reference_resolver, + no_cache=no_cache, + ), ) - return frozenset(scopes.stale_warning_scope.models_by_name) diff --git a/src/sqlbuild/compiler/scopes/_helpers/cache.py b/src/sqlbuild/compiler/scopes/_helpers/cache.py index 80424a7afc..11cf2a132b 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/cache.py +++ b/src/sqlbuild/compiler/scopes/_helpers/cache.py @@ -16,12 +16,13 @@ LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.scopes._helpers.lookup import build_lookup from sqlbuild.compiler.scopes._helpers.paths import normalize_path from sqlbuild.compiler.scopes.constants import ( GLOBAL_DECLARATION_DIRECTORIES, GLOBAL_NAMED_DECLARATION_DIRECTORIES, - SCOPE_CACHE_DIRECTORY, + SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME, SCOPE_CACHE_MAX_BYTES, SCOPE_CACHE_SCHEMA_VERSION, @@ -575,7 +576,9 @@ def scope_index_fingerprint(*, project_dir: Path) -> str: def read_cached_scope_index(*, project_dir: Path, fingerprint: str) -> ScopeIndex | None: """Read a verified index; every storage or validation fault is a cache miss.""" - path: Path = project_dir / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + path: Path = ( + compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ) try: if path.stat().st_size > SCOPE_CACHE_MAX_BYTES: return None @@ -612,7 +615,7 @@ def write_cached_scope_index(*, project_dir: Path, fingerprint: str, index: Scop "schema_version": SCOPE_CACHE_SCHEMA_VERSION, } ) - directory: Path = project_dir / SCOPE_CACHE_DIRECTORY + directory: Path = compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME directory.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp(prefix=".scope-index-", dir=directory) temporary_path: Path = Path(temporary_name) diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index 0fdb41dd57..a78506c9bf 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -9,7 +9,8 @@ SCOPE_METADATA_SCHEMA_VERSION: int = 3 SCOPE_CACHE_SCHEMA_VERSION: int = 3 SCOPE_FINGERPRINT_ALGORITHM_VERSION: int = 2 -SCOPE_CACHE_DIRECTORY: Path = Path("target/cache/compiler/declaration-scopes-v2") +SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" +SCOPE_CACHE_DIRECTORY: Path = Path("target/cache/compiler") / SCOPE_CACHE_DIRECTORY_NAME SCOPE_CACHE_FILENAME: str = "scope-index.json" SCOPE_CACHE_MAX_BYTES: int = 16 * 1024 * 1024 SCOPE_SOURCE_SUFFIXES: frozenset[str] = frozenset({".sql", ".yml", ".yaml", ".py"}) diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index 2bf34d2a19..ee84e2c18d 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -2,6 +2,7 @@ from __future__ import annotations +from functools import partial from pathlib import Path from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -13,6 +14,8 @@ from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.scopes._helpers.builder import build_tolerant_scope_index from sqlbuild.compiler.scopes._helpers.cache import ( read_cached_scope_index, @@ -57,23 +60,27 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S ) adapter: BaseAdapter = resolve_adapter(adapter_name=adapter_name) sql_lexical_syntax = adapter.sql_lexical_syntax - compile_inputs: CompileProjectInputs = build_compile_inputs( - discovered_inputs=discovered, - adapter_context=CompileAdapterContext( - value_renderer=adapter, - collection_rendering=resolve_effective_collection_rendering( - project_config=discovered.project_config, - declaration_override=None, + compile_inputs: CompileProjectInputs = compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial( + build_compile_inputs, + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, + declaration_override=None, + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, ), - python_functions_inherit_default_namespace=( - adapter.python_functions_inherit_default_namespace() - ), - sql_lexical_syntax=adapter.sql_lexical_syntax, + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, ), - resolved_connection={}, - no_sql_validation=True, - defer_model_sql_validation=True, - no_cache=True, ) index: ScopeIndex = scope_index_with_compile_usages(inputs=compile_inputs) except (OSError, UnicodeError, ValueError, ImportError): diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 1ec801614e..60699505be 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -26,6 +26,7 @@ finding_payload, run_custom_hosts, ) +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( @@ -119,7 +120,9 @@ def evaluate_custom_rules_cached( ) for rule in custom } - cache_path: Path = project_dir / CUSTOM_RULES_CACHE_FILE + cache_path: Path = rules_bulk_cache_path( + project_dir=project_dir, file_name=CUSTOM_RULES_CACHE_FILE + ) stored: dict[str, _RuleCache] = _read_cache(cache_path) results: dict[tuple[str, str], tuple[dict[str, object], ...]] = {} plan: dict[str, list[str] | None] = {} diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 872498fa0f..a2a4ff7470 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -34,6 +34,7 @@ custom_rule_test_evidence, ) from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.native_memo import ( native_payload_digests, native_request_identity, @@ -41,6 +42,7 @@ write_native_response, ) from sqlbuild.rule_engine.constants import ( + NATIVE_RULES_CACHE_FILE, RULES_NATIVE_API_VERSION, TYPE_PROOF_RULE_CODES, ) @@ -223,6 +225,11 @@ def _native_request( "initial_findings": [_finding_payload(finding) for finding in initial_findings], "defer_suppressions": True, "custom_rules": custom_payloads, + "rules_cache_path": str( + rules_bulk_cache_path( + project_dir=project_dir.resolve(), file_name=NATIVE_RULES_CACHE_FILE + ) + ), } diff --git a/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py b/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py new file mode 100644 index 0000000000..6f326cc15c --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py @@ -0,0 +1,22 @@ +"""Locations of the Rules caches, namespaced by compiler engine.""" + +from pathlib import Path + +from sqlbuild.compiler.frontier.constants import TARGET_DIRECTORY_NAME +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name +from sqlbuild.rule_engine.constants import ( + RULES_BULK_CACHE_DIRECTORY_NAME, + RULES_CACHE_DIRECTORY_NAME, +) + + +def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: + """Return one bulk Rules cache file owned by the active compiler engine.""" + + return ( + project_dir + / TARGET_DIRECTORY_NAME + / engine_cache_name(RULES_CACHE_DIRECTORY_NAME) + / RULES_BULK_CACHE_DIRECTORY_NAME + / file_name + ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py index 4620c174fe..af631e553e 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -11,6 +11,7 @@ import orjson from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION @@ -39,7 +40,11 @@ def read_native_response(*, project_dir: Path, identity: str) -> str | None: """Return the stored response for this exact request identity, if any.""" try: - payload: object = orjson.loads((project_dir / NATIVE_RULES_MEMO_FILE).read_bytes()) + payload: object = orjson.loads( + rules_bulk_cache_path( + project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE + ).read_bytes() + ) except (OSError, orjson.JSONDecodeError): return None if not isinstance(payload, dict) or payload.get("identity") != identity: @@ -51,7 +56,7 @@ def read_native_response(*, project_dir: Path, identity: str) -> str | None: def write_native_response(*, project_dir: Path, identity: str, response: str) -> None: """Atomically replace the stored response with the latest evaluated request.""" - path: Path = project_dir / NATIVE_RULES_MEMO_FILE + path: Path = rules_bulk_cache_path(project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE) path.parent.mkdir(parents=True, exist_ok=True) temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") temporary.write_bytes(orjson.dumps({"identity": identity, "response": response})) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index b2962166c9..9f5b2c8c32 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -58,10 +58,11 @@ native_catalogue, start_custom_rules, ) +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory -from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES +from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( @@ -792,7 +793,7 @@ def _sql_file_rule_identity( def _sql_rule_cache_path(project_dir: Path) -> Path: - return project_dir / "target" / "rules-cache" / "bulk" / "sql.json" + return rules_bulk_cache_path(project_dir=project_dir, file_name=SQL_RULES_CACHE_FILE) def _read_sql_rule_cache(project_dir: Path) -> dict[str, dict[str, object]]: diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index c3fe200ed5..f7efc3453e 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -12,9 +12,13 @@ CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULE_PROJECT_SUBJECT: str = "" CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" -CUSTOM_RULES_CACHE_FILE: str = "target/rules-cache/bulk/custom-rules.json" +RULES_CACHE_DIRECTORY_NAME: str = "rules-cache" +RULES_BULK_CACHE_DIRECTORY_NAME: str = "bulk" +CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" NATIVE_RULES_MEMO_VERSION: str = "native-rules-response-v2" -NATIVE_RULES_MEMO_FILE: str = "target/rules-cache/bulk/native-response.json" +NATIVE_RULES_MEMO_FILE: str = "native-response.json" +SQL_RULES_CACHE_FILE: str = "sql.json" +NATIVE_RULES_CACHE_FILE: str = "native.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" CUSTOM_HOST_ABANDONED_SUFFIX: str = ".abandoned" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index df9edf5609..e1a9dba792 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -795,3 +795,37 @@ class SharedCacheKeyTestCase: description: str schedule: Callable[[pytest.MonkeyPatch], None] expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class CompilerEngineParityTestCase: + """Two engines compiling separate copies of one project.""" + + description: str + left_engine: str + right_engine: str + expected_engines: tuple[str, str] + expected_exit_codes: tuple[int, int] + + +@dataclass(frozen=True) +class CompilerEngineStoreTestCase: + """A sequence of engine compiles in one project and which of them reuse a stored compile.""" + + description: str + engines: tuple[str, ...] + expected_reused: tuple[bool, ...] + expected_python_stores: tuple[str, ...] + expected_native_stores: tuple[str, ...] + + +@dataclass(frozen=True) +class CompilerEngineRulesStoreTestCase: + """Compile plus Rules under each engine and the store files each engine must own.""" + + description: str + rules_selector: str + stores: tuple[str, ...] + native_marker: str + expected_python_files: tuple[str, ...] + expected_native_files: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 32277c374e..dd23c1fc04 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import itertools import json import os @@ -4141,3 +4142,56 @@ def json_report_keys(stdout: str) -> tuple[str, ...]: """Parse a JSON compile report and return its top-level keys.""" return tuple(cast(dict[str, object], json.loads(stdout))) + + +_COMPILER_ENGINE_LINE: re.Pattern[str] = re.compile(r'\n "compiler_engine": "([a-z]+)",') + + +def copy_compile_project(*, source: Path, destination: Path) -> Path: + """Copy a prepared project to a new directory and return the copy.""" + + _ = shutil.copytree(source, destination) + return destination + + +def engine_reuse_compile(*, project_dir: Path, engine: str) -> CompileReuseRun: + """Compile with reuse enabled under one engine selected by the hidden flag.""" + + return run_reuse_compile(project_dir=project_dir, global_args=("--compiler-engine", engine)) + + +def engine_compile_and_rules(*, project_dir: Path, engine: str, rules_selector: str) -> int: + """Compile and run Rules under one engine; return the Rules exit code.""" + + _ = engine_reuse_compile(project_dir=project_dir, engine=engine) + return run_installed_sqb( + project_dir=project_dir, + args=("--compiler-engine", engine, "rules", "run", rules_selector), + env=COMPILE_REUSE_ENV, + ).returncode + + +def report_engine(run: CompileReuseRun) -> str: + """Return the engine named in a JSON compile report.""" + + match: re.Match[str] | None = _COMPILER_ENGINE_LINE.search(run.report) + assert match is not None + return match.group(1) + + +def report_without_engine(run: CompileReuseRun) -> str: + """Return a JSON compile report without its timings and engine fields.""" + + return _COMPILER_ENGINE_LINE.sub("", run.report) + + +def store_digests(*, project_dir: Path, stores: tuple[str, ...]) -> dict[str, str]: + """Hash every file below the given project-relative store directories.""" + + files: list[Path] = [] + for store in stores: + files.extend(filter(Path.is_file, sorted((project_dir / store).rglob("*")))) + return { + path.relative_to(project_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in files + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py new file mode 100644 index 0000000000..b8fcbd0d16 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py @@ -0,0 +1,151 @@ +"""Both compiler engines produce identical compiles and never share on-disk compile stores.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + CompilerEngineParityTestCase, + CompilerEngineRulesStoreTestCase, + CompilerEngineStoreTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + engine_compile_and_rules, + engine_reuse_compile, + prepare_compile_reuse_project, + report_engine, + report_without_engine, + store_digests, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineParityTestCase( + description="python_and_native", + left_engine="python", + right_engine="native", + expected_engines=("python", "native"), + expected_exit_codes=(0, 0), + ) + ], + ids=lambda case: case.description, +) +def test_given_same_project_when_compiling_with_each_engine_then_outputs_are_identical( + test_case: CompilerEngineParityTestCase, tmp_path: Path +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + left: CompileReuseRun = engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "left"), + engine=test_case.left_engine, + ) + right: CompileReuseRun = engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "right"), + engine=test_case.right_engine, + ) + + assert (left.returncode, right.returncode) == test_case.expected_exit_codes + assert (report_engine(left), report_engine(right)) == test_case.expected_engines + assert report_without_engine(right) == report_without_engine(left) + assert right.compiled == left.compiled + assert left.compiled + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineStoreTestCase( + description="python_native_native_python", + engines=("python", "native", "native", "python"), + expected_reused=(False, False, True, True), + expected_python_stores=("target/cache/compiler", "target/rules-cache"), + expected_native_stores=( + "target/cache/compiler-native-v1", + "target/rules-cache-native-v1", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_compile_by_one_engine_when_other_engine_compiles_then_no_store_is_shared( + test_case: CompilerEngineStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=project_dir) + + first: CompileReuseRun = engine_reuse_compile( + project_dir=project_dir, engine=test_case.engines[0] + ) + python_stores: dict[str, str] = store_digests( + project_dir=project_dir, stores=test_case.expected_python_stores + ) + later: list[CompileReuseRun] = [ + engine_reuse_compile(project_dir=project_dir, engine=engine) + for engine in test_case.engines[1:-1] + ] + python_stores_after_native: dict[str, str] = store_digests( + project_dir=project_dir, stores=test_case.expected_python_stores + ) + last: CompileReuseRun = engine_reuse_compile( + project_dir=project_dir, engine=test_case.engines[-1] + ) + runs: list[CompileReuseRun] = [first, *later, last] + + assert tuple(run.reused for run in runs) == test_case.expected_reused + assert python_stores + assert python_stores_after_native == python_stores + assert store_digests(project_dir=project_dir, stores=test_case.expected_native_stores) + assert {report_without_engine(run) for run in runs[2:]} == {report_without_engine(last)} + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineRulesStoreTestCase( + description="built_in_rules_after_compile", + rules_selector="SQBR", + stores=("target/cache", "target/rules-cache", "target/rules-cache-native-v1"), + native_marker="-native-v1/", + expected_python_files=( + "target/rules-cache/bulk/native.json", + "target/rules-cache/bulk/sql.json", + ), + expected_native_files=( + "target/rules-cache-native-v1/bulk/native.json", + "target/rules-cache-native-v1/bulk/sql.json", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_python_compile_and_rules_when_native_runs_them_then_native_writes_only_its_stores( + test_case: CompilerEngineRulesStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=project_dir) + + python_exit: int = engine_compile_and_rules( + project_dir=project_dir, engine="python", rules_selector=test_case.rules_selector + ) + python_files: dict[str, str] = store_digests(project_dir=project_dir, stores=test_case.stores) + native_exit: int = engine_compile_and_rules( + project_dir=project_dir, engine="native", rules_selector=test_case.rules_selector + ) + all_files: dict[str, str] = store_digests(project_dir=project_dir, stores=test_case.stores) + native_files: set[str] = set(all_files) - set(python_files) + + assert native_exit == python_exit + assert {path: all_files.get(path) for path in python_files} == python_files + assert all(test_case.native_marker in path for path in native_files), sorted(native_files) + assert set(test_case.expected_python_files) <= set(python_files) + assert set(test_case.expected_native_files) <= native_files + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py index 51a4626013..3b871fc8f4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py @@ -103,3 +103,33 @@ class UntypedCursorParsingTestCase: argv: tuple[str, ...] expected_start: str | None expected_end: str | None + + +@dataclass(frozen=True) +class CompilerEngineOptionTestCase: + """One compiler engine selection through the hidden flag and the environment.""" + + description: str + argv: tuple[str, ...] + environment_value: str + expected_engine: str | None + + +@dataclass(frozen=True) +class RejectedCompilerEngineTestCase: + """An unsupported engine selection and the usage error that names accepted values.""" + + description: str + argv: tuple[str, ...] + environment_value: str + expected_exit_code: int + expected_error: str + + +@dataclass(frozen=True) +class HiddenOptionTestCase: + """A help page and an option that must not appear on it.""" + + description: str + option: str + expected_listed: bool diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py new file mode 100644 index 0000000000..6e8c4c3bbe --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py @@ -0,0 +1,113 @@ +"""The hidden --compiler-engine flag overrides SQLBUILD_COMPILER_ENGINE and both are validated.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.cli.commands._helpers.entry.parser import build_cli_parser +from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation +from sqlbuild.cli.entry.models import ParsedCliInvocation +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( + CompilerEngineOptionTestCase, + HiddenOptionTestCase, + RejectedCompilerEngineTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineOptionTestCase( + description="default_leaves_engine_to_environment", + argv=("compile",), + environment_value="", + expected_engine=None, + ), + CompilerEngineOptionTestCase( + description="flag_before_subcommand", + argv=("--compiler-engine", "native", "compile"), + environment_value="", + expected_engine="native", + ), + CompilerEngineOptionTestCase( + description="flag_after_subcommand", + argv=("plan", "--compiler-engine", "python"), + environment_value="", + expected_engine="python", + ), + CompilerEngineOptionTestCase( + description="flag_overrides_invalid_environment", + argv=("compile", "--compiler-engine", "python"), + environment_value="rust", + expected_engine="python", + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_selection_when_parsing_then_flag_value_is_recorded( + test_case: CompilerEngineOptionTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.environment_value) + + parsed: ParsedCliInvocation = parse_cli_invocation( + argv=test_case.argv, parser=build_cli_parser() + ) + + assert parsed.args is not None + assert parsed.args.compiler_engine == test_case.expected_engine + + +@pytest.mark.parametrize( + "test_case", + [ + RejectedCompilerEngineTestCase( + description="unknown_flag_value", + argv=("compile", "--compiler-engine", "rust"), + environment_value="", + expected_exit_code=2, + expected_error="invalid choice: 'rust' (choose from 'python', 'native')", + ), + RejectedCompilerEngineTestCase( + description="unknown_environment_value", + argv=("compile",), + environment_value="rust", + expected_exit_code=2, + expected_error="SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'rust')", + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_engine_when_parsing_then_usage_error_names_accepted_values( + test_case: RejectedCompilerEngineTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.environment_value) + + parsed: ParsedCliInvocation = parse_cli_invocation( + argv=test_case.argv, parser=build_cli_parser() + ) + + assert parsed.args is None + assert parsed.exit_code == test_case.expected_exit_code + assert test_case.expected_error in capsys.readouterr().err + + +@pytest.mark.parametrize( + "test_case", + [ + HiddenOptionTestCase( + description="root_help", option="--compiler-engine", expected_listed=False + ) + ], + ids=lambda case: case.description, +) +def test_given_root_help_when_rendering_then_compiler_engine_flag_stays_hidden( + test_case: HiddenOptionTestCase, +) -> None: + assert (test_case.option in build_cli_parser().format_help()) is test_case.expected_listed + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index ea3686abfc..90ea98eb84 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -91,3 +91,22 @@ class ChangedPathsTestCase: change: Callable[[Path], object] expected_paths: tuple[str, ...] store: Callable[[Path], dict[str, StoredProjectFile]] = digested_project_files + + +@dataclass(frozen=True) +class EngineReuseDigestTestCase: + """Two engine environments for one invocation and whether their reuse keys may match.""" + + description: str + first_environment: dict[str, str] + second_environment: dict[str, str] + expected_same_digest: bool + + +@dataclass(frozen=True) +class EngineReuseStoreTestCase: + """One engine and the store directory that holds its compile reuse entry.""" + + description: str + engine: str + expected_entry_directory: str diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index efacd4c1c0..dd03ca35e2 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -16,13 +16,20 @@ is_racy, snapshot_project_files, ) +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( + environment_digest, + invocation_digest, + tracked_environment_names, +) from sqlbuild.cli.compile_reuse.constants import RACY_WINDOW_NS from sqlbuild.cli.compile_reuse.models import ( + CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult, StoredProjectFile, ) from sqlbuild.cli.compile_reuse.types import FileStamp +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR _LATER_SNAPSHOT_NS: int = 10 * RACY_WINDOW_NS @@ -266,3 +273,42 @@ def described_settings_inputs( def _path_names(paths: tuple[str, ...]) -> tuple[str, ...]: return tuple(Path(path).name for path in paths) + + +def orders_reuse_request(project_dir: Path) -> CompileReuseRequest: + """Return a plain JSON compile reuse request for a project.""" + + return CompileReuseRequest( + project_dir=project_dir, + selected_target=None, + json_output=True, + no_color=True, + manifest=False, + dag_path=None, + defer_to=None, + no_cache=False, + no_sql_validation=False, + lineage_mode="fast", + select=(), + exclude=(), + cli_vars=None, + profiling=False, + debug=False, + ) + + +def engine_reuse_key( + *, environment: dict[str, str], project_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[str, str]: + """Return the invocation and environment digests computed under one engine environment.""" + + monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) + monkeypatch.delenv(STAGE_CAPTURE_DIR_ENV_VAR, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + return ( + invocation_digest( + request=orders_reuse_request(project_dir), project_dir=str(project_dir), use_color=False + ), + environment_digest(names=tracked_environment_names(template_names=())), + ) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py new file mode 100644 index 0000000000..a8cd940fe7 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py @@ -0,0 +1,105 @@ +"""Compile reuse never replays a compile recorded by the other compiler engine.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_reuse._helpers.attempt import attempt_reuse +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + EngineReuseDigestTestCase, + EngineReuseStoreTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import ( + engine_reuse_key, + orders_reuse_request, + write_file, +) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineReuseDigestTestCase( + description="python_then_native", + first_environment={COMPILER_ENGINE_ENV_VAR: "python"}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + expected_same_digest=False, + ), + EngineReuseDigestTestCase( + description="unset_then_native", + first_environment={}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + expected_same_digest=False, + ), + EngineReuseDigestTestCase( + description="unset_then_explicit_python", + first_environment={}, + second_environment={COMPILER_ENGINE_ENV_VAR: "python"}, + expected_same_digest=True, + ), + EngineReuseDigestTestCase( + description="native_twice", + first_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + expected_same_digest=True, + ), + EngineReuseDigestTestCase( + description="stage_capture_directory_is_not_a_compile_input", + first_environment={}, + second_environment={STAGE_CAPTURE_DIR_ENV_VAR: "/tmp/captures"}, + expected_same_digest=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_same_invocation_when_engine_environment_changes_then_reuse_key_follows_engine( + test_case: EngineReuseDigestTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + first: tuple[str, str] = engine_reuse_key( + environment=test_case.first_environment, project_dir=tmp_path, monkeypatch=monkeypatch + ) + second: tuple[str, str] = engine_reuse_key( + environment=test_case.second_environment, project_dir=tmp_path, monkeypatch=monkeypatch + ) + + assert (first == second) is test_case.expected_same_digest + + +@pytest.mark.parametrize( + "test_case", + [ + EngineReuseStoreTestCase( + description="python", + engine="python", + expected_entry_directory="target/cache/compiler/project-reuse-v1", + ), + EngineReuseStoreTestCase( + description="native", + engine="native", + expected_entry_directory="target/cache/compiler-native-v1/project-reuse-v1", + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_attempting_reuse_then_entry_lives_in_that_engine_store( + test_case: EngineReuseStoreTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_file(tmp_path / "sqlbuild_project.toml", 'name = "orders"\nadapter = "duckdb"\n') + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine) + monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, "0") + + attempt: CompileReuseAttempt = attempt_reuse(request=orders_reuse_request(tmp_path)) + + assert attempt.entry_path is not None + assert attempt.entry_path.parent.relative_to(tmp_path).as_posix() == ( + test_case.expected_entry_directory + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/__init__.py b/tests/unit/src/sqlbuild/compiler/frontier/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py new file mode 100644 index 0000000000..7049921840 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -0,0 +1,98 @@ +"""Test case types for compiler engine selection, store namespaces, and stage captures.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage + + +@dataclass(frozen=True) +class EngineResolutionTestCase: + """One SQLBUILD_COMPILER_ENGINE value and the engine it selects.""" + + description: str + raw_value: str + expected_engine: CompilerEngine + + +@dataclass(frozen=True) +class EngineErrorTestCase: + """One unsupported engine value and the exact error naming accepted values.""" + + description: str + raw_value: str + expected_message: str + + +@dataclass(frozen=True) +class EngineOverrideTestCase: + """An engine variable before an override, and what the override shows and restores.""" + + description: str + previous: str + override: CompilerEngine + expected_inside: CompilerEngine + expected_after: str + + +@dataclass(frozen=True) +class EngineCacheNameTestCase: + """One engine and the store directory name it owns.""" + + description: str + engine: CompilerEngine + base: str + expected_name: str + + +@dataclass(frozen=True) +class EngineStorePathsTestCase: + """One engine and the project-relative compiler and Rules store paths it uses.""" + + description: str + engine: CompilerEngine + expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class StageCaptureTestCase: + """One value whose canonical capture must match an exact JSON-compatible shape.""" + + description: str + value: Callable[[], object] + expected_capture: object + + +@dataclass(frozen=True) +class StageCaptureOrderTestCase: + """Two values that differ only in ordering and whether their captures must be identical.""" + + description: str + first: Callable[[], object] + second: Callable[[], object] + expected_identical: bool + + +@dataclass(frozen=True) +class UnorderedAttributeTestCase: + """Two fill orders for a memo map and an ordered map, and whether captures match.""" + + description: str + first_memo_keys: tuple[str, ...] + second_memo_keys: tuple[str, ...] + first_ordered_keys: tuple[str, ...] + second_ordered_keys: tuple[str, ...] + expected_identical: bool + + +@dataclass(frozen=True) +class FrontierCaptureTestCase: + """Frontier stages compiled in order on one engine and the capture files they write.""" + + description: str + engine: CompilerEngine + stages: tuple[CompilerStage, ...] + expected_results: tuple[object, ...] + expected_files: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py new file mode 100644 index 0000000000..5259371578 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -0,0 +1,87 @@ +"""Shared values and helpers for compiler frontier tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path, PurePosixPath + +from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path + +HELPERS_MODULE: str = "tests.unit.src.sqlbuild.compiler.frontier.helpers" + + +class OrderStatus(StrEnum): + """A neutral enum captured by name.""" + + PLACED = "placed" + + +@dataclass(frozen=True) +class OrderLine: + """A neutral dataclass captured field by field.""" + + order_id: int + status: OrderStatus + path: PurePosixPath + + +def order_total(amount_cents: int) -> int: + """A neutral callable captured by its qualified name.""" + + return amount_cents + + +class LinkedCustomer: + """A plain object that can point back at itself.""" + + def __init__(self) -> None: + self.name: str = "customer" + self.peer: LinkedCustomer | None = None + + +def linked_customer() -> LinkedCustomer: + """Return a customer whose peer is itself.""" + + customer: LinkedCustomer = LinkedCustomer() + customer.peer = customer + return customer + + +class MemoCatalog: + """A plain object with one memo map and one ordered map.""" + + def __init__(self, *, memo: dict[str, int], ordered: dict[str, int]) -> None: + self.memo: dict[str, int] = memo + self.ordered: dict[str, int] = ordered + + +def memo_catalog(*, memo_keys: tuple[str, ...], ordered_keys: tuple[str, ...]) -> MemoCatalog: + """Return a catalog whose maps were filled in the given key orders.""" + + return MemoCatalog( + memo={key: len(key) for key in memo_keys}, + ordered={key: len(key) for key in ordered_keys}, + ) + + +def order_line() -> OrderLine: + """Return one neutral order line.""" + + return OrderLine(order_id=7, status=OrderStatus.PLACED, path=PurePosixPath("models/a.sql")) + + +def store_paths(project_dir: Path) -> tuple[str, ...]: + """Return every engine-owned store path relative to the project directory.""" + + paths: tuple[Path, ...] = ( + compiler_cache_directory(project_dir), + compile_cache_root(project_dir=project_dir, target_config=None, no_cache=False) + or project_dir, + compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME, + rules_bulk_cache_path(project_dir=project_dir, file_name="sql.json"), + ) + return tuple(path.relative_to(project_dir).as_posix() for path in paths) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py b/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py new file mode 100644 index 0000000000..cf5b751b56 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py @@ -0,0 +1,86 @@ +"""On-disk compiler stores are namespaced by engine; Python keeps its existing paths.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( + EngineCacheNameTestCase, + EngineStorePathsTestCase, +) +from tests.unit.src.sqlbuild.compiler.frontier.helpers import store_paths + + +@pytest.mark.parametrize( + "test_case", + [ + EngineCacheNameTestCase( + description="python_keeps_name", + engine=CompilerEngine.PYTHON, + base="compiler", + expected_name="compiler", + ), + EngineCacheNameTestCase( + description="native_gets_versioned_suffix", + engine=CompilerEngine.NATIVE, + base="compiler", + expected_name="compiler-native-v1", + ), + EngineCacheNameTestCase( + description="native_rules_cache", + engine=CompilerEngine.NATIVE, + base="rules-cache", + expected_name="rules-cache-native-v1", + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_naming_store_then_only_native_is_suffixed( + test_case: EngineCacheNameTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + + assert engine_cache_name(test_case.base) == test_case.expected_name + + +@pytest.mark.parametrize( + "test_case", + [ + EngineStorePathsTestCase( + description="python_paths_unchanged", + engine=CompilerEngine.PYTHON, + expected_paths=( + "target/cache/compiler", + "target/cache/compiler", + "target/cache/compiler/declaration-scopes-v2", + "target/rules-cache/bulk/sql.json", + ), + ), + EngineStorePathsTestCase( + description="native_paths_separate", + engine=CompilerEngine.NATIVE, + expected_paths=( + "target/cache/compiler-native-v1", + "target/cache/compiler-native-v1", + "target/cache/compiler-native-v1/declaration-scopes-v2", + "target/rules-cache-native-v1/bulk/sql.json", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_resolving_stores_then_each_engine_owns_its_paths( + test_case: EngineStorePathsTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + + assert store_paths(tmp_path) == test_case.expected_paths + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py new file mode 100644 index 0000000000..a4c9d19334 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py @@ -0,0 +1,155 @@ +"""The compiler engine switch reads SQLBUILD_COMPILER_ENGINE and rejects unknown values.""" + +from __future__ import annotations + +import os + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.exceptions import CompilerEngineError +from sqlbuild.compiler.frontier.main.compiler_engine_override import compiler_engine_override +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( + EngineErrorTestCase, + EngineOverrideTestCase, + EngineResolutionTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineResolutionTestCase( + description="empty", raw_value="", expected_engine=CompilerEngine.PYTHON + ), + EngineResolutionTestCase( + description="python", raw_value="python", expected_engine=CompilerEngine.PYTHON + ), + EngineResolutionTestCase( + description="native", raw_value="native", expected_engine=CompilerEngine.NATIVE + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_variable_when_resolving_then_returns_selected_engine( + test_case: EngineResolutionTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.raw_value) + + assert resolve_compiler_engine() is test_case.expected_engine + + +@pytest.mark.parametrize( + "test_case", + [ + EngineResolutionTestCase( + description="unset", raw_value="", expected_engine=CompilerEngine.PYTHON + ) + ], + ids=lambda case: case.description, +) +def test_given_no_engine_variable_when_resolving_then_python_is_the_default( + test_case: EngineResolutionTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) + + assert resolve_compiler_engine() is test_case.expected_engine + + +@pytest.mark.parametrize( + "test_case", + [ + EngineErrorTestCase( + description="unknown", + raw_value="rust", + expected_message="SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'rust')", + ), + EngineErrorTestCase( + description="wrong_case", + raw_value="Native", + expected_message=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'Native')" + ), + ), + EngineErrorTestCase( + description="padded", + raw_value=" python", + expected_message=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native (got ' python')" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_engine_when_resolving_then_error_names_accepted_and_current_values( + test_case: EngineErrorTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.raw_value) + + with pytest.raises(CompilerEngineError) as error: + _ = resolve_compiler_engine() + + assert str(error.value) == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + EngineOverrideTestCase( + description="python_restored", + previous="python", + override=CompilerEngine.NATIVE, + expected_inside=CompilerEngine.NATIVE, + expected_after="python", + ), + EngineOverrideTestCase( + description="empty_restored", + previous="", + override=CompilerEngine.NATIVE, + expected_inside=CompilerEngine.NATIVE, + expected_after="", + ), + ], + ids=lambda case: case.description, +) +def test_given_override_when_leaving_then_engine_variable_is_restored( + test_case: EngineOverrideTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.previous) + + with compiler_engine_override(test_case.override): + inside: CompilerEngine = resolve_compiler_engine() + + assert inside is test_case.expected_inside + assert os.environ[COMPILER_ENGINE_ENV_VAR] == test_case.expected_after + + +@pytest.mark.parametrize( + "test_case", + [ + EngineOverrideTestCase( + description="unset_removed", + previous="", + override=CompilerEngine.NATIVE, + expected_inside=CompilerEngine.NATIVE, + expected_after="", + ) + ], + ids=lambda case: case.description, +) +def test_given_unset_variable_when_override_ends_then_variable_is_removed( + test_case: EngineOverrideTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) + + with compiler_engine_override(test_case.override): + inside: CompilerEngine = resolve_compiler_engine() + + assert inside is test_case.expected_inside + assert os.environ.get(COMPILER_ENGINE_ENV_VAR, "") == test_case.expected_after + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py new file mode 100644 index 0000000000..6b2c2384f9 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -0,0 +1,205 @@ +"""Stage captures render frontier objects as deterministic, comparable JSON.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, + STAGE_CAPTURE_UNORDERED_ATTRIBUTES, +) +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage +from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( + FrontierCaptureTestCase, + StageCaptureOrderTestCase, + StageCaptureTestCase, + UnorderedAttributeTestCase, +) +from tests.unit.src.sqlbuild.compiler.frontier.helpers import ( + HELPERS_MODULE, + linked_customer, + memo_catalog, + order_line, + order_total, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StageCaptureTestCase( + description="dataclass_enum_and_path", + value=order_line, + expected_capture={ + "__type__": f"{HELPERS_MODULE}:OrderLine", + "order_id": 7, + "status": {"__enum__": f"{HELPERS_MODULE}:OrderStatus.PLACED"}, + "path": {"__path__": "models/a.sql"}, + }, + ), + StageCaptureTestCase( + description="callable_becomes_qualified_name", + value=lambda: {"total": order_total}, + expected_capture={"total": {"__callable__": f"{HELPERS_MODULE}:order_total"}}, + ), + StageCaptureTestCase( + description="sets_are_sorted", + value=lambda: frozenset({"orders", "customers", "products"}), + expected_capture={"__set__": ["customers", "orders", "products"]}, + ), + StageCaptureTestCase( + description="non_string_keys_are_pairs_in_insertion_order", + value=lambda: {2: "second", 1: "first"}, + expected_capture={"__mapping__": [[2, "second"], [1, "first"]]}, + ), + StageCaptureTestCase( + description="cycles_are_cut", + value=linked_customer, + expected_capture={ + "__type__": f"{HELPERS_MODULE}:LinkedCustomer", + "name": "customer", + "peer": {"__cycle__": f"{HELPERS_MODULE}:LinkedCustomer"}, + }, + ), + StageCaptureTestCase( + description="non_finite_float_and_bytes", + value=lambda: (float("inf"), b"orders"), + expected_capture=[ + {"__float__": "inf"}, + { + "__bytes__": 6, + "sha256": "1c168adb00d208e42f93314529f1fa9c0427eb63233ceda95a5db52b7012a719", + }, + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_frontier_value_when_rendering_capture_then_json_is_canonical( + test_case: StageCaptureTestCase, +) -> None: + assert json.loads(render_stage_capture(test_case.value())) == test_case.expected_capture + + +@pytest.mark.parametrize( + "test_case", + [ + StageCaptureOrderTestCase( + description="mapping_insertion_order_is_visible", + first=lambda: {"orders": 1, "customers": 2}, + second=lambda: {"customers": 2, "orders": 1}, + expected_identical=False, + ), + StageCaptureOrderTestCase( + description="non_string_key_insertion_order_is_visible", + first=lambda: {1: "orders", 2: "customers"}, + second=lambda: {2: "customers", 1: "orders"}, + expected_identical=False, + ), + StageCaptureOrderTestCase( + description="tuple_order_is_visible", + first=lambda: ("orders", "customers"), + second=lambda: ("customers", "orders"), + expected_identical=False, + ), + StageCaptureOrderTestCase( + description="set_order_is_not_visible", + first=lambda: frozenset(("orders", "customers", "products")), + second=lambda: frozenset(("products", "customers", "orders")), + expected_identical=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_reordered_value_when_rendering_capture_then_only_unordered_collections_match( + test_case: StageCaptureOrderTestCase, +) -> None: + assert ( + render_stage_capture(test_case.first()) == render_stage_capture(test_case.second()) + ) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + UnorderedAttributeTestCase( + description="memo_fill_order_is_not_visible", + first_memo_keys=("orders", "customers"), + second_memo_keys=("customers", "orders"), + first_ordered_keys=("orders", "customers"), + second_ordered_keys=("orders", "customers"), + expected_identical=True, + ), + UnorderedAttributeTestCase( + description="other_attribute_order_stays_visible", + first_memo_keys=("orders", "customers"), + second_memo_keys=("orders", "customers"), + first_ordered_keys=("orders", "customers"), + second_ordered_keys=("customers", "orders"), + expected_identical=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_declared_memo_attribute_when_rendering_capture_then_only_it_is_sorted( + test_case: UnorderedAttributeTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + STAGE_CAPTURE_UNORDERED_ATTRIBUTES, f"{HELPERS_MODULE}:MemoCatalog", frozenset({"memo"}) + ) + first: str = render_stage_capture( + memo_catalog(memo_keys=test_case.first_memo_keys, ordered_keys=test_case.first_ordered_keys) + ) + second: str = render_stage_capture( + memo_catalog( + memo_keys=test_case.second_memo_keys, ordered_keys=test_case.second_ordered_keys + ) + ) + + assert (first == second) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + FrontierCaptureTestCase( + description=engine.value, + engine=engine, + stages=(CompilerStage.DISCOVERED_PROJECT_INPUTS, CompilerStage.COMPILED_PROJECT), + expected_results=( + "discovered_project_inputs-result", + "compiled_project-result", + ), + expected_files=("discovered_project_inputs.json", "compiled_project.json"), + ) + for engine in CompilerEngine + ], + ids=lambda case: case.description, +) +def test_given_capture_directory_when_compiling_frontier_then_each_stage_is_captured_in_order( + test_case: FrontierCaptureTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + monkeypatch.setenv(STAGE_CAPTURE_DIR_ENV_VAR, str(tmp_path)) + + results: tuple[object, ...] = tuple( + compile_frontier(until=stage, python_stage=lambda stage=stage: f"{stage.value}-result") + for stage in test_case.stages + ) + + captures: list[Path] = sorted(tmp_path.iterdir()) + assert results == test_case.expected_results + assert tuple(path.name.split("-", 1)[1] for path in captures) == test_case.expected_files + assert tuple(json.loads(path.read_text(encoding="utf-8")) for path in captures) == ( + test_case.expected_results + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 819d317d9d..6738ebdc7d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.rule_engine._helpers.host import custom_host, custom_host_pool +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from sqlbuild.rule_engine.models import CustomRulesOutcome @@ -75,8 +76,10 @@ def test_given_partitioned_hosts_when_evaluating_then_findings_and_cache_match_o detects_duplicates=test_case.detects_duplicates, ) project: CompiledProject = orders_project(model_count=test_case.model_count) - cache_path: Path = tmp_path / CUSTOM_RULES_CACHE_FILE - cache_glob: str = CUSTOM_RULES_CACHE_FILE + cache_path: Path = rules_bulk_cache_path( + project_dir=tmp_path, file_name=CUSTOM_RULES_CACHE_FILE + ) + cache_glob: str = cache_path.relative_to(tmp_path).as_posix() cache_enabled: bool = test_case.cache_enabled single_launches: list[str] = use_hosts(monkeypatch=monkeypatch, hosts=1) From 0b077614345c0425b6f23ad3d3ba31529c472517 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:31:28 +0100 Subject: [PATCH 2/6] test(compiler): add compiler engine differential harness and corpora Compare exit codes, normalised stdout and stderr, the compiled tree, manifest, DAG and per-model query fingerprints between two engines over the e2e fixtures, website examples, seeded generated projects, a failure corpus and dense projects, with optional per-stage captures. --- scripts/compiler_differential/__init__.py | 1 + .../_helpers/__init__.py | 0 .../compiler_differential/_helpers/compare.py | 118 ++++ .../_helpers/comparison.py | 264 ++++++++ .../compiler_differential/_helpers/corpus.py | 141 ++++ .../_helpers/execution.py | 176 +++++ .../_helpers/failure_cases.py | 504 +++++++++++++++ .../_helpers/normalize.py | 52 ++ .../compiler_differential/_helpers/options.py | 42 ++ .../_helpers/project_comparison.py | 135 ++++ .../compiler_differential/_helpers/report.py | 39 ++ scripts/compiler_differential/_helpers/run.py | 65 ++ .../classes/dense_project.py | 37 ++ .../classes/project_builder.py | 604 ++++++++++++++++++ scripts/compiler_differential/constants.py | 186 ++++++ scripts/compiler_differential/exceptions.py | 5 + .../compiler_differential/main/__init__.py | 1 + .../main/differential.py | 91 +++ .../main/failure_cases.py | 10 + .../main/generate_project.py | 10 + scripts/compiler_differential/models.py | 159 +++++ scripts/run_compiler_differential.py | 15 + .../scripts/compiler_differential/__init__.py | 0 .../compiler_differential/_test_types.py | 27 + .../scripts/compiler_differential/helpers.py | 87 +++ .../test_compiler_differential.py | 143 +++++ .../scripts/compiler_differential/__init__.py | 0 .../_helpers/__init__.py | 0 .../_helpers/_test_types.py | 101 +++ .../_helpers/test_compare.py | 128 ++++ .../_helpers/test_corpus.py | 69 ++ .../_helpers/test_normalize.py | 117 ++++ .../_helpers/test_options.py | 107 ++++ .../compiler_differential/main/__init__.py | 0 .../compiler_differential/main/_test_types.py | 47 ++ .../compiler_differential/main/helpers.py | 27 + .../main/test_failure_cases.py | 63 ++ .../main/test_generate_project.py | 90 +++ 38 files changed, 3661 insertions(+) create mode 100644 scripts/compiler_differential/__init__.py create mode 100644 scripts/compiler_differential/_helpers/__init__.py create mode 100644 scripts/compiler_differential/_helpers/compare.py create mode 100644 scripts/compiler_differential/_helpers/comparison.py create mode 100644 scripts/compiler_differential/_helpers/corpus.py create mode 100644 scripts/compiler_differential/_helpers/execution.py create mode 100644 scripts/compiler_differential/_helpers/failure_cases.py create mode 100644 scripts/compiler_differential/_helpers/normalize.py create mode 100644 scripts/compiler_differential/_helpers/options.py create mode 100644 scripts/compiler_differential/_helpers/project_comparison.py create mode 100644 scripts/compiler_differential/_helpers/report.py create mode 100644 scripts/compiler_differential/_helpers/run.py create mode 100644 scripts/compiler_differential/classes/dense_project.py create mode 100644 scripts/compiler_differential/classes/project_builder.py create mode 100644 scripts/compiler_differential/constants.py create mode 100644 scripts/compiler_differential/exceptions.py create mode 100644 scripts/compiler_differential/main/__init__.py create mode 100644 scripts/compiler_differential/main/differential.py create mode 100644 scripts/compiler_differential/main/failure_cases.py create mode 100644 scripts/compiler_differential/main/generate_project.py create mode 100644 scripts/compiler_differential/models.py create mode 100644 scripts/run_compiler_differential.py create mode 100644 tests/e2e/scripts/compiler_differential/__init__.py create mode 100644 tests/e2e/scripts/compiler_differential/_test_types.py create mode 100644 tests/e2e/scripts/compiler_differential/helpers.py create mode 100644 tests/e2e/scripts/compiler_differential/test_compiler_differential.py create mode 100644 tests/unit/scripts/compiler_differential/__init__.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/__init__.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/_test_types.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/test_compare.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/test_corpus.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/test_normalize.py create mode 100644 tests/unit/scripts/compiler_differential/_helpers/test_options.py create mode 100644 tests/unit/scripts/compiler_differential/main/__init__.py create mode 100644 tests/unit/scripts/compiler_differential/main/_test_types.py create mode 100644 tests/unit/scripts/compiler_differential/main/helpers.py create mode 100644 tests/unit/scripts/compiler_differential/main/test_failure_cases.py create mode 100644 tests/unit/scripts/compiler_differential/main/test_generate_project.py diff --git a/scripts/compiler_differential/__init__.py b/scripts/compiler_differential/__init__.py new file mode 100644 index 0000000000..63dca5118f --- /dev/null +++ b/scripts/compiler_differential/__init__.py @@ -0,0 +1 @@ +"""Differential harness comparing the Python and native compiler engines.""" diff --git a/scripts/compiler_differential/_helpers/__init__.py b/scripts/compiler_differential/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/compiler_differential/_helpers/compare.py b/scripts/compiler_differential/_helpers/compare.py new file mode 100644 index 0000000000..3ee796c4c3 --- /dev/null +++ b/scripts/compiler_differential/_helpers/compare.py @@ -0,0 +1,118 @@ +"""Locate the first difference between two JSON documents, texts, or file trees.""" + +from __future__ import annotations + +import json +from typing import cast + +from scripts.compiler_differential.constants import MISSING_VALUE, VALUE_PREVIEW_CHARACTERS +from scripts.compiler_differential.models import Divergence + + +def first_json_difference(*, left: object, right: object, pointer: str = "") -> Divergence | None: + """Return the first JSON pointer whose value, type, or object key order differs.""" + + if type(left) is not type(right): + return _divergence(location=pointer or "/", left=left, right=right) + left_object: dict[str, object] | None = as_json_object(left) + right_object: dict[str, object] | None = as_json_object(right) + if left_object is not None and right_object is not None: + return _first_object_difference(left=left_object, right=right_object, pointer=pointer) + if isinstance(left, list) and isinstance(right, list): + for index, (left_item, right_item) in enumerate(zip(left, right, strict=False)): + found: Divergence | None = first_json_difference( + left=left_item, right=right_item, pointer=f"{pointer}/{index}" + ) + if found is not None: + return found + if len(left) != len(right): + index: int = min(len(left), len(right)) + return _divergence( + location=f"{pointer}/{index}", + left=left[index] if index < len(left) else MISSING_VALUE, + right=right[index] if index < len(right) else MISSING_VALUE, + ) + return None + if left != right: + return _divergence(location=pointer or "/", left=left, right=right) + return None + + +def first_text_difference(*, left: str, right: str) -> Divergence | None: + """Return the first differing line, numbered from one.""" + + if left == right: + return None + left_lines: list[str] = left.split("\n") + right_lines: list[str] = right.split("\n") + for index in range(max(len(left_lines), len(right_lines))): + left_line: str = left_lines[index] if index < len(left_lines) else MISSING_VALUE + right_line: str = right_lines[index] if index < len(right_lines) else MISSING_VALUE + if left_line != right_line: + return Divergence( + location=f"line {index + 1}", left=_preview(left_line), right=_preview(right_line) + ) + return Divergence(location="end of file", left="", right="") + + +def first_document_difference(*, left: str, right: str) -> Divergence | None: + """Compare two documents as JSON when both parse, otherwise line by line.""" + + try: + left_payload: object = json.loads(left) + right_payload: object = json.loads(right) + except json.JSONDecodeError: + return first_text_difference(left=left, right=right) + return first_json_difference(left=left_payload, right=right_payload) + + +def as_json_object(value: object) -> dict[str, object] | None: + """Return a decoded JSON object with string keys, or None for any other value.""" + + return cast(dict[str, object], value) if isinstance(value, dict) else None + + +def json_pointer_token(key: str) -> str: + """Escape one object key as an RFC 6901 reference token.""" + + return key.replace("~", "~0").replace("/", "~1") + + +def _first_object_difference( + *, left: dict[str, object], right: dict[str, object], pointer: str +) -> Divergence | None: + for key in (*left, *(key for key in right if key not in left)): + location: str = f"{pointer}/{json_pointer_token(key)}" + if key not in left or key not in right: + return _divergence( + location=location, + left=left.get(key, MISSING_VALUE), + right=right.get(key, MISSING_VALUE), + ) + found: Divergence | None = first_json_difference( + left=left[key], right=right[key], pointer=location + ) + if found is not None: + return found + if list(left) != list(right): + return Divergence( + location=f"{pointer or '/'} (key order)", + left=_preview(list(left)), + right=_preview(list(right)), + ) + return None + + +def _divergence(*, location: str, left: object, right: object) -> Divergence: + return Divergence(location=location, left=_preview(left), right=_preview(right)) + + +def _preview(value: object) -> str: + rendered: str = ( + value + if isinstance(value, str) and value == MISSING_VALUE + else json.dumps(value, ensure_ascii=False) + ) + if len(rendered) <= VALUE_PREVIEW_CHARACTERS: + return rendered + return rendered[: VALUE_PREVIEW_CHARACTERS - 3] + "..." diff --git a/scripts/compiler_differential/_helpers/comparison.py b/scripts/compiler_differential/_helpers/comparison.py new file mode 100644 index 0000000000..cfd54bfbae --- /dev/null +++ b/scripts/compiler_differential/_helpers/comparison.py @@ -0,0 +1,264 @@ +"""Compare everything two engines produced for one project and report first differences.""" + +from __future__ import annotations + +import json +from collections.abc import Callable + +from scripts.compiler_differential._helpers.compare import ( + as_json_object, + first_document_difference, + first_json_difference, + first_text_difference, +) +from scripts.compiler_differential._helpers.normalize import ( + normalize_artifact_text, + normalize_stderr, + strip_manifest_metadata, + strip_report_fields, +) +from scripts.compiler_differential.constants import ( + ERROR_SEVERITY, + MISSING_VALUE, + MODEL_RESOURCE_TYPE, + PLAN_LABEL, +) +from scripts.compiler_differential.models import ( + CommandOutcome, + Difference, + Divergence, + EngineRun, +) + + +def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> list[Difference]: + """Return the first difference in every artifact that differs between two engine runs.""" + + found: list[tuple[str, Divergence | None]] = [] + for left_outcome, right_outcome in zip(left.outcomes, right.outcomes, strict=True): + found.extend(_command_differences(left=left_outcome, right=right_outcome)) + found.extend(_tree_differences(left=left.compiled, right=right.compiled)) + found.append( + ( + "target/manifest.json", + _optional_json_difference( + left=left.manifest, right=right.manifest, strip=strip_manifest_metadata + ), + ) + ) + found.append( + ( + "target/sqlbuild_dag.json", + _optional_json_difference(left=left.dag, right=right.dag, strip=None), + ) + ) + found.extend( + (f"query fingerprint of model {model}", divergence) + for model, divergence in _fingerprint_differences(left=left, right=right) + ) + found.extend(_capture_differences(left=left.captures, right=right.captures)) + return [ + Difference( + project=project, + artifact=artifact, + location=divergence.location, + left=divergence.left, + right=divergence.right, + ) + for artifact, divergence in found + if divergence is not None + ] + + +def diagnostic_codes(*, outcome: CommandOutcome, errors_only: bool) -> tuple[str, ...]: + """Return the diagnostic codes of a JSON compile report, in report order.""" + + payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} + diagnostics: object = payload.get("diagnostics") + found: list[str] = [] + for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else (): + diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic) + if diagnostic is not None and ( + not errors_only or diagnostic.get("severity") == ERROR_SEVERITY + ): + found.append(str(diagnostic.get("code"))) + return tuple(found) + + +def _command_differences( + *, left: CommandOutcome, right: CommandOutcome +) -> list[tuple[str, Divergence | None]]: + label: str = f"`{left.label}`" + exit_codes: Divergence | None = ( + None + if (left.exit_code, left.timed_out) == (right.exit_code, right.timed_out) + else Divergence( + location="exit code", + left=_exit_label(left), + right=_exit_label(right), + ) + ) + return [ + (f"{label} exit code", exit_codes), + (f"{label} stdout", _stdout_difference(left=left.stdout, right=right.stdout)), + ( + f"{label} stderr", + first_text_difference( + left=normalize_stderr(left.stderr), right=normalize_stderr(right.stderr) + ), + ), + ] + + +def _exit_label(outcome: CommandOutcome) -> str: + return "timed out" if outcome.timed_out else str(outcome.exit_code) + + +def _stdout_difference(*, left: str, right: str) -> Divergence | None: + normalized_left: str = normalize_artifact_text(left) + normalized_right: str = normalize_artifact_text(right) + try: + left_payload: object = json.loads(normalized_left) + right_payload: object = json.loads(normalized_right) + except json.JSONDecodeError: + return first_text_difference(left=normalized_left, right=normalized_right) + return first_json_difference( + left=strip_report_fields(left_payload), right=strip_report_fields(right_payload) + ) + + +def _tree_differences( + *, left: dict[str, bytes], right: dict[str, bytes] +) -> list[tuple[str, Divergence | None]]: + differences: list[tuple[str, Divergence | None]] = [] + for path in sorted(set(left) | set(right)): + artifact: str = f"target/compiled/{path}" + if path not in left or path not in right: + differences.append( + ( + artifact, + _presence_divergence(left_present=path in left, right_present=path in right), + ) + ) + continue + differences.append( + ( + artifact, + first_text_difference( + left=normalize_artifact_text(left[path].decode("utf-8", "surrogateescape")), + right=normalize_artifact_text(right[path].decode("utf-8", "surrogateescape")), + ), + ) + ) + return differences + + +def _optional_json_difference( + *, left: str | None, right: str | None, strip: Callable[[object], object] | None +) -> Divergence | None: + if left is None or right is None: + if left is None and right is None: + return None + return _presence_divergence(left_present=left is not None, right_present=right is not None) + try: + left_payload: object = json.loads(normalize_artifact_text(left)) + right_payload: object = json.loads(normalize_artifact_text(right)) + except json.JSONDecodeError: + return first_text_difference(left=left, right=right) + if strip is not None: + left_payload = strip(left_payload) + right_payload = strip(right_payload) + return first_json_difference(left=left_payload, right=right_payload) + + +def _fingerprint_differences( + *, left: EngineRun, right: EngineRun +) -> list[tuple[str, Divergence | None]]: + left_fingerprints: dict[str, str] = _model_fingerprints(left) + right_fingerprints: dict[str, str] = _model_fingerprints(right) + return [ + ( + model, + None + if left_fingerprints.get(model) == right_fingerprints.get(model) + else Divergence( + location="fingerprint", + left=left_fingerprints.get(model, MISSING_VALUE), + right=right_fingerprints.get(model, MISSING_VALUE), + ), + ) + for model in sorted(set(left_fingerprints) | set(right_fingerprints)) + ] + + +def _model_fingerprints(run: EngineRun) -> dict[str, str]: + fingerprints: dict[str, str] = {} + manifest: dict[str, object] = as_json_object(_loads(run.manifest)) or {} + nodes: dict[str, object] = as_json_object(manifest.get("nodes")) or {} + for raw_node in nodes.values(): + node: dict[str, object] = as_json_object(raw_node) or {} + checksum: dict[str, object] | None = as_json_object(node.get("checksum")) + if checksum is not None and node.get("resource_type") == MODEL_RESOURCE_TYPE: + fingerprints[f"{node.get('name')} (manifest query hash)"] = str( + checksum.get("checksum") + ) + for outcome in run.outcomes: + plan: dict[str, object] = ( + as_json_object(_loads(outcome.stdout)) or {} if outcome.label == PLAN_LABEL else {} + ) + models: object = plan.get("models") + for raw_model in models if isinstance(models, list) else (): + model: dict[str, object] | None = as_json_object(raw_model) + if model is not None: + fingerprints[f"{model.get('name')} (plan version hash)"] = str( + model.get("expected_version_hash") + ) + return fingerprints + + +def _loads(text: str | None) -> object: + if text is None: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +def _capture_differences( + *, left: dict[str, dict[str, str]], right: dict[str, dict[str, str]] +) -> list[tuple[str, Divergence | None]]: + differences: list[tuple[str, Divergence | None]] = [] + for command in sorted(set(left) | set(right)): + left_files: dict[str, str] = left.get(command, {}) + right_files: dict[str, str] = right.get(command, {}) + for name in sorted(set(left_files) | set(right_files)): + artifact: str = f"stage capture {command}/{name}" + if name not in left_files or name not in right_files: + differences.append( + ( + artifact, + _presence_divergence( + left_present=name in left_files, right_present=name in right_files + ), + ) + ) + continue + differences.append( + ( + artifact, + first_document_difference( + left=normalize_artifact_text(left_files[name]), + right=normalize_artifact_text(right_files[name]), + ), + ) + ) + return differences + + +def _presence_divergence(*, left_present: bool, right_present: bool) -> Divergence: + return Divergence( + location="file", + left="present" if left_present else MISSING_VALUE, + right="present" if right_present else MISSING_VALUE, + ) diff --git a/scripts/compiler_differential/_helpers/corpus.py b/scripts/compiler_differential/_helpers/corpus.py new file mode 100644 index 0000000000..531a0e554f --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus.py @@ -0,0 +1,141 @@ +"""Assemble the projects the differential harness compares.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from scripts.compiler_differential._helpers.failure_cases import all_failure_cases +from scripts.compiler_differential.classes.dense_project import DenseProject +from scripts.compiler_differential.classes.project_builder import ProjectBuilder +from scripts.compiler_differential.constants import ( + COLD_COMPILE, + CORPUS_DENSE, + CORPUS_EXAMPLES, + CORPUS_FAILURES, + CORPUS_FIXTURES, + CORPUS_SEEDS, + DUCKDB_ADAPTER, + EXAMPLE_ROOT, + EXPECT_SUCCESS, + FIXTURE_EXPECTED_OUTCOMES, + FIXTURE_PROJECT_SUBDIRECTORIES, + FIXTURE_ROOT, + PLAN, + PROJECT_CONFIG_FILE, + WARM_COMPILE, +) +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialCommand, + ExpectedOutcome, + GeneratedProject, +) + + +def build_corpus( + *, + repo_root: Path, + corpora: tuple[str, ...], + seeds: range, + dense_models: int, + projects: tuple[Path, ...], + project_expectation: ExpectedOutcome, +) -> list[CorpusProject]: + """Return the selected corpus entries in a stable order.""" + + entries: list[CorpusProject] = [ + _authored_project( + name=f"project/{path.name}", + source_dir=path, + subdirectory=None, + expected=project_expectation, + ) + for path in projects + ] + if CORPUS_FIXTURES in corpora: + entries.extend( + _authored_project( + name=f"fixture/{path.name}", + source_dir=path, + subdirectory=FIXTURE_PROJECT_SUBDIRECTORIES.get(path.name), + expected=FIXTURE_EXPECTED_OUTCOMES.get(path.name, EXPECT_SUCCESS), + ) + for path in _project_directories(repo_root / FIXTURE_ROOT) + ) + if CORPUS_EXAMPLES in corpora: + entries.extend( + _authored_project( + name=f"example/{path.name}", + source_dir=path, + subdirectory=None, + expected=EXPECT_SUCCESS, + ) + for path in _project_directories(repo_root / EXAMPLE_ROOT) + ) + if CORPUS_SEEDS in corpora: + entries.extend(_seed_project(seed) for seed in seeds) + if CORPUS_FAILURES in corpora: + entries.extend( + CorpusProject( + name=f"failure/{case.name}", + commands=(COLD_COMPILE,), + expected=ExpectedOutcome(error_code=case.expected_code), + writer=case.write, + ) + for case in all_failure_cases() + ) + if CORPUS_DENSE in corpora: + entries.extend( + CorpusProject( + name=f"dense/{dense_models}{'-custom-rules' if custom_rules else ''}", + commands=(COLD_COMPILE, WARM_COMPILE), + expected=EXPECT_SUCCESS, + writer=DenseProject(model_count=dense_models, custom_rules=custom_rules).write, + ) + for custom_rules in (False, True) + ) + return entries + + +def _project_directories(root: Path) -> list[Path]: + return sorted( + path + for path in root.iterdir() + if path.is_dir() + and ((path / PROJECT_CONFIG_FILE).is_file() or path.name in FIXTURE_PROJECT_SUBDIRECTORIES) + ) + + +def _authored_project( + *, name: str, source_dir: Path, subdirectory: str | None, expected: ExpectedOutcome +) -> CorpusProject: + project_dir: Path = source_dir if subdirectory is None else source_dir / subdirectory + return CorpusProject( + name=name, + commands=(COLD_COMPILE, WARM_COMPILE, *_plan_commands(project_dir)), + expected=expected, + source_dir=source_dir, + project_subdirectory=subdirectory, + ) + + +def _plan_commands(project_dir: Path) -> tuple[DifferentialCommand, ...]: + config_path: Path = project_dir / PROJECT_CONFIG_FILE + if not config_path.is_file(): + return () + try: + adapter: object = tomllib.loads(config_path.read_text(encoding="utf-8")).get("adapter") + except tomllib.TOMLDecodeError: + return () + return (PLAN,) if adapter == DUCKDB_ADAPTER else () + + +def _seed_project(seed: int) -> CorpusProject: + generated: GeneratedProject = ProjectBuilder(seed).build() + return CorpusProject( + name=f"seed/{seed}", + commands=(COLD_COMPILE, PLAN), + expected=ExpectedOutcome(error_code=generated.expected_error_code), + writer=generated.write, + ) diff --git a/scripts/compiler_differential/_helpers/execution.py b/scripts/compiler_differential/_helpers/execution.py new file mode 100644 index 0000000000..06bd80c6c2 --- /dev/null +++ b/scripts/compiler_differential/_helpers/execution.py @@ -0,0 +1,176 @@ +"""Copy a project and run every differential command under one engine in fresh processes.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +from scripts.compiler_differential.constants import ( + CAPTURES_DIRECTORY, + COMMAND_TIMEOUT_SECONDS, + COMPILED_DIRECTORY, + DAG_FILE, + ENGINE_ENV_VAR, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + MANIFEST_FILE, + PROJECT_DIRECTORY, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.models import ( + CommandOutcome, + CorpusProject, + DifferentialCommand, + DifferentialOptions, + EngineRun, +) + +_COPY_IGNORED: tuple[str, ...] = ( + "target", + "logs", + "__pycache__", + ".sqlbuild", + "*.duckdb", + "*.duckdb.wal", +) + + +def run_engine( + *, + project: CorpusProject, + source_dir: Path, + case_dir: Path, + engine: str, + options: DifferentialOptions, +) -> EngineRun: + """Run the project's commands under one engine in a fresh copy at a fixed path.""" + + workspace: Path = case_dir / PROJECT_DIRECTORY + shutil.rmtree(workspace, ignore_errors=True) + _ = shutil.copytree(source_dir, workspace, ignore=shutil.ignore_patterns(*_COPY_IGNORED)) + project_dir: Path = ( + workspace + if project.project_subdirectory is None + else workspace / project.project_subdirectory + ) + capture_root: Path = case_dir / f"{engine}-{CAPTURES_DIRECTORY}" + shutil.rmtree(capture_root, ignore_errors=True) + outcomes: list[CommandOutcome] = [] + for index, command in enumerate(project.commands): + capture_dir: Path | None = ( + capture_root / f"{index}-{command.label}" if options.stage_captures else None + ) + outcomes.append( + _run_command( + command=command, + project_dir=project_dir, + environment=_command_environment( + engine=engine, options=options, capture_dir=capture_dir + ), + python=options.python, + ) + ) + run: EngineRun = EngineRun( + engine=engine, + outcomes=tuple(outcomes), + compiled=_compiled_tree(project_dir / COMPILED_DIRECTORY), + manifest=_optional_text(project_dir / MANIFEST_FILE), + dag=_optional_text(project_dir / DAG_FILE), + captures=_captures(capture_root), + ) + retained: Path = case_dir / engine + shutil.rmtree(retained, ignore_errors=True) + _ = workspace.rename(retained) + return run + + +def harness_environment() -> dict[str, str]: + """Return this process's environment without engine, capture, and dbt settings.""" + + return { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } + + +def _command_environment( + *, engine: str, options: DifferentialOptions, capture_dir: Path | None +) -> dict[str, str]: + environment: dict[str, str] = { + **harness_environment(), + ENGINE_ENV_VAR: engine, + **options.engine_environment.get(engine, {}), + } + if capture_dir is not None: + environment[STAGE_CAPTURE_ENV_VAR] = str(capture_dir) + return environment + + +def _run_command( + *, command: DifferentialCommand, project_dir: Path, environment: dict[str, str], python: Path +) -> CommandOutcome: + try: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", SQB_ENTRY, "--no-color", *command.arguments], + cwd=project_dir, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="surrogateescape", + timeout=COMMAND_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as error: + return CommandOutcome( + label=command.label, + exit_code=-1, + stdout=_timeout_text(error.stdout), + stderr=_timeout_text(error.stderr), + timed_out=True, + ) + return CommandOutcome( + label=command.label, + exit_code=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + + +def _timeout_text(value: bytes | str | None) -> str: + if value is None: + return "" + return value if isinstance(value, str) else value.decode("utf-8", "surrogateescape") + + +def _compiled_tree(root: Path) -> dict[str, bytes]: + if not root.is_dir(): + return {} + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def _optional_text(path: Path) -> str | None: + return path.read_text(encoding="utf-8", errors="surrogateescape") if path.is_file() else None + + +def _captures(root: Path) -> dict[str, dict[str, str]]: + captures: dict[str, dict[str, str]] = {} + for command_dir in sorted(root.iterdir()) if root.is_dir() else (): + captures[command_dir.name] = _capture_files(command_dir) + return captures + + +def _capture_files(command_dir: Path) -> dict[str, str]: + return { + capture.name: capture.read_text(encoding="utf-8") + for capture in sorted(command_dir.iterdir()) + if capture.is_file() + } diff --git a/scripts/compiler_differential/_helpers/failure_cases.py b/scripts/compiler_differential/_helpers/failure_cases.py new file mode 100644 index 0000000000..a49d3df4e5 --- /dev/null +++ b/scripts/compiler_differential/_helpers/failure_cases.py @@ -0,0 +1,504 @@ +"""Minimal failing projects, each derived from the shared base project.""" + +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_FILES, + FAILURE_BASE_MART, + FAILURE_BASE_STAGING, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, + FAILURE_STAGING_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n' + + +def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: + return FailureCase( + files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code + ) + + +def _staging(sql: str) -> dict[str, str]: + return {FAILURE_STAGING_PATH: sql} + + +def _mart_body(body: str) -> dict[str, str]: + return {FAILURE_MART_PATH: 'MODEL (\n description "Order totals per customer",\n);\n\n' + body} + + +def _rules(*codes: str) -> dict[str, str]: + return { + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + _RULES_CONFIG.format(codes='", "'.join(codes)) + } + + +def all_failure_cases() -> tuple[FailureCase, ...]: + """Return every failure case in a stable order.""" + + return ( + _case( + name="config-toml-syntax", + expected_code="D001", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + "[defaults\nmaterialized = 'table'\n" + }, + ), + _case( + name="config-unknown-default-key", + expected_code="D001", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + '\n[defaults]\ndescription = "Shared"\n' + }, + ), + _case( + name="config-unknown-adapter", + expected_code="C601", + files={FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace('"duckdb"', '"warehouse9000"')}, + ), + _case( + name="model-header-syntax", + expected_code="D002", + files=_staging( + FAILURE_BASE_STAGING.replace( + '"Staged orders",', '"Staged orders",\n materialized (,' + ) + ), + ), + _case( + name="model-sql-syntax", + expected_code="P001", + files=_staging(FAILURE_BASE_STAGING.replace("SELECT order_id,", "SELECT order_id,, ")), + ), + _case( + name="test-parse", + expected_code="D003", + files={"tests/unit/test_stg_orders.sql": "TEST(\n\nSELECT 1\n"}, + ), + _case( + name="audit-parse", + expected_code="D004", + files={ + "models/marts/_sqlbuild/audits/singular/broken.sql": "AUDIT (name);\n\nSELECT 1\n" + }, + ), + _case( + name="schema-unterminated-declaration", + expected_code="D013", + files={"models/staging/_sqlbuild/_schemas/orders.sql": "SCHEMA (\n name orders,\n"}, + ), + _case( + name="source-parse", + expected_code="D006", + files={FAILURE_SOURCES_PATH: "sources:\n - name: raw_orders\n columns: [\n"}, + ), + _case( + name="duplicate-model", + expected_code="D007", + files={"models/marts/stg_orders.sql": FAILURE_BASE_STAGING}, + ), + _case( + name="seed-without-header", + expected_code="D008", + files={"seeds/lookups.csv": ""}, + ), + _case( + name="scenario-parse", + expected_code="D009", + files={"tests/scenarios/orders.sql": "SCENARIO (\n\nSELECT 1\n"}, + ), + _case( + name="python-node-missing-description", + expected_code="P010", + files={ + "python/tasks/exports.py": ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\ndef export_orders(ctx: TaskContext):\n return None\n" + ) + }, + ), + _case( + name="enum-parse", + expected_code="D013", + files={"enums/order_status.sql": "ENUM (\n name order_status,\n members [,\n);\n"}, + ), + _case( + name="hook-without-header", + expected_code="D014", + files={"hooks/sql/record_refresh.sql": "SELECT 1\n"}, + ), + _case( + name="invalid-resource-name", + expected_code="D016", + files={"models/marts/order-totals.sql": FAILURE_BASE_MART}, + ), + _case( + name="built-in-audit-shadow", + expected_code="P003", + files={ + "models/staging/_sqlbuild/_audits/generic/not_null.sql": ( + 'AUDIT (name "not_null");\n\nSELECT 1 WHERE FALSE\n' + ) + }, + ), + _case( + name="singular-audit-one-resource", + expected_code="P004", + files={ + "models/marts/_sqlbuild/audits/singular/positive_totals.sql": ( + 'AUDIT (name "positive_totals");\n\n' + 'SELECT customer_id FROM __ref("customer_totals") WHERE total_amount < 0\n' + ) + }, + ), + _case( + name="macro-generated-reference", + expected_code="P006", + files={ + "models/marts/_sqlbuild/_macros/orders.py": ( + "def staged_orders() -> str:\n" + ' """Return the staged orders relation."""\n' + " return '__ref(\"stg_orders\")'\n" + ), + **_mart_body( + "SELECT customer_id, SUM(amount) AS total_amount\n" + "FROM @staged_orders()\nGROUP BY customer_id\n" + ), + }, + ), + _case( + name="missing-description", + expected_code="P010", + files=_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + ), + _case( + name="unknown-ref", + expected_code="P001", + files=_mart_body( + 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_payments")\n' + "GROUP BY customer_id\n" + ), + ), + _case( + name="unknown-source", + expected_code="P001", + files=_staging( + FAILURE_BASE_STAGING.replace('__source("raw_orders")', '__source("raw_payments")') + ), + ), + _case( + name="unknown-macro", + expected_code="P001", + files=_staging( + FAILURE_BASE_STAGING.replace("amount, status", '@cents("amount") AS amount, status') + ), + ), + _case( + name="failing-macro", + expected_code="P001", + files={ + "models/staging/_sqlbuild/_macros/cents.py": ( + "def cents(expression: str) -> str:\n" + ' """Refuse every input."""\n' + ' raise ValueError(f"cannot convert {expression}")\n' + ), + **_staging( + FAILURE_BASE_STAGING.replace( + "amount, status", '@cents("amount") AS amount, status' + ) + ), + }, + ), + _case( + name="unknown-enum-member", + expected_code="P001", + files={ + "models/staging/_sqlbuild/_enums/order_status.sql": ( + "ENUM (\n name order_status,\n members [PLACED, SHIPPED],\n);\n" + ), + **_staging( + FAILURE_BASE_STAGING + 'WHERE status = @enum("order_status").RETURNED\n' + ), + }, + ), + _case( + name="unknown-constant", + expected_code="P001", + files=_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + ), + _case( + name="invalid-decimal-constant", + expected_code="D013", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + 'CONSTANT (name minimum_amount, type decimal, value "ten");\n' + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="duplicate-declaration", + expected_code="P001", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name minimum_amount, value 1);\n" + "CONSTANT (name minimum_amount, value 2);\n" + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="inaccessible-declaration", + expected_code="P001", + files={ + "models/marts/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name minimum_amount, value 1);\n" + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="over-broad-global", + expected_code="S024", + files={ + "constants/limits.sql": "CONSTANT (name minimum_amount, value 1);\n", + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="unused-declaration", + expected_code="S010", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name spare, value 1);\n" + ) + }, + ), + _case( + name="macro-import-cycle", + expected_code="P001", + files={ + "macros/first.py": ( + "from macros.second import second_value\n\n\n" + "def first_value() -> str:\n" + ' """First value."""\n' + " return second_value()\n" + ), + "macros/second.py": ( + "from macros.first import first_value\n\n\n" + "def second_value() -> str:\n" + ' """Second value."""\n' + " return first_value()\n" + ), + }, + ), + _case( + name="contract-missing-column", + expected_code="K001", + files=_staging( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n" + " discount (type DOUBLE),\n" + " ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="contract-type-mismatch", + expected_code="K002", + files=_staging( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type VARCHAR),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="set-operation-width", + expected_code="B216", + files=_mart_body( + 'SELECT customer_id, amount FROM __ref("stg_orders")\n' + 'UNION ALL\nSELECT customer_id FROM __ref("stg_orders")\n' + ), + ), + _case( + name="comparison-type", + expected_code="B217", + files=_mart_body( + "SELECT customer_id, SUM(amount) AS total_amount\n" + "FROM __ref(\"stg_orders\")\nWHERE amount = DATE '2026-01-01'\n" + "GROUP BY customer_id\n" + ), + ), + _case( + name="unknown-column", + expected_code="B002", + files=_mart_body( + 'SELECT customer_id, SUM(discount) AS total_amount\nFROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" + ), + ), + _case( + name="ungrouped-column", + expected_code="B230", + files=_mart_body( + "SELECT customer_id, status, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" + ), + ), + _case( + name="test-expected-unknown-column", + expected_code="B302", + files={ + "tests/unit/test_stg_orders.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + "__expected__stg_orders AS (\n SELECT 1 AS order_id, 2 AS discount\n)\n" + "SELECT 1\n" + ) + }, + ), + _case( + name="rule-unused-cte", + expected_code="SQBRSQL005", + files={ + **_rules("SQBRSQL005"), + **_mart_body( + "WITH spare AS (SELECT 1 AS one)\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ), + }, + ), + _case( + name="rule-custom-finding", + expected_code="XSQBRDIFF001", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[rules]\nselect = ["XSQBR"]\n\n' + + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n", + "rules/no_staging.py": ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n\n" + '@rule(code="XSQBRDIFF001", message="Models avoid the stg_ prefix", ' + 'remediation="Rename the model.")\n' + "def no_staging_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + ' if not model.name.startswith("stg_"):\n' + " return []\n" + " return [ctx.finding(subject=model)]\n" + ), + }, + ), + _case( + name="multiple-errors-order", + expected_code="P010", + files={ + **_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + ' description "Order totals per customer",\n', "" + ), + "models/marts/order_counts.sql": ( + "MODEL ();\n\nSELECT customer_id, COUNT(*) AS order_count\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ), + }, + ), + _case( + name="contract-extra-column", + expected_code="K005", + files=_staging( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="contract-unknown-type", + expected_code="K002", + files=_staging( + 'MODEL (\n description "Staged orders",\n' + " columns (\n order_id (type MYSTERY_TYPE),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="cursor-unknown-column", + expected_code="B300", + files=_mart_body( + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ) + | { + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " cursor placed_at,\n cursor_type timestamp,\n cursor_grain day,\n" + " cursor_inputs (\n stg_orders order_id,\n ),\n);\n\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ) + }, + ), + _case( + name="cursor-model-without-inputs", + expected_code="P011", + files={ + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " cursor placed_at,\n cursor_type timestamp,\n cursor_grain day,\n);\n\n" + "SELECT TIMESTAMP '2026-01-01 00:00:00' AS placed_at, 1 AS customer_id\n" + ) + }, + ), + _case( + name="invalid-declaration-name", + expected_code="D016", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name MinimumAmount, value 1);\n" + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("MinimumAmount")\n'), + }, + ), + _case( + name="python-node-syntax", + expected_code="D011", + files={"python/tasks/exports.py": "def broken(:\n pass\n"}, + ), + _case( + name="hook-unknown-argument", + expected_code="P001", + files={ + "hooks/sql/record_refresh.sql": ( + "HOOK (\n description \"Record a refresh\"\n);\n\nSELECT @'label' AS label\n" + ), + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + '"Order totals per customer",', + '"Order totals per customer",\n post_hooks [sql("record_refresh", tag: "x")],', + ), + }, + ), + _case( + name="model-private-enum-misuse", + expected_code="P001", + files=_staging( + 'MODEL (\n description "Staged orders",\n' + " enums (\n _state [OPEN, CLOSED],\n ),\n);\n\n" + "SELECT order_id, customer_id, amount, status\n" + 'FROM __source("raw_orders")\nWHERE status = @enum("_state").PENDING\n' + ), + ), + _case( + name="variable-not-defined", + expected_code="P001", + files=_staging( + FAILURE_BASE_STAGING.replace("status\n", "status, '@@region' AS region\n", 1) + ), + ), + ) diff --git a/scripts/compiler_differential/_helpers/normalize.py b/scripts/compiler_differential/_helpers/normalize.py new file mode 100644 index 0000000000..a81a9f8987 --- /dev/null +++ b/scripts/compiler_differential/_helpers/normalize.py @@ -0,0 +1,52 @@ +"""Remove run-specific noise that legitimately differs between two identical compiles.""" + +from __future__ import annotations + +from scripts.compiler_differential._helpers.compare import as_json_object +from scripts.compiler_differential.constants import ( + ELAPSED_TIME_MASK, + ELAPSED_TIME_PATTERN, + ENGINE_NAMESPACE_PATTERN, + INVOCATION_ID_MASK, + INVOCATION_ID_PATTERN, + MANIFEST_VOLATILE_METADATA, + STRIPPED_REPORT_FIELDS, +) + + +def normalize_artifact_text(text: str) -> str: + """Mask invocation ids and engine store suffixes, which differ by design.""" + + return ENGINE_NAMESPACE_PATTERN.sub("", INVOCATION_ID_PATTERN.sub(INVOCATION_ID_MASK, text)) + + +def normalize_stderr(text: str) -> str: + """Mask elapsed times as well as the artifact noise in progress and diagnostic output.""" + + return ELAPSED_TIME_PATTERN.sub(ELAPSED_TIME_MASK, normalize_artifact_text(text)) + + +def strip_report_fields(payload: object) -> object: + """Drop the timings and engine fields of a top-level command report.""" + + report: dict[str, object] | None = as_json_object(payload) + if report is None: + return payload + return {key: value for key, value in report.items() if key not in STRIPPED_REPORT_FIELDS} + + +def strip_manifest_metadata(payload: object) -> object: + """Drop the manifest generation time and invocation id.""" + + manifest: dict[str, object] | None = as_json_object(payload) + metadata: dict[str, object] | None = ( + None if manifest is None else as_json_object(manifest.get("metadata")) + ) + if manifest is None or metadata is None: + return payload + return { + **manifest, + "metadata": { + key: value for key, value in metadata.items() if key not in MANIFEST_VOLATILE_METADATA + }, + } diff --git a/scripts/compiler_differential/_helpers/options.py b/scripts/compiler_differential/_helpers/options.py new file mode 100644 index 0000000000..10d465f56a --- /dev/null +++ b/scripts/compiler_differential/_helpers/options.py @@ -0,0 +1,42 @@ +"""Parse harness option values.""" + +from __future__ import annotations + +import argparse +import re + +from scripts.compiler_differential.constants import ( + EXPECT_FAILURE_PREFIX, + EXPECT_SUCCESS, + EXPECT_SUCCESS_VALUE, +) +from scripts.compiler_differential.exceptions import DifferentialUsageError +from scripts.compiler_differential.models import ExpectedOutcome + +_DIAGNOSTIC_CODE: re.Pattern[str] = re.compile(r"[A-Z][A-Z0-9]*[0-9]") + + +def parse_engine_environment(values: list[str]) -> dict[str, dict[str, str]]: + """Parse repeated `ENGINE:NAME=VALUE` options into per-engine environment overrides.""" + + environment: dict[str, dict[str, str]] = {} + for value in values: + engine, separator, assignment = value.partition(":") + name, equals, setting = assignment.partition("=") + if not separator or not equals or not engine or not name: + raise DifferentialUsageError(f"--engine-env expects ENGINE:NAME=VALUE, got {value!r}") + environment.setdefault(engine, {})[name] = setting + return environment + + +def parse_expected_outcome(value: str) -> ExpectedOutcome: + """Parse `success` or `failure:` into the outcome every --project must produce.""" + + if value == EXPECT_SUCCESS_VALUE: + return EXPECT_SUCCESS + code: str = value.removeprefix(EXPECT_FAILURE_PREFIX) + if code == value or not _DIAGNOSTIC_CODE.fullmatch(code): + raise argparse.ArgumentTypeError( + f"expected {EXPECT_SUCCESS_VALUE} or {EXPECT_FAILURE_PREFIX}, got {value!r}" + ) + return ExpectedOutcome(error_code=code) diff --git a/scripts/compiler_differential/_helpers/project_comparison.py b/scripts/compiler_differential/_helpers/project_comparison.py new file mode 100644 index 0000000000..3741cdb565 --- /dev/null +++ b/scripts/compiler_differential/_helpers/project_comparison.py @@ -0,0 +1,135 @@ +"""Run one corpus project under both engines and compare what they produced.""" + +from __future__ import annotations + +import shutil +import time +from pathlib import Path + +from scripts.compiler_differential._helpers.comparison import compare_engine_runs, diagnostic_codes +from scripts.compiler_differential._helpers.execution import run_engine +from scripts.compiler_differential.models import ( + CommandOutcome, + CorpusProject, + Difference, + DifferentialOptions, + EngineRun, + ProjectComparison, +) + +_SOURCE_DIRECTORY: str = "source" +_CORPUS_ARTIFACT: str = "corpus expectation" +_EXPECTATION_LABELS: tuple[str, str] = ("expected", "actual") + + +def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> ProjectComparison: + """Compare both engines on one project against its expectation; any timeout fails.""" + + started: float = time.monotonic() + case_dir: Path = options.work_dir / project.name.replace("/", "__") + shutil.rmtree(case_dir, ignore_errors=True) + case_dir.mkdir(parents=True) + source_dir: Path = _materialized_source(project=project, case_dir=case_dir) + left_engine, right_engine = options.engines + left: EngineRun = run_engine( + project=project, + source_dir=source_dir, + case_dir=case_dir, + engine=left_engine, + options=options, + ) + right: EngineRun = run_engine( + project=project, + source_dir=source_dir, + case_dir=case_dir, + engine=right_engine, + options=options, + ) + differences: list[Difference] = [ + *_timeout_differences(project=project, runs=(left, right)), + *_corpus_differences(project=project, run=left), + *compare_engine_runs(project=project.name, left=left, right=right), + ] + return ProjectComparison( + project=project.name, + differences=tuple(differences), + seconds=time.monotonic() - started, + ) + + +def _materialized_source(*, project: CorpusProject, case_dir: Path) -> Path: + if project.source_dir is not None: + return project.source_dir + source_dir: Path = case_dir / _SOURCE_DIRECTORY + if project.writer is not None: + project.writer(source_dir) + return source_dir + + +def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Difference]: + first: CommandOutcome = run.outcomes[0] + errors: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=True) + expected_code: str | None = project.expected.error_code + if expected_code is not None: + codes: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=False) + met: bool = first.exit_code != 0 and bool(errors) and expected_code in codes + return ( + [] + if met + else [ + _expectation_difference( + project=project, + location="expected failure", + expected=f"non-zero exit reporting {expected_code}", + actual=f"exit {first.exit_code}: {', '.join(codes) or ''}", + ) + ] + ) + problems: list[str] = [ + f"`{outcome.label}` exit {outcome.exit_code}" + for outcome in run.outcomes + if outcome.exit_code != 0 + ] + if errors: + problems.append(f"compile errors {', '.join(errors)}") + if not problems: + return [] + return [ + _expectation_difference( + project=project, + location="expected success", + expected="every command exits 0 and compile reports no errors", + actual="; ".join(problems), + ) + ] + + +def _timeout_differences( + *, project: CorpusProject, runs: tuple[EngineRun, ...] +) -> list[Difference]: + differences: list[Difference] = [] + for run in runs: + differences.extend( + _expectation_difference( + project=project, + location=f"{run.engine} `{outcome.label}`", + expected="completes within the command timeout", + actual="timed out", + ) + for outcome in run.outcomes + if outcome.timed_out + ) + return differences + + +def _expectation_difference( + *, project: CorpusProject, location: str, expected: str, actual: str +) -> Difference: + return Difference( + project=project.name, + artifact=_CORPUS_ARTIFACT, + location=location, + left=expected, + right=actual, + labels=_EXPECTATION_LABELS, + ) diff --git a/scripts/compiler_differential/_helpers/report.py b/scripts/compiler_differential/_helpers/report.py new file mode 100644 index 0000000000..fccf230595 --- /dev/null +++ b/scripts/compiler_differential/_helpers/report.py @@ -0,0 +1,39 @@ +"""Render differential results for terminals and CI logs.""" + +from __future__ import annotations + +from scripts.compiler_differential.models import Difference, ProjectComparison + + +def format_comparison(*, comparison: ProjectComparison, engines: tuple[str, str]) -> str: + """Render one project's verdict and the first difference in every differing artifact.""" + + status: str = "OK " if not comparison.differences else "DIFF" + lines: list[str] = [f"{status} {comparison.project} ({comparison.seconds:.1f}s)"] + for difference in comparison.differences: + left_label, right_label = difference.labels or engines + width: int = max(len(left_label), len(right_label)) + lines.append(f" - {difference.artifact} at {difference.location}") + lines.append(f" {left_label:<{width}}: {difference.left}") + lines.append(f" {right_label:<{width}}: {difference.right}") + return "\n".join(lines) + + +def format_summary( + *, comparisons: list[ProjectComparison], engines: tuple[str, str], seconds: float +) -> str: + """Summarize the run, naming the first difference of the first differing project.""" + + differing: list[ProjectComparison] = [item for item in comparisons if item.differences] + left_engine, right_engine = engines + if not differing: + return ( + f"Compiler differential passed: {len(comparisons)} projects identical " + f"({left_engine} vs {right_engine}) in {seconds:.1f}s" + ) + first: Difference = differing[0].differences[0] + return ( + f"Compiler differential FAILED: {len(differing)} of {len(comparisons)} projects differ " + f"({left_engine} vs {right_engine}) in {seconds:.1f}s\n" + f"First difference: {first.project}: {first.artifact} at {first.location}" + ) diff --git a/scripts/compiler_differential/_helpers/run.py b/scripts/compiler_differential/_helpers/run.py new file mode 100644 index 0000000000..0d200afe02 --- /dev/null +++ b/scripts/compiler_differential/_helpers/run.py @@ -0,0 +1,65 @@ +"""Build run options and compare a corpus project by project.""" + +from __future__ import annotations + +import argparse +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from scripts.compiler_differential._helpers.corpus import build_corpus +from scripts.compiler_differential._helpers.options import parse_engine_environment +from scripts.compiler_differential._helpers.project_comparison import compare_project +from scripts.compiler_differential._helpers.report import format_comparison +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialOptions, + ProjectComparison, +) + + +def differential_options(*, args: argparse.Namespace, work_dir: Path) -> DifferentialOptions: + """Return run options from parsed command-line arguments.""" + + return DifferentialOptions( + engines=(args.engines[0], args.engines[1]), + work_dir=work_dir, + jobs=args.jobs, + stage_captures=args.stage_captures, + python=args.python, + engine_environment=parse_engine_environment(args.engine_env), + ) + + +def selected_corpus(*, args: argparse.Namespace, repo_root: Path) -> list[CorpusProject]: + """Return the projects selected by the parsed command-line arguments.""" + + return build_corpus( + repo_root=repo_root, + corpora=tuple(args.corpus), + seeds=range(args.seed_start, args.seed_start + args.seeds), + dense_models=args.dense_models, + projects=tuple(path.absolute() for path in args.project), + project_expectation=args.expect, + ) + + +def compare_corpus( + *, corpus: list[CorpusProject], options: DifferentialOptions +) -> list[ProjectComparison]: + """Compare every project, printing each result as it completes.""" + + print( + f"Comparing {len(corpus)} projects ({options.engines[0]} vs {options.engines[1]}, " + f"{options.jobs} jobs) in {options.work_dir}", + file=sys.stderr, + ) + comparisons: list[ProjectComparison] = [] + with ThreadPoolExecutor(max_workers=options.jobs) as pool: + for comparison in pool.map( + lambda project: compare_project(project=project, options=options), corpus + ): + comparisons.append(comparison) + print(format_comparison(comparison=comparison, engines=options.engines)) + sys.stdout.flush() + return comparisons diff --git a/scripts/compiler_differential/classes/dense_project.py b/scripts/compiler_differential/classes/dense_project.py new file mode 100644 index 0000000000..1d15205931 --- /dev/null +++ b/scripts/compiler_differential/classes/dense_project.py @@ -0,0 +1,37 @@ +"""Write the dense benchmark project, optionally with a custom Rule.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.compiler_differential.constants import ( + CUSTOM_RULE_FILE, + CUSTOM_RULE_SOURCE, + CUSTOM_RULE_THRESHOLDS, + PROJECT_CONFIG_FILE, +) + + +class DenseProject: + """A dense compile benchmark project of one size.""" + + def __init__(self, *, model_count: int, custom_rules: bool) -> None: + self._model_count: int = model_count + self._custom_rules: bool = custom_rules + + def write(self, project_dir: Path) -> None: + """Write the project; with custom rules, add one Rule every model is checked by.""" + + write_dense_compile_project(project_dir=project_dir, model_count=self._model_count) + if not self._custom_rules: + return + rule_path: Path = project_dir / CUSTOM_RULE_FILE + rule_path.parent.mkdir(parents=True, exist_ok=True) + _ = rule_path.write_text(CUSTOM_RULE_SOURCE, encoding="utf-8", newline="\n") + config_path: Path = project_dir / PROJECT_CONFIG_FILE + _ = config_path.write_text( + config_path.read_text(encoding="utf-8") + CUSTOM_RULE_THRESHOLDS, + encoding="utf-8", + newline="\n", + ) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py new file mode 100644 index 0000000000..beb7b1d465 --- /dev/null +++ b/scripts/compiler_differential/classes/project_builder.py @@ -0,0 +1,604 @@ +"""Build one seeded random project file by file.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from dataclasses import replace + +from scripts.compiler_differential.constants import ( + GENERATOR_CONSTANT_KIND, + GENERATOR_CROSS_DOMAIN_SHARE, + GENERATOR_DOMAINS, + GENERATOR_ENUM_KIND, + GENERATOR_FLOAT_VALUES, + GENERATOR_HALF, + GENERATOR_HOOK_KIND, + GENERATOR_INVALID_CODES, + GENERATOR_INVALID_SHARE, + GENERATOR_LAYERS, + GENERATOR_MACRO_KIND, + GENERATOR_NON_ASCII_DESCRIPTION_SHARE, + GENERATOR_NON_ASCII_LABELS, + GENERATOR_REGION_ENV_VAR, + GENERATOR_ROLES, + GENERATOR_SOURCE_COLUMNS, + GENERATOR_STATUS_VALUES, + GENERATOR_SUFFIXES, + GENERATOR_VARIABLE_SHARE, +) +from scripts.compiler_differential.models import DeclarationPlan, GeneratedProject, ModelPlan + +_STAGING: str = GENERATOR_LAYERS[0] +_MARTS: str = GENERATOR_LAYERS[1] + + +class ProjectBuilder: + """Accumulate the files of one generated project from a seeded random source.""" + + def __init__(self, seed: int) -> None: + self._seed: int = seed + self._random: random.Random = random.Random(seed) + self._files: dict[str, str] = {} + self._features: set[str] = set() + self._models: list[ModelPlan] = [] + self._sources: list[str] = [] + self._seeds: list[str] = [] + self._declarations: list[DeclarationPlan] = [] + self._model_lines: dict[str, list[str]] = {} + self._model_filters: dict[str, list[str]] = {} + self._private_constants: dict[str, list[str]] = {} + self._hook_consumers: list[ModelPlan] = [] + + def build(self) -> GeneratedProject: + """Return the finished project, optionally carrying one deliberate invalid input.""" + + domains: list[str] = self._random.sample(GENERATOR_DOMAINS, k=self._random.randint(1, 3)) + self._write_sources() + self._write_seeds() + self._plan_models(domains=domains) + self._plan_enums() + for index in range(self._random.randint(1, 3)): + self._plan_constant(index=index) + for index in range(self._random.randint(1, 3)): + self._plan_macro(index=index) + invalid_kind: str | None = ( + self._random.choice(sorted(GENERATOR_INVALID_CODES)) + if self._random.random() < GENERATOR_INVALID_SHARE + else None + ) + self._write_declarations() + self._write_models() + self._write_hooks() + self._write_python_nodes() + self._write_tests_and_scenarios() + self._write_singular_audit() + self._write_config(domains=domains) + if invalid_kind is not None: + self._inject_invalid(kind=invalid_kind) + return GeneratedProject( + files=dict(sorted(self._files.items())), + seed=self._seed, + expected_error_code=( + None if invalid_kind is None else GENERATOR_INVALID_CODES[invalid_kind] + ), + features=tuple(sorted(self._features)), + ) + + def _chance(self, share: float) -> bool: + return self._random.random() < share + + def _label(self) -> str: + return self._random.choice(GENERATOR_NON_ASCII_LABELS) + + def _write_config(self, *, domains: list[str]) -> None: + lines: list[str] = [ + f'name = "generated_{self._seed}"', + 'adapter = "duckdb"', + 'default_target = "dev"', + "", + "[connection]", + 'database = "generated.duckdb"', + "", + "[vars]", + f'region = "{self._random.choice(("north", "south", "east"))}"', + f'discount_rate = "{self._random.choice(GENERATOR_FLOAT_VALUES)}"', + f'site_label = "{self._label()}"', + "", + "[targets.dev]", + f'schema = "{self._random.choice(("analytics", "main"))}"', + "", + "[settings]", + f'default_audit_severity = "{self._random.choice(("warn", "error"))}"', + "", + "[defaults]", + f'materialized = "{self._random.choice(("table", "view"))}"', + "", + '[path_defaults."*/staging"]', + 'materialized = "view"', + 'tags = ["staging"]', + ] + for domain in domains: + if self._chance(GENERATOR_HALF): + self._features.add("path_defaults_literal") + lines.extend( + [ + "", + f'[path_defaults."{domain}/marts"]', + f'materialized = "{self._random.choice(("table", "view"))}"', + f'tags = ["{domain}", "marts"]', + ] + ) + self._features.add("path_defaults_wildcard") + self._files["sqlbuild_project.toml"] = "\n".join(lines) + "\n" + + def _write_sources(self) -> None: + entries: list[str] = [] + columns: str = "\n".join( + f" - name: {column}\n type: {column_type}" + for column, column_type in GENERATOR_SOURCE_COLUMNS + ) + for index in range(self._random.randint(1, 3)): + name: str = f"raw_events_{index}" + self._sources.append(name) + rows: str = " UNION ALL ".join( + f"SELECT {row} AS id, CAST({row * 7 + index} AS DOUBLE) AS amount, " + f"'{GENERATOR_STATUS_VALUES[row % len(GENERATOR_STATUS_VALUES)][1]}' AS status, " + f"TIMESTAMP '2026-01-0{row + 1} 00:00:00' AS created_at" + for row in range(1, 4) + ) + entries.append( + f" - name: {name}\n" + f" description: Generated event feed {index}.\n" + f' expression: "({rows})"\n' + f" columns:\n{columns}" + ) + self._files["sources/raw.yml"] = "sources:\n" + "\n".join(entries) + "\n" + + def _write_seeds(self) -> None: + schemas: list[str] = [] + for index in range(self._random.randint(0, 2)): + name: str = f"lookup_{index}" + self._seeds.append(name) + self._files[f"seeds/{name}.csv"] = f'id,label\n1,"{self._label()}"\n2,plain\n' + schemas.append( + f" - name: {name}\n" + f" description: Generated lookup {index}.\n" + " columns:\n" + " - name: id\n type: INTEGER\n" + " - name: label\n type: VARCHAR" + ) + if schemas: + self._features.add("seeds") + self._files["seeds/seeds.yml"] = "seeds:\n" + "\n".join(schemas) + "\n" + + def _plan_models(self, *, domains: list[str]) -> None: + staging: list[ModelPlan] = [] + for domain in domains: + staging.extend( + ModelPlan( + name=f"stg_{domain}_{index}", + domain=domain, + layer=_STAGING, + inputs=(f"source:{self._random.choice(self._sources)}",), + description=self._description(f"Staged {domain} events {index}"), + ) + for index in range(self._random.randint(1, 3)) + ) + self._models.extend(staging) + for domain in domains: + own: list[ModelPlan] = [model for model in staging if model.domain == domain] + for index in range(self._random.randint(1, 3)): + self._models.append( + ModelPlan( + name=f"{domain}_summary_{index}", + domain=domain, + layer=_MARTS, + inputs=self._mart_inputs(own=own, staging=staging), + description=self._description(f"Summary of {domain} activity {index}"), + ) + ) + + def _mart_inputs(self, *, own: list[ModelPlan], staging: list[ModelPlan]) -> tuple[str, ...]: + upstream: list[ModelPlan] = [self._random.choice(own)] + other: ModelPlan = self._random.choice(staging) + if other not in upstream and self._chance(GENERATOR_CROSS_DOMAIN_SHARE): + upstream.append(other) + self._features.add("cross_domain_ref") + inputs: list[str] = [f"ref:{model.name}" for model in upstream] + if self._seeds and self._chance(GENERATOR_HALF): + inputs.append(f"seed:{self._random.choice(self._seeds)}") + return tuple(inputs) + + def _description(self, text: str) -> str: + if not self._chance(GENERATOR_NON_ASCII_DESCRIPTION_SHARE): + return text + self._features.add("non_ascii_description") + return f"{text} ({self._label()})" + + def _consumers(self) -> tuple[ModelPlan, ...]: + count: int = self._random.randint(1, min(3, len(self._models))) + return tuple( + sorted(self._random.sample(self._models, k=count), key=lambda model: model.name) + ) + + def _plan_enums(self) -> None: + status: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_ENUM_KIND, + name="event_status", + consumers=self._consumers(), + body=( + "ENUM (\n name event_status,\n members (\n" + + "".join(f' {member} "{value}",\n' for member, value in GENERATOR_STATUS_VALUES) + + " ),\n);\n" + ), + ) + self._declarations.append(status) + for model in status.consumers: + member: str = self._random.choice(GENERATOR_STATUS_VALUES)[0] + self._model_filters.setdefault(model.name, []).append( + f'status <> @enum("event_status").{member}' + ) + if not self._chance(GENERATOR_HALF): + return + self._features.add("integer_enum") + shared_name: bool = self._chance(GENERATOR_HALF) + name: str = "adjust_0" if shared_name else "event_priority" + if shared_name: + self._features.add("name_shared_across_namespaces") + priority: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_ENUM_KIND, + name=name, + consumers=self._consumers(), + body=f"ENUM (\n name {name},\n members (LOW 1, HIGH 3),\n);\n", + ) + self._declarations.append(priority) + for model in priority.consumers: + self._model_lines.setdefault(model.name, []).append(f'@enum("{name}").HIGH AS priority') + + def _plan_constant(self, *, index: int) -> None: + name: str = f"threshold_{index}" + writers: dict[str, Callable[[str], tuple[str, str, str]]] = { + "text": self._text_constant, + "bool": self._bool_constant, + "decimal": self._decimal_constant, + "int": self._int_constant, + "float": self._float_constant, + } + parts: tuple[str, str, str] = writers[self._random.choice(sorted(writers))](name) + body: str = parts[0] + literal: str = parts[1] + expression: str = parts[2] + declaration: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_CONSTANT_KIND, name=name, consumers=self._consumers(), body=body + ) + if len(declaration.consumers) == 1 and self._chance(GENERATOR_HALF): + self._features.add("model_private_value") + declaration = replace(declaration, name=f"_{name}", private=True) + expression = expression.replace(f'"{name}"', f'"_{name}"') + self._private_constants.setdefault(declaration.consumers[0].name, []).append( + f" _{name} {literal},\n" + ) + self._declarations.append(declaration) + for model in declaration.consumers: + self._model_lines.setdefault(model.name, []).append(expression) + + def _text_constant(self, name: str) -> tuple[str, str, str]: + self._features.add("non_ascii_constant") + literal: str = f'"{self._label()}"' + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'@const("{name}") AS {name}_label', + ) + + def _bool_constant(self, name: str) -> tuple[str, str, str]: + literal: str = self._random.choice(("true", "false")) + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'@const("{name}") AS {name}_flag', + ) + + def _decimal_constant(self, name: str) -> tuple[str, str, str]: + self._features.add("decimal_constant") + return ( + f'CONSTANT (\n name {name},\n type decimal,\n value "2.4700",\n);\n', + "2.47", + f'amount * @const("{name}") AS {name}_amount', + ) + + def _int_constant(self, name: str) -> tuple[str, str, str]: + literal: str = str(self._random.randint(-5, 500)) + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'amount + @const("{name}") AS {name}_amount', + ) + + def _float_constant(self, name: str) -> tuple[str, str, str]: + self._features.add("float_constant") + literal: str = self._random.choice(GENERATOR_FLOAT_VALUES) + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'amount * @const("{name}") AS {name}_amount', + ) + + def _plan_macro(self, *, index: int) -> None: + name: str = f"adjust_{index}" + writers: dict[str, Callable[[str], tuple[str, str]]] = { + "ctx": self._context_macro, + "env": self._environment_macro, + "nested": self._nested_macro, + "plain": self._plain_macro, + } + parts: tuple[str, str] = writers[self._random.choice(sorted(writers))](name) + declaration: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_MACRO_KIND, name=name, consumers=self._consumers(), body=parts[0] + ) + self._declarations.append(declaration) + for model in declaration.consumers: + self._model_lines.setdefault(model.name, []).append(parts[1]) + + def _context_macro(self, name: str) -> tuple[str, str]: + self._features.add("macro_reads_ctx") + return ( + f"def {name}(ctx, expression: str) -> str:\n" + ' """Scale an amount by the configured discount."""\n' + ' rate = ctx.vars["discount_rate"]\n' + ' return f"({expression} * (1 - {rate}))"\n', + f'@{name}("amount") AS {name}_amount', + ) + + def _environment_macro(self, name: str) -> tuple[str, str]: + self._features.add("macro_reads_env") + return ( + "import os\n\n\n" + f"def {name}(expression: str) -> str:\n" + ' """Tag an amount with the deployment region."""\n' + f' region = os.environ.get("{GENERATOR_REGION_ENV_VAR}", "east")\n' + " return f\"({expression} + LENGTH('{region}'))\"\n", + f'@{name}("amount") AS {name}_amount', + ) + + def _nested_macro(self, name: str) -> tuple[str, str]: + self._features.add("nested_macro") + return ( + f"def {name}_round(expression: str) -> str:\n" + ' """Round an amount to two places."""\n' + ' return f"ROUND({expression}, 2)"\n\n\n' + f"def {name}(expression: str) -> str:\n" + ' """Double and round an amount."""\n' + f' return {name}_round(f"({{expression}} * 2)")\n', + f'@{name}_round(@{name}("amount")) AS {name}_amount', + ) + + def _plain_macro(self, name: str) -> tuple[str, str]: + offset: str = name.rsplit("_", 1)[1] + return ( + f"def {name}(expression: str) -> str:\n" + ' """Offset an amount."""\n' + f' return f"({{expression}} + {offset} + 1)"\n', + f'@{name}("amount") AS {name}_amount', + ) + + def _write_declarations(self) -> None: + for declaration in self._declarations: + if declaration.private: + continue + directory: str = self._placement(declaration) + suffix: str = GENERATOR_SUFFIXES[declaration.kind] + self._files[f"{directory}/{declaration.name}{suffix}"] = declaration.body + + def _placement(self, declaration: DeclarationPlan) -> str: + role: str = GENERATOR_ROLES[declaration.kind] + folders: set[str] = {model.folder for model in declaration.consumers} + domains: set[str] = {model.domain for model in declaration.consumers} + if len(folders) == 1: + self._features.add("exact_owner_scope") + return f"{folders.pop()}/_sqlbuild/_{role}" + if len(domains) == 1: + self._features.add("tree_scope") + return f"models/{domains.pop()}/_sqlbuild/{role}" + self._features.add("project_scope") + return role + + def _write_models(self) -> None: + for model in self._models: + self._files[f"{model.folder}/{model.name}.sql"] = self._model_sql(model) + + def _model_sql(self, model: ModelPlan) -> str: + header: list[str] = [f' description "{model.description}"'] + if self._chance(GENERATOR_HALF): + header.append(" columns (\n id (audits [not_null]),\n )") + self._features.add("column_audits") + private: list[str] = self._private_constants.get(model.name, []) + if private: + header.append(" constants (\n" + "".join(private) + " )") + header.extend(self._model_hooks(model)) + columns: list[str] = ["t.id", "t.amount", "t.status", "t.created_at"] + columns.extend( + line.replace("amount", "t.amount", 1) for line in self._model_lines.get(model.name, []) + ) + joins: list[str] = [] + for position, extra in enumerate(model.inputs[1:], start=1): + joins.append(f"LEFT JOIN {_relation(extra)} j{position} ON t.id = j{position}.id") + columns.extend( + f"j{position}.label AS lookup_label_{position}" + for _ in range(int(extra.startswith("seed:"))) + ) + if self._chance(GENERATOR_VARIABLE_SHARE): + self._features.add("project_variable") + columns.append("'@@site_label' AS site_label") + filters: list[str] = [ + condition.replace("status", "t.status", 1) + for condition in self._model_filters.get(model.name, []) + ] + where: str = f"\nWHERE {' AND '.join(filters)}" if filters else "" + return ( + "MODEL (\n" + + ",\n".join(header) + + ",\n);\n\nSELECT\n " + + ",\n ".join(columns) + + f"\nFROM {_relation(model.inputs[0])} t" + + "".join(f"\n{join}" for join in joins) + + where + + "\n" + ) + + def _model_hooks(self, model: ModelPlan) -> list[str]: + if model.layer != _MARTS or not self._chance(GENERATOR_HALF): + return [] + self._features.add("sql_hooks") + self._hook_consumers.append(model) + return [ + " post_hooks [\n" + ' sql("record_refresh", relation: "@@CTX:destination.qualified", ' + f'label: "{self._label()}"),\n' + " inline_sql(\"SELECT '@@CTX:destination.qualified' AS refreshed\"),\n" + " ]" + ] + + def _write_hooks(self) -> None: + if not self._hook_consumers: + return + hook: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_HOOK_KIND, name="record_refresh", consumers=tuple(self._hook_consumers) + ) + self._files[f"{self._placement(hook)}/sql/record_refresh.sql"] = ( + 'HOOK (\n description "Record a refresh for one relation"\n);\n\n' + "SELECT @'relation' AS refreshed_relation, @'label' AS refresh_label\n" + ) + + def _write_python_nodes(self) -> None: + if not self._chance(GENERATOR_HALF): + return + self._features.add("python_nodes") + self._files["python/tasks/exports.py"] = ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\n" + "def collect_exports(ctx: TaskContext):\n" + ' """Collect generated exports."""\n' + ' return ctx.result(payload={"rows": 3})\n\n\n' + "@task(depends_on=collect_exports)\n" + "def publish_exports(ctx: TaskContext):\n" + ' """Publish generated exports."""\n' + ' return ctx.result(metadata={"published": True})\n' + ) + + def _write_tests_and_scenarios(self) -> None: + model: ModelPlan = self._random.choice( + [model for model in self._models if model.layer == _STAGING] + ) + source: str = model.inputs[0].split(":", 1)[1] + uses_enum: bool = model.name in self._model_filters + if uses_enum: + self._features.add("relationship_grant") + expected_status: str = '@enum("event_status").PLACED' if uses_enum else "'placed'" + self._features.add("unit_test") + self._files[f"tests/unit/{model.domain}/test_{model.name}.sql"] = ( + "TEST();\n\nWITH\n" + f"__source__{source} AS (\n" + " SELECT 1 AS id, CAST(10 AS DOUBLE) AS amount, 'placed' AS status,\n" + " TIMESTAMP '2026-01-01 00:00:00' AS created_at\n" + "),\n" + f"__expected__{model.name} AS (\n" + f" SELECT 1 AS id, {expected_status} AS status\n" + ")\n" + "SELECT 1\n" + ) + if not self._chance(GENERATOR_HALF): + return + self._features.add("scenario") + self._files[f"tests/scenarios/{model.name}_scenario.sql"] = ( + 'SCENARIO (\n description "Generated scenario for staged events"\n);\n\n' + "WITH\n" + f"__source__{source} AS (\n" + " SELECT 2 AS id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status,\n" + " TIMESTAMP '2026-01-02 00:00:00' AS created_at\n" + "),\n" + f"__expected__{model.name} AS (\n" + " SELECT 2 AS id\n" + ")\n" + "SELECT 1\n" + ) + + def _write_singular_audit(self) -> None: + by_folder: dict[str, list[ModelPlan]] = {} + for model in self._models: + if model.layer == _MARTS: + by_folder.setdefault(model.folder, []).append(model) + candidates: list[list[ModelPlan]] = [ + group for group in by_folder.values() if len(group) > 1 + ] + if not candidates or not self._chance(GENERATOR_HALF): + return + self._features.add("singular_audit") + first, second = self._random.sample(self._random.choice(candidates), k=2) + self._files[f"{first.folder}/_sqlbuild/audits/singular/{first.name}_matches.sql"] = ( + f'AUDIT (name "{first.name}_matches");\n\n' + f'SELECT a.id\nFROM __ref("{first.name}") a\n' + f'LEFT JOIN __ref("{second.name}") b ON a.id = b.id\nWHERE b.id IS NULL\n' + ) + + def _inject_invalid(self, *, kind: str) -> None: + self._features.add(f"invalid:{kind}") + model: ModelPlan = self._random.choice(self._models) + path: str = f"{model.folder}/{model.name}.sql" + injections: dict[str, Callable[[ModelPlan], str]] = { + "unknown_ref": lambda _: '(SELECT 1 FROM __ref("missing_model")) AS missing', + "unknown_macro": lambda _: '@not_a_macro("id") AS bogus', + "failing_macro": self._failing_macro_column, + "unknown_enum_member": lambda _: '@enum("event_status").MISSING AS bogus', + "unknown_constant": lambda _: '@const("no_such_value") AS bogus', + "missing_description": self._drop_description, + "duplicate_macro": self._duplicate_macro, + "header_syntax": self._break_header, + "over_broad_constant": self._over_broad_constant_column, + } + column: str = injections[kind](model) + if column: + self._files[path] = self._files[path].replace("SELECT\n", f"SELECT\n {column},\n", 1) + + def _failing_macro_column(self, model: ModelPlan) -> str: + self._files[f"{model.folder}/_sqlbuild/_macros/explode.py"] = ( + "def explode(expression: str) -> str:\n" + ' """Refuse every input."""\n' + ' raise ValueError(f"cannot adjust {expression}")\n' + ) + return '@explode("id") AS bogus' + + def _drop_description(self, model: ModelPlan) -> str: + path: str = f"{model.folder}/{model.name}.sql" + self._files[path] = self._files[path].replace(f' description "{model.description}",\n', "") + self._files[path] = self._files[path].replace(f' description "{model.description}"\n', "") + return "" + + def _duplicate_macro(self, model: ModelPlan) -> str: + _ = model + self._files["macros/duplicate_adjust_0.py"] = ( + "def adjust_0(expression: str) -> str:\n" + ' """Duplicate a macro name."""\n' + " return expression\n" + ) + return "" + + def _break_header(self, model: ModelPlan) -> str: + path: str = f"{model.folder}/{model.name}.sql" + self._files[path] = self._files[path].replace( + "MODEL (\n", "MODEL (\n materialized (,\n", 1 + ) + return "" + + def _over_broad_constant_column(self, model: ModelPlan) -> str: + _ = model + self._files["constants/unused_threshold.sql"] = ( + "CONSTANT (name unused_threshold, value 3);\n" + ) + return '@const("unused_threshold") AS narrow' + + +def _relation(reference: str) -> str: + kind, name = reference.split(":", 1) + return f'__{kind}("{name}")' diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py new file mode 100644 index 0000000000..012ee68423 --- /dev/null +++ b/scripts/compiler_differential/constants.py @@ -0,0 +1,186 @@ +"""Stable constants for the compiler engine differential harness.""" + +import re + +from scripts.compiler_differential.models import DifferentialCommand, ExpectedOutcome + +ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" +STAGE_CAPTURE_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +DEFAULT_ENGINES: tuple[str, str] = ("python", "native") +SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( + {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR} +) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +PROJECT_CONFIG_FILE: str = "sqlbuild_project.toml" +DUCKDB_ADAPTER: str = "duckdb" +PROJECT_DIRECTORY: str = "project" +CAPTURES_DIRECTORY: str = "captures" +TARGET_DIRECTORY: str = "target" +COMPILED_DIRECTORY: str = "target/compiled" +MANIFEST_FILE: str = "target/manifest.json" +DAG_FILE: str = "target/sqlbuild_dag.json" +STRIPPED_REPORT_FIELDS: frozenset[str] = frozenset({"compile_timings", "compiler_engine"}) +MANIFEST_VOLATILE_METADATA: frozenset[str] = frozenset({"generated_at", "invocation_id"}) +INVOCATION_ID_PATTERN: re.Pattern[str] = re.compile(r"\b\d{8}T\d{6}Z_[0-9a-f]{12}\b") +INVOCATION_ID_MASK: str = "" +ENGINE_NAMESPACE_PATTERN: re.Pattern[str] = re.compile(r"-native-v\d+\b") +ELAPSED_TIME_PATTERN: re.Pattern[str] = re.compile(r"\b\d+(?:\.\d+)?\s?m?s\b") +ELAPSED_TIME_MASK: str = "" +COMMAND_TIMEOUT_SECONDS: float = 900.0 +VALUE_PREVIEW_CHARACTERS: int = 160 +MISSING_VALUE: str = "" +DEFAULT_SEED_COUNT: int = 12 +PLAN_LABEL: str = "plan" +ERROR_SEVERITY: str = "error" +MODEL_RESOURCE_TYPE: str = "model" +DEFAULT_DENSE_MODELS: int = 3000 +FIXTURE_ROOT: str = "tests/e2e/fixtures" +EXAMPLE_ROOT: str = "website/examples" +FIXTURE_PROJECT_SUBDIRECTORIES: dict[str, str] = {"dbt_interop": "sqlbuild_project"} +EXPECT_SUCCESS: ExpectedOutcome = ExpectedOutcome() +EXPECT_SUCCESS_VALUE: str = "success" +EXPECT_FAILURE_PREFIX: str = "failure:" +FIXTURE_EXPECTED_OUTCOMES: dict[str, ExpectedOutcome] = { + "dbt_interop": ExpectedOutcome(error_code="C214") +} +CORPUS_FIXTURES: str = "fixtures" +CORPUS_EXAMPLES: str = "examples" +CORPUS_SEEDS: str = "seeds" +CORPUS_FAILURES: str = "failures" +CORPUS_DENSE: str = "dense" +CORPUS_NAMES: tuple[str, ...] = ( + CORPUS_FIXTURES, + CORPUS_EXAMPLES, + CORPUS_SEEDS, + CORPUS_FAILURES, + CORPUS_DENSE, +) +PER_PULL_REQUEST_CORPORA: tuple[str, ...] = ( + CORPUS_FIXTURES, + CORPUS_EXAMPLES, + CORPUS_SEEDS, + CORPUS_FAILURES, +) +CUSTOM_RULE_FILE: str = "rules/order_names.py" +CUSTOM_RULE_SOURCE: str = """from sqlbuild.rules import Finding, Model, RuleContext, rule + + +@rule( + code="XSQBRDIFF001", + message="Model names stay below sixty characters", + remediation="Rename the model.", +) +def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: + if len(model.name) < 60: + return [] + return [ctx.finding(subject=model)] +""" +CUSTOM_RULE_THRESHOLDS: str = "\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" +COLD_COMPILE: DifferentialCommand = DifferentialCommand( + label="compile", arguments=("compile", "--json", "--manifest", "--dag") +) +WARM_COMPILE: DifferentialCommand = DifferentialCommand( + label="compile-warm", arguments=("compile", "--json") +) +PLAN: DifferentialCommand = DifferentialCommand(label=PLAN_LABEL, arguments=("plan", "--json")) + +GENERATOR_DOMAINS: tuple[str, ...] = ("sales", "inventory", "support", "fulfillment", "billing") +GENERATOR_LAYERS: tuple[str, ...] = ("staging", "marts") +GENERATOR_SOURCE_COLUMNS: tuple[tuple[str, str], ...] = ( + ("id", "INTEGER"), + ("amount", "DOUBLE"), + ("status", "VARCHAR"), + ("created_at", "TIMESTAMP"), +) +GENERATOR_STATUS_VALUES: tuple[tuple[str, str], ...] = ( + ("PLACED", "placed"), + ("SHIPPED", "expédié"), + ("CANCELLED", "取消"), + ("RETURNED", "returned ✓"), +) +GENERATOR_NON_ASCII_LABELS: tuple[str, ...] = ( + "Zürich – depot", + "東京 warehouse", + "café ☕ counter", + "naïve résumé", + "Ελληνικά orders", +) +GENERATOR_FLOAT_VALUES: tuple[str, ...] = ("0.15", "1e-07", "12345.678", "0.1", "2.5e+20", "-3.75") +GENERATOR_REGION_ENV_VAR: str = "SQB_DIFFERENTIAL_REGION" +GENERATOR_INVALID_SHARE: float = 0.25 +GENERATOR_INVALID_CODES: dict[str, str] = { + "unknown_ref": "P001", + "unknown_macro": "P001", + "failing_macro": "P001", + "unknown_enum_member": "P001", + "unknown_constant": "P001", + "missing_description": "P010", + "duplicate_macro": "P001", + "header_syntax": "D002", + "over_broad_constant": "S024", +} +GENERATOR_HALF: float = 0.5 +GENERATOR_CROSS_DOMAIN_SHARE: float = 0.4 +GENERATOR_NON_ASCII_DESCRIPTION_SHARE: float = 0.3 +GENERATOR_VARIABLE_SHARE: float = 0.3 + +FAILURE_BASE_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "duckdb"\n\n[connection]\ndatabase = "failure.duckdb"\n' +) +FAILURE_BASE_SOURCES: str = """sources: + - name: raw_orders + description: Orders feed. + expression: >- + (SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, + 'placed' AS status) + columns: + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: DOUBLE + - name: status + type: VARCHAR +""" +FAILURE_BASE_STAGING: str = """MODEL ( + description "Staged orders", +); + +SELECT order_id, customer_id, amount, status +FROM __source("raw_orders") +""" +FAILURE_BASE_MART: str = """MODEL ( + description "Order totals per customer", +); + +SELECT customer_id, SUM(amount) AS total_amount +FROM __ref("stg_orders") +GROUP BY customer_id +""" +FAILURE_BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": FAILURE_BASE_CONFIG, + "sources/raw.yml": FAILURE_BASE_SOURCES, + "models/staging/stg_orders.sql": FAILURE_BASE_STAGING, + "models/marts/customer_totals.sql": FAILURE_BASE_MART, +} +FAILURE_STAGING_PATH: str = "models/staging/stg_orders.sql" +FAILURE_MART_PATH: str = "models/marts/customer_totals.sql" +FAILURE_CONFIG_PATH: str = "sqlbuild_project.toml" +FAILURE_SOURCES_PATH: str = "sources/raw.yml" +GENERATOR_ENUM_KIND: str = "enum" +GENERATOR_CONSTANT_KIND: str = "constant" +GENERATOR_MACRO_KIND: str = "macro" +GENERATOR_HOOK_KIND: str = "hook" +GENERATOR_ROLES: dict[str, str] = { + GENERATOR_ENUM_KIND: "enums", + GENERATOR_CONSTANT_KIND: "constants", + GENERATOR_MACRO_KIND: "macros", + GENERATOR_HOOK_KIND: "hooks", +} +GENERATOR_SUFFIXES: dict[str, str] = { + GENERATOR_ENUM_KIND: ".sql", + GENERATOR_CONSTANT_KIND: ".sql", + GENERATOR_MACRO_KIND: ".py", +} diff --git a/scripts/compiler_differential/exceptions.py b/scripts/compiler_differential/exceptions.py new file mode 100644 index 0000000000..21f24d38ee --- /dev/null +++ b/scripts/compiler_differential/exceptions.py @@ -0,0 +1,5 @@ +"""Compiler differential harness errors.""" + + +class DifferentialUsageError(ValueError): + """Raised when the harness is invoked with an invalid option value.""" diff --git a/scripts/compiler_differential/main/__init__.py b/scripts/compiler_differential/main/__init__.py new file mode 100644 index 0000000000..e716df0fcd --- /dev/null +++ b/scripts/compiler_differential/main/__init__.py @@ -0,0 +1 @@ +"""Public compiler differential entry points.""" diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py new file mode 100644 index 0000000000..ea25888d9d --- /dev/null +++ b/scripts/compiler_differential/main/differential.py @@ -0,0 +1,91 @@ +"""Compare compile and plan output of two compiler engines across a project corpus.""" + +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +import time +from pathlib import Path + +from scripts.compiler_differential._helpers.options import parse_expected_outcome +from scripts.compiler_differential._helpers.report import format_summary +from scripts.compiler_differential._helpers.run import ( + compare_corpus, + differential_options, + selected_corpus, +) +from scripts.compiler_differential.constants import ( + CORPUS_NAMES, + DEFAULT_DENSE_MODELS, + DEFAULT_ENGINES, + DEFAULT_SEED_COUNT, + EXPECT_SUCCESS, + PER_PULL_REQUEST_CORPORA, +) +from scripts.compiler_differential.models import DifferentialOptions, ProjectComparison + + +def run_compiler_differential(argv: list[str] | None = None) -> int: + """Run the selected corpus under both engines; exit 1 when any artifact differs.""" + + args: argparse.Namespace = _parse_args(argv) + repo_root: Path = Path(__file__).resolve().parents[3] + started: float = time.monotonic() + with tempfile.TemporaryDirectory(prefix="sqlbuild-differential-") as temporary: + work_dir: Path = (args.work_dir or Path(temporary)).absolute() + options: DifferentialOptions = differential_options(args=args, work_dir=work_dir) + comparisons: list[ProjectComparison] = compare_corpus( + corpus=selected_corpus(args=args, repo_root=repo_root), options=options + ) + print( + format_summary( + comparisons=comparisons, + engines=options.engines, + seconds=time.monotonic() - started, + ) + ) + return 1 if any(comparison.differences for comparison in comparisons) else 0 + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + prog="python -m scripts.run_compiler_differential", + description="Compare sqb compile and plan output between two compiler engines.", + ) + parser.add_argument( + "--corpus", + nargs="*", + choices=CORPUS_NAMES, + default=list(PER_PULL_REQUEST_CORPORA), + help="corpora to compare (default: the per-PR corpora; pass none with --project)", + ) + parser.add_argument("--project", nargs="*", type=Path, default=[], help="extra project dirs") + parser.add_argument( + "--expect", + type=parse_expected_outcome, + default=EXPECT_SUCCESS, + metavar="success|failure:CODE", + help="outcome every --project must produce on the first engine (default: success)", + ) + parser.add_argument("--seeds", type=int, default=DEFAULT_SEED_COUNT) + parser.add_argument("--seed-start", type=int, default=0) + parser.add_argument("--dense-models", type=int, default=DEFAULT_DENSE_MODELS) + parser.add_argument("--engines", nargs=2, default=list(DEFAULT_ENGINES), metavar="ENGINE") + parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1) + parser.add_argument( + "--stage-captures", + action="store_true", + help="also dump and diff every frontier object to localise a mismatch to a stage", + ) + parser.add_argument( + "--engine-env", + action="append", + default=[], + metavar="ENGINE:NAME=VALUE", + help="extra environment for one engine's processes", + ) + parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here") + parser.add_argument("--python", type=Path, default=Path(sys.executable)) + return parser.parse_args(argv) diff --git a/scripts/compiler_differential/main/failure_cases.py b/scripts/compiler_differential/main/failure_cases.py new file mode 100644 index 0000000000..5d6352bdf7 --- /dev/null +++ b/scripts/compiler_differential/main/failure_cases.py @@ -0,0 +1,10 @@ +"""The failure corpus: one minimal failing project per diagnostic family member.""" + +from scripts.compiler_differential._helpers.failure_cases import all_failure_cases +from scripts.compiler_differential.models import FailureCase + + +def failure_cases() -> tuple[FailureCase, ...]: + """Return every failure case in a stable order.""" + + return all_failure_cases() diff --git a/scripts/compiler_differential/main/generate_project.py b/scripts/compiler_differential/main/generate_project.py new file mode 100644 index 0000000000..5dd0c654d9 --- /dev/null +++ b/scripts/compiler_differential/main/generate_project.py @@ -0,0 +1,10 @@ +"""Generate one deterministic random project for a seed.""" + +from scripts.compiler_differential.classes.project_builder import ProjectBuilder +from scripts.compiler_differential.models import GeneratedProject + + +def generate_project(*, seed: int) -> GeneratedProject: + """Return the same project files and expectation for the same seed on every platform.""" + + return ProjectBuilder(seed).build() diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py new file mode 100644 index 0000000000..7da950905d --- /dev/null +++ b/scripts/compiler_differential/models.py @@ -0,0 +1,159 @@ +"""Typed records for the compiler engine differential harness.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True) +class DifferentialCommand: + """One sqb invocation run identically under both engines.""" + + label: str + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class ExpectedOutcome: + """What a corpus project must produce: success, or failure with one diagnostic code.""" + + error_code: str | None = None + + +@dataclass(frozen=True) +class CorpusProject: + """One project compared across engines.""" + + name: str + commands: tuple[DifferentialCommand, ...] + expected: ExpectedOutcome + source_dir: Path | None = None + writer: Callable[[Path], None] | None = None + project_subdirectory: str | None = None + + +@dataclass(frozen=True) +class CommandOutcome: + """What one sqb invocation printed and returned.""" + + label: str + exit_code: int + stdout: str + stderr: str + timed_out: bool = False + + +@dataclass(frozen=True) +class EngineRun: + """Everything one engine produced for one corpus project.""" + + engine: str + outcomes: tuple[CommandOutcome, ...] + compiled: dict[str, bytes] + manifest: str | None + dag: str | None + captures: dict[str, dict[str, str]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class Divergence: + """Where two values first differ, with short previews of both sides.""" + + location: str + left: str + right: str + + +@dataclass(frozen=True) +class Difference: + """The first observed divergence in one compared artifact.""" + + project: str + artifact: str + location: str + left: str + right: str + labels: tuple[str, str] | None = None + + +@dataclass(frozen=True) +class ProjectComparison: + """All differences found for one corpus project.""" + + project: str + differences: tuple[Difference, ...] + seconds: float + + +@dataclass(frozen=True) +class DifferentialOptions: + """Resolved harness settings.""" + + engines: tuple[str, str] + work_dir: Path + jobs: int + stage_captures: bool + python: Path + engine_environment: dict[str, dict[str, str]] + + +@dataclass(frozen=True) +class WritableProject: + """Project files that can be written below a fresh directory.""" + + files: dict[str, str] + + def write(self, project_dir: Path) -> None: + """Write every UTF-8 file, creating parent directories.""" + + for relative_path, contents in sorted(self.files.items()): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8", newline="\n") + + +@dataclass(frozen=True) +class GeneratedProject(WritableProject): + """A deterministic random project and the error it was generated to produce, if any.""" + + seed: int + expected_error_code: str | None + features: tuple[str, ...] + + +@dataclass(frozen=True) +class FailureCase(WritableProject): + """One minimal project that must fail compile with one diagnostic code.""" + + name: str + expected_code: str + + +@dataclass(frozen=True) +class ModelPlan: + """One generated SQL model and the relations it reads.""" + + name: str + domain: str + layer: str + inputs: tuple[str, ...] + description: str + + @property + def folder(self) -> str: + """Return the model's folder relative to the project root.""" + + return f"models/{self.domain}/{self.layer}" + + +@dataclass(frozen=True) +class DeclarationPlan: + """One generated enum, constant, macro, or named hook and the models that use it.""" + + kind: str + name: str + consumers: tuple[ModelPlan, ...] + body: str = "" + private: bool = False diff --git a/scripts/run_compiler_differential.py b/scripts/run_compiler_differential.py new file mode 100644 index 0000000000..0ffb26aada --- /dev/null +++ b/scripts/run_compiler_differential.py @@ -0,0 +1,15 @@ +"""Direct wrapper for the compiler engine differential harness.""" + +from __future__ import annotations + +from scripts.compiler_differential.main.differential import run_compiler_differential + + +def main(argv: list[str] | None = None) -> int: + """Compare compile and plan output between the Python and native compiler engines.""" + + return run_compiler_differential(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/e2e/scripts/compiler_differential/__init__.py b/tests/e2e/scripts/compiler_differential/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py new file mode 100644 index 0000000000..9e0b02f814 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -0,0 +1,27 @@ +"""Test case types for the compiler differential harness end-to-end tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class HarnessRunTestCase: + """One harness run over a fixture and the report lines it must and must not print.""" + + description: str + extra_arguments: tuple[str, ...] + expected_exit_code: int + expected_lines: tuple[str, ...] + expected_patterns: tuple[str, ...] + expected_absent: tuple[str, ...] + + +@dataclass(frozen=True) +class ProjectExpectationTestCase: + """A broken project run with one --expect value and the verdict the harness must reach.""" + + description: str + extra_arguments: tuple[str, ...] + expected_exit_code: int + expected_lines: tuple[str, ...] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py new file mode 100644 index 0000000000..a5488ddd3c --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -0,0 +1,87 @@ +"""Harness invocations and a perturbation that makes one engine's output differ.""" + +from __future__ import annotations + +from pathlib import Path + +NATIVE_ONLY_STDERR_LINE: str = "native engine perturbation active" +WAFFLE_SHOP_FIXTURE: Path = Path(__file__).resolve().parents[2] / "fixtures" / "waffle_shop" +_SITECUSTOMIZE: str = f''' +import dataclasses +import sys + +import sqlbuild.compiler.frontier.main._compile_frontier as frontier +from sqlbuild.compiler.frontier.types import CompilerStage + +_original = frontier.native_frontier + + +def _perturbed(*, until, python_stage): + result = _original(until=until, python_stage=python_stage) + if until is not CompilerStage.COMPILED_PROJECT or not result.models: + return result + first = result.models[0] + changed = dataclasses.replace( + first, query_sql=f"SELECT * FROM ({{first.query_sql}}) AS perturbed_by_test" + ) + return dataclasses.replace(result, models=(changed, *result.models[1:])) + + +frontier.native_frontier = _perturbed +print("{NATIVE_ONLY_STDERR_LINE}", file=sys.stderr) +''' + +_BROKEN_REF_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "broken_orders"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "broken_orders.duckdb"\n' + ), + "models/order_totals.sql": ( + 'MODEL (\n description "Order totals",\n);\n\nSELECT * FROM __ref("nope")\n' + ), +} + + +def write_broken_ref_project(directory: Path) -> Path: + """Write a project whose only model references a model that does not exist.""" + + for relative_path, contents in _BROKEN_REF_FILES.items(): + path: Path = directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8", newline="\n") + return directory + + +def write_native_perturbation(directory: Path) -> Path: + """Write a sitecustomize module that changes the first compiled model on the native path.""" + + directory.mkdir(parents=True, exist_ok=True) + _ = (directory / "sitecustomize.py").write_text(_SITECUSTOMIZE, encoding="utf-8") + return directory + + +def harness_arguments( + *, work_dir: Path, extra: tuple[str, ...], project: Path = WAFFLE_SHOP_FIXTURE +) -> list[str]: + """Compare only one project (the waffle shop fixture by default), keeping run directories.""" + + return [ + "--corpus", + "--project", + str(project), + "--jobs", + "1", + "--work-dir", + str(work_dir), + *extra, + ] + + +def perturbation_arguments(directory: Path) -> tuple[str, ...]: + """Return the options that load the perturbation into native-engine processes only.""" + + return ( + "--stage-captures", + "--engine-env", + f"native:PYTHONPATH={write_native_perturbation(directory)}", + ) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py new file mode 100644 index 0000000000..54e646dd33 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -0,0 +1,143 @@ +"""The differential harness passes identical engines and pinpoints a perturbed one.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from scripts.compiler_differential.main.differential import run_compiler_differential +from tests.e2e.scripts.compiler_differential._test_types import ( + HarnessRunTestCase, + ProjectExpectationTestCase, +) +from tests.e2e.scripts.compiler_differential.helpers import ( + NATIVE_ONLY_STDERR_LINE, + harness_arguments, + perturbation_arguments, + write_broken_ref_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="identical_engines", + extra_arguments=(), + expected_exit_code=0, + expected_lines=( + "OK project/waffle_shop", + "Compiler differential passed: 1 projects identical (python vs native)", + ), + expected_patterns=(), + expected_absent=("DIFF",), + ) + ], + ids=lambda case: case.description, +) +def test_given_identical_engines_when_comparing_fixture_then_harness_passes( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments(work_dir=tmp_path / "work", extra=test_case.extra_arguments) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert not any(text in output for text in test_case.expected_absent), output + + +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="perturbed_native_compiled_project", + extra_arguments=(), + expected_exit_code=1, + expected_lines=( + "DIFF project/waffle_shop", + "- `compile` stdout at /resources/models/0/query_sql", + 'native: "SELECT * FROM (', + "- `compile` stderr at line 1", + f'native: "{NATIVE_ONLY_STDERR_LINE}"', + "- stage capture 0-compile/003-compiled_project.json at /models/0/query_sql", + "First difference: project/waffle_shop: `compile` stdout at", + ), + expected_patterns=( + r"- target/compiled/models/\S+\.sql at line \d+", + r"- query fingerprint of model \w+ \(manifest query hash\)", + r"- query fingerprint of model \w+ \(plan version hash\)", + ), + expected_absent=("discovered_project_inputs.json", "compile_project_inputs.json"), + ) + ], + ids=lambda case: case.description, +) +def test_given_perturbed_native_engine_when_comparing_then_every_artifact_is_located( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=(*perturbation_arguments(tmp_path / "perturbation"), *test_case.extra_arguments), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output + assert not any(text in output for text in test_case.expected_absent), output + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectExpectationTestCase( + description="success_expected_by_default", + extra_arguments=(), + expected_exit_code=1, + expected_lines=( + "DIFF project/broken_orders", + "- corpus expectation at expected success", + ), + ), + ProjectExpectationTestCase( + description="declared_failure_code", + extra_arguments=("--expect", "failure:P001"), + expected_exit_code=0, + expected_lines=("OK project/broken_orders",), + ), + ProjectExpectationTestCase( + description="wrong_failure_code", + extra_arguments=("--expect", "failure:P010"), + expected_exit_code=1, + expected_lines=( + "DIFF project/broken_orders", + "- corpus expectation at expected failure", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_fails( + test_case: ProjectExpectationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=test_case.extra_arguments, + project=write_broken_ref_project(tmp_path / "broken_orders"), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/__init__.py b/tests/unit/scripts/compiler_differential/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/__init__.py b/tests/unit/scripts/compiler_differential/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/_test_types.py new file mode 100644 index 0000000000..d605e21b48 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/_test_types.py @@ -0,0 +1,101 @@ +"""Test case types for the differential harness comparison helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.compiler_differential.models import ExpectedOutcome + + +@dataclass(frozen=True) +class JsonDifferenceTestCase: + """Two JSON documents and where the harness must report their first difference.""" + + description: str + left: object + right: object + expected_location: str | None + + +@dataclass(frozen=True) +class TextDifferenceTestCase: + """Two texts and the first differing line the harness must report.""" + + description: str + left: str + right: str + expected_location: str | None + + +@dataclass(frozen=True) +class PreviewTestCase: + """A long differing value and the bounded preview the report shows.""" + + description: str + left: object + right: object + expected_max_length: int + expected_suffix: str + + +@dataclass(frozen=True) +class NormalizationTestCase: + """Raw output and the normalized text the harness compares.""" + + description: str + raw: str + expected_text: str + + +@dataclass(frozen=True) +class PayloadStripTestCase: + """A decoded report or manifest and what remains once volatile fields are removed.""" + + description: str + payload: object + expected_payload: object + + +@dataclass(frozen=True) +class EngineEnvironmentTestCase: + """Repeated --engine-env values and the per-engine overrides they produce.""" + + description: str + values: tuple[str, ...] + expected_environment: dict[str, dict[str, str]] + + +@dataclass(frozen=True) +class EngineEnvironmentErrorTestCase: + """A malformed --engine-env value and the usage error it raises.""" + + description: str + value: str + expected_message: str + + +@dataclass(frozen=True) +class ExpectedOutcomeTestCase: + """One --expect value and the outcome it parses to.""" + + description: str + value: str + expected_outcome: ExpectedOutcome + + +@dataclass(frozen=True) +class ExpectedOutcomeErrorTestCase: + """One malformed --expect value.""" + + description: str + value: str + expected_message: str + + +@dataclass(frozen=True) +class CorpusExpectationTestCase: + """A corpus selection and the expectation each named entry must declare.""" + + description: str + corpora: tuple[str, ...] + expected_outcomes: dict[str, ExpectedOutcome] diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_compare.py b/tests/unit/scripts/compiler_differential/_helpers/test_compare.py new file mode 100644 index 0000000000..44c711296c --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_compare.py @@ -0,0 +1,128 @@ +"""First-difference reports name the exact JSON pointer or line that diverged.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential._helpers.compare import ( + first_json_difference, + first_text_difference, +) +from scripts.compiler_differential.models import Divergence +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + JsonDifferenceTestCase, + PreviewTestCase, + TextDifferenceTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JsonDifferenceTestCase( + description="identical", + left={"orders": [1, 2]}, + right={"orders": [1, 2]}, + expected_location=None, + ), + JsonDifferenceTestCase( + description="nested_value", + left={"resources": {"models": [{"name": "orders", "sql": "SELECT 1"}]}}, + right={"resources": {"models": [{"name": "orders", "sql": "SELECT 2"}]}}, + expected_location="/resources/models/0/sql", + ), + JsonDifferenceTestCase( + description="first_of_several", + left={"a": 1, "b": [1, 2, 3]}, + right={"a": 1, "b": [1, 9, 8]}, + expected_location="/b/1", + ), + JsonDifferenceTestCase( + description="extra_list_item", + left={"diagnostics": [{"code": "P010"}]}, + right={"diagnostics": [{"code": "P010"}, {"code": "S024"}]}, + expected_location="/diagnostics/1", + ), + JsonDifferenceTestCase( + description="missing_key", + left={"summary": {"models": 2, "seeds": 1}}, + right={"summary": {"models": 2}}, + expected_location="/summary/seeds", + ), + JsonDifferenceTestCase( + description="key_order", + left={"name": "orders", "path": "models/orders.sql"}, + right={"path": "models/orders.sql", "name": "orders"}, + expected_location="/ (key order)", + ), + JsonDifferenceTestCase( + description="integer_versus_float", + left={"ratio": 1}, + right={"ratio": 1.0}, + expected_location="/ratio", + ), + JsonDifferenceTestCase( + description="escaped_pointer_token", + left={"models/orders.sql": {"a~b": 1}}, + right={"models/orders.sql": {"a~b": 2}}, + expected_location="/models~1orders.sql/a~0b", + ), + ], + ids=lambda case: case.description, +) +def test_given_two_documents_when_comparing_then_first_difference_pointer_is_reported( + test_case: JsonDifferenceTestCase, +) -> None: + found: Divergence | None = first_json_difference(left=test_case.left, right=test_case.right) + + assert getattr(found, "location", None) == test_case.expected_location + + +@pytest.mark.parametrize( + "test_case", + [ + TextDifferenceTestCase( + description="identical", left="a\nb\n", right="a\nb\n", expected_location=None + ), + TextDifferenceTestCase( + description="second_line", left="a\nb\nc", right="a\nx\nc", expected_location="line 2" + ), + TextDifferenceTestCase( + description="trailing_line", left="a\nb", right="a\nb\nc", expected_location="line 3" + ), + ], + ids=lambda case: case.description, +) +def test_given_two_texts_when_comparing_then_first_differing_line_is_reported( + test_case: TextDifferenceTestCase, +) -> None: + found: Divergence | None = first_text_difference(left=test_case.left, right=test_case.right) + + assert getattr(found, "location", None) == test_case.expected_location + + +@pytest.mark.parametrize( + "test_case", + [ + PreviewTestCase( + description="long_sql", + left={"sql": "SELECT " + "a, " * 200}, + right={"sql": "SELECT " + "b, " * 200}, + expected_max_length=160, + expected_suffix="...", + ) + ], + ids=lambda case: case.description, +) +def test_given_long_differing_values_when_reporting_then_previews_are_bounded( + test_case: PreviewTestCase, +) -> None: + found: Divergence | None = first_json_difference(left=test_case.left, right=test_case.right) + + assert found is not None + assert len(found.left) <= test_case.expected_max_length + assert found.left.endswith(test_case.expected_suffix) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py b/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py new file mode 100644 index 0000000000..f35dfd35e1 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py @@ -0,0 +1,69 @@ +"""Every corpus entry declares the outcome both engines must produce.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.corpus import build_corpus +from scripts.compiler_differential.models import CorpusProject, ExpectedOutcome +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + CorpusExpectationTestCase, +) + +_REPO_ROOT: Path = Path(__file__).resolve().parents[5] + + +@pytest.mark.parametrize( + "test_case", + [ + CorpusExpectationTestCase( + description="fixtures_succeed_except_dbt_interop", + corpora=("fixtures",), + expected_outcomes={ + "fixture/waffle_shop": ExpectedOutcome(), + "fixture/dbt_interop": ExpectedOutcome(error_code="C214"), + }, + ), + CorpusExpectationTestCase( + description="examples_succeed", + corpora=("examples",), + expected_outcomes={"example/waffle-shop": ExpectedOutcome()}, + ), + CorpusExpectationTestCase( + description="dense_succeeds", + corpora=("dense",), + expected_outcomes={ + "dense/20": ExpectedOutcome(), + "dense/20-custom-rules": ExpectedOutcome(), + }, + ), + CorpusExpectationTestCase( + description="extra_projects_take_the_declared_expectation", + corpora=(), + expected_outcomes={"project/orders": ExpectedOutcome(error_code="P001")}, + ), + ], + ids=lambda case: case.description, +) +def test_given_corpus_selection_when_building_then_entries_declare_expected_outcomes( + test_case: CorpusExpectationTestCase, +) -> None: + corpus: list[CorpusProject] = build_corpus( + repo_root=_REPO_ROOT, + corpora=test_case.corpora, + seeds=range(0), + dense_models=20, + projects=(Path("/projects/orders"),), + project_expectation=ExpectedOutcome(error_code="P001"), + ) + outcomes: dict[str, ExpectedOutcome] = {entry.name: entry.expected for entry in corpus} + + assert { + name: outcomes.get(name) for name in test_case.expected_outcomes + } == test_case.expected_outcomes + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py b/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py new file mode 100644 index 0000000000..a89a893598 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py @@ -0,0 +1,117 @@ +"""Normalization removes only run-specific noise before engines are compared.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential._helpers.normalize import ( + normalize_artifact_text, + normalize_stderr, + strip_manifest_metadata, + strip_report_fields, +) +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + NormalizationTestCase, + PayloadStripTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NormalizationTestCase( + description="phase_timing", + raw="Project compile OK (0.23s)", + expected_text="Project compile OK ()", + ), + NormalizationTestCase( + description="rules_timing", + raw="Evaluated rules. (0.02s; built-in 0.00s, custom 12ms)", + expected_text="Evaluated rules. (; built-in , custom )", + ), + NormalizationTestCase( + description="engine_store_path", + raw="wrote target/cache/compiler-native-v1/facts", + expected_text="wrote target/cache/compiler/facts", + ), + ], + ids=lambda case: case.description, +) +def test_given_stderr_when_normalizing_then_elapsed_times_are_masked( + test_case: NormalizationTestCase, +) -> None: + assert normalize_stderr(test_case.raw) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + NormalizationTestCase( + description="invocation_id", + raw='"run_id": "20261006T172510Z_dee386e8ffa7"', + expected_text='"run_id": ""', + ), + NormalizationTestCase( + description="durations_are_kept_in_artifacts", + raw="SELECT '5s' AS label", + expected_text="SELECT '5s' AS label", + ), + ], + ids=lambda case: case.description, +) +def test_given_artifact_text_when_normalizing_then_only_run_identity_is_masked( + test_case: NormalizationTestCase, +) -> None: + assert normalize_artifact_text(test_case.raw) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + PayloadStripTestCase( + description="compile_report", + payload={ + "summary": {"models": 2}, + "compile_timings": {"total_ms": 10}, + "compiler_engine": "native", + }, + expected_payload={"summary": {"models": 2}}, + ), + PayloadStripTestCase( + description="non_object_report", payload=[1, 2], expected_payload=[1, 2] + ), + ], + ids=lambda case: case.description, +) +def test_given_compile_report_when_stripping_then_only_timings_and_engine_are_removed( + test_case: PayloadStripTestCase, +) -> None: + assert strip_report_fields(test_case.payload) == test_case.expected_payload + + +@pytest.mark.parametrize( + "test_case", + [ + PayloadStripTestCase( + description="manifest_metadata", + payload={ + "metadata": { + "generated_at": "2026-10-06T17:25:10+00:00", + "invocation_id": "20261006T172510Z_dee386e8ffa7", + "project_name": "orders", + }, + "nodes": {}, + }, + expected_payload={"metadata": {"project_name": "orders"}, "nodes": {}}, + ) + ], + ids=lambda case: case.description, +) +def test_given_manifest_when_stripping_then_generation_identity_is_removed( + test_case: PayloadStripTestCase, +) -> None: + assert strip_manifest_metadata(test_case.payload) == test_case.expected_payload + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_options.py b/tests/unit/scripts/compiler_differential/_helpers/test_options.py new file mode 100644 index 0000000000..83a3bd61ac --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_options.py @@ -0,0 +1,107 @@ +"""Per-engine environment options parse into exact overrides.""" + +from __future__ import annotations + +import argparse + +import pytest + +from scripts.compiler_differential._helpers.options import ( + parse_engine_environment, + parse_expected_outcome, +) +from scripts.compiler_differential.exceptions import DifferentialUsageError +from scripts.compiler_differential.models import ExpectedOutcome +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + EngineEnvironmentErrorTestCase, + EngineEnvironmentTestCase, + ExpectedOutcomeErrorTestCase, + ExpectedOutcomeTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineEnvironmentTestCase( + description="grouped_by_engine", + values=("native:PYTHONPATH=/tmp/hook", "native:ORDERS_REGION=a=b", "python:EMPTY="), + expected_environment={ + "native": {"PYTHONPATH": "/tmp/hook", "ORDERS_REGION": "a=b"}, + "python": {"EMPTY": ""}, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_engine_assignments_when_parsing_then_values_group_by_engine( + test_case: EngineEnvironmentTestCase, +) -> None: + assert parse_engine_environment(list(test_case.values)) == test_case.expected_environment + + +@pytest.mark.parametrize( + "test_case", + [ + EngineEnvironmentErrorTestCase( + description=value.replace(":", "_").replace("=", "_") or "empty", + value=value, + expected_message="ENGINE:NAME=VALUE", + ) + for value in ("native", "native:NAME", ":NAME=1", "native:=1") + ], + ids=lambda case: case.description, +) +def test_given_malformed_assignment_when_parsing_then_usage_error_is_raised( + test_case: EngineEnvironmentErrorTestCase, +) -> None: + with pytest.raises(DifferentialUsageError, match=test_case.expected_message): + _ = parse_engine_environment([test_case.value]) + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedOutcomeTestCase( + description="success", value="success", expected_outcome=ExpectedOutcome() + ), + ExpectedOutcomeTestCase( + description="failure_code", + value="failure:P001", + expected_outcome=ExpectedOutcome(error_code="P001"), + ), + ExpectedOutcomeTestCase( + description="rule_code", + value="failure:SQBRSQL005", + expected_outcome=ExpectedOutcome(error_code="SQBRSQL005"), + ), + ], + ids=lambda case: case.description, +) +def test_given_expect_value_when_parsing_then_outcome_matches( + test_case: ExpectedOutcomeTestCase, +) -> None: + assert parse_expected_outcome(test_case.value) == test_case.expected_outcome + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedOutcomeErrorTestCase( + description=value.replace(":", "_") or "empty", + value=value, + expected_message="success or failure:", + ) + for value in ("", "failure", "failure:", "failure:p001", "P001", "ok") + ], + ids=lambda case: case.description, +) +def test_given_malformed_expect_value_when_parsing_then_argument_error_is_raised( + test_case: ExpectedOutcomeErrorTestCase, +) -> None: + with pytest.raises(argparse.ArgumentTypeError, match=test_case.expected_message): + _ = parse_expected_outcome(test_case.value) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/main/__init__.py b/tests/unit/scripts/compiler_differential/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/main/_test_types.py b/tests/unit/scripts/compiler_differential/main/_test_types.py new file mode 100644 index 0000000000..78f387505f --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/_test_types.py @@ -0,0 +1,47 @@ +"""Test case types for the generator and failure corpus entry points.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.compiler_differential.models import FailureCase + + +@dataclass(frozen=True) +class SeedDeterminismTestCase: + """One seed generated twice.""" + + description: str + seed: int + expected_identical: bool + + +@dataclass(frozen=True) +class SeedRangeTestCase: + """A seed range and what the generated projects must cover together.""" + + description: str + seeds: range + expected_features: frozenset[str] + expected_distinct_projects: int + expected_min_invalid: int + expected_max_invalid: int + + +@dataclass(frozen=True) +class FailureCorpusTestCase: + """The failure corpus and the breadth it must keep.""" + + description: str + expected_min_cases: int + expected_min_codes: int + expected_min_families: int + + +@dataclass(frozen=True) +class FailureCaseWriteTestCase: + """One failure case and the files its written project must contain.""" + + description: str + case: FailureCase + expected_files: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/main/helpers.py b/tests/unit/scripts/compiler_differential/main/helpers.py new file mode 100644 index 0000000000..daff4ea8e3 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/helpers.py @@ -0,0 +1,27 @@ +"""Helpers for generator and failure corpus tests.""" + +from __future__ import annotations + +from scripts.compiler_differential.main.generate_project import generate_project +from scripts.compiler_differential.models import GeneratedProject + + +def generated_projects(seeds: range) -> list[GeneratedProject]: + """Generate one project per seed.""" + + return [generate_project(seed=seed) for seed in seeds] + + +def project_features(projects: list[GeneratedProject]) -> frozenset[str]: + """Return every feature any project exercises.""" + + features: set[str] = set() + for project in projects: + features.update(project.features) + return frozenset(features) + + +def invalid_count(projects: list[GeneratedProject]) -> int: + """Count projects generated to fail with a known code.""" + + return sum(project.expected_error_code is not None for project in projects) diff --git a/tests/unit/scripts/compiler_differential/main/test_failure_cases.py b/tests/unit/scripts/compiler_differential/main/test_failure_cases.py new file mode 100644 index 0000000000..e8076319c3 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/test_failure_cases.py @@ -0,0 +1,63 @@ +"""The failure corpus stays broad, uniquely named, and writes complete projects.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.compiler_differential.main.failure_cases import failure_cases +from scripts.compiler_differential.models import FailureCase +from tests.unit.scripts.compiler_differential.main._test_types import ( + FailureCaseWriteTestCase, + FailureCorpusTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + FailureCorpusTestCase( + description="breadth", + expected_min_cases=45, + expected_min_codes=30, + expected_min_families=7, + ) + ], + ids=lambda case: case.description, +) +def test_given_failure_corpus_when_listing_then_names_are_unique_and_codes_are_broad( + test_case: FailureCorpusTestCase, +) -> None: + cases: tuple[FailureCase, ...] = failure_cases() + codes: set[str] = {case.expected_code for case in cases} + + assert len({case.name for case in cases}) == len(cases) + assert len(cases) >= test_case.expected_min_cases + assert len(codes) >= test_case.expected_min_codes + assert len({code.rstrip("0123456789") for code in codes}) >= test_case.expected_min_families + + +@pytest.mark.parametrize( + "test_case", + [ + FailureCaseWriteTestCase( + description=case.name, case=case, expected_files=tuple(sorted(case.files)) + ) + for case in failure_cases() + ], + ids=lambda case: case.description, +) +def test_given_failure_case_when_writing_then_every_file_exists( + test_case: FailureCaseWriteTestCase, tmp_path: Path +) -> None: + test_case.case.write(tmp_path) + + assert ( + tuple(sorted(path.relative_to(tmp_path).as_posix() for path in tmp_path.rglob("*.*"))) + == test_case.expected_files + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/main/test_generate_project.py b/tests/unit/scripts/compiler_differential/main/test_generate_project.py new file mode 100644 index 0000000000..bde93149c8 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/test_generate_project.py @@ -0,0 +1,90 @@ +"""Generated projects are deterministic per seed and cover the intended feature space.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential.main.generate_project import generate_project +from scripts.compiler_differential.models import GeneratedProject +from tests.unit.scripts.compiler_differential.main._test_types import ( + SeedDeterminismTestCase, + SeedRangeTestCase, +) +from tests.unit.scripts.compiler_differential.main.helpers import ( + generated_projects, + invalid_count, + project_features, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SeedDeterminismTestCase(description=f"seed_{seed}", seed=seed, expected_identical=True) + for seed in (0, 7, 41) + ], + ids=lambda case: case.description, +) +def test_given_same_seed_when_generating_twice_then_projects_are_identical( + test_case: SeedDeterminismTestCase, +) -> None: + first: GeneratedProject = generate_project(seed=test_case.seed) + second: GeneratedProject = generate_project(seed=test_case.seed) + + assert (first == second) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + SeedRangeTestCase( + description="first_sixty_seeds", + seeds=range(60), + expected_features=frozenset( + { + "exact_owner_scope", + "tree_scope", + "project_scope", + "model_private_value", + "relationship_grant", + "name_shared_across_namespaces", + "non_ascii_constant", + "float_constant", + "decimal_constant", + "integer_enum", + "macro_reads_ctx", + "macro_reads_env", + "nested_macro", + "path_defaults_literal", + "path_defaults_wildcard", + "sql_hooks", + "python_nodes", + "unit_test", + "scenario", + "singular_audit", + "column_audits", + } + ), + expected_distinct_projects=60, + expected_min_invalid=5, + expected_max_invalid=30, + ) + ], + ids=lambda case: case.description, +) +def test_given_seed_range_when_generating_then_features_and_failures_are_covered( + test_case: SeedRangeTestCase, +) -> None: + projects: list[GeneratedProject] = generated_projects(test_case.seeds) + + assert test_case.expected_features <= project_features(projects) + assert len({tuple(project.files.items()) for project in projects}) == ( + test_case.expected_distinct_projects + ) + assert ( + test_case.expected_min_invalid <= invalid_count(projects) <= test_case.expected_max_invalid + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 5b8a5b5a910d3f99aa34dbd64e5e654843a70789 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:31:28 +0100 Subject: [PATCH 3/6] ci: run the compiler engine differential per pull request and nightly --- .github/workflows/compiler-differential.yml | 44 +++++++++++++++++++++ .github/workflows/verify.yml | 44 +++++++++++++++++++-- Makefile | 19 ++++++++- 3 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/compiler-differential.yml diff --git a/.github/workflows/compiler-differential.yml b/.github/workflows/compiler-differential.yml new file mode 100644 index 0000000000..67604d4878 --- /dev/null +++ b/.github/workflows/compiler-differential.yml @@ -0,0 +1,44 @@ +name: Compiler Differential + +on: + schedule: + - cron: "43 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: compiler-differential-${{ github.ref }} + cancel-in-progress: false + +jobs: + dense: + name: Dense compiler differential (${{ matrix.models }} models) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + models: [3000, 10000] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Compare dense projects with and without custom rules + env: + MODELS: ${{ matrix.models }} + run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}" + diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 7cc298cad3..bad18bf43d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-24.04 outputs: checks: ${{ steps.scope.outputs.checks }} + compiler_differential: ${{ steps.scope.outputs.compiler_differential }} duckdb: ${{ steps.scope.outputs.duckdb }} postgres: ${{ steps.scope.outputs.postgres }} performance: ${{ steps.scope.outputs.performance }} @@ -43,11 +44,12 @@ jobs: echo "rules_performance=$5" echo "checks=$6" echo "wheels=$7" + echo "compiler_differential=$8" } >> "${GITHUB_OUTPUT}" } if [[ "${EVENT_NAME}" != "pull_request" ]]; then - write_scope true true true true true true true + write_scope true true true true true true true true exit 0 fi @@ -57,6 +59,7 @@ jobs: format_performance=false rules_performance=false wheels=false + compiler_differential=false broad=false changed=false # A change that only touches the website, root-level Markdown, the README demo GIFs or the @@ -77,6 +80,11 @@ jobs: wheels=true ;; esac + case "${path}" in + scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|.github/workflows/verify.yml|Makefile) + compiler_differential=true + ;; + esac case "${path}" in crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) broad=true @@ -132,7 +140,7 @@ jobs: done < <(git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}") if [[ "${changed}" != "true" ]]; then - write_scope true true true true true true true + write_scope true true true true true true true true exit 0 fi @@ -145,6 +153,7 @@ jobs: duckdb=true postgres=true format_performance=true + compiler_differential=true fi write_scope \ "${duckdb}" \ @@ -153,7 +162,8 @@ jobs: "${format_performance}" \ "${rules_performance}" \ "${checks}" \ - "${wheels}" + "${wheels}" \ + "${compiler_differential}" lockfile: name: Lockfile consistency @@ -258,6 +268,31 @@ jobs: - name: Run DuckDB E2E shard run: make ${{ matrix.target }} + compiler_differential: + name: Compiler engine differential + needs: [changes, lockfile] + if: needs.changes.outputs.compiler_differential == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Compare fixtures, examples, generated seeds and the failure corpus + run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures + format_performance: name: Formatter performance guards needs: [changes, lockfile] @@ -675,6 +710,7 @@ jobs: lockfile, checks, e2e_shards, + compiler_differential, format_performance, performance, cold_compile_performance, @@ -694,6 +730,7 @@ jobs: LOCKFILE_RESULT: ${{ needs.lockfile.result }} CHECKS_RESULT: ${{ needs.checks.result }} E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} + COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }} PERFORMANCE_RESULT: ${{ needs.performance.result }} COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }} @@ -715,6 +752,7 @@ jobs: fi for result in \ "${E2E_SHARDS_RESULT}" \ + "${COMPILER_DIFFERENTIAL_RESULT}" \ "${FORMAT_PERFORMANCE_RESULT}" \ "${PERFORMANCE_RESULT}" \ "${COLD_COMPILE_PERFORMANCE_RESULT}" \ diff --git a/Makefile b/Makefile index 16496970d5..9e4a3f80ff 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ SHELL := /bin/bash test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ test-e2e-duckdb-integrations test-e2e-performance \ test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ - test-e2e-dense-compile-performance test-e2e-varied-cache-performance + test-e2e-dense-compile-performance test-e2e-varied-cache-performance \ + compiler-differential compiler-differential-dense format: uv run ruff format . @@ -130,6 +131,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ tests/e2e/scripts/compile_performance_ratio \ + tests/e2e/scripts/compiler_differential \ tests/e2e/scripts/release_performance \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ @@ -285,6 +287,21 @@ test-e2e-varied-cache-performance: -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes +COMPILER_DIFFERENTIAL_JOBS ?= 4 +COMPILER_DIFFERENTIAL_SEEDS ?= 12 +COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000 +COMPILER_DIFFERENTIAL_ARGS ?= + +compiler-differential: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \ + --jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \ + $(COMPILER_DIFFERENTIAL_ARGS) + +compiler-differential-dense: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \ + --corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \ + --jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS) + skills: uv run python -m scripts.generate_docs_skill From 2bacbac0e41ce88b0edf7192eda8ba05004dc9b0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:43:58 +0100 Subject: [PATCH 4/6] fix(compiler): defer stage-capture annotations for Python 3.12 itertools.count is not subscriptable at runtime before Python 3.13, so the ClassVar annotation failed at import on 3.12. --- .../compiler/frontier/classes/stage_capture_sequence.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py index 0ec43b4b13..786ba5da74 100644 --- a/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py @@ -1,5 +1,7 @@ """Process-wide ordering of stage captures.""" +from __future__ import annotations + import itertools import threading from typing import ClassVar From 7e8a7caa6d7fc4590042ae7825766f60dbd82c5b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:51:34 +0100 Subject: [PATCH 5/6] test(compile): leave the engine label out of semantic compile fingerprints --- .../main/semantic_compile_fingerprint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py b/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py index b61fd19578..8a2adcb219 100644 --- a/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py +++ b/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py @@ -14,6 +14,7 @@ def semantic_compile_fingerprint(*, payload: dict[str, object], compiled_dir: Pa semantic_payload: dict[str, object] = dict(payload) semantic_payload.pop("compile_timings", None) + semantic_payload.pop("compiler_engine", None) semantic_payload.pop("version", None) fingerprint: Any = hashlib.sha256(orjson.dumps(semantic_payload, option=orjson.OPT_SORT_KEYS)) for artifact_path in sorted(path for path in compiled_dir.rglob("*") if path.is_file()): From f489772b272f01d01aae8f67f87def170f717f22 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:36:18 +0100 Subject: [PATCH 6/6] test(cli): match the engine choice error across Python versions --- .../cli/commands/_helpers/entry/test_compiler_engine_option.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py index 6e8c4c3bbe..273ba2609d 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py @@ -66,7 +66,7 @@ def test_given_engine_selection_when_parsing_then_flag_value_is_recorded( argv=("compile", "--compiler-engine", "rust"), environment_value="", expected_exit_code=2, - expected_error="invalid choice: 'rust' (choose from 'python', 'native')", + expected_error="argument --compiler-engine: invalid choice: 'rust'", ), RejectedCompilerEngineTestCase( description="unknown_environment_value",