diff --git a/.github/workflows/compiler-differential.yml b/.github/workflows/compiler-differential.yml new file mode 100644 index 0000000000..67604d4878 --- /dev/null +++ b/.github/workflows/compiler-differential.yml @@ -0,0 +1,44 @@ +name: Compiler Differential + +on: + schedule: + - cron: "43 3 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: compiler-differential-${{ github.ref }} + cancel-in-progress: false + +jobs: + dense: + name: Dense compiler differential (${{ matrix.models }} models) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + models: [3000, 10000] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Compare dense projects with and without custom rules + env: + MODELS: ${{ matrix.models }} + run: make compiler-differential-dense COMPILER_DIFFERENTIAL_DENSE_MODELS="${MODELS}" + diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 7cc298cad3..bad18bf43d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-24.04 outputs: checks: ${{ steps.scope.outputs.checks }} + compiler_differential: ${{ steps.scope.outputs.compiler_differential }} duckdb: ${{ steps.scope.outputs.duckdb }} postgres: ${{ steps.scope.outputs.postgres }} performance: ${{ steps.scope.outputs.performance }} @@ -43,11 +44,12 @@ jobs: echo "rules_performance=$5" echo "checks=$6" echo "wheels=$7" + echo "compiler_differential=$8" } >> "${GITHUB_OUTPUT}" } if [[ "${EVENT_NAME}" != "pull_request" ]]; then - write_scope true true true true true true true + write_scope true true true true true true true true exit 0 fi @@ -57,6 +59,7 @@ jobs: format_performance=false rules_performance=false wheels=false + compiler_differential=false broad=false changed=false # A change that only touches the website, root-level Markdown, the README demo GIFs or the @@ -77,6 +80,11 @@ jobs: wheels=true ;; esac + case "${path}" in + scripts/compiler_differential/*|scripts/run_compiler_differential.py|tests/*/scripts/compiler_differential/*|.github/workflows/verify.yml|Makefile) + compiler_differential=true + ;; + esac case "${path}" in crates/*|Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) broad=true @@ -132,7 +140,7 @@ jobs: done < <(git diff --name-only -z "${BASE_SHA}...${HEAD_SHA}") if [[ "${changed}" != "true" ]]; then - write_scope true true true true true true true + write_scope true true true true true true true true exit 0 fi @@ -145,6 +153,7 @@ jobs: duckdb=true postgres=true format_performance=true + compiler_differential=true fi write_scope \ "${duckdb}" \ @@ -153,7 +162,8 @@ jobs: "${format_performance}" \ "${rules_performance}" \ "${checks}" \ - "${wheels}" + "${wheels}" \ + "${compiler_differential}" lockfile: name: Lockfile consistency @@ -258,6 +268,31 @@ jobs: - name: Run DuckDB E2E shard run: make ${{ matrix.target }} + compiler_differential: + name: Compiler engine differential + needs: [changes, lockfile] + if: needs.changes.outputs.compiler_differential == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Compare fixtures, examples, generated seeds and the failure corpus + run: make compiler-differential COMPILER_DIFFERENTIAL_ARGS=--stage-captures + format_performance: name: Formatter performance guards needs: [changes, lockfile] @@ -675,6 +710,7 @@ jobs: lockfile, checks, e2e_shards, + compiler_differential, format_performance, performance, cold_compile_performance, @@ -694,6 +730,7 @@ jobs: LOCKFILE_RESULT: ${{ needs.lockfile.result }} CHECKS_RESULT: ${{ needs.checks.result }} E2E_SHARDS_RESULT: ${{ needs.e2e_shards.result }} + COMPILER_DIFFERENTIAL_RESULT: ${{ needs.compiler_differential.result }} FORMAT_PERFORMANCE_RESULT: ${{ needs.format_performance.result }} PERFORMANCE_RESULT: ${{ needs.performance.result }} COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }} @@ -715,6 +752,7 @@ jobs: fi for result in \ "${E2E_SHARDS_RESULT}" \ + "${COMPILER_DIFFERENTIAL_RESULT}" \ "${FORMAT_PERFORMANCE_RESULT}" \ "${PERFORMANCE_RESULT}" \ "${COLD_COMPILE_PERFORMANCE_RESULT}" \ diff --git a/Makefile b/Makefile index 16496970d5..9e4a3f80ff 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,8 @@ SHELL := /bin/bash test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \ test-e2e-duckdb-integrations test-e2e-performance \ test-e2e-cold-compile-performance test-e2e-cache-compile-performance \ - test-e2e-dense-compile-performance test-e2e-varied-cache-performance + test-e2e-dense-compile-performance test-e2e-varied-cache-performance \ + compiler-differential compiler-differential-dense format: uv run ruff format . @@ -130,6 +131,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ tests/e2e/scripts/compile_performance_ratio \ + tests/e2e/scripts/compiler_differential \ tests/e2e/scripts/release_performance \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ @@ -285,6 +287,21 @@ test-e2e-varied-cache-performance: -vv -rP --log-level=INFO --log-cli-level=INFO --color=yes +COMPILER_DIFFERENTIAL_JOBS ?= 4 +COMPILER_DIFFERENTIAL_SEEDS ?= 12 +COMPILER_DIFFERENTIAL_DENSE_MODELS ?= 3000 +COMPILER_DIFFERENTIAL_ARGS ?= + +compiler-differential: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \ + --jobs $(COMPILER_DIFFERENTIAL_JOBS) --seeds $(COMPILER_DIFFERENTIAL_SEEDS) \ + $(COMPILER_DIFFERENTIAL_ARGS) + +compiler-differential-dense: + env PYTHONUNBUFFERED=1 uv run python -m scripts.run_compiler_differential \ + --corpus dense --dense-models $(COMPILER_DIFFERENTIAL_DENSE_MODELS) \ + --jobs $(COMPILER_DIFFERENTIAL_JOBS) $(COMPILER_DIFFERENTIAL_ARGS) + skills: uv run python -m scripts.generate_docs_skill diff --git a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs index ed1749bf96..80efa9b7dc 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/cache.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/cache.rs @@ -16,21 +16,24 @@ pub(crate) struct RuleCacheEntry { } pub(crate) struct Cache { - root: PathBuf, + native_rules_path: PathBuf, } impl Cache { - pub(crate) fn open(project_dir: &Path) -> Result { - Ok(Self { - root: project_dir.join("target/rules-cache"), - }) + pub(crate) fn open(project_dir: &Path, rules_cache_path: &str) -> Result { + let native_rules_path = if rules_cache_path.is_empty() { + project_dir.join("target/rules-cache/bulk/native.json") + } else { + PathBuf::from(rules_cache_path) + }; + Ok(Self { native_rules_path }) } pub(crate) fn native_rules_bucket( &self, ruleset_identity: &str, ) -> Result { - read_bucket(&self.root.join("bulk/native.json"), ruleset_identity) + read_bucket(&self.native_rules_path, ruleset_identity) } pub(crate) fn put_native_rules_bucket( @@ -38,11 +41,7 @@ impl Cache { ruleset_identity: &str, bucket: &RuleCacheBucket, ) -> Result<(), String> { - write_bucket( - &self.root.join("bulk/native.json"), - ruleset_identity, - bucket, - ) + write_bucket(&self.native_rules_path, ruleset_identity, bucket) } } diff --git a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs index 6652778977..ad6787b3bd 100644 --- a/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs +++ b/crates/sqlbuild-rules/src/engine/_helpers/evaluation.rs @@ -320,7 +320,7 @@ pub(crate) fn evaluate_request(request: EvaluateRequest) -> Result Result<(), String> { + let test_cases = [test_types::ConfiguredCachePathTestCase { + description: "an engine-owned cache file replaces the default bulk cache", + cache_file: "target/rules-cache-native-v1/bulk/native.json", + expected_default_cache_exists: false, + expected_warm_hits: 3, + }]; + for test_case in &test_cases { + let project_dir = TempDir::new().map_err(|error| error.to_string())?; + let mut rest = split_request( + &project_dir, + &json!({"select": ["SQBR"], "cache": {"enabled": true}}), + ); + let cache_path = project_dir.path().join(test_case.cache_file); + rest["rules_cache_path"] = json!(cache_path); + let models = orders_models(); + + let cold = evaluate_split(&rest, &models)?; + let warm = evaluate_split(&rest, &models)?; + + assert!(cache_path.is_file(), "{}", test_case.description); + assert_eq!( + project_dir + .path() + .join("target/rules-cache/bulk/native.json") + .exists(), + test_case.expected_default_cache_exists, + "{}", + test_case.description + ); + assert_eq!( + warm["cache_hits"], test_case.expected_warm_hits, + "{}", + test_case.description + ); + assert_eq!(warm["faults"], cold["faults"], "{}", test_case.description); + } + Ok(()) +} + #[test] fn given_corrupt_model_cache_when_evaluating_then_every_model_reevaluates() -> Result<(), String> { let test_cases = [test_types::ModelCacheReuseTestCase { diff --git a/crates/sqlbuild-rules/src/engine/tests/test_types.rs b/crates/sqlbuild-rules/src/engine/tests/test_types.rs index 8dbb4535ad..8f825fa48a 100644 --- a/crates/sqlbuild-rules/src/engine/tests/test_types.rs +++ b/crates/sqlbuild-rules/src/engine/tests/test_types.rs @@ -242,6 +242,13 @@ pub(crate) struct ModelCacheReuseTestCase { pub(crate) expected_misses: u64, } +pub(crate) struct ConfiguredCachePathTestCase { + pub(crate) description: &'static str, + pub(crate) cache_file: &'static str, + pub(crate) expected_default_cache_exists: bool, + pub(crate) expected_warm_hits: u64, +} + pub(crate) struct CachePruningTestCase { pub(crate) description: &'static str, pub(crate) expected_paths: &'static [&'static str], diff --git a/crates/sqlbuild-rules/src/models.rs b/crates/sqlbuild-rules/src/models.rs index b9bfbd3af2..e7576c976c 100644 --- a/crates/sqlbuild-rules/src/models.rs +++ b/crates/sqlbuild-rules/src/models.rs @@ -729,6 +729,7 @@ pub struct EvaluateRequest { pub initial_findings: Vec, pub defer_suppressions: bool, pub custom_rules: Vec, + pub rules_cache_path: String, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -765,6 +766,7 @@ impl Default for EvaluateRequest { initial_findings: vec![], defer_suppressions: false, custom_rules: vec![], + rules_cache_path: String::new(), } } } diff --git a/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py b/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py index b61fd19578..8a2adcb219 100644 --- a/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py +++ b/scripts/cold_compile_performance/main/semantic_compile_fingerprint.py @@ -14,6 +14,7 @@ def semantic_compile_fingerprint(*, payload: dict[str, object], compiled_dir: Pa semantic_payload: dict[str, object] = dict(payload) semantic_payload.pop("compile_timings", None) + semantic_payload.pop("compiler_engine", None) semantic_payload.pop("version", None) fingerprint: Any = hashlib.sha256(orjson.dumps(semantic_payload, option=orjson.OPT_SORT_KEYS)) for artifact_path in sorted(path for path in compiled_dir.rglob("*") if path.is_file()): diff --git a/scripts/compiler_differential/__init__.py b/scripts/compiler_differential/__init__.py new file mode 100644 index 0000000000..63dca5118f --- /dev/null +++ b/scripts/compiler_differential/__init__.py @@ -0,0 +1 @@ +"""Differential harness comparing the Python and native compiler engines.""" diff --git a/scripts/compiler_differential/_helpers/__init__.py b/scripts/compiler_differential/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/compiler_differential/_helpers/compare.py b/scripts/compiler_differential/_helpers/compare.py new file mode 100644 index 0000000000..3ee796c4c3 --- /dev/null +++ b/scripts/compiler_differential/_helpers/compare.py @@ -0,0 +1,118 @@ +"""Locate the first difference between two JSON documents, texts, or file trees.""" + +from __future__ import annotations + +import json +from typing import cast + +from scripts.compiler_differential.constants import MISSING_VALUE, VALUE_PREVIEW_CHARACTERS +from scripts.compiler_differential.models import Divergence + + +def first_json_difference(*, left: object, right: object, pointer: str = "") -> Divergence | None: + """Return the first JSON pointer whose value, type, or object key order differs.""" + + if type(left) is not type(right): + return _divergence(location=pointer or "/", left=left, right=right) + left_object: dict[str, object] | None = as_json_object(left) + right_object: dict[str, object] | None = as_json_object(right) + if left_object is not None and right_object is not None: + return _first_object_difference(left=left_object, right=right_object, pointer=pointer) + if isinstance(left, list) and isinstance(right, list): + for index, (left_item, right_item) in enumerate(zip(left, right, strict=False)): + found: Divergence | None = first_json_difference( + left=left_item, right=right_item, pointer=f"{pointer}/{index}" + ) + if found is not None: + return found + if len(left) != len(right): + index: int = min(len(left), len(right)) + return _divergence( + location=f"{pointer}/{index}", + left=left[index] if index < len(left) else MISSING_VALUE, + right=right[index] if index < len(right) else MISSING_VALUE, + ) + return None + if left != right: + return _divergence(location=pointer or "/", left=left, right=right) + return None + + +def first_text_difference(*, left: str, right: str) -> Divergence | None: + """Return the first differing line, numbered from one.""" + + if left == right: + return None + left_lines: list[str] = left.split("\n") + right_lines: list[str] = right.split("\n") + for index in range(max(len(left_lines), len(right_lines))): + left_line: str = left_lines[index] if index < len(left_lines) else MISSING_VALUE + right_line: str = right_lines[index] if index < len(right_lines) else MISSING_VALUE + if left_line != right_line: + return Divergence( + location=f"line {index + 1}", left=_preview(left_line), right=_preview(right_line) + ) + return Divergence(location="end of file", left="", right="") + + +def first_document_difference(*, left: str, right: str) -> Divergence | None: + """Compare two documents as JSON when both parse, otherwise line by line.""" + + try: + left_payload: object = json.loads(left) + right_payload: object = json.loads(right) + except json.JSONDecodeError: + return first_text_difference(left=left, right=right) + return first_json_difference(left=left_payload, right=right_payload) + + +def as_json_object(value: object) -> dict[str, object] | None: + """Return a decoded JSON object with string keys, or None for any other value.""" + + return cast(dict[str, object], value) if isinstance(value, dict) else None + + +def json_pointer_token(key: str) -> str: + """Escape one object key as an RFC 6901 reference token.""" + + return key.replace("~", "~0").replace("/", "~1") + + +def _first_object_difference( + *, left: dict[str, object], right: dict[str, object], pointer: str +) -> Divergence | None: + for key in (*left, *(key for key in right if key not in left)): + location: str = f"{pointer}/{json_pointer_token(key)}" + if key not in left or key not in right: + return _divergence( + location=location, + left=left.get(key, MISSING_VALUE), + right=right.get(key, MISSING_VALUE), + ) + found: Divergence | None = first_json_difference( + left=left[key], right=right[key], pointer=location + ) + if found is not None: + return found + if list(left) != list(right): + return Divergence( + location=f"{pointer or '/'} (key order)", + left=_preview(list(left)), + right=_preview(list(right)), + ) + return None + + +def _divergence(*, location: str, left: object, right: object) -> Divergence: + return Divergence(location=location, left=_preview(left), right=_preview(right)) + + +def _preview(value: object) -> str: + rendered: str = ( + value + if isinstance(value, str) and value == MISSING_VALUE + else json.dumps(value, ensure_ascii=False) + ) + if len(rendered) <= VALUE_PREVIEW_CHARACTERS: + return rendered + return rendered[: VALUE_PREVIEW_CHARACTERS - 3] + "..." diff --git a/scripts/compiler_differential/_helpers/comparison.py b/scripts/compiler_differential/_helpers/comparison.py new file mode 100644 index 0000000000..cfd54bfbae --- /dev/null +++ b/scripts/compiler_differential/_helpers/comparison.py @@ -0,0 +1,264 @@ +"""Compare everything two engines produced for one project and report first differences.""" + +from __future__ import annotations + +import json +from collections.abc import Callable + +from scripts.compiler_differential._helpers.compare import ( + as_json_object, + first_document_difference, + first_json_difference, + first_text_difference, +) +from scripts.compiler_differential._helpers.normalize import ( + normalize_artifact_text, + normalize_stderr, + strip_manifest_metadata, + strip_report_fields, +) +from scripts.compiler_differential.constants import ( + ERROR_SEVERITY, + MISSING_VALUE, + MODEL_RESOURCE_TYPE, + PLAN_LABEL, +) +from scripts.compiler_differential.models import ( + CommandOutcome, + Difference, + Divergence, + EngineRun, +) + + +def compare_engine_runs(*, project: str, left: EngineRun, right: EngineRun) -> list[Difference]: + """Return the first difference in every artifact that differs between two engine runs.""" + + found: list[tuple[str, Divergence | None]] = [] + for left_outcome, right_outcome in zip(left.outcomes, right.outcomes, strict=True): + found.extend(_command_differences(left=left_outcome, right=right_outcome)) + found.extend(_tree_differences(left=left.compiled, right=right.compiled)) + found.append( + ( + "target/manifest.json", + _optional_json_difference( + left=left.manifest, right=right.manifest, strip=strip_manifest_metadata + ), + ) + ) + found.append( + ( + "target/sqlbuild_dag.json", + _optional_json_difference(left=left.dag, right=right.dag, strip=None), + ) + ) + found.extend( + (f"query fingerprint of model {model}", divergence) + for model, divergence in _fingerprint_differences(left=left, right=right) + ) + found.extend(_capture_differences(left=left.captures, right=right.captures)) + return [ + Difference( + project=project, + artifact=artifact, + location=divergence.location, + left=divergence.left, + right=divergence.right, + ) + for artifact, divergence in found + if divergence is not None + ] + + +def diagnostic_codes(*, outcome: CommandOutcome, errors_only: bool) -> tuple[str, ...]: + """Return the diagnostic codes of a JSON compile report, in report order.""" + + payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} + diagnostics: object = payload.get("diagnostics") + found: list[str] = [] + for raw_diagnostic in diagnostics if isinstance(diagnostics, list) else (): + diagnostic: dict[str, object] | None = as_json_object(raw_diagnostic) + if diagnostic is not None and ( + not errors_only or diagnostic.get("severity") == ERROR_SEVERITY + ): + found.append(str(diagnostic.get("code"))) + return tuple(found) + + +def _command_differences( + *, left: CommandOutcome, right: CommandOutcome +) -> list[tuple[str, Divergence | None]]: + label: str = f"`{left.label}`" + exit_codes: Divergence | None = ( + None + if (left.exit_code, left.timed_out) == (right.exit_code, right.timed_out) + else Divergence( + location="exit code", + left=_exit_label(left), + right=_exit_label(right), + ) + ) + return [ + (f"{label} exit code", exit_codes), + (f"{label} stdout", _stdout_difference(left=left.stdout, right=right.stdout)), + ( + f"{label} stderr", + first_text_difference( + left=normalize_stderr(left.stderr), right=normalize_stderr(right.stderr) + ), + ), + ] + + +def _exit_label(outcome: CommandOutcome) -> str: + return "timed out" if outcome.timed_out else str(outcome.exit_code) + + +def _stdout_difference(*, left: str, right: str) -> Divergence | None: + normalized_left: str = normalize_artifact_text(left) + normalized_right: str = normalize_artifact_text(right) + try: + left_payload: object = json.loads(normalized_left) + right_payload: object = json.loads(normalized_right) + except json.JSONDecodeError: + return first_text_difference(left=normalized_left, right=normalized_right) + return first_json_difference( + left=strip_report_fields(left_payload), right=strip_report_fields(right_payload) + ) + + +def _tree_differences( + *, left: dict[str, bytes], right: dict[str, bytes] +) -> list[tuple[str, Divergence | None]]: + differences: list[tuple[str, Divergence | None]] = [] + for path in sorted(set(left) | set(right)): + artifact: str = f"target/compiled/{path}" + if path not in left or path not in right: + differences.append( + ( + artifact, + _presence_divergence(left_present=path in left, right_present=path in right), + ) + ) + continue + differences.append( + ( + artifact, + first_text_difference( + left=normalize_artifact_text(left[path].decode("utf-8", "surrogateescape")), + right=normalize_artifact_text(right[path].decode("utf-8", "surrogateescape")), + ), + ) + ) + return differences + + +def _optional_json_difference( + *, left: str | None, right: str | None, strip: Callable[[object], object] | None +) -> Divergence | None: + if left is None or right is None: + if left is None and right is None: + return None + return _presence_divergence(left_present=left is not None, right_present=right is not None) + try: + left_payload: object = json.loads(normalize_artifact_text(left)) + right_payload: object = json.loads(normalize_artifact_text(right)) + except json.JSONDecodeError: + return first_text_difference(left=left, right=right) + if strip is not None: + left_payload = strip(left_payload) + right_payload = strip(right_payload) + return first_json_difference(left=left_payload, right=right_payload) + + +def _fingerprint_differences( + *, left: EngineRun, right: EngineRun +) -> list[tuple[str, Divergence | None]]: + left_fingerprints: dict[str, str] = _model_fingerprints(left) + right_fingerprints: dict[str, str] = _model_fingerprints(right) + return [ + ( + model, + None + if left_fingerprints.get(model) == right_fingerprints.get(model) + else Divergence( + location="fingerprint", + left=left_fingerprints.get(model, MISSING_VALUE), + right=right_fingerprints.get(model, MISSING_VALUE), + ), + ) + for model in sorted(set(left_fingerprints) | set(right_fingerprints)) + ] + + +def _model_fingerprints(run: EngineRun) -> dict[str, str]: + fingerprints: dict[str, str] = {} + manifest: dict[str, object] = as_json_object(_loads(run.manifest)) or {} + nodes: dict[str, object] = as_json_object(manifest.get("nodes")) or {} + for raw_node in nodes.values(): + node: dict[str, object] = as_json_object(raw_node) or {} + checksum: dict[str, object] | None = as_json_object(node.get("checksum")) + if checksum is not None and node.get("resource_type") == MODEL_RESOURCE_TYPE: + fingerprints[f"{node.get('name')} (manifest query hash)"] = str( + checksum.get("checksum") + ) + for outcome in run.outcomes: + plan: dict[str, object] = ( + as_json_object(_loads(outcome.stdout)) or {} if outcome.label == PLAN_LABEL else {} + ) + models: object = plan.get("models") + for raw_model in models if isinstance(models, list) else (): + model: dict[str, object] | None = as_json_object(raw_model) + if model is not None: + fingerprints[f"{model.get('name')} (plan version hash)"] = str( + model.get("expected_version_hash") + ) + return fingerprints + + +def _loads(text: str | None) -> object: + if text is None: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +def _capture_differences( + *, left: dict[str, dict[str, str]], right: dict[str, dict[str, str]] +) -> list[tuple[str, Divergence | None]]: + differences: list[tuple[str, Divergence | None]] = [] + for command in sorted(set(left) | set(right)): + left_files: dict[str, str] = left.get(command, {}) + right_files: dict[str, str] = right.get(command, {}) + for name in sorted(set(left_files) | set(right_files)): + artifact: str = f"stage capture {command}/{name}" + if name not in left_files or name not in right_files: + differences.append( + ( + artifact, + _presence_divergence( + left_present=name in left_files, right_present=name in right_files + ), + ) + ) + continue + differences.append( + ( + artifact, + first_document_difference( + left=normalize_artifact_text(left_files[name]), + right=normalize_artifact_text(right_files[name]), + ), + ) + ) + return differences + + +def _presence_divergence(*, left_present: bool, right_present: bool) -> Divergence: + return Divergence( + location="file", + left="present" if left_present else MISSING_VALUE, + right="present" if right_present else MISSING_VALUE, + ) diff --git a/scripts/compiler_differential/_helpers/corpus.py b/scripts/compiler_differential/_helpers/corpus.py new file mode 100644 index 0000000000..531a0e554f --- /dev/null +++ b/scripts/compiler_differential/_helpers/corpus.py @@ -0,0 +1,141 @@ +"""Assemble the projects the differential harness compares.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from scripts.compiler_differential._helpers.failure_cases import all_failure_cases +from scripts.compiler_differential.classes.dense_project import DenseProject +from scripts.compiler_differential.classes.project_builder import ProjectBuilder +from scripts.compiler_differential.constants import ( + COLD_COMPILE, + CORPUS_DENSE, + CORPUS_EXAMPLES, + CORPUS_FAILURES, + CORPUS_FIXTURES, + CORPUS_SEEDS, + DUCKDB_ADAPTER, + EXAMPLE_ROOT, + EXPECT_SUCCESS, + FIXTURE_EXPECTED_OUTCOMES, + FIXTURE_PROJECT_SUBDIRECTORIES, + FIXTURE_ROOT, + PLAN, + PROJECT_CONFIG_FILE, + WARM_COMPILE, +) +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialCommand, + ExpectedOutcome, + GeneratedProject, +) + + +def build_corpus( + *, + repo_root: Path, + corpora: tuple[str, ...], + seeds: range, + dense_models: int, + projects: tuple[Path, ...], + project_expectation: ExpectedOutcome, +) -> list[CorpusProject]: + """Return the selected corpus entries in a stable order.""" + + entries: list[CorpusProject] = [ + _authored_project( + name=f"project/{path.name}", + source_dir=path, + subdirectory=None, + expected=project_expectation, + ) + for path in projects + ] + if CORPUS_FIXTURES in corpora: + entries.extend( + _authored_project( + name=f"fixture/{path.name}", + source_dir=path, + subdirectory=FIXTURE_PROJECT_SUBDIRECTORIES.get(path.name), + expected=FIXTURE_EXPECTED_OUTCOMES.get(path.name, EXPECT_SUCCESS), + ) + for path in _project_directories(repo_root / FIXTURE_ROOT) + ) + if CORPUS_EXAMPLES in corpora: + entries.extend( + _authored_project( + name=f"example/{path.name}", + source_dir=path, + subdirectory=None, + expected=EXPECT_SUCCESS, + ) + for path in _project_directories(repo_root / EXAMPLE_ROOT) + ) + if CORPUS_SEEDS in corpora: + entries.extend(_seed_project(seed) for seed in seeds) + if CORPUS_FAILURES in corpora: + entries.extend( + CorpusProject( + name=f"failure/{case.name}", + commands=(COLD_COMPILE,), + expected=ExpectedOutcome(error_code=case.expected_code), + writer=case.write, + ) + for case in all_failure_cases() + ) + if CORPUS_DENSE in corpora: + entries.extend( + CorpusProject( + name=f"dense/{dense_models}{'-custom-rules' if custom_rules else ''}", + commands=(COLD_COMPILE, WARM_COMPILE), + expected=EXPECT_SUCCESS, + writer=DenseProject(model_count=dense_models, custom_rules=custom_rules).write, + ) + for custom_rules in (False, True) + ) + return entries + + +def _project_directories(root: Path) -> list[Path]: + return sorted( + path + for path in root.iterdir() + if path.is_dir() + and ((path / PROJECT_CONFIG_FILE).is_file() or path.name in FIXTURE_PROJECT_SUBDIRECTORIES) + ) + + +def _authored_project( + *, name: str, source_dir: Path, subdirectory: str | None, expected: ExpectedOutcome +) -> CorpusProject: + project_dir: Path = source_dir if subdirectory is None else source_dir / subdirectory + return CorpusProject( + name=name, + commands=(COLD_COMPILE, WARM_COMPILE, *_plan_commands(project_dir)), + expected=expected, + source_dir=source_dir, + project_subdirectory=subdirectory, + ) + + +def _plan_commands(project_dir: Path) -> tuple[DifferentialCommand, ...]: + config_path: Path = project_dir / PROJECT_CONFIG_FILE + if not config_path.is_file(): + return () + try: + adapter: object = tomllib.loads(config_path.read_text(encoding="utf-8")).get("adapter") + except tomllib.TOMLDecodeError: + return () + return (PLAN,) if adapter == DUCKDB_ADAPTER else () + + +def _seed_project(seed: int) -> CorpusProject: + generated: GeneratedProject = ProjectBuilder(seed).build() + return CorpusProject( + name=f"seed/{seed}", + commands=(COLD_COMPILE, PLAN), + expected=ExpectedOutcome(error_code=generated.expected_error_code), + writer=generated.write, + ) diff --git a/scripts/compiler_differential/_helpers/execution.py b/scripts/compiler_differential/_helpers/execution.py new file mode 100644 index 0000000000..06bd80c6c2 --- /dev/null +++ b/scripts/compiler_differential/_helpers/execution.py @@ -0,0 +1,176 @@ +"""Copy a project and run every differential command under one engine in fresh processes.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +from scripts.compiler_differential.constants import ( + CAPTURES_DIRECTORY, + COMMAND_TIMEOUT_SECONDS, + COMPILED_DIRECTORY, + DAG_FILE, + ENGINE_ENV_VAR, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + MANIFEST_FILE, + PROJECT_DIRECTORY, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.models import ( + CommandOutcome, + CorpusProject, + DifferentialCommand, + DifferentialOptions, + EngineRun, +) + +_COPY_IGNORED: tuple[str, ...] = ( + "target", + "logs", + "__pycache__", + ".sqlbuild", + "*.duckdb", + "*.duckdb.wal", +) + + +def run_engine( + *, + project: CorpusProject, + source_dir: Path, + case_dir: Path, + engine: str, + options: DifferentialOptions, +) -> EngineRun: + """Run the project's commands under one engine in a fresh copy at a fixed path.""" + + workspace: Path = case_dir / PROJECT_DIRECTORY + shutil.rmtree(workspace, ignore_errors=True) + _ = shutil.copytree(source_dir, workspace, ignore=shutil.ignore_patterns(*_COPY_IGNORED)) + project_dir: Path = ( + workspace + if project.project_subdirectory is None + else workspace / project.project_subdirectory + ) + capture_root: Path = case_dir / f"{engine}-{CAPTURES_DIRECTORY}" + shutil.rmtree(capture_root, ignore_errors=True) + outcomes: list[CommandOutcome] = [] + for index, command in enumerate(project.commands): + capture_dir: Path | None = ( + capture_root / f"{index}-{command.label}" if options.stage_captures else None + ) + outcomes.append( + _run_command( + command=command, + project_dir=project_dir, + environment=_command_environment( + engine=engine, options=options, capture_dir=capture_dir + ), + python=options.python, + ) + ) + run: EngineRun = EngineRun( + engine=engine, + outcomes=tuple(outcomes), + compiled=_compiled_tree(project_dir / COMPILED_DIRECTORY), + manifest=_optional_text(project_dir / MANIFEST_FILE), + dag=_optional_text(project_dir / DAG_FILE), + captures=_captures(capture_root), + ) + retained: Path = case_dir / engine + shutil.rmtree(retained, ignore_errors=True) + _ = workspace.rename(retained) + return run + + +def harness_environment() -> dict[str, str]: + """Return this process's environment without engine, capture, and dbt settings.""" + + return { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } + + +def _command_environment( + *, engine: str, options: DifferentialOptions, capture_dir: Path | None +) -> dict[str, str]: + environment: dict[str, str] = { + **harness_environment(), + ENGINE_ENV_VAR: engine, + **options.engine_environment.get(engine, {}), + } + if capture_dir is not None: + environment[STAGE_CAPTURE_ENV_VAR] = str(capture_dir) + return environment + + +def _run_command( + *, command: DifferentialCommand, project_dir: Path, environment: dict[str, str], python: Path +) -> CommandOutcome: + try: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", SQB_ENTRY, "--no-color", *command.arguments], + cwd=project_dir, + env=environment, + capture_output=True, + text=True, + encoding="utf-8", + errors="surrogateescape", + timeout=COMMAND_TIMEOUT_SECONDS, + check=False, + ) + except subprocess.TimeoutExpired as error: + return CommandOutcome( + label=command.label, + exit_code=-1, + stdout=_timeout_text(error.stdout), + stderr=_timeout_text(error.stderr), + timed_out=True, + ) + return CommandOutcome( + label=command.label, + exit_code=completed.returncode, + stdout=completed.stdout, + stderr=completed.stderr, + ) + + +def _timeout_text(value: bytes | str | None) -> str: + if value is None: + return "" + return value if isinstance(value, str) else value.decode("utf-8", "surrogateescape") + + +def _compiled_tree(root: Path) -> dict[str, bytes]: + if not root.is_dir(): + return {} + return { + path.relative_to(root).as_posix(): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def _optional_text(path: Path) -> str | None: + return path.read_text(encoding="utf-8", errors="surrogateescape") if path.is_file() else None + + +def _captures(root: Path) -> dict[str, dict[str, str]]: + captures: dict[str, dict[str, str]] = {} + for command_dir in sorted(root.iterdir()) if root.is_dir() else (): + captures[command_dir.name] = _capture_files(command_dir) + return captures + + +def _capture_files(command_dir: Path) -> dict[str, str]: + return { + capture.name: capture.read_text(encoding="utf-8") + for capture in sorted(command_dir.iterdir()) + if capture.is_file() + } diff --git a/scripts/compiler_differential/_helpers/failure_cases.py b/scripts/compiler_differential/_helpers/failure_cases.py new file mode 100644 index 0000000000..a49d3df4e5 --- /dev/null +++ b/scripts/compiler_differential/_helpers/failure_cases.py @@ -0,0 +1,504 @@ +"""Minimal failing projects, each derived from the shared base project.""" + +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_FILES, + FAILURE_BASE_MART, + FAILURE_BASE_STAGING, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, + FAILURE_STAGING_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_RULES_CONFIG: str = '\n[rules]\nselect = ["{codes}"]\n' + + +def _case(*, name: str, expected_code: str, files: dict[str, str]) -> FailureCase: + return FailureCase( + files={**FAILURE_BASE_FILES, **files}, name=name, expected_code=expected_code + ) + + +def _staging(sql: str) -> dict[str, str]: + return {FAILURE_STAGING_PATH: sql} + + +def _mart_body(body: str) -> dict[str, str]: + return {FAILURE_MART_PATH: 'MODEL (\n description "Order totals per customer",\n);\n\n' + body} + + +def _rules(*codes: str) -> dict[str, str]: + return { + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + _RULES_CONFIG.format(codes='", "'.join(codes)) + } + + +def all_failure_cases() -> tuple[FailureCase, ...]: + """Return every failure case in a stable order.""" + + return ( + _case( + name="config-toml-syntax", + expected_code="D001", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + "[defaults\nmaterialized = 'table'\n" + }, + ), + _case( + name="config-unknown-default-key", + expected_code="D001", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + '\n[defaults]\ndescription = "Shared"\n' + }, + ), + _case( + name="config-unknown-adapter", + expected_code="C601", + files={FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG.replace('"duckdb"', '"warehouse9000"')}, + ), + _case( + name="model-header-syntax", + expected_code="D002", + files=_staging( + FAILURE_BASE_STAGING.replace( + '"Staged orders",', '"Staged orders",\n materialized (,' + ) + ), + ), + _case( + name="model-sql-syntax", + expected_code="P001", + files=_staging(FAILURE_BASE_STAGING.replace("SELECT order_id,", "SELECT order_id,, ")), + ), + _case( + name="test-parse", + expected_code="D003", + files={"tests/unit/test_stg_orders.sql": "TEST(\n\nSELECT 1\n"}, + ), + _case( + name="audit-parse", + expected_code="D004", + files={ + "models/marts/_sqlbuild/audits/singular/broken.sql": "AUDIT (name);\n\nSELECT 1\n" + }, + ), + _case( + name="schema-unterminated-declaration", + expected_code="D013", + files={"models/staging/_sqlbuild/_schemas/orders.sql": "SCHEMA (\n name orders,\n"}, + ), + _case( + name="source-parse", + expected_code="D006", + files={FAILURE_SOURCES_PATH: "sources:\n - name: raw_orders\n columns: [\n"}, + ), + _case( + name="duplicate-model", + expected_code="D007", + files={"models/marts/stg_orders.sql": FAILURE_BASE_STAGING}, + ), + _case( + name="seed-without-header", + expected_code="D008", + files={"seeds/lookups.csv": ""}, + ), + _case( + name="scenario-parse", + expected_code="D009", + files={"tests/scenarios/orders.sql": "SCENARIO (\n\nSELECT 1\n"}, + ), + _case( + name="python-node-missing-description", + expected_code="P010", + files={ + "python/tasks/exports.py": ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\ndef export_orders(ctx: TaskContext):\n return None\n" + ) + }, + ), + _case( + name="enum-parse", + expected_code="D013", + files={"enums/order_status.sql": "ENUM (\n name order_status,\n members [,\n);\n"}, + ), + _case( + name="hook-without-header", + expected_code="D014", + files={"hooks/sql/record_refresh.sql": "SELECT 1\n"}, + ), + _case( + name="invalid-resource-name", + expected_code="D016", + files={"models/marts/order-totals.sql": FAILURE_BASE_MART}, + ), + _case( + name="built-in-audit-shadow", + expected_code="P003", + files={ + "models/staging/_sqlbuild/_audits/generic/not_null.sql": ( + 'AUDIT (name "not_null");\n\nSELECT 1 WHERE FALSE\n' + ) + }, + ), + _case( + name="singular-audit-one-resource", + expected_code="P004", + files={ + "models/marts/_sqlbuild/audits/singular/positive_totals.sql": ( + 'AUDIT (name "positive_totals");\n\n' + 'SELECT customer_id FROM __ref("customer_totals") WHERE total_amount < 0\n' + ) + }, + ), + _case( + name="macro-generated-reference", + expected_code="P006", + files={ + "models/marts/_sqlbuild/_macros/orders.py": ( + "def staged_orders() -> str:\n" + ' """Return the staged orders relation."""\n' + " return '__ref(\"stg_orders\")'\n" + ), + **_mart_body( + "SELECT customer_id, SUM(amount) AS total_amount\n" + "FROM @staged_orders()\nGROUP BY customer_id\n" + ), + }, + ), + _case( + name="missing-description", + expected_code="P010", + files=_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + ), + _case( + name="unknown-ref", + expected_code="P001", + files=_mart_body( + 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_payments")\n' + "GROUP BY customer_id\n" + ), + ), + _case( + name="unknown-source", + expected_code="P001", + files=_staging( + FAILURE_BASE_STAGING.replace('__source("raw_orders")', '__source("raw_payments")') + ), + ), + _case( + name="unknown-macro", + expected_code="P001", + files=_staging( + FAILURE_BASE_STAGING.replace("amount, status", '@cents("amount") AS amount, status') + ), + ), + _case( + name="failing-macro", + expected_code="P001", + files={ + "models/staging/_sqlbuild/_macros/cents.py": ( + "def cents(expression: str) -> str:\n" + ' """Refuse every input."""\n' + ' raise ValueError(f"cannot convert {expression}")\n' + ), + **_staging( + FAILURE_BASE_STAGING.replace( + "amount, status", '@cents("amount") AS amount, status' + ) + ), + }, + ), + _case( + name="unknown-enum-member", + expected_code="P001", + files={ + "models/staging/_sqlbuild/_enums/order_status.sql": ( + "ENUM (\n name order_status,\n members [PLACED, SHIPPED],\n);\n" + ), + **_staging( + FAILURE_BASE_STAGING + 'WHERE status = @enum("order_status").RETURNED\n' + ), + }, + ), + _case( + name="unknown-constant", + expected_code="P001", + files=_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + ), + _case( + name="invalid-decimal-constant", + expected_code="D013", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + 'CONSTANT (name minimum_amount, type decimal, value "ten");\n' + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="duplicate-declaration", + expected_code="P001", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name minimum_amount, value 1);\n" + "CONSTANT (name minimum_amount, value 2);\n" + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="inaccessible-declaration", + expected_code="P001", + files={ + "models/marts/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name minimum_amount, value 1);\n" + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="over-broad-global", + expected_code="S024", + files={ + "constants/limits.sql": "CONSTANT (name minimum_amount, value 1);\n", + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("minimum_amount")\n'), + }, + ), + _case( + name="unused-declaration", + expected_code="S010", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name spare, value 1);\n" + ) + }, + ), + _case( + name="macro-import-cycle", + expected_code="P001", + files={ + "macros/first.py": ( + "from macros.second import second_value\n\n\n" + "def first_value() -> str:\n" + ' """First value."""\n' + " return second_value()\n" + ), + "macros/second.py": ( + "from macros.first import first_value\n\n\n" + "def second_value() -> str:\n" + ' """Second value."""\n' + " return first_value()\n" + ), + }, + ), + _case( + name="contract-missing-column", + expected_code="K001", + files=_staging( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n" + " discount (type DOUBLE),\n" + " ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="contract-type-mismatch", + expected_code="K002", + files=_staging( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type VARCHAR),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="set-operation-width", + expected_code="B216", + files=_mart_body( + 'SELECT customer_id, amount FROM __ref("stg_orders")\n' + 'UNION ALL\nSELECT customer_id FROM __ref("stg_orders")\n' + ), + ), + _case( + name="comparison-type", + expected_code="B217", + files=_mart_body( + "SELECT customer_id, SUM(amount) AS total_amount\n" + "FROM __ref(\"stg_orders\")\nWHERE amount = DATE '2026-01-01'\n" + "GROUP BY customer_id\n" + ), + ), + _case( + name="unknown-column", + expected_code="B002", + files=_mart_body( + 'SELECT customer_id, SUM(discount) AS total_amount\nFROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" + ), + ), + _case( + name="ungrouped-column", + expected_code="B230", + files=_mart_body( + "SELECT customer_id, status, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" + ), + ), + _case( + name="test-expected-unknown-column", + expected_code="B302", + files={ + "tests/unit/test_stg_orders.sql": ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status\n),\n" + "__expected__stg_orders AS (\n SELECT 1 AS order_id, 2 AS discount\n)\n" + "SELECT 1\n" + ) + }, + ), + _case( + name="rule-unused-cte", + expected_code="SQBRSQL005", + files={ + **_rules("SQBRSQL005"), + **_mart_body( + "WITH spare AS (SELECT 1 AS one)\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ), + }, + ), + _case( + name="rule-custom-finding", + expected_code="XSQBRDIFF001", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[rules]\nselect = ["XSQBR"]\n\n' + + "[rules.thresholds]\nmin_custom_rule_test_cases = 0\n", + "rules/no_staging.py": ( + "from sqlbuild.rules import Finding, Model, RuleContext, rule\n\n\n" + '@rule(code="XSQBRDIFF001", message="Models avoid the stg_ prefix", ' + 'remediation="Rename the model.")\n' + "def no_staging_prefix(*, model: Model, ctx: RuleContext) -> list[Finding]:\n" + ' if not model.name.startswith("stg_"):\n' + " return []\n" + " return [ctx.finding(subject=model)]\n" + ), + }, + ), + _case( + name="multiple-errors-order", + expected_code="P010", + files={ + **_staging(FAILURE_BASE_STAGING.replace(' description "Staged orders",\n', "")), + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + ' description "Order totals per customer",\n', "" + ), + "models/marts/order_counts.sql": ( + "MODEL ();\n\nSELECT customer_id, COUNT(*) AS order_count\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ), + }, + ), + _case( + name="contract-extra-column", + expected_code="K005", + files=_staging( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="contract-unknown-type", + expected_code="K002", + files=_staging( + 'MODEL (\n description "Staged orders",\n' + " columns (\n order_id (type MYSTERY_TYPE),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + ), + _case( + name="cursor-unknown-column", + expected_code="B300", + files=_mart_body( + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ) + | { + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " cursor placed_at,\n cursor_type timestamp,\n cursor_grain day,\n" + " cursor_inputs (\n stg_orders order_id,\n ),\n);\n\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + 'FROM __ref("stg_orders")\nGROUP BY customer_id\n' + ) + }, + ), + _case( + name="cursor-model-without-inputs", + expected_code="P011", + files={ + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " cursor placed_at,\n cursor_type timestamp,\n cursor_grain day,\n);\n\n" + "SELECT TIMESTAMP '2026-01-01 00:00:00' AS placed_at, 1 AS customer_id\n" + ) + }, + ), + _case( + name="invalid-declaration-name", + expected_code="D016", + files={ + "models/staging/_sqlbuild/_constants/limits.sql": ( + "CONSTANT (name MinimumAmount, value 1);\n" + ), + **_staging(FAILURE_BASE_STAGING + 'WHERE amount > @const("MinimumAmount")\n'), + }, + ), + _case( + name="python-node-syntax", + expected_code="D011", + files={"python/tasks/exports.py": "def broken(:\n pass\n"}, + ), + _case( + name="hook-unknown-argument", + expected_code="P001", + files={ + "hooks/sql/record_refresh.sql": ( + "HOOK (\n description \"Record a refresh\"\n);\n\nSELECT @'label' AS label\n" + ), + FAILURE_MART_PATH: FAILURE_BASE_MART.replace( + '"Order totals per customer",', + '"Order totals per customer",\n post_hooks [sql("record_refresh", tag: "x")],', + ), + }, + ), + _case( + name="model-private-enum-misuse", + expected_code="P001", + files=_staging( + 'MODEL (\n description "Staged orders",\n' + " enums (\n _state [OPEN, CLOSED],\n ),\n);\n\n" + "SELECT order_id, customer_id, amount, status\n" + 'FROM __source("raw_orders")\nWHERE status = @enum("_state").PENDING\n' + ), + ), + _case( + name="variable-not-defined", + expected_code="P001", + files=_staging( + FAILURE_BASE_STAGING.replace("status\n", "status, '@@region' AS region\n", 1) + ), + ), + ) diff --git a/scripts/compiler_differential/_helpers/normalize.py b/scripts/compiler_differential/_helpers/normalize.py new file mode 100644 index 0000000000..a81a9f8987 --- /dev/null +++ b/scripts/compiler_differential/_helpers/normalize.py @@ -0,0 +1,52 @@ +"""Remove run-specific noise that legitimately differs between two identical compiles.""" + +from __future__ import annotations + +from scripts.compiler_differential._helpers.compare import as_json_object +from scripts.compiler_differential.constants import ( + ELAPSED_TIME_MASK, + ELAPSED_TIME_PATTERN, + ENGINE_NAMESPACE_PATTERN, + INVOCATION_ID_MASK, + INVOCATION_ID_PATTERN, + MANIFEST_VOLATILE_METADATA, + STRIPPED_REPORT_FIELDS, +) + + +def normalize_artifact_text(text: str) -> str: + """Mask invocation ids and engine store suffixes, which differ by design.""" + + return ENGINE_NAMESPACE_PATTERN.sub("", INVOCATION_ID_PATTERN.sub(INVOCATION_ID_MASK, text)) + + +def normalize_stderr(text: str) -> str: + """Mask elapsed times as well as the artifact noise in progress and diagnostic output.""" + + return ELAPSED_TIME_PATTERN.sub(ELAPSED_TIME_MASK, normalize_artifact_text(text)) + + +def strip_report_fields(payload: object) -> object: + """Drop the timings and engine fields of a top-level command report.""" + + report: dict[str, object] | None = as_json_object(payload) + if report is None: + return payload + return {key: value for key, value in report.items() if key not in STRIPPED_REPORT_FIELDS} + + +def strip_manifest_metadata(payload: object) -> object: + """Drop the manifest generation time and invocation id.""" + + manifest: dict[str, object] | None = as_json_object(payload) + metadata: dict[str, object] | None = ( + None if manifest is None else as_json_object(manifest.get("metadata")) + ) + if manifest is None or metadata is None: + return payload + return { + **manifest, + "metadata": { + key: value for key, value in metadata.items() if key not in MANIFEST_VOLATILE_METADATA + }, + } diff --git a/scripts/compiler_differential/_helpers/options.py b/scripts/compiler_differential/_helpers/options.py new file mode 100644 index 0000000000..10d465f56a --- /dev/null +++ b/scripts/compiler_differential/_helpers/options.py @@ -0,0 +1,42 @@ +"""Parse harness option values.""" + +from __future__ import annotations + +import argparse +import re + +from scripts.compiler_differential.constants import ( + EXPECT_FAILURE_PREFIX, + EXPECT_SUCCESS, + EXPECT_SUCCESS_VALUE, +) +from scripts.compiler_differential.exceptions import DifferentialUsageError +from scripts.compiler_differential.models import ExpectedOutcome + +_DIAGNOSTIC_CODE: re.Pattern[str] = re.compile(r"[A-Z][A-Z0-9]*[0-9]") + + +def parse_engine_environment(values: list[str]) -> dict[str, dict[str, str]]: + """Parse repeated `ENGINE:NAME=VALUE` options into per-engine environment overrides.""" + + environment: dict[str, dict[str, str]] = {} + for value in values: + engine, separator, assignment = value.partition(":") + name, equals, setting = assignment.partition("=") + if not separator or not equals or not engine or not name: + raise DifferentialUsageError(f"--engine-env expects ENGINE:NAME=VALUE, got {value!r}") + environment.setdefault(engine, {})[name] = setting + return environment + + +def parse_expected_outcome(value: str) -> ExpectedOutcome: + """Parse `success` or `failure:` into the outcome every --project must produce.""" + + if value == EXPECT_SUCCESS_VALUE: + return EXPECT_SUCCESS + code: str = value.removeprefix(EXPECT_FAILURE_PREFIX) + if code == value or not _DIAGNOSTIC_CODE.fullmatch(code): + raise argparse.ArgumentTypeError( + f"expected {EXPECT_SUCCESS_VALUE} or {EXPECT_FAILURE_PREFIX}, got {value!r}" + ) + return ExpectedOutcome(error_code=code) diff --git a/scripts/compiler_differential/_helpers/project_comparison.py b/scripts/compiler_differential/_helpers/project_comparison.py new file mode 100644 index 0000000000..3741cdb565 --- /dev/null +++ b/scripts/compiler_differential/_helpers/project_comparison.py @@ -0,0 +1,135 @@ +"""Run one corpus project under both engines and compare what they produced.""" + +from __future__ import annotations + +import shutil +import time +from pathlib import Path + +from scripts.compiler_differential._helpers.comparison import compare_engine_runs, diagnostic_codes +from scripts.compiler_differential._helpers.execution import run_engine +from scripts.compiler_differential.models import ( + CommandOutcome, + CorpusProject, + Difference, + DifferentialOptions, + EngineRun, + ProjectComparison, +) + +_SOURCE_DIRECTORY: str = "source" +_CORPUS_ARTIFACT: str = "corpus expectation" +_EXPECTATION_LABELS: tuple[str, str] = ("expected", "actual") + + +def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> ProjectComparison: + """Compare both engines on one project against its expectation; any timeout fails.""" + + started: float = time.monotonic() + case_dir: Path = options.work_dir / project.name.replace("/", "__") + shutil.rmtree(case_dir, ignore_errors=True) + case_dir.mkdir(parents=True) + source_dir: Path = _materialized_source(project=project, case_dir=case_dir) + left_engine, right_engine = options.engines + left: EngineRun = run_engine( + project=project, + source_dir=source_dir, + case_dir=case_dir, + engine=left_engine, + options=options, + ) + right: EngineRun = run_engine( + project=project, + source_dir=source_dir, + case_dir=case_dir, + engine=right_engine, + options=options, + ) + differences: list[Difference] = [ + *_timeout_differences(project=project, runs=(left, right)), + *_corpus_differences(project=project, run=left), + *compare_engine_runs(project=project.name, left=left, right=right), + ] + return ProjectComparison( + project=project.name, + differences=tuple(differences), + seconds=time.monotonic() - started, + ) + + +def _materialized_source(*, project: CorpusProject, case_dir: Path) -> Path: + if project.source_dir is not None: + return project.source_dir + source_dir: Path = case_dir / _SOURCE_DIRECTORY + if project.writer is not None: + project.writer(source_dir) + return source_dir + + +def _corpus_differences(*, project: CorpusProject, run: EngineRun) -> list[Difference]: + first: CommandOutcome = run.outcomes[0] + errors: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=True) + expected_code: str | None = project.expected.error_code + if expected_code is not None: + codes: tuple[str, ...] = diagnostic_codes(outcome=first, errors_only=False) + met: bool = first.exit_code != 0 and bool(errors) and expected_code in codes + return ( + [] + if met + else [ + _expectation_difference( + project=project, + location="expected failure", + expected=f"non-zero exit reporting {expected_code}", + actual=f"exit {first.exit_code}: {', '.join(codes) or ''}", + ) + ] + ) + problems: list[str] = [ + f"`{outcome.label}` exit {outcome.exit_code}" + for outcome in run.outcomes + if outcome.exit_code != 0 + ] + if errors: + problems.append(f"compile errors {', '.join(errors)}") + if not problems: + return [] + return [ + _expectation_difference( + project=project, + location="expected success", + expected="every command exits 0 and compile reports no errors", + actual="; ".join(problems), + ) + ] + + +def _timeout_differences( + *, project: CorpusProject, runs: tuple[EngineRun, ...] +) -> list[Difference]: + differences: list[Difference] = [] + for run in runs: + differences.extend( + _expectation_difference( + project=project, + location=f"{run.engine} `{outcome.label}`", + expected="completes within the command timeout", + actual="timed out", + ) + for outcome in run.outcomes + if outcome.timed_out + ) + return differences + + +def _expectation_difference( + *, project: CorpusProject, location: str, expected: str, actual: str +) -> Difference: + return Difference( + project=project.name, + artifact=_CORPUS_ARTIFACT, + location=location, + left=expected, + right=actual, + labels=_EXPECTATION_LABELS, + ) diff --git a/scripts/compiler_differential/_helpers/report.py b/scripts/compiler_differential/_helpers/report.py new file mode 100644 index 0000000000..fccf230595 --- /dev/null +++ b/scripts/compiler_differential/_helpers/report.py @@ -0,0 +1,39 @@ +"""Render differential results for terminals and CI logs.""" + +from __future__ import annotations + +from scripts.compiler_differential.models import Difference, ProjectComparison + + +def format_comparison(*, comparison: ProjectComparison, engines: tuple[str, str]) -> str: + """Render one project's verdict and the first difference in every differing artifact.""" + + status: str = "OK " if not comparison.differences else "DIFF" + lines: list[str] = [f"{status} {comparison.project} ({comparison.seconds:.1f}s)"] + for difference in comparison.differences: + left_label, right_label = difference.labels or engines + width: int = max(len(left_label), len(right_label)) + lines.append(f" - {difference.artifact} at {difference.location}") + lines.append(f" {left_label:<{width}}: {difference.left}") + lines.append(f" {right_label:<{width}}: {difference.right}") + return "\n".join(lines) + + +def format_summary( + *, comparisons: list[ProjectComparison], engines: tuple[str, str], seconds: float +) -> str: + """Summarize the run, naming the first difference of the first differing project.""" + + differing: list[ProjectComparison] = [item for item in comparisons if item.differences] + left_engine, right_engine = engines + if not differing: + return ( + f"Compiler differential passed: {len(comparisons)} projects identical " + f"({left_engine} vs {right_engine}) in {seconds:.1f}s" + ) + first: Difference = differing[0].differences[0] + return ( + f"Compiler differential FAILED: {len(differing)} of {len(comparisons)} projects differ " + f"({left_engine} vs {right_engine}) in {seconds:.1f}s\n" + f"First difference: {first.project}: {first.artifact} at {first.location}" + ) diff --git a/scripts/compiler_differential/_helpers/run.py b/scripts/compiler_differential/_helpers/run.py new file mode 100644 index 0000000000..0d200afe02 --- /dev/null +++ b/scripts/compiler_differential/_helpers/run.py @@ -0,0 +1,65 @@ +"""Build run options and compare a corpus project by project.""" + +from __future__ import annotations + +import argparse +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from scripts.compiler_differential._helpers.corpus import build_corpus +from scripts.compiler_differential._helpers.options import parse_engine_environment +from scripts.compiler_differential._helpers.project_comparison import compare_project +from scripts.compiler_differential._helpers.report import format_comparison +from scripts.compiler_differential.models import ( + CorpusProject, + DifferentialOptions, + ProjectComparison, +) + + +def differential_options(*, args: argparse.Namespace, work_dir: Path) -> DifferentialOptions: + """Return run options from parsed command-line arguments.""" + + return DifferentialOptions( + engines=(args.engines[0], args.engines[1]), + work_dir=work_dir, + jobs=args.jobs, + stage_captures=args.stage_captures, + python=args.python, + engine_environment=parse_engine_environment(args.engine_env), + ) + + +def selected_corpus(*, args: argparse.Namespace, repo_root: Path) -> list[CorpusProject]: + """Return the projects selected by the parsed command-line arguments.""" + + return build_corpus( + repo_root=repo_root, + corpora=tuple(args.corpus), + seeds=range(args.seed_start, args.seed_start + args.seeds), + dense_models=args.dense_models, + projects=tuple(path.absolute() for path in args.project), + project_expectation=args.expect, + ) + + +def compare_corpus( + *, corpus: list[CorpusProject], options: DifferentialOptions +) -> list[ProjectComparison]: + """Compare every project, printing each result as it completes.""" + + print( + f"Comparing {len(corpus)} projects ({options.engines[0]} vs {options.engines[1]}, " + f"{options.jobs} jobs) in {options.work_dir}", + file=sys.stderr, + ) + comparisons: list[ProjectComparison] = [] + with ThreadPoolExecutor(max_workers=options.jobs) as pool: + for comparison in pool.map( + lambda project: compare_project(project=project, options=options), corpus + ): + comparisons.append(comparison) + print(format_comparison(comparison=comparison, engines=options.engines)) + sys.stdout.flush() + return comparisons diff --git a/scripts/compiler_differential/classes/dense_project.py b/scripts/compiler_differential/classes/dense_project.py new file mode 100644 index 0000000000..1d15205931 --- /dev/null +++ b/scripts/compiler_differential/classes/dense_project.py @@ -0,0 +1,37 @@ +"""Write the dense benchmark project, optionally with a custom Rule.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.compiler_differential.constants import ( + CUSTOM_RULE_FILE, + CUSTOM_RULE_SOURCE, + CUSTOM_RULE_THRESHOLDS, + PROJECT_CONFIG_FILE, +) + + +class DenseProject: + """A dense compile benchmark project of one size.""" + + def __init__(self, *, model_count: int, custom_rules: bool) -> None: + self._model_count: int = model_count + self._custom_rules: bool = custom_rules + + def write(self, project_dir: Path) -> None: + """Write the project; with custom rules, add one Rule every model is checked by.""" + + write_dense_compile_project(project_dir=project_dir, model_count=self._model_count) + if not self._custom_rules: + return + rule_path: Path = project_dir / CUSTOM_RULE_FILE + rule_path.parent.mkdir(parents=True, exist_ok=True) + _ = rule_path.write_text(CUSTOM_RULE_SOURCE, encoding="utf-8", newline="\n") + config_path: Path = project_dir / PROJECT_CONFIG_FILE + _ = config_path.write_text( + config_path.read_text(encoding="utf-8") + CUSTOM_RULE_THRESHOLDS, + encoding="utf-8", + newline="\n", + ) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py new file mode 100644 index 0000000000..beb7b1d465 --- /dev/null +++ b/scripts/compiler_differential/classes/project_builder.py @@ -0,0 +1,604 @@ +"""Build one seeded random project file by file.""" + +from __future__ import annotations + +import random +from collections.abc import Callable +from dataclasses import replace + +from scripts.compiler_differential.constants import ( + GENERATOR_CONSTANT_KIND, + GENERATOR_CROSS_DOMAIN_SHARE, + GENERATOR_DOMAINS, + GENERATOR_ENUM_KIND, + GENERATOR_FLOAT_VALUES, + GENERATOR_HALF, + GENERATOR_HOOK_KIND, + GENERATOR_INVALID_CODES, + GENERATOR_INVALID_SHARE, + GENERATOR_LAYERS, + GENERATOR_MACRO_KIND, + GENERATOR_NON_ASCII_DESCRIPTION_SHARE, + GENERATOR_NON_ASCII_LABELS, + GENERATOR_REGION_ENV_VAR, + GENERATOR_ROLES, + GENERATOR_SOURCE_COLUMNS, + GENERATOR_STATUS_VALUES, + GENERATOR_SUFFIXES, + GENERATOR_VARIABLE_SHARE, +) +from scripts.compiler_differential.models import DeclarationPlan, GeneratedProject, ModelPlan + +_STAGING: str = GENERATOR_LAYERS[0] +_MARTS: str = GENERATOR_LAYERS[1] + + +class ProjectBuilder: + """Accumulate the files of one generated project from a seeded random source.""" + + def __init__(self, seed: int) -> None: + self._seed: int = seed + self._random: random.Random = random.Random(seed) + self._files: dict[str, str] = {} + self._features: set[str] = set() + self._models: list[ModelPlan] = [] + self._sources: list[str] = [] + self._seeds: list[str] = [] + self._declarations: list[DeclarationPlan] = [] + self._model_lines: dict[str, list[str]] = {} + self._model_filters: dict[str, list[str]] = {} + self._private_constants: dict[str, list[str]] = {} + self._hook_consumers: list[ModelPlan] = [] + + def build(self) -> GeneratedProject: + """Return the finished project, optionally carrying one deliberate invalid input.""" + + domains: list[str] = self._random.sample(GENERATOR_DOMAINS, k=self._random.randint(1, 3)) + self._write_sources() + self._write_seeds() + self._plan_models(domains=domains) + self._plan_enums() + for index in range(self._random.randint(1, 3)): + self._plan_constant(index=index) + for index in range(self._random.randint(1, 3)): + self._plan_macro(index=index) + invalid_kind: str | None = ( + self._random.choice(sorted(GENERATOR_INVALID_CODES)) + if self._random.random() < GENERATOR_INVALID_SHARE + else None + ) + self._write_declarations() + self._write_models() + self._write_hooks() + self._write_python_nodes() + self._write_tests_and_scenarios() + self._write_singular_audit() + self._write_config(domains=domains) + if invalid_kind is not None: + self._inject_invalid(kind=invalid_kind) + return GeneratedProject( + files=dict(sorted(self._files.items())), + seed=self._seed, + expected_error_code=( + None if invalid_kind is None else GENERATOR_INVALID_CODES[invalid_kind] + ), + features=tuple(sorted(self._features)), + ) + + def _chance(self, share: float) -> bool: + return self._random.random() < share + + def _label(self) -> str: + return self._random.choice(GENERATOR_NON_ASCII_LABELS) + + def _write_config(self, *, domains: list[str]) -> None: + lines: list[str] = [ + f'name = "generated_{self._seed}"', + 'adapter = "duckdb"', + 'default_target = "dev"', + "", + "[connection]", + 'database = "generated.duckdb"', + "", + "[vars]", + f'region = "{self._random.choice(("north", "south", "east"))}"', + f'discount_rate = "{self._random.choice(GENERATOR_FLOAT_VALUES)}"', + f'site_label = "{self._label()}"', + "", + "[targets.dev]", + f'schema = "{self._random.choice(("analytics", "main"))}"', + "", + "[settings]", + f'default_audit_severity = "{self._random.choice(("warn", "error"))}"', + "", + "[defaults]", + f'materialized = "{self._random.choice(("table", "view"))}"', + "", + '[path_defaults."*/staging"]', + 'materialized = "view"', + 'tags = ["staging"]', + ] + for domain in domains: + if self._chance(GENERATOR_HALF): + self._features.add("path_defaults_literal") + lines.extend( + [ + "", + f'[path_defaults."{domain}/marts"]', + f'materialized = "{self._random.choice(("table", "view"))}"', + f'tags = ["{domain}", "marts"]', + ] + ) + self._features.add("path_defaults_wildcard") + self._files["sqlbuild_project.toml"] = "\n".join(lines) + "\n" + + def _write_sources(self) -> None: + entries: list[str] = [] + columns: str = "\n".join( + f" - name: {column}\n type: {column_type}" + for column, column_type in GENERATOR_SOURCE_COLUMNS + ) + for index in range(self._random.randint(1, 3)): + name: str = f"raw_events_{index}" + self._sources.append(name) + rows: str = " UNION ALL ".join( + f"SELECT {row} AS id, CAST({row * 7 + index} AS DOUBLE) AS amount, " + f"'{GENERATOR_STATUS_VALUES[row % len(GENERATOR_STATUS_VALUES)][1]}' AS status, " + f"TIMESTAMP '2026-01-0{row + 1} 00:00:00' AS created_at" + for row in range(1, 4) + ) + entries.append( + f" - name: {name}\n" + f" description: Generated event feed {index}.\n" + f' expression: "({rows})"\n' + f" columns:\n{columns}" + ) + self._files["sources/raw.yml"] = "sources:\n" + "\n".join(entries) + "\n" + + def _write_seeds(self) -> None: + schemas: list[str] = [] + for index in range(self._random.randint(0, 2)): + name: str = f"lookup_{index}" + self._seeds.append(name) + self._files[f"seeds/{name}.csv"] = f'id,label\n1,"{self._label()}"\n2,plain\n' + schemas.append( + f" - name: {name}\n" + f" description: Generated lookup {index}.\n" + " columns:\n" + " - name: id\n type: INTEGER\n" + " - name: label\n type: VARCHAR" + ) + if schemas: + self._features.add("seeds") + self._files["seeds/seeds.yml"] = "seeds:\n" + "\n".join(schemas) + "\n" + + def _plan_models(self, *, domains: list[str]) -> None: + staging: list[ModelPlan] = [] + for domain in domains: + staging.extend( + ModelPlan( + name=f"stg_{domain}_{index}", + domain=domain, + layer=_STAGING, + inputs=(f"source:{self._random.choice(self._sources)}",), + description=self._description(f"Staged {domain} events {index}"), + ) + for index in range(self._random.randint(1, 3)) + ) + self._models.extend(staging) + for domain in domains: + own: list[ModelPlan] = [model for model in staging if model.domain == domain] + for index in range(self._random.randint(1, 3)): + self._models.append( + ModelPlan( + name=f"{domain}_summary_{index}", + domain=domain, + layer=_MARTS, + inputs=self._mart_inputs(own=own, staging=staging), + description=self._description(f"Summary of {domain} activity {index}"), + ) + ) + + def _mart_inputs(self, *, own: list[ModelPlan], staging: list[ModelPlan]) -> tuple[str, ...]: + upstream: list[ModelPlan] = [self._random.choice(own)] + other: ModelPlan = self._random.choice(staging) + if other not in upstream and self._chance(GENERATOR_CROSS_DOMAIN_SHARE): + upstream.append(other) + self._features.add("cross_domain_ref") + inputs: list[str] = [f"ref:{model.name}" for model in upstream] + if self._seeds and self._chance(GENERATOR_HALF): + inputs.append(f"seed:{self._random.choice(self._seeds)}") + return tuple(inputs) + + def _description(self, text: str) -> str: + if not self._chance(GENERATOR_NON_ASCII_DESCRIPTION_SHARE): + return text + self._features.add("non_ascii_description") + return f"{text} ({self._label()})" + + def _consumers(self) -> tuple[ModelPlan, ...]: + count: int = self._random.randint(1, min(3, len(self._models))) + return tuple( + sorted(self._random.sample(self._models, k=count), key=lambda model: model.name) + ) + + def _plan_enums(self) -> None: + status: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_ENUM_KIND, + name="event_status", + consumers=self._consumers(), + body=( + "ENUM (\n name event_status,\n members (\n" + + "".join(f' {member} "{value}",\n' for member, value in GENERATOR_STATUS_VALUES) + + " ),\n);\n" + ), + ) + self._declarations.append(status) + for model in status.consumers: + member: str = self._random.choice(GENERATOR_STATUS_VALUES)[0] + self._model_filters.setdefault(model.name, []).append( + f'status <> @enum("event_status").{member}' + ) + if not self._chance(GENERATOR_HALF): + return + self._features.add("integer_enum") + shared_name: bool = self._chance(GENERATOR_HALF) + name: str = "adjust_0" if shared_name else "event_priority" + if shared_name: + self._features.add("name_shared_across_namespaces") + priority: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_ENUM_KIND, + name=name, + consumers=self._consumers(), + body=f"ENUM (\n name {name},\n members (LOW 1, HIGH 3),\n);\n", + ) + self._declarations.append(priority) + for model in priority.consumers: + self._model_lines.setdefault(model.name, []).append(f'@enum("{name}").HIGH AS priority') + + def _plan_constant(self, *, index: int) -> None: + name: str = f"threshold_{index}" + writers: dict[str, Callable[[str], tuple[str, str, str]]] = { + "text": self._text_constant, + "bool": self._bool_constant, + "decimal": self._decimal_constant, + "int": self._int_constant, + "float": self._float_constant, + } + parts: tuple[str, str, str] = writers[self._random.choice(sorted(writers))](name) + body: str = parts[0] + literal: str = parts[1] + expression: str = parts[2] + declaration: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_CONSTANT_KIND, name=name, consumers=self._consumers(), body=body + ) + if len(declaration.consumers) == 1 and self._chance(GENERATOR_HALF): + self._features.add("model_private_value") + declaration = replace(declaration, name=f"_{name}", private=True) + expression = expression.replace(f'"{name}"', f'"_{name}"') + self._private_constants.setdefault(declaration.consumers[0].name, []).append( + f" _{name} {literal},\n" + ) + self._declarations.append(declaration) + for model in declaration.consumers: + self._model_lines.setdefault(model.name, []).append(expression) + + def _text_constant(self, name: str) -> tuple[str, str, str]: + self._features.add("non_ascii_constant") + literal: str = f'"{self._label()}"' + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'@const("{name}") AS {name}_label', + ) + + def _bool_constant(self, name: str) -> tuple[str, str, str]: + literal: str = self._random.choice(("true", "false")) + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'@const("{name}") AS {name}_flag', + ) + + def _decimal_constant(self, name: str) -> tuple[str, str, str]: + self._features.add("decimal_constant") + return ( + f'CONSTANT (\n name {name},\n type decimal,\n value "2.4700",\n);\n', + "2.47", + f'amount * @const("{name}") AS {name}_amount', + ) + + def _int_constant(self, name: str) -> tuple[str, str, str]: + literal: str = str(self._random.randint(-5, 500)) + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'amount + @const("{name}") AS {name}_amount', + ) + + def _float_constant(self, name: str) -> tuple[str, str, str]: + self._features.add("float_constant") + literal: str = self._random.choice(GENERATOR_FLOAT_VALUES) + return ( + f"CONSTANT (name {name}, value {literal});\n", + literal, + f'amount * @const("{name}") AS {name}_amount', + ) + + def _plan_macro(self, *, index: int) -> None: + name: str = f"adjust_{index}" + writers: dict[str, Callable[[str], tuple[str, str]]] = { + "ctx": self._context_macro, + "env": self._environment_macro, + "nested": self._nested_macro, + "plain": self._plain_macro, + } + parts: tuple[str, str] = writers[self._random.choice(sorted(writers))](name) + declaration: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_MACRO_KIND, name=name, consumers=self._consumers(), body=parts[0] + ) + self._declarations.append(declaration) + for model in declaration.consumers: + self._model_lines.setdefault(model.name, []).append(parts[1]) + + def _context_macro(self, name: str) -> tuple[str, str]: + self._features.add("macro_reads_ctx") + return ( + f"def {name}(ctx, expression: str) -> str:\n" + ' """Scale an amount by the configured discount."""\n' + ' rate = ctx.vars["discount_rate"]\n' + ' return f"({expression} * (1 - {rate}))"\n', + f'@{name}("amount") AS {name}_amount', + ) + + def _environment_macro(self, name: str) -> tuple[str, str]: + self._features.add("macro_reads_env") + return ( + "import os\n\n\n" + f"def {name}(expression: str) -> str:\n" + ' """Tag an amount with the deployment region."""\n' + f' region = os.environ.get("{GENERATOR_REGION_ENV_VAR}", "east")\n' + " return f\"({expression} + LENGTH('{region}'))\"\n", + f'@{name}("amount") AS {name}_amount', + ) + + def _nested_macro(self, name: str) -> tuple[str, str]: + self._features.add("nested_macro") + return ( + f"def {name}_round(expression: str) -> str:\n" + ' """Round an amount to two places."""\n' + ' return f"ROUND({expression}, 2)"\n\n\n' + f"def {name}(expression: str) -> str:\n" + ' """Double and round an amount."""\n' + f' return {name}_round(f"({{expression}} * 2)")\n', + f'@{name}_round(@{name}("amount")) AS {name}_amount', + ) + + def _plain_macro(self, name: str) -> tuple[str, str]: + offset: str = name.rsplit("_", 1)[1] + return ( + f"def {name}(expression: str) -> str:\n" + ' """Offset an amount."""\n' + f' return f"({{expression}} + {offset} + 1)"\n', + f'@{name}("amount") AS {name}_amount', + ) + + def _write_declarations(self) -> None: + for declaration in self._declarations: + if declaration.private: + continue + directory: str = self._placement(declaration) + suffix: str = GENERATOR_SUFFIXES[declaration.kind] + self._files[f"{directory}/{declaration.name}{suffix}"] = declaration.body + + def _placement(self, declaration: DeclarationPlan) -> str: + role: str = GENERATOR_ROLES[declaration.kind] + folders: set[str] = {model.folder for model in declaration.consumers} + domains: set[str] = {model.domain for model in declaration.consumers} + if len(folders) == 1: + self._features.add("exact_owner_scope") + return f"{folders.pop()}/_sqlbuild/_{role}" + if len(domains) == 1: + self._features.add("tree_scope") + return f"models/{domains.pop()}/_sqlbuild/{role}" + self._features.add("project_scope") + return role + + def _write_models(self) -> None: + for model in self._models: + self._files[f"{model.folder}/{model.name}.sql"] = self._model_sql(model) + + def _model_sql(self, model: ModelPlan) -> str: + header: list[str] = [f' description "{model.description}"'] + if self._chance(GENERATOR_HALF): + header.append(" columns (\n id (audits [not_null]),\n )") + self._features.add("column_audits") + private: list[str] = self._private_constants.get(model.name, []) + if private: + header.append(" constants (\n" + "".join(private) + " )") + header.extend(self._model_hooks(model)) + columns: list[str] = ["t.id", "t.amount", "t.status", "t.created_at"] + columns.extend( + line.replace("amount", "t.amount", 1) for line in self._model_lines.get(model.name, []) + ) + joins: list[str] = [] + for position, extra in enumerate(model.inputs[1:], start=1): + joins.append(f"LEFT JOIN {_relation(extra)} j{position} ON t.id = j{position}.id") + columns.extend( + f"j{position}.label AS lookup_label_{position}" + for _ in range(int(extra.startswith("seed:"))) + ) + if self._chance(GENERATOR_VARIABLE_SHARE): + self._features.add("project_variable") + columns.append("'@@site_label' AS site_label") + filters: list[str] = [ + condition.replace("status", "t.status", 1) + for condition in self._model_filters.get(model.name, []) + ] + where: str = f"\nWHERE {' AND '.join(filters)}" if filters else "" + return ( + "MODEL (\n" + + ",\n".join(header) + + ",\n);\n\nSELECT\n " + + ",\n ".join(columns) + + f"\nFROM {_relation(model.inputs[0])} t" + + "".join(f"\n{join}" for join in joins) + + where + + "\n" + ) + + def _model_hooks(self, model: ModelPlan) -> list[str]: + if model.layer != _MARTS or not self._chance(GENERATOR_HALF): + return [] + self._features.add("sql_hooks") + self._hook_consumers.append(model) + return [ + " post_hooks [\n" + ' sql("record_refresh", relation: "@@CTX:destination.qualified", ' + f'label: "{self._label()}"),\n' + " inline_sql(\"SELECT '@@CTX:destination.qualified' AS refreshed\"),\n" + " ]" + ] + + def _write_hooks(self) -> None: + if not self._hook_consumers: + return + hook: DeclarationPlan = DeclarationPlan( + kind=GENERATOR_HOOK_KIND, name="record_refresh", consumers=tuple(self._hook_consumers) + ) + self._files[f"{self._placement(hook)}/sql/record_refresh.sql"] = ( + 'HOOK (\n description "Record a refresh for one relation"\n);\n\n' + "SELECT @'relation' AS refreshed_relation, @'label' AS refresh_label\n" + ) + + def _write_python_nodes(self) -> None: + if not self._chance(GENERATOR_HALF): + return + self._features.add("python_nodes") + self._files["python/tasks/exports.py"] = ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\n" + "def collect_exports(ctx: TaskContext):\n" + ' """Collect generated exports."""\n' + ' return ctx.result(payload={"rows": 3})\n\n\n' + "@task(depends_on=collect_exports)\n" + "def publish_exports(ctx: TaskContext):\n" + ' """Publish generated exports."""\n' + ' return ctx.result(metadata={"published": True})\n' + ) + + def _write_tests_and_scenarios(self) -> None: + model: ModelPlan = self._random.choice( + [model for model in self._models if model.layer == _STAGING] + ) + source: str = model.inputs[0].split(":", 1)[1] + uses_enum: bool = model.name in self._model_filters + if uses_enum: + self._features.add("relationship_grant") + expected_status: str = '@enum("event_status").PLACED' if uses_enum else "'placed'" + self._features.add("unit_test") + self._files[f"tests/unit/{model.domain}/test_{model.name}.sql"] = ( + "TEST();\n\nWITH\n" + f"__source__{source} AS (\n" + " SELECT 1 AS id, CAST(10 AS DOUBLE) AS amount, 'placed' AS status,\n" + " TIMESTAMP '2026-01-01 00:00:00' AS created_at\n" + "),\n" + f"__expected__{model.name} AS (\n" + f" SELECT 1 AS id, {expected_status} AS status\n" + ")\n" + "SELECT 1\n" + ) + if not self._chance(GENERATOR_HALF): + return + self._features.add("scenario") + self._files[f"tests/scenarios/{model.name}_scenario.sql"] = ( + 'SCENARIO (\n description "Generated scenario for staged events"\n);\n\n' + "WITH\n" + f"__source__{source} AS (\n" + " SELECT 2 AS id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status,\n" + " TIMESTAMP '2026-01-02 00:00:00' AS created_at\n" + "),\n" + f"__expected__{model.name} AS (\n" + " SELECT 2 AS id\n" + ")\n" + "SELECT 1\n" + ) + + def _write_singular_audit(self) -> None: + by_folder: dict[str, list[ModelPlan]] = {} + for model in self._models: + if model.layer == _MARTS: + by_folder.setdefault(model.folder, []).append(model) + candidates: list[list[ModelPlan]] = [ + group for group in by_folder.values() if len(group) > 1 + ] + if not candidates or not self._chance(GENERATOR_HALF): + return + self._features.add("singular_audit") + first, second = self._random.sample(self._random.choice(candidates), k=2) + self._files[f"{first.folder}/_sqlbuild/audits/singular/{first.name}_matches.sql"] = ( + f'AUDIT (name "{first.name}_matches");\n\n' + f'SELECT a.id\nFROM __ref("{first.name}") a\n' + f'LEFT JOIN __ref("{second.name}") b ON a.id = b.id\nWHERE b.id IS NULL\n' + ) + + def _inject_invalid(self, *, kind: str) -> None: + self._features.add(f"invalid:{kind}") + model: ModelPlan = self._random.choice(self._models) + path: str = f"{model.folder}/{model.name}.sql" + injections: dict[str, Callable[[ModelPlan], str]] = { + "unknown_ref": lambda _: '(SELECT 1 FROM __ref("missing_model")) AS missing', + "unknown_macro": lambda _: '@not_a_macro("id") AS bogus', + "failing_macro": self._failing_macro_column, + "unknown_enum_member": lambda _: '@enum("event_status").MISSING AS bogus', + "unknown_constant": lambda _: '@const("no_such_value") AS bogus', + "missing_description": self._drop_description, + "duplicate_macro": self._duplicate_macro, + "header_syntax": self._break_header, + "over_broad_constant": self._over_broad_constant_column, + } + column: str = injections[kind](model) + if column: + self._files[path] = self._files[path].replace("SELECT\n", f"SELECT\n {column},\n", 1) + + def _failing_macro_column(self, model: ModelPlan) -> str: + self._files[f"{model.folder}/_sqlbuild/_macros/explode.py"] = ( + "def explode(expression: str) -> str:\n" + ' """Refuse every input."""\n' + ' raise ValueError(f"cannot adjust {expression}")\n' + ) + return '@explode("id") AS bogus' + + def _drop_description(self, model: ModelPlan) -> str: + path: str = f"{model.folder}/{model.name}.sql" + self._files[path] = self._files[path].replace(f' description "{model.description}",\n', "") + self._files[path] = self._files[path].replace(f' description "{model.description}"\n', "") + return "" + + def _duplicate_macro(self, model: ModelPlan) -> str: + _ = model + self._files["macros/duplicate_adjust_0.py"] = ( + "def adjust_0(expression: str) -> str:\n" + ' """Duplicate a macro name."""\n' + " return expression\n" + ) + return "" + + def _break_header(self, model: ModelPlan) -> str: + path: str = f"{model.folder}/{model.name}.sql" + self._files[path] = self._files[path].replace( + "MODEL (\n", "MODEL (\n materialized (,\n", 1 + ) + return "" + + def _over_broad_constant_column(self, model: ModelPlan) -> str: + _ = model + self._files["constants/unused_threshold.sql"] = ( + "CONSTANT (name unused_threshold, value 3);\n" + ) + return '@const("unused_threshold") AS narrow' + + +def _relation(reference: str) -> str: + kind, name = reference.split(":", 1) + return f'__{kind}("{name}")' diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py new file mode 100644 index 0000000000..012ee68423 --- /dev/null +++ b/scripts/compiler_differential/constants.py @@ -0,0 +1,186 @@ +"""Stable constants for the compiler engine differential harness.""" + +import re + +from scripts.compiler_differential.models import DifferentialCommand, ExpectedOutcome + +ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" +STAGE_CAPTURE_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +DEFAULT_ENGINES: tuple[str, str] = ("python", "native") +SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( + {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR} +) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +PROJECT_CONFIG_FILE: str = "sqlbuild_project.toml" +DUCKDB_ADAPTER: str = "duckdb" +PROJECT_DIRECTORY: str = "project" +CAPTURES_DIRECTORY: str = "captures" +TARGET_DIRECTORY: str = "target" +COMPILED_DIRECTORY: str = "target/compiled" +MANIFEST_FILE: str = "target/manifest.json" +DAG_FILE: str = "target/sqlbuild_dag.json" +STRIPPED_REPORT_FIELDS: frozenset[str] = frozenset({"compile_timings", "compiler_engine"}) +MANIFEST_VOLATILE_METADATA: frozenset[str] = frozenset({"generated_at", "invocation_id"}) +INVOCATION_ID_PATTERN: re.Pattern[str] = re.compile(r"\b\d{8}T\d{6}Z_[0-9a-f]{12}\b") +INVOCATION_ID_MASK: str = "" +ENGINE_NAMESPACE_PATTERN: re.Pattern[str] = re.compile(r"-native-v\d+\b") +ELAPSED_TIME_PATTERN: re.Pattern[str] = re.compile(r"\b\d+(?:\.\d+)?\s?m?s\b") +ELAPSED_TIME_MASK: str = "" +COMMAND_TIMEOUT_SECONDS: float = 900.0 +VALUE_PREVIEW_CHARACTERS: int = 160 +MISSING_VALUE: str = "" +DEFAULT_SEED_COUNT: int = 12 +PLAN_LABEL: str = "plan" +ERROR_SEVERITY: str = "error" +MODEL_RESOURCE_TYPE: str = "model" +DEFAULT_DENSE_MODELS: int = 3000 +FIXTURE_ROOT: str = "tests/e2e/fixtures" +EXAMPLE_ROOT: str = "website/examples" +FIXTURE_PROJECT_SUBDIRECTORIES: dict[str, str] = {"dbt_interop": "sqlbuild_project"} +EXPECT_SUCCESS: ExpectedOutcome = ExpectedOutcome() +EXPECT_SUCCESS_VALUE: str = "success" +EXPECT_FAILURE_PREFIX: str = "failure:" +FIXTURE_EXPECTED_OUTCOMES: dict[str, ExpectedOutcome] = { + "dbt_interop": ExpectedOutcome(error_code="C214") +} +CORPUS_FIXTURES: str = "fixtures" +CORPUS_EXAMPLES: str = "examples" +CORPUS_SEEDS: str = "seeds" +CORPUS_FAILURES: str = "failures" +CORPUS_DENSE: str = "dense" +CORPUS_NAMES: tuple[str, ...] = ( + CORPUS_FIXTURES, + CORPUS_EXAMPLES, + CORPUS_SEEDS, + CORPUS_FAILURES, + CORPUS_DENSE, +) +PER_PULL_REQUEST_CORPORA: tuple[str, ...] = ( + CORPUS_FIXTURES, + CORPUS_EXAMPLES, + CORPUS_SEEDS, + CORPUS_FAILURES, +) +CUSTOM_RULE_FILE: str = "rules/order_names.py" +CUSTOM_RULE_SOURCE: str = """from sqlbuild.rules import Finding, Model, RuleContext, rule + + +@rule( + code="XSQBRDIFF001", + message="Model names stay below sixty characters", + remediation="Rename the model.", +) +def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: + if len(model.name) < 60: + return [] + return [ctx.finding(subject=model)] +""" +CUSTOM_RULE_THRESHOLDS: str = "\n[rules.thresholds]\nmin_custom_rule_test_cases = 0\n" +COLD_COMPILE: DifferentialCommand = DifferentialCommand( + label="compile", arguments=("compile", "--json", "--manifest", "--dag") +) +WARM_COMPILE: DifferentialCommand = DifferentialCommand( + label="compile-warm", arguments=("compile", "--json") +) +PLAN: DifferentialCommand = DifferentialCommand(label=PLAN_LABEL, arguments=("plan", "--json")) + +GENERATOR_DOMAINS: tuple[str, ...] = ("sales", "inventory", "support", "fulfillment", "billing") +GENERATOR_LAYERS: tuple[str, ...] = ("staging", "marts") +GENERATOR_SOURCE_COLUMNS: tuple[tuple[str, str], ...] = ( + ("id", "INTEGER"), + ("amount", "DOUBLE"), + ("status", "VARCHAR"), + ("created_at", "TIMESTAMP"), +) +GENERATOR_STATUS_VALUES: tuple[tuple[str, str], ...] = ( + ("PLACED", "placed"), + ("SHIPPED", "expédié"), + ("CANCELLED", "取消"), + ("RETURNED", "returned ✓"), +) +GENERATOR_NON_ASCII_LABELS: tuple[str, ...] = ( + "Zürich – depot", + "東京 warehouse", + "café ☕ counter", + "naïve résumé", + "Ελληνικά orders", +) +GENERATOR_FLOAT_VALUES: tuple[str, ...] = ("0.15", "1e-07", "12345.678", "0.1", "2.5e+20", "-3.75") +GENERATOR_REGION_ENV_VAR: str = "SQB_DIFFERENTIAL_REGION" +GENERATOR_INVALID_SHARE: float = 0.25 +GENERATOR_INVALID_CODES: dict[str, str] = { + "unknown_ref": "P001", + "unknown_macro": "P001", + "failing_macro": "P001", + "unknown_enum_member": "P001", + "unknown_constant": "P001", + "missing_description": "P010", + "duplicate_macro": "P001", + "header_syntax": "D002", + "over_broad_constant": "S024", +} +GENERATOR_HALF: float = 0.5 +GENERATOR_CROSS_DOMAIN_SHARE: float = 0.4 +GENERATOR_NON_ASCII_DESCRIPTION_SHARE: float = 0.3 +GENERATOR_VARIABLE_SHARE: float = 0.3 + +FAILURE_BASE_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "duckdb"\n\n[connection]\ndatabase = "failure.duckdb"\n' +) +FAILURE_BASE_SOURCES: str = """sources: + - name: raw_orders + description: Orders feed. + expression: >- + (SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, + 'placed' AS status) + columns: + - name: order_id + type: INTEGER + - name: customer_id + type: INTEGER + - name: amount + type: DOUBLE + - name: status + type: VARCHAR +""" +FAILURE_BASE_STAGING: str = """MODEL ( + description "Staged orders", +); + +SELECT order_id, customer_id, amount, status +FROM __source("raw_orders") +""" +FAILURE_BASE_MART: str = """MODEL ( + description "Order totals per customer", +); + +SELECT customer_id, SUM(amount) AS total_amount +FROM __ref("stg_orders") +GROUP BY customer_id +""" +FAILURE_BASE_FILES: dict[str, str] = { + "sqlbuild_project.toml": FAILURE_BASE_CONFIG, + "sources/raw.yml": FAILURE_BASE_SOURCES, + "models/staging/stg_orders.sql": FAILURE_BASE_STAGING, + "models/marts/customer_totals.sql": FAILURE_BASE_MART, +} +FAILURE_STAGING_PATH: str = "models/staging/stg_orders.sql" +FAILURE_MART_PATH: str = "models/marts/customer_totals.sql" +FAILURE_CONFIG_PATH: str = "sqlbuild_project.toml" +FAILURE_SOURCES_PATH: str = "sources/raw.yml" +GENERATOR_ENUM_KIND: str = "enum" +GENERATOR_CONSTANT_KIND: str = "constant" +GENERATOR_MACRO_KIND: str = "macro" +GENERATOR_HOOK_KIND: str = "hook" +GENERATOR_ROLES: dict[str, str] = { + GENERATOR_ENUM_KIND: "enums", + GENERATOR_CONSTANT_KIND: "constants", + GENERATOR_MACRO_KIND: "macros", + GENERATOR_HOOK_KIND: "hooks", +} +GENERATOR_SUFFIXES: dict[str, str] = { + GENERATOR_ENUM_KIND: ".sql", + GENERATOR_CONSTANT_KIND: ".sql", + GENERATOR_MACRO_KIND: ".py", +} diff --git a/scripts/compiler_differential/exceptions.py b/scripts/compiler_differential/exceptions.py new file mode 100644 index 0000000000..21f24d38ee --- /dev/null +++ b/scripts/compiler_differential/exceptions.py @@ -0,0 +1,5 @@ +"""Compiler differential harness errors.""" + + +class DifferentialUsageError(ValueError): + """Raised when the harness is invoked with an invalid option value.""" diff --git a/scripts/compiler_differential/main/__init__.py b/scripts/compiler_differential/main/__init__.py new file mode 100644 index 0000000000..e716df0fcd --- /dev/null +++ b/scripts/compiler_differential/main/__init__.py @@ -0,0 +1 @@ +"""Public compiler differential entry points.""" diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py new file mode 100644 index 0000000000..ea25888d9d --- /dev/null +++ b/scripts/compiler_differential/main/differential.py @@ -0,0 +1,91 @@ +"""Compare compile and plan output of two compiler engines across a project corpus.""" + +from __future__ import annotations + +import argparse +import os +import sys +import tempfile +import time +from pathlib import Path + +from scripts.compiler_differential._helpers.options import parse_expected_outcome +from scripts.compiler_differential._helpers.report import format_summary +from scripts.compiler_differential._helpers.run import ( + compare_corpus, + differential_options, + selected_corpus, +) +from scripts.compiler_differential.constants import ( + CORPUS_NAMES, + DEFAULT_DENSE_MODELS, + DEFAULT_ENGINES, + DEFAULT_SEED_COUNT, + EXPECT_SUCCESS, + PER_PULL_REQUEST_CORPORA, +) +from scripts.compiler_differential.models import DifferentialOptions, ProjectComparison + + +def run_compiler_differential(argv: list[str] | None = None) -> int: + """Run the selected corpus under both engines; exit 1 when any artifact differs.""" + + args: argparse.Namespace = _parse_args(argv) + repo_root: Path = Path(__file__).resolve().parents[3] + started: float = time.monotonic() + with tempfile.TemporaryDirectory(prefix="sqlbuild-differential-") as temporary: + work_dir: Path = (args.work_dir or Path(temporary)).absolute() + options: DifferentialOptions = differential_options(args=args, work_dir=work_dir) + comparisons: list[ProjectComparison] = compare_corpus( + corpus=selected_corpus(args=args, repo_root=repo_root), options=options + ) + print( + format_summary( + comparisons=comparisons, + engines=options.engines, + seconds=time.monotonic() - started, + ) + ) + return 1 if any(comparison.differences for comparison in comparisons) else 0 + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + prog="python -m scripts.run_compiler_differential", + description="Compare sqb compile and plan output between two compiler engines.", + ) + parser.add_argument( + "--corpus", + nargs="*", + choices=CORPUS_NAMES, + default=list(PER_PULL_REQUEST_CORPORA), + help="corpora to compare (default: the per-PR corpora; pass none with --project)", + ) + parser.add_argument("--project", nargs="*", type=Path, default=[], help="extra project dirs") + parser.add_argument( + "--expect", + type=parse_expected_outcome, + default=EXPECT_SUCCESS, + metavar="success|failure:CODE", + help="outcome every --project must produce on the first engine (default: success)", + ) + parser.add_argument("--seeds", type=int, default=DEFAULT_SEED_COUNT) + parser.add_argument("--seed-start", type=int, default=0) + parser.add_argument("--dense-models", type=int, default=DEFAULT_DENSE_MODELS) + parser.add_argument("--engines", nargs=2, default=list(DEFAULT_ENGINES), metavar="ENGINE") + parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1) + parser.add_argument( + "--stage-captures", + action="store_true", + help="also dump and diff every frontier object to localise a mismatch to a stage", + ) + parser.add_argument( + "--engine-env", + action="append", + default=[], + metavar="ENGINE:NAME=VALUE", + help="extra environment for one engine's processes", + ) + parser.add_argument("--work-dir", type=Path, default=None, help="keep run directories here") + parser.add_argument("--python", type=Path, default=Path(sys.executable)) + return parser.parse_args(argv) diff --git a/scripts/compiler_differential/main/failure_cases.py b/scripts/compiler_differential/main/failure_cases.py new file mode 100644 index 0000000000..5d6352bdf7 --- /dev/null +++ b/scripts/compiler_differential/main/failure_cases.py @@ -0,0 +1,10 @@ +"""The failure corpus: one minimal failing project per diagnostic family member.""" + +from scripts.compiler_differential._helpers.failure_cases import all_failure_cases +from scripts.compiler_differential.models import FailureCase + + +def failure_cases() -> tuple[FailureCase, ...]: + """Return every failure case in a stable order.""" + + return all_failure_cases() diff --git a/scripts/compiler_differential/main/generate_project.py b/scripts/compiler_differential/main/generate_project.py new file mode 100644 index 0000000000..5dd0c654d9 --- /dev/null +++ b/scripts/compiler_differential/main/generate_project.py @@ -0,0 +1,10 @@ +"""Generate one deterministic random project for a seed.""" + +from scripts.compiler_differential.classes.project_builder import ProjectBuilder +from scripts.compiler_differential.models import GeneratedProject + + +def generate_project(*, seed: int) -> GeneratedProject: + """Return the same project files and expectation for the same seed on every platform.""" + + return ProjectBuilder(seed).build() diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py new file mode 100644 index 0000000000..7da950905d --- /dev/null +++ b/scripts/compiler_differential/models.py @@ -0,0 +1,159 @@ +"""Typed records for the compiler engine differential harness.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True) +class DifferentialCommand: + """One sqb invocation run identically under both engines.""" + + label: str + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class ExpectedOutcome: + """What a corpus project must produce: success, or failure with one diagnostic code.""" + + error_code: str | None = None + + +@dataclass(frozen=True) +class CorpusProject: + """One project compared across engines.""" + + name: str + commands: tuple[DifferentialCommand, ...] + expected: ExpectedOutcome + source_dir: Path | None = None + writer: Callable[[Path], None] | None = None + project_subdirectory: str | None = None + + +@dataclass(frozen=True) +class CommandOutcome: + """What one sqb invocation printed and returned.""" + + label: str + exit_code: int + stdout: str + stderr: str + timed_out: bool = False + + +@dataclass(frozen=True) +class EngineRun: + """Everything one engine produced for one corpus project.""" + + engine: str + outcomes: tuple[CommandOutcome, ...] + compiled: dict[str, bytes] + manifest: str | None + dag: str | None + captures: dict[str, dict[str, str]] = field(default_factory=dict) + + +@dataclass(frozen=True) +class Divergence: + """Where two values first differ, with short previews of both sides.""" + + location: str + left: str + right: str + + +@dataclass(frozen=True) +class Difference: + """The first observed divergence in one compared artifact.""" + + project: str + artifact: str + location: str + left: str + right: str + labels: tuple[str, str] | None = None + + +@dataclass(frozen=True) +class ProjectComparison: + """All differences found for one corpus project.""" + + project: str + differences: tuple[Difference, ...] + seconds: float + + +@dataclass(frozen=True) +class DifferentialOptions: + """Resolved harness settings.""" + + engines: tuple[str, str] + work_dir: Path + jobs: int + stage_captures: bool + python: Path + engine_environment: dict[str, dict[str, str]] + + +@dataclass(frozen=True) +class WritableProject: + """Project files that can be written below a fresh directory.""" + + files: dict[str, str] + + def write(self, project_dir: Path) -> None: + """Write every UTF-8 file, creating parent directories.""" + + for relative_path, contents in sorted(self.files.items()): + path: Path = project_dir / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8", newline="\n") + + +@dataclass(frozen=True) +class GeneratedProject(WritableProject): + """A deterministic random project and the error it was generated to produce, if any.""" + + seed: int + expected_error_code: str | None + features: tuple[str, ...] + + +@dataclass(frozen=True) +class FailureCase(WritableProject): + """One minimal project that must fail compile with one diagnostic code.""" + + name: str + expected_code: str + + +@dataclass(frozen=True) +class ModelPlan: + """One generated SQL model and the relations it reads.""" + + name: str + domain: str + layer: str + inputs: tuple[str, ...] + description: str + + @property + def folder(self) -> str: + """Return the model's folder relative to the project root.""" + + return f"models/{self.domain}/{self.layer}" + + +@dataclass(frozen=True) +class DeclarationPlan: + """One generated enum, constant, macro, or named hook and the models that use it.""" + + kind: str + name: str + consumers: tuple[ModelPlan, ...] + body: str = "" + private: bool = False diff --git a/scripts/run_compiler_differential.py b/scripts/run_compiler_differential.py new file mode 100644 index 0000000000..0ffb26aada --- /dev/null +++ b/scripts/run_compiler_differential.py @@ -0,0 +1,15 @@ +"""Direct wrapper for the compiler engine differential harness.""" + +from __future__ import annotations + +from scripts.compiler_differential.main.differential import run_compiler_differential + + +def main(argv: list[str] | None = None) -> int: + """Compare compile and plan output between the Python and native compiler engines.""" + + return run_compiler_differential(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/sqlbuild/cli/commands/_helpers/compile/output.py b/src/sqlbuild/cli/commands/_helpers/compile/output.py index 7a873ed3de..232054aa0c 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/output.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/output.py @@ -38,6 +38,7 @@ ) from sqlbuild.compiler.discovery.constants import SQL_HOOK_OUTPUT_FIELDS from sqlbuild.compiler.discovery.main.serialize_hook_entries import serialize_hook_entries +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine from sqlbuild.compiler.lineage.models import ProjectColumnLineage from sqlbuild.compiler.pipeline.models import ProjectGraph from sqlbuild.compiler.python_nodes.main.hook_identities import build_hook_identities @@ -208,6 +209,7 @@ def format_compile_json( sql_validation_enabled=sql_validation_enabled, ), "compile_timings": timings_ms, + "compiler_engine": resolve_compiler_engine().value, "lineage_mode": lineage_mode.value, "resources": _resources(graph=graph, lineage=lineage), "artifacts": _artifacts(written=written, manifest=manifest), diff --git a/src/sqlbuild/cli/commands/_helpers/debug/checks.py b/src/sqlbuild/cli/commands/_helpers/debug/checks.py index 85cade7924..aa01e5c13c 100644 --- a/src/sqlbuild/cli/commands/_helpers/debug/checks.py +++ b/src/sqlbuild/cli/commands/_helpers/debug/checks.py @@ -41,6 +41,7 @@ ) from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveredProvider +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( resolve_effective_adapter_name, ) @@ -272,6 +273,7 @@ def _build_runtime_lines() -> list[DebugLine]: sqlbuild_version = "unknown" return [ DebugLine(label="sqlbuild version", message=sqlbuild_version), + DebugLine(label="compiler engine", message=resolve_compiler_engine().value), DebugLine(label="python version", message=platform.python_version()), DebugLine(label="python path", message=sys.executable), DebugLine(label="os info", message=platform.platform()), diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parser.py b/src/sqlbuild/cli/commands/_helpers/entry/parser.py index ba08ac6710..17e9c79555 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parser.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parser.py @@ -29,6 +29,7 @@ ) from sqlbuild.cli.commands.types import CliCommand, CompileLineageMode from sqlbuild.compiler.contract_adoption.types import ContractAction +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_OPTION, COMPILER_ENGINE_VALUES from sqlbuild.compiler.lineage.types import ColumnLineageMode from sqlbuild.compiler.scopes.types import DeclarationKind @@ -98,6 +99,13 @@ def _add_global_args(*, parser: argparse.ArgumentParser, suppress_defaults: bool default=argparse.SUPPRESS if suppress_defaults else False, help="enable verbose output and mirror internal diagnostics to stderr", ) + parser.add_argument( + COMPILER_ENGINE_OPTION, + dest="compiler_engine", + choices=COMPILER_ENGINE_VALUES, + default=argparse.SUPPRESS if suppress_defaults else None, + help=argparse.SUPPRESS, + ) def _add_global_args_to_subcommands( diff --git a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py index 56bc7f1221..70575c1dce 100644 --- a/src/sqlbuild/cli/commands/_helpers/entry/parsing.py +++ b/src/sqlbuild/cli/commands/_helpers/entry/parsing.py @@ -25,6 +25,8 @@ SelectorFileSummary, SelectorInputs, ) +from sqlbuild.compiler.frontier.exceptions import CompilerEngineError +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine from sqlbuild.compiler.scopes.types import DeclarationKind from sqlbuild.integrations.dbt.types import DbtInteropCommand @@ -115,6 +117,11 @@ def parse_cli_invocation( if unknown_args: parser.error(f"unrecognized arguments: {' '.join(unknown_args)}") args.verbose = args.verbose or args.debug + if args.compiler_engine is None: + try: + _ = resolve_compiler_engine() + except CompilerEngineError as error: + parser.error(str(error)) if ( args.debug and args.command == CliCommand.DBT diff --git a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py index 1454a47f27..eef67adaca 100644 --- a/src/sqlbuild/cli/commands/_helpers/lineage/cache.py +++ b/src/sqlbuild/cli/commands/_helpers/lineage/cache.py @@ -16,11 +16,14 @@ from sqlbuild.cli.commands.constants import TARGET_DIRECTORY_NAME from sqlbuild.cli.commands.models import LineageNode, RelationLineageIndex from sqlbuild.compiler.compile.models import CompiledObjectKey, CompiledProject +from sqlbuild.compiler.frontier.constants import CACHE_DIRECTORY_NAME +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name from sqlbuild.compiler.pipeline.models import ProjectGraph _CACHE_SCHEMA_VERSION: int = 1 _CACHE_ALGORITHM_VERSION: str = "relation-lineage-graph-v4" -_CACHE_RELATIVE_PATH: Path = Path("cache/lineage/v1/structural-graph.sqlite3") +_LINEAGE_CACHE_DIRECTORY_NAME: str = "lineage" +_CACHE_FILE_RELATIVE_PATH: Path = Path("v1/structural-graph.sqlite3") _CACHE_MAX_BYTES: int = 50_000_000 _CACHE_MAX_ROWS: int = 1_000_000 _SQLITE_TIMEOUT_SECONDS: float = 5.0 @@ -176,7 +179,13 @@ def _is_excluded(*, path: Path, project_dir: Path) -> bool: def _cache_path(*, project_dir: Path) -> Path: - return project_dir / TARGET_DIRECTORY_NAME / _CACHE_RELATIVE_PATH + return ( + project_dir + / TARGET_DIRECTORY_NAME + / CACHE_DIRECTORY_NAME + / engine_cache_name(_LINEAGE_CACHE_DIRECTORY_NAME) + / _CACHE_FILE_RELATIVE_PATH + ) def _relation_index(*, graph: ProjectGraph) -> RelationLineageIndex: diff --git a/src/sqlbuild/cli/commands/classes/cli_namespace.py b/src/sqlbuild/cli/commands/classes/cli_namespace.py index b07cb18c1c..474c482885 100644 --- a/src/sqlbuild/cli/commands/classes/cli_namespace.py +++ b/src/sqlbuild/cli/commands/classes/cli_namespace.py @@ -48,6 +48,7 @@ "start_cursor_int": None, "end_cursor_int": None, "no_color": False, + "compiler_engine": None, "fail_fast": False, "full_refresh": False, "as_target": None, @@ -217,6 +218,7 @@ class CliNamespace: start_cursor_int: str | None end_cursor_int: str | None no_color: bool + compiler_engine: str | None fail_fast: bool full_refresh: bool as_target: str | None diff --git a/src/sqlbuild/cli/commands/main/entrypoint/entry.py b/src/sqlbuild/cli/commands/main/entrypoint/entry.py index f294f909a1..da0ef22774 100644 --- a/src/sqlbuild/cli/commands/main/entrypoint/entry.py +++ b/src/sqlbuild/cli/commands/main/entrypoint/entry.py @@ -14,6 +14,8 @@ CliEntrypointHandlers, ParsedCliInvocation, ) +from sqlbuild.compiler.frontier.main.compiler_engine_override import compiler_engine_override +from sqlbuild.compiler.frontier.types import CompilerEngine from sqlbuild.diagnostics.main.diagnostics_context import diagnostics_context from sqlbuild.presentation.main.supports_color import supports_color from sqlbuild.runtime.observability.main.invocation_external_context_scope import ( @@ -61,13 +63,16 @@ def _main_with_dependencies( dispatch_with_compute_logs, ) - return dispatch_with_compute_logs( - args=args, - identity=identity, - operation=lambda: dispatch_and_handle_errors( + with compiler_engine_override( + None if args.compiler_engine is None else CompilerEngine(args.compiler_engine) + ): + return dispatch_with_compute_logs( args=args, - invocation=invocation, - handlers=handlers, - use_color=use_color, - ), - ) + identity=identity, + operation=lambda: dispatch_and_handle_errors( + args=args, + invocation=invocation, + handlers=handlers, + use_color=use_color, + ), + ) diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py index d7a0e729b3..dac7203b29 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/attempt.py @@ -45,7 +45,7 @@ PROJECT_CONFIG_FILENAMES, REUSE_DISABLE_ENV_VAR, REUSE_DISABLE_VALUE, - REUSE_ENTRY_DIRECTORY_PARTS, + REUSE_ENTRY_DIRECTORY_NAME, REUSE_ENTRY_SUFFIX, REUSE_HIT_MESSAGE, TOTAL_TIMING, @@ -62,6 +62,7 @@ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.presentation.main.supports_color import supports_color @@ -81,7 +82,8 @@ def attempt_reuse(*, request: CompileReuseRequest) -> CompileReuseAttempt: attempt: CompileReuseAttempt = replace( bypass, outcome=CompileReuseOutcome.MISS, - entry_path=Path(project_dir, *REUSE_ENTRY_DIRECTORY_PARTS) + entry_path=compiler_cache_directory(Path(project_dir)) + / REUSE_ENTRY_DIRECTORY_NAME / f"{entry_slot_name(selected_target=request.selected_target)}{REUSE_ENTRY_SUFFIX}", runtime=runtime_identity(), search_path=search_path_stamps(project_dir=project_dir), diff --git a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py index b3eddc3024..3c792b9ba5 100644 --- a/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py +++ b/src/sqlbuild/cli/compile_reuse/_helpers/runtime_identity.py @@ -17,8 +17,10 @@ PROJECT_ROOT_PATH_MTIME_NS, REUSE_FORMAT_VERSION, TRACKED_ENVIRONMENT_PREFIXES, + UNTRACKED_ENVIRONMENT_NAMES, ) from sqlbuild.cli.compile_reuse.models import CompileReuseRequest, SettingsEnvironmentInputs +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine def runtime_identity() -> dict[str, str]: @@ -96,10 +98,12 @@ def module_stamps_unchanged(*, stamps: tuple[tuple[str, int, int], ...]) -> bool def tracked_environment_names(*, template_names: tuple[str, ...]) -> tuple[str, ...]: - """Return SQLBuild-prefixed variables plus the variables templates read.""" + """Return tracked SQLBuild variables (not engine or capture settings) plus template reads.""" prefixed: set[str] = { - name for name in os.environ if name.startswith(TRACKED_ENVIRONMENT_PREFIXES) + name + for name in os.environ + if name.startswith(TRACKED_ENVIRONMENT_PREFIXES) and name not in UNTRACKED_ENVIRONMENT_NAMES } return tuple(sorted(prefixed.union(template_names))) @@ -155,6 +159,7 @@ def invocation_digest(*, request: CompileReuseRequest, project_dir: str, use_col "exclude": list(request.exclude), "cli_vars": request.cli_vars, "sys_path": list(sys.path), + "compiler_engine": resolve_compiler_engine().value, } encoded: bytes = json.dumps( payload, sort_keys=True, separators=(",", ":"), default=repr diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 95f701e062..e8896df728 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -7,9 +7,19 @@ from sqlbuild.cli.compile_reuse.classes.compile_artifact_write_registry import ( CompileArtifactWriteRegistry, ) +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, +) REUSE_FORMAT_VERSION: int = 1 -REUSE_ENTRY_DIRECTORY_PARTS: tuple[str, ...] = ("target", "cache", "compiler", "project-reuse-v1") +REUSE_ENTRY_DIRECTORY_NAME: str = "project-reuse-v1" +REUSE_ENTRY_DIRECTORY_PARTS: tuple[str, ...] = ( + "target", + "cache", + "compiler", + REUSE_ENTRY_DIRECTORY_NAME, +) REUSE_ENTRY_MAGIC: bytes = b"SQBREUSE1\n" REUSE_ENTRY_SUFFIX: str = ".entry" REUSE_STDOUT_SUFFIX: str = ".stdout" @@ -25,6 +35,9 @@ REUSE_DISABLE_VALUE: str = "1" REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") +UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( + {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} +) MISSING_ENVIRONMENT_VALUE: str = "\0missing" MISSING_FILE_DIGEST: str = "\0missing" PROJECT_CONFIG_FILENAMES: tuple[str, ...] = ("sqlbuild_project.toml", "sqlbuild_project.yml") diff --git a/src/sqlbuild/compiler/discovery/main/discover.py b/src/sqlbuild/compiler/discovery/main/discover.py index ccf9207c03..6154330a7f 100644 --- a/src/sqlbuild/compiler/discovery/main/discover.py +++ b/src/sqlbuild/compiler/discovery/main/discover.py @@ -2,6 +2,7 @@ from __future__ import annotations +from functools import partial from pathlib import Path from sqlbuild.compiler.discovery._helpers.filesystem.aggregation import ( @@ -26,6 +27,8 @@ from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs, DiscoveryCacheRequest from sqlbuild.compiler.discovery.types import DeclarationFilesReuse from sqlbuild.compiler.fact_cache.classes.fact_cache_store import FactCacheStore +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.runtime.observability.classes.operation_lifecycle import OperationLifecycle from sqlbuild.spec.contracts.models import LocalConfig, ProjectConfig @@ -41,12 +44,16 @@ def discover_project_inputs( """Load all raw project inputs from disk before semantic resolution.""" with OperationLifecycle(operation_kind="project", operation_name="project_discovery"): - return _discover_project_inputs( - project_dir=project_dir, - sql_analysis_enabled_override=sql_analysis_enabled_override, - extract_output_column_locations=extract_output_column_locations, - cache_request=cache_request, - declaration_reuse=declaration_reuse, + return compile_frontier( + until=CompilerStage.DISCOVERED_PROJECT_INPUTS, + python_stage=partial( + _discover_project_inputs, + project_dir=project_dir, + sql_analysis_enabled_override=sql_analysis_enabled_override, + extract_output_column_locations=extract_output_column_locations, + cache_request=cache_request, + declaration_reuse=declaration_reuse, + ), ) diff --git a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py b/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py index 39067cf45e..27ffbcebd1 100644 --- a/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py +++ b/src/sqlbuild/compiler/fact_cache/main/_compile_cache_root.py @@ -9,6 +9,7 @@ COMPILE_CACHE_DISABLE_ENV_VAR, COMPILE_CACHE_DISABLE_VALUE, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.spec.contracts.models import TargetConfig @@ -24,4 +25,4 @@ def compile_cache_root( or os.environ.get(COMPILE_CACHE_DISABLE_ENV_VAR) == COMPILE_CACHE_DISABLE_VALUE ): return None - return project_dir / "target" / "cache" / "compiler" + return compiler_cache_directory(project_dir) diff --git a/src/sqlbuild/compiler/frontier/__init__.py b/src/sqlbuild/compiler/frontier/__init__.py new file mode 100644 index 0000000000..86ae110a66 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/__init__.py @@ -0,0 +1 @@ +"""Compiler engine selection and the native frontier seam.""" diff --git a/src/sqlbuild/compiler/frontier/_helpers/__init__.py b/src/sqlbuild/compiler/frontier/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sqlbuild/compiler/frontier/_helpers/engine.py b/src/sqlbuild/compiler/frontier/_helpers/engine.py new file mode 100644 index 0000000000..875cbbd141 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/_helpers/engine.py @@ -0,0 +1,26 @@ +"""Engine parsing and environment lookup shared by the frontier entry points.""" + +import os + +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + COMPILER_ENGINE_VALUES, + DEFAULT_COMPILER_ENGINE, +) +from sqlbuild.compiler.frontier.exceptions import CompilerEngineError +from sqlbuild.compiler.frontier.types import CompilerEngine + + +def active_compiler_engine() -> CompilerEngine: + """Return the engine selected by SQLBUILD_COMPILER_ENGINE, defaulting to Python.""" + + raw_value: str | None = os.environ.get(COMPILER_ENGINE_ENV_VAR) + if not raw_value: + return DEFAULT_COMPILER_ENGINE + try: + return CompilerEngine(raw_value) + except ValueError as error: + raise CompilerEngineError( + f"{COMPILER_ENGINE_ENV_VAR} must be one of {', '.join(COMPILER_ENGINE_VALUES)} " + f"(got {raw_value!r})" + ) from error diff --git a/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py new file mode 100644 index 0000000000..4c8aebc06a --- /dev/null +++ b/src/sqlbuild/compiler/frontier/_helpers/native_frontier.py @@ -0,0 +1,12 @@ +"""The native side of the compiler frontier.""" + +from collections.abc import Callable + +from sqlbuild.compiler.frontier.types import CompilerStage + + +def native_frontier[T](*, until: CompilerStage, python_stage: Callable[[], T]) -> T: + """Produce the frontier object for `until` natively; no stage is native yet.""" + + _ = until + return python_stage() diff --git a/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py b/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py new file mode 100644 index 0000000000..da8a53de2a --- /dev/null +++ b/src/sqlbuild/compiler/frontier/_helpers/stage_capture.py @@ -0,0 +1,37 @@ +"""Dump frontier objects as canonical JSON so two engines can be compared stage by stage.""" + +import json +import os +from pathlib import Path + +from sqlbuild.compiler.frontier.classes.stage_capture_encoder import StageCaptureEncoder +from sqlbuild.compiler.frontier.classes.stage_capture_sequence import StageCaptureSequence +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_DIR_ENV_VAR, STAGE_CAPTURE_SUFFIX +from sqlbuild.compiler.frontier.types import CompilerStage + + +def write_stage_capture(*, stage: CompilerStage, value: object) -> None: + """Write one frontier object when SQLBUILD_COMPILER_STAGE_CAPTURE_DIR names a directory.""" + + directory: str | None = os.environ.get(STAGE_CAPTURE_DIR_ENV_VAR) + if not directory: + return + path: Path = Path(directory) / ( + f"{StageCaptureSequence.next_value():03d}-{stage.value}{STAGE_CAPTURE_SUFFIX}" + ) + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(render_stage_capture(value), encoding="utf-8") + + +def render_stage_capture(value: object) -> str: + """Render a value as indented JSON that keeps insertion order and sorts only sets.""" + + return ( + json.dumps( + StageCaptureEncoder().encode(value), + indent=1, + ensure_ascii=False, + allow_nan=False, + ) + + "\n" + ) diff --git a/src/sqlbuild/compiler/frontier/classes/__init__.py b/src/sqlbuild/compiler/frontier/classes/__init__.py new file mode 100644 index 0000000000..27eb23ee2d --- /dev/null +++ b/src/sqlbuild/compiler/frontier/classes/__init__.py @@ -0,0 +1 @@ +"""Compiler frontier runtime classes.""" diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py new file mode 100644 index 0000000000..b7d31c891a --- /dev/null +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_encoder.py @@ -0,0 +1,160 @@ +"""Convert frontier objects into canonical JSON-compatible values.""" + +from __future__ import annotations + +import dataclasses +import datetime +import decimal +import enum +import functools +import hashlib +import json +import math +import re +import types +from collections.abc import Mapping +from collections.abc import Set as AbstractSet +from pathlib import PurePath +from typing import Any + +from sqlbuild.compiler.frontier.constants import ( + STAGE_CAPTURE_SKIPPED_SLOTS, + STAGE_CAPTURE_UNORDERED_ATTRIBUTES, +) + +_CALLABLE_TYPES: tuple[type, ...] = ( + types.FunctionType, + types.BuiltinFunctionType, + types.MethodType, + types.BuiltinMethodType, + type, +) + + +class StageCaptureEncoder: + """Encode one object graph deterministically, replacing callables with qualified names.""" + + def __init__(self) -> None: + self._active: set[int] = set() + + def encode(self, value: object) -> object: + """Return a JSON-compatible value that keeps insertion order and sorts only sets.""" + + if isinstance(value, enum.Enum): + return {"__enum__": f"{qualified_name(type(value))}.{value.name}"} + if value is None or isinstance(value, (bool, int, str)): + return value + if isinstance(value, float): + return value if math.isfinite(value) else {"__float__": repr(value)} + scalar: object | None = _encode_scalar(value) + if scalar is not None: + return scalar + identity: int = id(value) + if identity in self._active: + return {"__cycle__": qualified_name(type(value))} + self._active.add(identity) + try: + return self._encode_container(value) + finally: + self._active.discard(identity) + + def _encode_container(self, value: object) -> object: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return { + "__type__": qualified_name(type(value)), + **{ + field.name: self.encode(getattr(value, field.name, None)) + for field in dataclasses.fields(value) + }, + } + if isinstance(value, Mapping): + return self._encode_mapping(value) + if isinstance(value, AbstractSet): + return {"__set__": sorted((self.encode(item) for item in value), key=_sort_key)} + if isinstance(value, (list, tuple)): + return [self.encode(item) for item in value] + if isinstance(value, functools.partial): + return { + "__partial__": self.encode(value.func), + "args": self.encode(value.args), + "keywords": self.encode(value.keywords), + } + attributes: dict[str, object] | None = _instance_attributes(value) + type_name: str = qualified_name(type(value)) + if attributes is None: + return {"__opaque__": type_name} + unordered: frozenset[str] = STAGE_CAPTURE_UNORDERED_ATTRIBUTES.get(type_name, frozenset()) + return { + "__type__": type_name, + **{ + name: self._encode_attribute(value=item, unordered=name in unordered) + for name, item in attributes.items() + }, + } + + def _encode_attribute(self, *, value: object, unordered: bool) -> object: + if not unordered or not isinstance(value, Mapping): + return self.encode(value) + return {"__unordered_mapping__": sorted(self._mapping_pairs(value), key=_sort_key)} + + def _encode_mapping(self, value: Mapping[Any, object]) -> object: + if all(isinstance(key, str) for key in value): + return {str(key): self.encode(item) for key, item in value.items()} + return {"__mapping__": self._mapping_pairs(value)} + + def _mapping_pairs(self, value: Mapping[Any, object]) -> list[list[object]]: + return [[self.encode(key), self.encode(item)] for key, item in value.items()] + + +def qualified_name(value: object) -> str: + """Return `module:qualname` for a callable or class.""" + + module: object = getattr(value, "__module__", None) + name: object = getattr(value, "__qualname__", None) or getattr(value, "__name__", None) + if name is None: + return qualified_name(type(value)) + return f"{module}:{name}" if isinstance(module, str) else str(name) + + +def _encode_scalar(value: object) -> object | None: + if isinstance(value, PurePath): + return {"__path__": value.as_posix()} + if isinstance(value, decimal.Decimal): + return {"__decimal__": str(value)} + if isinstance(value, (datetime.date, datetime.time)): + return {"__datetime__": value.isoformat()} + if isinstance(value, (bytes, bytearray, memoryview)): + data: bytes = bytes(value) + return {"__bytes__": len(data), "sha256": hashlib.sha256(data).hexdigest()} + if isinstance(value, re.Pattern): + return {"__pattern__": str(value.pattern)} + if isinstance(value, types.ModuleType): + return {"__module__": value.__name__} + if isinstance(value, _CALLABLE_TYPES): + return {"__callable__": qualified_name(value)} + return None + + +def _sort_key(item: object) -> str: + return json.dumps(item, sort_keys=True, ensure_ascii=False) + + +def _instance_attributes(value: object) -> dict[str, object] | None: + instance_dict: object = getattr(value, "__dict__", None) + attributes: dict[str, object] = ( + {str(name): item for name, item in instance_dict.items()} + if isinstance(instance_dict, dict) + else {} + ) + for cls in type(value).__mro__: + slots: object = cls.__dict__.get("__slots__", ()) + for name in (slots,) if isinstance(slots, str) else slots: + if ( + isinstance(name, str) + and name not in STAGE_CAPTURE_SKIPPED_SLOTS + and hasattr(value, name) + ): + attributes[name] = getattr(value, name) + if not attributes and not isinstance(instance_dict, dict): + return None + return attributes diff --git a/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py new file mode 100644 index 0000000000..786ba5da74 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/classes/stage_capture_sequence.py @@ -0,0 +1,21 @@ +"""Process-wide ordering of stage captures.""" + +from __future__ import annotations + +import itertools +import threading +from typing import ClassVar + + +class StageCaptureSequence: + """Number stage captures in the order this process produced them.""" + + _counter: ClassVar[itertools.count[int]] = itertools.count(1) + _lock: ClassVar[threading.Lock] = threading.Lock() + + @classmethod + def next_value(cls) -> int: + """Return the next capture number.""" + + with cls._lock: + return next(cls._counter) diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py new file mode 100644 index 0000000000..b74f1abdf5 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -0,0 +1,20 @@ +"""Compiler engine switch, cache namespace, and stage capture constants.""" + +from sqlbuild.compiler.frontier.types import CompilerEngine + +COMPILER_ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" +COMPILER_ENGINE_OPTION: str = "--compiler-engine" +DEFAULT_COMPILER_ENGINE: CompilerEngine = CompilerEngine.PYTHON +COMPILER_ENGINE_VALUES: tuple[str, ...] = tuple(engine.value for engine in CompilerEngine) +NATIVE_CACHE_NAMESPACE_SUFFIX: str = "-native-v1" +TARGET_DIRECTORY_NAME: str = "target" +CACHE_DIRECTORY_NAME: str = "cache" +COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" +STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +STAGE_CAPTURE_SUFFIX: str = ".json" +STAGE_CAPTURE_SKIPPED_SLOTS: frozenset[str] = frozenset({"__dict__", "__weakref__"}) +STAGE_CAPTURE_UNORDERED_ATTRIBUTES: dict[str, frozenset[str]] = { + "sqlbuild.compiler.sql_analysis.classes.binding_catalog:BindingCatalog": frozenset( + {"schemas", "analysis_shapes", "expression_shapes", "shared_analyses"} + ) +} diff --git a/src/sqlbuild/compiler/frontier/exceptions.py b/src/sqlbuild/compiler/frontier/exceptions.py new file mode 100644 index 0000000000..81f0c64dc3 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/exceptions.py @@ -0,0 +1,5 @@ +"""Compiler engine selection errors.""" + + +class CompilerEngineError(ValueError): + """Raised when the requested compiler engine is not a supported value.""" diff --git a/src/sqlbuild/compiler/frontier/main/__init__.py b/src/sqlbuild/compiler/frontier/main/__init__.py new file mode 100644 index 0000000000..c834fc5400 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/__init__.py @@ -0,0 +1 @@ +"""Public compiler engine and frontier entry points.""" diff --git a/src/sqlbuild/compiler/frontier/main/_compile_frontier.py b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py new file mode 100644 index 0000000000..a872314978 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/_compile_frontier.py @@ -0,0 +1,20 @@ +"""The single seam between the Python compiler stages and the native frontier.""" + +from collections.abc import Callable + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier._helpers.native_frontier import native_frontier +from sqlbuild.compiler.frontier._helpers.stage_capture import write_stage_capture +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage + + +def compile_frontier[T](*, until: CompilerStage, python_stage: Callable[[], T]) -> T: + """Produce the frontier object for `until` on the active engine, capturing it on request.""" + + result: T = ( + python_stage() + if active_compiler_engine() is CompilerEngine.PYTHON + else native_frontier(until=until, python_stage=python_stage) + ) + write_stage_capture(stage=until, value=result) + return result diff --git a/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py b/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py new file mode 100644 index 0000000000..aafb64585e --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/compiler_cache_directory.py @@ -0,0 +1,21 @@ +"""Project-local compiler store root for the active engine.""" + +from pathlib import Path + +from sqlbuild.compiler.frontier.constants import ( + CACHE_DIRECTORY_NAME, + COMPILER_CACHE_DIRECTORY_NAME, + TARGET_DIRECTORY_NAME, +) +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name + + +def compiler_cache_directory(project_dir: Path) -> Path: + """Return the compiler store root owned by the active engine.""" + + return ( + project_dir + / TARGET_DIRECTORY_NAME + / CACHE_DIRECTORY_NAME + / engine_cache_name(COMPILER_CACHE_DIRECTORY_NAME) + ) diff --git a/src/sqlbuild/compiler/frontier/main/compiler_engine_override.py b/src/sqlbuild/compiler/frontier/main/compiler_engine_override.py new file mode 100644 index 0000000000..fbfc87cd3d --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/compiler_engine_override.py @@ -0,0 +1,26 @@ +"""Select the compiler engine for one CLI invocation.""" + +import os +from collections.abc import Iterator +from contextlib import contextmanager + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.types import CompilerEngine + + +@contextmanager +def compiler_engine_override(engine: CompilerEngine | None) -> Iterator[None]: + """Select an engine for one invocation so worker threads and subprocesses see it too.""" + + if engine is None: + yield + return + previous: str | None = os.environ.get(COMPILER_ENGINE_ENV_VAR) + os.environ[COMPILER_ENGINE_ENV_VAR] = engine.value + try: + yield + finally: + if previous is None: + _ = os.environ.pop(COMPILER_ENGINE_ENV_VAR, None) + else: + os.environ[COMPILER_ENGINE_ENV_VAR] = previous diff --git a/src/sqlbuild/compiler/frontier/main/engine_cache_name.py b/src/sqlbuild/compiler/frontier/main/engine_cache_name.py new file mode 100644 index 0000000000..da1eca742c --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/engine_cache_name.py @@ -0,0 +1,13 @@ +"""Engine namespacing for on-disk compiler stores.""" + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier.constants import NATIVE_CACHE_NAMESPACE_SUFFIX +from sqlbuild.compiler.frontier.types import CompilerEngine + + +def engine_cache_name(base: str) -> str: + """Name an on-disk store so engines never read each other's entries.""" + + if active_compiler_engine() is CompilerEngine.PYTHON: + return base + return f"{base}{NATIVE_CACHE_NAMESPACE_SUFFIX}" diff --git a/src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py b/src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py new file mode 100644 index 0000000000..bddadba080 --- /dev/null +++ b/src/sqlbuild/compiler/frontier/main/resolve_compiler_engine.py @@ -0,0 +1,10 @@ +"""Public compiler engine resolution entry point.""" + +from sqlbuild.compiler.frontier._helpers.engine import active_compiler_engine +from sqlbuild.compiler.frontier.types import CompilerEngine + + +def resolve_compiler_engine() -> CompilerEngine: + """Return the active compiler engine, raising CompilerEngineError for an unknown value.""" + + return active_compiler_engine() diff --git a/src/sqlbuild/compiler/frontier/types.py b/src/sqlbuild/compiler/frontier/types.py new file mode 100644 index 0000000000..f7d4daa79d --- /dev/null +++ b/src/sqlbuild/compiler/frontier/types.py @@ -0,0 +1,18 @@ +"""Compiler engine and frontier stage identities.""" + +from enum import StrEnum + + +class CompilerEngine(StrEnum): + """Which compiler implementation produces the frontier objects.""" + + PYTHON = "python" + NATIVE = "native" + + +class CompilerStage(StrEnum): + """Frontier objects handed from one compiler stage to the next, in pipeline order.""" + + DISCOVERED_PROJECT_INPUTS = "discovered_project_inputs" + COMPILE_PROJECT_INPUTS = "compile_project_inputs" + COMPILED_PROJECT = "compiled_project" diff --git a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py index 850c2996c4..1a5c79e100 100644 --- a/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py +++ b/src/sqlbuild/compiler/pipeline/_helpers/analysis_selection.py @@ -4,10 +4,13 @@ from dataclasses import replace +from sqlbuild.adapter.contract.models import ExpressionInferenceProfile +from sqlbuild.compiler.compile.main._assemble_project import assemble_project from sqlbuild.compiler.compile.models import ( CompileAnalysisSelection, CompiledObjectKey, CompiledProject, + CompileProjectInputs, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import CompilePipelineOptions @@ -99,3 +102,27 @@ def resolve_configured_rule_selection( policies=PlannerPolicies(auto_load_sources=analysis_selection.auto_load_sources), ) return frozenset(model.key for model in scopes.stale_warning_scope.models_by_name.values()) + + +def resolve_analysis_model_names( + *, + compile_inputs: CompileProjectInputs, + inference_profile: ExpressionInferenceProfile, + selection: CompileAnalysisSelection | None, +) -> frozenset[str] | None: + """Return the models selected for deep analysis, or None to analyse every model.""" + + if selection is None or (not selection.select and not selection.exclude): + return None + structural_project: CompiledProject = assemble_project( + inputs=compile_inputs, + inference_profile=inference_profile, + skip_column_inference=True, + analysis_model_names=frozenset(), + ) + scopes: PlannerScopeResolution = resolve_planner_scopes( + project=structural_project, + selection=PlannerSelection(select=selection.select, exclude=selection.exclude), + policies=PlannerPolicies(auto_load_sources=selection.auto_load_sources), + ) + return frozenset(scopes.stale_warning_scope.models_by_name) diff --git a/src/sqlbuild/compiler/pipeline/main/compiled_project.py b/src/sqlbuild/compiler/pipeline/main/compiled_project.py index 62d0b230ac..748d8db834 100644 --- a/src/sqlbuild/compiler/pipeline/main/compiled_project.py +++ b/src/sqlbuild/compiler/pipeline/main/compiled_project.py @@ -3,6 +3,7 @@ from __future__ import annotations from dataclasses import replace +from functools import partial from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.adapter.contract.models import ExpressionInferenceProfile @@ -21,7 +22,10 @@ TABLE_PROMOTION_MODE_SETTING_KEY, ) from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.lineage.types import ColumnLineageMode +from sqlbuild.compiler.pipeline._helpers.analysis_selection import resolve_analysis_model_names from sqlbuild.compiler.pipeline._helpers.target_defaults import apply_target_defaults from sqlbuild.compiler.pipeline._helpers.target_validation import ( validate_managed_loader_target_isolation, @@ -29,14 +33,6 @@ validate_named_target_schema_strategy, validate_project_targets, ) -from sqlbuild.compiler.planner.main.selection._resolve_planner_scopes import ( - resolve_planner_scopes, -) -from sqlbuild.compiler.planner.models import ( - PlannerPolicies, - PlannerScopeResolution, - PlannerSelection, -) from sqlbuild.compiler.profiling.main.record import record_compile_timing from sqlbuild.compiler.references.types import ExternalSqlReferenceResolver from sqlbuild.spec.contracts.main.resolve_effective_adapter_name import ( @@ -62,6 +58,37 @@ def build_compiled_project( ) -> CompiledProject: """Build one compiled project with adapter defaults and target validation applied.""" + return compile_frontier( + until=CompilerStage.COMPILED_PROJECT, + python_stage=partial( + _build_python_compiled_project, + discovered_inputs=discovered_inputs, + adapter=adapter, + selected_target=selected_target, + no_sql_validation=no_sql_validation, + skip_column_inference=skip_column_inference, + column_lineage_mode=column_lineage_mode, + cli_vars=cli_vars, + external_sql_reference_resolver=external_sql_reference_resolver, + resolved_connection=resolved_connection, + analysis_selection=analysis_selection, + ), + ) + + +def _build_python_compiled_project( + *, + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None, + no_sql_validation: bool, + skip_column_inference: bool, + column_lineage_mode: ColumnLineageMode, + cli_vars: dict[str, object] | None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, + resolved_connection: dict[str, object] | None, + analysis_selection: CompileAnalysisSelection | None, +) -> CompiledProject: validate_named_target_schema_strategy( discovered_inputs=discovered_inputs, adapter=adapter, @@ -74,22 +101,11 @@ def build_compiled_project( ) no_cache: bool = analysis_selection is not None and analysis_selection.no_cache with record_compile_timing("attachment_ms"): - compile_inputs: CompileProjectInputs = build_compile_inputs( + compile_inputs: CompileProjectInputs = _frontier_compile_inputs( discovered_inputs=discovered_inputs, - adapter_context=CompileAdapterContext( - value_renderer=adapter, - collection_rendering=resolve_effective_collection_rendering( - project_config=discovered_inputs.project_config, - declaration_override=None, - ), - python_functions_inherit_default_namespace=( - adapter.python_functions_inherit_default_namespace() - ), - sql_lexical_syntax=adapter.sql_lexical_syntax, - ), + adapter=adapter, selected_target=selected_target, no_sql_validation=no_sql_validation, - defer_model_sql_validation=True, cli_vars=cli_vars, resolved_connection=resolved_connection, external_sql_reference_resolver=external_sql_reference_resolver, @@ -98,7 +114,7 @@ def build_compiled_project( if analysis_selection is not None and analysis_selection.on_inputs_ready is not None: analysis_selection.on_inputs_ready(inputs=compile_inputs) inference_profile: ExpressionInferenceProfile = adapter.expression_inference_profile() - analysis_model_names: frozenset[str] | None = _resolve_analysis_model_names( + analysis_model_names: frozenset[str] | None = resolve_analysis_model_names( compile_inputs=compile_inputs, inference_profile=inference_profile, selection=analysis_selection, @@ -146,23 +162,39 @@ def build_compiled_project( ) -def _resolve_analysis_model_names( +def _frontier_compile_inputs( *, - compile_inputs: CompileProjectInputs, - inference_profile: ExpressionInferenceProfile, - selection: CompileAnalysisSelection | None, -) -> frozenset[str] | None: - if selection is None or (not selection.select and not selection.exclude): - return None - structural_project: CompiledProject = assemble_project( - inputs=compile_inputs, - inference_profile=inference_profile, - skip_column_inference=True, - analysis_model_names=frozenset(), - ) - scopes: PlannerScopeResolution = resolve_planner_scopes( - project=structural_project, - selection=PlannerSelection(select=selection.select, exclude=selection.exclude), - policies=PlannerPolicies(auto_load_sources=selection.auto_load_sources), + discovered_inputs: DiscoveredProjectInputs, + adapter: BaseAdapter, + selected_target: str | None, + no_sql_validation: bool, + cli_vars: dict[str, object] | None, + resolved_connection: dict[str, object] | None, + external_sql_reference_resolver: ExternalSqlReferenceResolver | None, + no_cache: bool, +) -> CompileProjectInputs: + return compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial( + build_compile_inputs, + discovered_inputs=discovered_inputs, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered_inputs.project_config, + declaration_override=None, + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, + ), + selected_target=selected_target, + no_sql_validation=no_sql_validation, + defer_model_sql_validation=True, + cli_vars=cli_vars, + resolved_connection=resolved_connection, + external_sql_reference_resolver=external_sql_reference_resolver, + no_cache=no_cache, + ), ) - return frozenset(scopes.stale_warning_scope.models_by_name) diff --git a/src/sqlbuild/compiler/scopes/_helpers/cache.py b/src/sqlbuild/compiler/scopes/_helpers/cache.py index 80424a7afc..11cf2a132b 100644 --- a/src/sqlbuild/compiler/scopes/_helpers/cache.py +++ b/src/sqlbuild/compiler/scopes/_helpers/cache.py @@ -16,12 +16,13 @@ LOCAL_CONFIG_FILENAME, PROJECT_CONFIG_FILENAME, ) +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory from sqlbuild.compiler.scopes._helpers.lookup import build_lookup from sqlbuild.compiler.scopes._helpers.paths import normalize_path from sqlbuild.compiler.scopes.constants import ( GLOBAL_DECLARATION_DIRECTORIES, GLOBAL_NAMED_DECLARATION_DIRECTORIES, - SCOPE_CACHE_DIRECTORY, + SCOPE_CACHE_DIRECTORY_NAME, SCOPE_CACHE_FILENAME, SCOPE_CACHE_MAX_BYTES, SCOPE_CACHE_SCHEMA_VERSION, @@ -575,7 +576,9 @@ def scope_index_fingerprint(*, project_dir: Path) -> str: def read_cached_scope_index(*, project_dir: Path, fingerprint: str) -> ScopeIndex | None: """Read a verified index; every storage or validation fault is a cache miss.""" - path: Path = project_dir / SCOPE_CACHE_DIRECTORY / SCOPE_CACHE_FILENAME + path: Path = ( + compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME / SCOPE_CACHE_FILENAME + ) try: if path.stat().st_size > SCOPE_CACHE_MAX_BYTES: return None @@ -612,7 +615,7 @@ def write_cached_scope_index(*, project_dir: Path, fingerprint: str, index: Scop "schema_version": SCOPE_CACHE_SCHEMA_VERSION, } ) - directory: Path = project_dir / SCOPE_CACHE_DIRECTORY + directory: Path = compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME directory.mkdir(parents=True, exist_ok=True) descriptor, temporary_name = tempfile.mkstemp(prefix=".scope-index-", dir=directory) temporary_path: Path = Path(temporary_name) diff --git a/src/sqlbuild/compiler/scopes/constants.py b/src/sqlbuild/compiler/scopes/constants.py index 0fdb41dd57..a78506c9bf 100644 --- a/src/sqlbuild/compiler/scopes/constants.py +++ b/src/sqlbuild/compiler/scopes/constants.py @@ -9,7 +9,8 @@ SCOPE_METADATA_SCHEMA_VERSION: int = 3 SCOPE_CACHE_SCHEMA_VERSION: int = 3 SCOPE_FINGERPRINT_ALGORITHM_VERSION: int = 2 -SCOPE_CACHE_DIRECTORY: Path = Path("target/cache/compiler/declaration-scopes-v2") +SCOPE_CACHE_DIRECTORY_NAME: str = "declaration-scopes-v2" +SCOPE_CACHE_DIRECTORY: Path = Path("target/cache/compiler") / SCOPE_CACHE_DIRECTORY_NAME SCOPE_CACHE_FILENAME: str = "scope-index.json" SCOPE_CACHE_MAX_BYTES: int = 16 * 1024 * 1024 SCOPE_SOURCE_SUFFIXES: frozenset[str] = frozenset({".sql", ".yml", ".yaml", ".py"}) diff --git a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py index 2bf34d2a19..ee84e2c18d 100644 --- a/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py +++ b/src/sqlbuild/compiler/scopes/main/load_or_build_scope_index.py @@ -2,6 +2,7 @@ from __future__ import annotations +from functools import partial from pathlib import Path from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter @@ -13,6 +14,8 @@ from sqlbuild.compiler.compile.models import CompileAdapterContext, CompileProjectInputs from sqlbuild.compiler.discovery.main.discover import discover_project_inputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerStage from sqlbuild.compiler.scopes._helpers.builder import build_tolerant_scope_index from sqlbuild.compiler.scopes._helpers.cache import ( read_cached_scope_index, @@ -57,23 +60,27 @@ def load_or_build_scope_index(*, project_dir: Path, no_cache: bool = False) -> S ) adapter: BaseAdapter = resolve_adapter(adapter_name=adapter_name) sql_lexical_syntax = adapter.sql_lexical_syntax - compile_inputs: CompileProjectInputs = build_compile_inputs( - discovered_inputs=discovered, - adapter_context=CompileAdapterContext( - value_renderer=adapter, - collection_rendering=resolve_effective_collection_rendering( - project_config=discovered.project_config, - declaration_override=None, + compile_inputs: CompileProjectInputs = compile_frontier( + until=CompilerStage.COMPILE_PROJECT_INPUTS, + python_stage=partial( + build_compile_inputs, + discovered_inputs=discovered, + adapter_context=CompileAdapterContext( + value_renderer=adapter, + collection_rendering=resolve_effective_collection_rendering( + project_config=discovered.project_config, + declaration_override=None, + ), + python_functions_inherit_default_namespace=( + adapter.python_functions_inherit_default_namespace() + ), + sql_lexical_syntax=adapter.sql_lexical_syntax, ), - python_functions_inherit_default_namespace=( - adapter.python_functions_inherit_default_namespace() - ), - sql_lexical_syntax=adapter.sql_lexical_syntax, + resolved_connection={}, + no_sql_validation=True, + defer_model_sql_validation=True, + no_cache=True, ), - resolved_connection={}, - no_sql_validation=True, - defer_model_sql_validation=True, - no_cache=True, ) index: ScopeIndex = scope_index_with_compile_usages(inputs=compile_inputs) except (OSError, UnicodeError, ValueError, ImportError): diff --git a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py index 1ec801614e..60699505be 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/custom_rules.py @@ -26,6 +26,7 @@ finding_payload, run_custom_hosts, ) +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.classes.fact_digests import FactDigests from sqlbuild.rule_engine.classes.rule_context import RuleFactViews, build_rule_fact_views from sqlbuild.rule_engine.constants import ( @@ -119,7 +120,9 @@ def evaluate_custom_rules_cached( ) for rule in custom } - cache_path: Path = project_dir / CUSTOM_RULES_CACHE_FILE + cache_path: Path = rules_bulk_cache_path( + project_dir=project_dir, file_name=CUSTOM_RULES_CACHE_FILE + ) stored: dict[str, _RuleCache] = _read_cache(cache_path) results: dict[tuple[str, str], tuple[dict[str, object], ...]] = {} plan: dict[str, list[str] | None] = {} diff --git a/src/sqlbuild/rule_engine/_helpers/engine/native.py b/src/sqlbuild/rule_engine/_helpers/engine/native.py index 872498fa0f..a2a4ff7470 100644 --- a/src/sqlbuild/rule_engine/_helpers/engine/native.py +++ b/src/sqlbuild/rule_engine/_helpers/engine/native.py @@ -34,6 +34,7 @@ custom_rule_test_evidence, ) from sqlbuild.rule_engine._helpers.engine.custom_rules import evaluate_custom_rules_cached +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.native_memo import ( native_payload_digests, native_request_identity, @@ -41,6 +42,7 @@ write_native_response, ) from sqlbuild.rule_engine.constants import ( + NATIVE_RULES_CACHE_FILE, RULES_NATIVE_API_VERSION, TYPE_PROOF_RULE_CODES, ) @@ -223,6 +225,11 @@ def _native_request( "initial_findings": [_finding_payload(finding) for finding in initial_findings], "defer_suppressions": True, "custom_rules": custom_payloads, + "rules_cache_path": str( + rules_bulk_cache_path( + project_dir=project_dir.resolve(), file_name=NATIVE_RULES_CACHE_FILE + ) + ), } diff --git a/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py b/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py new file mode 100644 index 0000000000..6f326cc15c --- /dev/null +++ b/src/sqlbuild/rule_engine/_helpers/run/cache_paths.py @@ -0,0 +1,22 @@ +"""Locations of the Rules caches, namespaced by compiler engine.""" + +from pathlib import Path + +from sqlbuild.compiler.frontier.constants import TARGET_DIRECTORY_NAME +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name +from sqlbuild.rule_engine.constants import ( + RULES_BULK_CACHE_DIRECTORY_NAME, + RULES_CACHE_DIRECTORY_NAME, +) + + +def rules_bulk_cache_path(*, project_dir: Path, file_name: str) -> Path: + """Return one bulk Rules cache file owned by the active compiler engine.""" + + return ( + project_dir + / TARGET_DIRECTORY_NAME + / engine_cache_name(RULES_CACHE_DIRECTORY_NAME) + / RULES_BULK_CACHE_DIRECTORY_NAME + / file_name + ) diff --git a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py index 4620c174fe..af631e553e 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/native_memo.py +++ b/src/sqlbuild/rule_engine/_helpers/run/native_memo.py @@ -11,6 +11,7 @@ import orjson from sqlbuild.compiler.fact_cache.main.code_identity import compiled_code_identity +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import NATIVE_RULES_MEMO_FILE, NATIVE_RULES_MEMO_VERSION @@ -39,7 +40,11 @@ def read_native_response(*, project_dir: Path, identity: str) -> str | None: """Return the stored response for this exact request identity, if any.""" try: - payload: object = orjson.loads((project_dir / NATIVE_RULES_MEMO_FILE).read_bytes()) + payload: object = orjson.loads( + rules_bulk_cache_path( + project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE + ).read_bytes() + ) except (OSError, orjson.JSONDecodeError): return None if not isinstance(payload, dict) or payload.get("identity") != identity: @@ -51,7 +56,7 @@ def read_native_response(*, project_dir: Path, identity: str) -> str | None: def write_native_response(*, project_dir: Path, identity: str, response: str) -> None: """Atomically replace the stored response with the latest evaluated request.""" - path: Path = project_dir / NATIVE_RULES_MEMO_FILE + path: Path = rules_bulk_cache_path(project_dir=project_dir, file_name=NATIVE_RULES_MEMO_FILE) path.parent.mkdir(parents=True, exist_ok=True) temporary: Path = path.with_suffix(f".tmp-{os.getpid()}-{threading.get_ident()}") temporary.write_bytes(orjson.dumps({"identity": identity, "response": response})) diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index b2962166c9..9f5b2c8c32 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -58,10 +58,11 @@ native_catalogue, start_custom_rules, ) +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine._helpers.run.findings import group_unevaluated_findings from sqlbuild.rule_engine._helpers.run.literal_duplicates import with_duplicate_literal_hints from sqlbuild.rule_engine._helpers.run.memory import release_freed_memory -from sqlbuild.rule_engine.constants import TYPE_PROOF_RULE_CODES +from sqlbuild.rule_engine.constants import SQL_RULES_CACHE_FILE, TYPE_PROOF_RULE_CODES from sqlbuild.rule_engine.exceptions import RulesError from sqlbuild.rule_engine.main.load_config import load_rules_config from sqlbuild.rule_engine.models import ( @@ -792,7 +793,7 @@ def _sql_file_rule_identity( def _sql_rule_cache_path(project_dir: Path) -> Path: - return project_dir / "target" / "rules-cache" / "bulk" / "sql.json" + return rules_bulk_cache_path(project_dir=project_dir, file_name=SQL_RULES_CACHE_FILE) def _read_sql_rule_cache(project_dir: Path) -> dict[str, dict[str, object]]: diff --git a/src/sqlbuild/rule_engine/constants.py b/src/sqlbuild/rule_engine/constants.py index c3fe200ed5..f7efc3453e 100644 --- a/src/sqlbuild/rule_engine/constants.py +++ b/src/sqlbuild/rule_engine/constants.py @@ -12,9 +12,13 @@ CUSTOM_HOST_MAX_TRACKED_READS: int = 100_000 CUSTOM_RULE_PROJECT_SUBJECT: str = "" CUSTOM_RULES_CACHE_VERSION: str = "custom-rules-reads-v1" -CUSTOM_RULES_CACHE_FILE: str = "target/rules-cache/bulk/custom-rules.json" +RULES_CACHE_DIRECTORY_NAME: str = "rules-cache" +RULES_BULK_CACHE_DIRECTORY_NAME: str = "bulk" +CUSTOM_RULES_CACHE_FILE: str = "custom-rules.json" NATIVE_RULES_MEMO_VERSION: str = "native-rules-response-v2" -NATIVE_RULES_MEMO_FILE: str = "target/rules-cache/bulk/native-response.json" +NATIVE_RULES_MEMO_FILE: str = "native-response.json" +SQL_RULES_CACHE_FILE: str = "sql.json" +NATIVE_RULES_CACHE_FILE: str = "native.json" CUSTOM_HOST_INPUT_TUPLE_SIZE: int = 2 SHARED_FACT_DATACLASS_TAG: str = "\0dataclass" CUSTOM_HOST_ABANDONED_SUFFIX: str = ".abandoned" diff --git a/tests/e2e/scripts/compiler_differential/__init__.py b/tests/e2e/scripts/compiler_differential/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py new file mode 100644 index 0000000000..9e0b02f814 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -0,0 +1,27 @@ +"""Test case types for the compiler differential harness end-to-end tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class HarnessRunTestCase: + """One harness run over a fixture and the report lines it must and must not print.""" + + description: str + extra_arguments: tuple[str, ...] + expected_exit_code: int + expected_lines: tuple[str, ...] + expected_patterns: tuple[str, ...] + expected_absent: tuple[str, ...] + + +@dataclass(frozen=True) +class ProjectExpectationTestCase: + """A broken project run with one --expect value and the verdict the harness must reach.""" + + description: str + extra_arguments: tuple[str, ...] + expected_exit_code: int + expected_lines: tuple[str, ...] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py new file mode 100644 index 0000000000..a5488ddd3c --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -0,0 +1,87 @@ +"""Harness invocations and a perturbation that makes one engine's output differ.""" + +from __future__ import annotations + +from pathlib import Path + +NATIVE_ONLY_STDERR_LINE: str = "native engine perturbation active" +WAFFLE_SHOP_FIXTURE: Path = Path(__file__).resolve().parents[2] / "fixtures" / "waffle_shop" +_SITECUSTOMIZE: str = f''' +import dataclasses +import sys + +import sqlbuild.compiler.frontier.main._compile_frontier as frontier +from sqlbuild.compiler.frontier.types import CompilerStage + +_original = frontier.native_frontier + + +def _perturbed(*, until, python_stage): + result = _original(until=until, python_stage=python_stage) + if until is not CompilerStage.COMPILED_PROJECT or not result.models: + return result + first = result.models[0] + changed = dataclasses.replace( + first, query_sql=f"SELECT * FROM ({{first.query_sql}}) AS perturbed_by_test" + ) + return dataclasses.replace(result, models=(changed, *result.models[1:])) + + +frontier.native_frontier = _perturbed +print("{NATIVE_ONLY_STDERR_LINE}", file=sys.stderr) +''' + +_BROKEN_REF_FILES: dict[str, str] = { + "sqlbuild_project.toml": ( + 'name = "broken_orders"\nadapter = "duckdb"\n\n' + '[connection]\ndatabase = "broken_orders.duckdb"\n' + ), + "models/order_totals.sql": ( + 'MODEL (\n description "Order totals",\n);\n\nSELECT * FROM __ref("nope")\n' + ), +} + + +def write_broken_ref_project(directory: Path) -> Path: + """Write a project whose only model references a model that does not exist.""" + + for relative_path, contents in _BROKEN_REF_FILES.items(): + path: Path = directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8", newline="\n") + return directory + + +def write_native_perturbation(directory: Path) -> Path: + """Write a sitecustomize module that changes the first compiled model on the native path.""" + + directory.mkdir(parents=True, exist_ok=True) + _ = (directory / "sitecustomize.py").write_text(_SITECUSTOMIZE, encoding="utf-8") + return directory + + +def harness_arguments( + *, work_dir: Path, extra: tuple[str, ...], project: Path = WAFFLE_SHOP_FIXTURE +) -> list[str]: + """Compare only one project (the waffle shop fixture by default), keeping run directories.""" + + return [ + "--corpus", + "--project", + str(project), + "--jobs", + "1", + "--work-dir", + str(work_dir), + *extra, + ] + + +def perturbation_arguments(directory: Path) -> tuple[str, ...]: + """Return the options that load the perturbation into native-engine processes only.""" + + return ( + "--stage-captures", + "--engine-env", + f"native:PYTHONPATH={write_native_perturbation(directory)}", + ) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py new file mode 100644 index 0000000000..54e646dd33 --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -0,0 +1,143 @@ +"""The differential harness passes identical engines and pinpoints a perturbed one.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from scripts.compiler_differential.main.differential import run_compiler_differential +from tests.e2e.scripts.compiler_differential._test_types import ( + HarnessRunTestCase, + ProjectExpectationTestCase, +) +from tests.e2e.scripts.compiler_differential.helpers import ( + NATIVE_ONLY_STDERR_LINE, + harness_arguments, + perturbation_arguments, + write_broken_ref_project, +) + + +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="identical_engines", + extra_arguments=(), + expected_exit_code=0, + expected_lines=( + "OK project/waffle_shop", + "Compiler differential passed: 1 projects identical (python vs native)", + ), + expected_patterns=(), + expected_absent=("DIFF",), + ) + ], + ids=lambda case: case.description, +) +def test_given_identical_engines_when_comparing_fixture_then_harness_passes( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments(work_dir=tmp_path / "work", extra=test_case.extra_arguments) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert not any(text in output for text in test_case.expected_absent), output + + +@pytest.mark.parametrize( + "test_case", + [ + HarnessRunTestCase( + description="perturbed_native_compiled_project", + extra_arguments=(), + expected_exit_code=1, + expected_lines=( + "DIFF project/waffle_shop", + "- `compile` stdout at /resources/models/0/query_sql", + 'native: "SELECT * FROM (', + "- `compile` stderr at line 1", + f'native: "{NATIVE_ONLY_STDERR_LINE}"', + "- stage capture 0-compile/003-compiled_project.json at /models/0/query_sql", + "First difference: project/waffle_shop: `compile` stdout at", + ), + expected_patterns=( + r"- target/compiled/models/\S+\.sql at line \d+", + r"- query fingerprint of model \w+ \(manifest query hash\)", + r"- query fingerprint of model \w+ \(plan version hash\)", + ), + expected_absent=("discovered_project_inputs.json", "compile_project_inputs.json"), + ) + ], + ids=lambda case: case.description, +) +def test_given_perturbed_native_engine_when_comparing_then_every_artifact_is_located( + test_case: HarnessRunTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=(*perturbation_arguments(tmp_path / "perturbation"), *test_case.extra_arguments), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + assert all(re.search(pattern, output) for pattern in test_case.expected_patterns), output + assert not any(text in output for text in test_case.expected_absent), output + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectExpectationTestCase( + description="success_expected_by_default", + extra_arguments=(), + expected_exit_code=1, + expected_lines=( + "DIFF project/broken_orders", + "- corpus expectation at expected success", + ), + ), + ProjectExpectationTestCase( + description="declared_failure_code", + extra_arguments=("--expect", "failure:P001"), + expected_exit_code=0, + expected_lines=("OK project/broken_orders",), + ), + ProjectExpectationTestCase( + description="wrong_failure_code", + extra_arguments=("--expect", "failure:P010"), + expected_exit_code=1, + expected_lines=( + "DIFF project/broken_orders", + "- corpus expectation at expected failure", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_fails( + test_case: ProjectExpectationTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=test_case.extra_arguments, + project=write_broken_ref_project(tmp_path / "broken_orders"), + ) + ) + + output: str = capsys.readouterr().out + assert exit_code == test_case.expected_exit_code, output + assert all(line in output for line in test_case.expected_lines), output + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index df9edf5609..e1a9dba792 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -795,3 +795,37 @@ class SharedCacheKeyTestCase: description: str schedule: Callable[[pytest.MonkeyPatch], None] expected_matches_uncached: bool = True + + +@dataclass(frozen=True) +class CompilerEngineParityTestCase: + """Two engines compiling separate copies of one project.""" + + description: str + left_engine: str + right_engine: str + expected_engines: tuple[str, str] + expected_exit_codes: tuple[int, int] + + +@dataclass(frozen=True) +class CompilerEngineStoreTestCase: + """A sequence of engine compiles in one project and which of them reuse a stored compile.""" + + description: str + engines: tuple[str, ...] + expected_reused: tuple[bool, ...] + expected_python_stores: tuple[str, ...] + expected_native_stores: tuple[str, ...] + + +@dataclass(frozen=True) +class CompilerEngineRulesStoreTestCase: + """Compile plus Rules under each engine and the store files each engine must own.""" + + description: str + rules_selector: str + stores: tuple[str, ...] + native_marker: str + expected_python_files: tuple[str, ...] + expected_native_files: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 32277c374e..dd23c1fc04 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import itertools import json import os @@ -4141,3 +4142,56 @@ def json_report_keys(stdout: str) -> tuple[str, ...]: """Parse a JSON compile report and return its top-level keys.""" return tuple(cast(dict[str, object], json.loads(stdout))) + + +_COMPILER_ENGINE_LINE: re.Pattern[str] = re.compile(r'\n "compiler_engine": "([a-z]+)",') + + +def copy_compile_project(*, source: Path, destination: Path) -> Path: + """Copy a prepared project to a new directory and return the copy.""" + + _ = shutil.copytree(source, destination) + return destination + + +def engine_reuse_compile(*, project_dir: Path, engine: str) -> CompileReuseRun: + """Compile with reuse enabled under one engine selected by the hidden flag.""" + + return run_reuse_compile(project_dir=project_dir, global_args=("--compiler-engine", engine)) + + +def engine_compile_and_rules(*, project_dir: Path, engine: str, rules_selector: str) -> int: + """Compile and run Rules under one engine; return the Rules exit code.""" + + _ = engine_reuse_compile(project_dir=project_dir, engine=engine) + return run_installed_sqb( + project_dir=project_dir, + args=("--compiler-engine", engine, "rules", "run", rules_selector), + env=COMPILE_REUSE_ENV, + ).returncode + + +def report_engine(run: CompileReuseRun) -> str: + """Return the engine named in a JSON compile report.""" + + match: re.Match[str] | None = _COMPILER_ENGINE_LINE.search(run.report) + assert match is not None + return match.group(1) + + +def report_without_engine(run: CompileReuseRun) -> str: + """Return a JSON compile report without its timings and engine fields.""" + + return _COMPILER_ENGINE_LINE.sub("", run.report) + + +def store_digests(*, project_dir: Path, stores: tuple[str, ...]) -> dict[str, str]: + """Hash every file below the given project-relative store directories.""" + + files: list[Path] = [] + for store in stores: + files.extend(filter(Path.is_file, sorted((project_dir / store).rglob("*")))) + return { + path.relative_to(project_dir).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in files + } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py new file mode 100644 index 0000000000..b8fcbd0d16 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compiler_engine_switch.py @@ -0,0 +1,151 @@ +"""Both compiler engines produce identical compiles and never share on-disk compile stores.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( + CompilerEngineParityTestCase, + CompilerEngineRulesStoreTestCase, + CompilerEngineStoreTestCase, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + CompileReuseRun, + copy_compile_project, + engine_compile_and_rules, + engine_reuse_compile, + prepare_compile_reuse_project, + report_engine, + report_without_engine, + store_digests, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineParityTestCase( + description="python_and_native", + left_engine="python", + right_engine="native", + expected_engines=("python", "native"), + expected_exit_codes=(0, 0), + ) + ], + ids=lambda case: case.description, +) +def test_given_same_project_when_compiling_with_each_engine_then_outputs_are_identical( + test_case: CompilerEngineParityTestCase, tmp_path: Path +) -> None: + prepared_project: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=prepared_project) + left: CompileReuseRun = engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "left"), + engine=test_case.left_engine, + ) + right: CompileReuseRun = engine_reuse_compile( + project_dir=copy_compile_project(source=prepared_project, destination=tmp_path / "right"), + engine=test_case.right_engine, + ) + + assert (left.returncode, right.returncode) == test_case.expected_exit_codes + assert (report_engine(left), report_engine(right)) == test_case.expected_engines + assert report_without_engine(right) == report_without_engine(left) + assert right.compiled == left.compiled + assert left.compiled + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineStoreTestCase( + description="python_native_native_python", + engines=("python", "native", "native", "python"), + expected_reused=(False, False, True, True), + expected_python_stores=("target/cache/compiler", "target/rules-cache"), + expected_native_stores=( + "target/cache/compiler-native-v1", + "target/rules-cache-native-v1", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_compile_by_one_engine_when_other_engine_compiles_then_no_store_is_shared( + test_case: CompilerEngineStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=project_dir) + + first: CompileReuseRun = engine_reuse_compile( + project_dir=project_dir, engine=test_case.engines[0] + ) + python_stores: dict[str, str] = store_digests( + project_dir=project_dir, stores=test_case.expected_python_stores + ) + later: list[CompileReuseRun] = [ + engine_reuse_compile(project_dir=project_dir, engine=engine) + for engine in test_case.engines[1:-1] + ] + python_stores_after_native: dict[str, str] = store_digests( + project_dir=project_dir, stores=test_case.expected_python_stores + ) + last: CompileReuseRun = engine_reuse_compile( + project_dir=project_dir, engine=test_case.engines[-1] + ) + runs: list[CompileReuseRun] = [first, *later, last] + + assert tuple(run.reused for run in runs) == test_case.expected_reused + assert python_stores + assert python_stores_after_native == python_stores + assert store_digests(project_dir=project_dir, stores=test_case.expected_native_stores) + assert {report_without_engine(run) for run in runs[2:]} == {report_without_engine(last)} + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineRulesStoreTestCase( + description="built_in_rules_after_compile", + rules_selector="SQBR", + stores=("target/cache", "target/rules-cache", "target/rules-cache-native-v1"), + native_marker="-native-v1/", + expected_python_files=( + "target/rules-cache/bulk/native.json", + "target/rules-cache/bulk/sql.json", + ), + expected_native_files=( + "target/rules-cache-native-v1/bulk/native.json", + "target/rules-cache-native-v1/bulk/sql.json", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_python_compile_and_rules_when_native_runs_them_then_native_writes_only_its_stores( + test_case: CompilerEngineRulesStoreTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "orders" + prepare_compile_reuse_project(project_dir=project_dir) + + python_exit: int = engine_compile_and_rules( + project_dir=project_dir, engine="python", rules_selector=test_case.rules_selector + ) + python_files: dict[str, str] = store_digests(project_dir=project_dir, stores=test_case.stores) + native_exit: int = engine_compile_and_rules( + project_dir=project_dir, engine="native", rules_selector=test_case.rules_selector + ) + all_files: dict[str, str] = store_digests(project_dir=project_dir, stores=test_case.stores) + native_files: set[str] = set(all_files) - set(python_files) + + assert native_exit == python_exit + assert {path: all_files.get(path) for path in python_files} == python_files + assert all(test_case.native_marker in path for path in native_files), sorted(native_files) + assert set(test_case.expected_python_files) <= set(python_files) + assert set(test_case.expected_native_files) <= native_files + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/__init__.py b/tests/unit/scripts/compiler_differential/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/__init__.py b/tests/unit/scripts/compiler_differential/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/_helpers/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/_test_types.py new file mode 100644 index 0000000000..d605e21b48 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/_test_types.py @@ -0,0 +1,101 @@ +"""Test case types for the differential harness comparison helpers.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.compiler_differential.models import ExpectedOutcome + + +@dataclass(frozen=True) +class JsonDifferenceTestCase: + """Two JSON documents and where the harness must report their first difference.""" + + description: str + left: object + right: object + expected_location: str | None + + +@dataclass(frozen=True) +class TextDifferenceTestCase: + """Two texts and the first differing line the harness must report.""" + + description: str + left: str + right: str + expected_location: str | None + + +@dataclass(frozen=True) +class PreviewTestCase: + """A long differing value and the bounded preview the report shows.""" + + description: str + left: object + right: object + expected_max_length: int + expected_suffix: str + + +@dataclass(frozen=True) +class NormalizationTestCase: + """Raw output and the normalized text the harness compares.""" + + description: str + raw: str + expected_text: str + + +@dataclass(frozen=True) +class PayloadStripTestCase: + """A decoded report or manifest and what remains once volatile fields are removed.""" + + description: str + payload: object + expected_payload: object + + +@dataclass(frozen=True) +class EngineEnvironmentTestCase: + """Repeated --engine-env values and the per-engine overrides they produce.""" + + description: str + values: tuple[str, ...] + expected_environment: dict[str, dict[str, str]] + + +@dataclass(frozen=True) +class EngineEnvironmentErrorTestCase: + """A malformed --engine-env value and the usage error it raises.""" + + description: str + value: str + expected_message: str + + +@dataclass(frozen=True) +class ExpectedOutcomeTestCase: + """One --expect value and the outcome it parses to.""" + + description: str + value: str + expected_outcome: ExpectedOutcome + + +@dataclass(frozen=True) +class ExpectedOutcomeErrorTestCase: + """One malformed --expect value.""" + + description: str + value: str + expected_message: str + + +@dataclass(frozen=True) +class CorpusExpectationTestCase: + """A corpus selection and the expectation each named entry must declare.""" + + description: str + corpora: tuple[str, ...] + expected_outcomes: dict[str, ExpectedOutcome] diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_compare.py b/tests/unit/scripts/compiler_differential/_helpers/test_compare.py new file mode 100644 index 0000000000..44c711296c --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_compare.py @@ -0,0 +1,128 @@ +"""First-difference reports name the exact JSON pointer or line that diverged.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential._helpers.compare import ( + first_json_difference, + first_text_difference, +) +from scripts.compiler_differential.models import Divergence +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + JsonDifferenceTestCase, + PreviewTestCase, + TextDifferenceTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + JsonDifferenceTestCase( + description="identical", + left={"orders": [1, 2]}, + right={"orders": [1, 2]}, + expected_location=None, + ), + JsonDifferenceTestCase( + description="nested_value", + left={"resources": {"models": [{"name": "orders", "sql": "SELECT 1"}]}}, + right={"resources": {"models": [{"name": "orders", "sql": "SELECT 2"}]}}, + expected_location="/resources/models/0/sql", + ), + JsonDifferenceTestCase( + description="first_of_several", + left={"a": 1, "b": [1, 2, 3]}, + right={"a": 1, "b": [1, 9, 8]}, + expected_location="/b/1", + ), + JsonDifferenceTestCase( + description="extra_list_item", + left={"diagnostics": [{"code": "P010"}]}, + right={"diagnostics": [{"code": "P010"}, {"code": "S024"}]}, + expected_location="/diagnostics/1", + ), + JsonDifferenceTestCase( + description="missing_key", + left={"summary": {"models": 2, "seeds": 1}}, + right={"summary": {"models": 2}}, + expected_location="/summary/seeds", + ), + JsonDifferenceTestCase( + description="key_order", + left={"name": "orders", "path": "models/orders.sql"}, + right={"path": "models/orders.sql", "name": "orders"}, + expected_location="/ (key order)", + ), + JsonDifferenceTestCase( + description="integer_versus_float", + left={"ratio": 1}, + right={"ratio": 1.0}, + expected_location="/ratio", + ), + JsonDifferenceTestCase( + description="escaped_pointer_token", + left={"models/orders.sql": {"a~b": 1}}, + right={"models/orders.sql": {"a~b": 2}}, + expected_location="/models~1orders.sql/a~0b", + ), + ], + ids=lambda case: case.description, +) +def test_given_two_documents_when_comparing_then_first_difference_pointer_is_reported( + test_case: JsonDifferenceTestCase, +) -> None: + found: Divergence | None = first_json_difference(left=test_case.left, right=test_case.right) + + assert getattr(found, "location", None) == test_case.expected_location + + +@pytest.mark.parametrize( + "test_case", + [ + TextDifferenceTestCase( + description="identical", left="a\nb\n", right="a\nb\n", expected_location=None + ), + TextDifferenceTestCase( + description="second_line", left="a\nb\nc", right="a\nx\nc", expected_location="line 2" + ), + TextDifferenceTestCase( + description="trailing_line", left="a\nb", right="a\nb\nc", expected_location="line 3" + ), + ], + ids=lambda case: case.description, +) +def test_given_two_texts_when_comparing_then_first_differing_line_is_reported( + test_case: TextDifferenceTestCase, +) -> None: + found: Divergence | None = first_text_difference(left=test_case.left, right=test_case.right) + + assert getattr(found, "location", None) == test_case.expected_location + + +@pytest.mark.parametrize( + "test_case", + [ + PreviewTestCase( + description="long_sql", + left={"sql": "SELECT " + "a, " * 200}, + right={"sql": "SELECT " + "b, " * 200}, + expected_max_length=160, + expected_suffix="...", + ) + ], + ids=lambda case: case.description, +) +def test_given_long_differing_values_when_reporting_then_previews_are_bounded( + test_case: PreviewTestCase, +) -> None: + found: Divergence | None = first_json_difference(left=test_case.left, right=test_case.right) + + assert found is not None + assert len(found.left) <= test_case.expected_max_length + assert found.left.endswith(test_case.expected_suffix) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py b/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py new file mode 100644 index 0000000000..f35dfd35e1 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_corpus.py @@ -0,0 +1,69 @@ +"""Every corpus entry declares the outcome both engines must produce.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.corpus import build_corpus +from scripts.compiler_differential.models import CorpusProject, ExpectedOutcome +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + CorpusExpectationTestCase, +) + +_REPO_ROOT: Path = Path(__file__).resolve().parents[5] + + +@pytest.mark.parametrize( + "test_case", + [ + CorpusExpectationTestCase( + description="fixtures_succeed_except_dbt_interop", + corpora=("fixtures",), + expected_outcomes={ + "fixture/waffle_shop": ExpectedOutcome(), + "fixture/dbt_interop": ExpectedOutcome(error_code="C214"), + }, + ), + CorpusExpectationTestCase( + description="examples_succeed", + corpora=("examples",), + expected_outcomes={"example/waffle-shop": ExpectedOutcome()}, + ), + CorpusExpectationTestCase( + description="dense_succeeds", + corpora=("dense",), + expected_outcomes={ + "dense/20": ExpectedOutcome(), + "dense/20-custom-rules": ExpectedOutcome(), + }, + ), + CorpusExpectationTestCase( + description="extra_projects_take_the_declared_expectation", + corpora=(), + expected_outcomes={"project/orders": ExpectedOutcome(error_code="P001")}, + ), + ], + ids=lambda case: case.description, +) +def test_given_corpus_selection_when_building_then_entries_declare_expected_outcomes( + test_case: CorpusExpectationTestCase, +) -> None: + corpus: list[CorpusProject] = build_corpus( + repo_root=_REPO_ROOT, + corpora=test_case.corpora, + seeds=range(0), + dense_models=20, + projects=(Path("/projects/orders"),), + project_expectation=ExpectedOutcome(error_code="P001"), + ) + outcomes: dict[str, ExpectedOutcome] = {entry.name: entry.expected for entry in corpus} + + assert { + name: outcomes.get(name) for name in test_case.expected_outcomes + } == test_case.expected_outcomes + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py b/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py new file mode 100644 index 0000000000..a89a893598 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_normalize.py @@ -0,0 +1,117 @@ +"""Normalization removes only run-specific noise before engines are compared.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential._helpers.normalize import ( + normalize_artifact_text, + normalize_stderr, + strip_manifest_metadata, + strip_report_fields, +) +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + NormalizationTestCase, + PayloadStripTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + NormalizationTestCase( + description="phase_timing", + raw="Project compile OK (0.23s)", + expected_text="Project compile OK ()", + ), + NormalizationTestCase( + description="rules_timing", + raw="Evaluated rules. (0.02s; built-in 0.00s, custom 12ms)", + expected_text="Evaluated rules. (; built-in , custom )", + ), + NormalizationTestCase( + description="engine_store_path", + raw="wrote target/cache/compiler-native-v1/facts", + expected_text="wrote target/cache/compiler/facts", + ), + ], + ids=lambda case: case.description, +) +def test_given_stderr_when_normalizing_then_elapsed_times_are_masked( + test_case: NormalizationTestCase, +) -> None: + assert normalize_stderr(test_case.raw) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + NormalizationTestCase( + description="invocation_id", + raw='"run_id": "20261006T172510Z_dee386e8ffa7"', + expected_text='"run_id": ""', + ), + NormalizationTestCase( + description="durations_are_kept_in_artifacts", + raw="SELECT '5s' AS label", + expected_text="SELECT '5s' AS label", + ), + ], + ids=lambda case: case.description, +) +def test_given_artifact_text_when_normalizing_then_only_run_identity_is_masked( + test_case: NormalizationTestCase, +) -> None: + assert normalize_artifact_text(test_case.raw) == test_case.expected_text + + +@pytest.mark.parametrize( + "test_case", + [ + PayloadStripTestCase( + description="compile_report", + payload={ + "summary": {"models": 2}, + "compile_timings": {"total_ms": 10}, + "compiler_engine": "native", + }, + expected_payload={"summary": {"models": 2}}, + ), + PayloadStripTestCase( + description="non_object_report", payload=[1, 2], expected_payload=[1, 2] + ), + ], + ids=lambda case: case.description, +) +def test_given_compile_report_when_stripping_then_only_timings_and_engine_are_removed( + test_case: PayloadStripTestCase, +) -> None: + assert strip_report_fields(test_case.payload) == test_case.expected_payload + + +@pytest.mark.parametrize( + "test_case", + [ + PayloadStripTestCase( + description="manifest_metadata", + payload={ + "metadata": { + "generated_at": "2026-10-06T17:25:10+00:00", + "invocation_id": "20261006T172510Z_dee386e8ffa7", + "project_name": "orders", + }, + "nodes": {}, + }, + expected_payload={"metadata": {"project_name": "orders"}, "nodes": {}}, + ) + ], + ids=lambda case: case.description, +) +def test_given_manifest_when_stripping_then_generation_identity_is_removed( + test_case: PayloadStripTestCase, +) -> None: + assert strip_manifest_metadata(test_case.payload) == test_case.expected_payload + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/test_options.py b/tests/unit/scripts/compiler_differential/_helpers/test_options.py new file mode 100644 index 0000000000..83a3bd61ac --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/test_options.py @@ -0,0 +1,107 @@ +"""Per-engine environment options parse into exact overrides.""" + +from __future__ import annotations + +import argparse + +import pytest + +from scripts.compiler_differential._helpers.options import ( + parse_engine_environment, + parse_expected_outcome, +) +from scripts.compiler_differential.exceptions import DifferentialUsageError +from scripts.compiler_differential.models import ExpectedOutcome +from tests.unit.scripts.compiler_differential._helpers._test_types import ( + EngineEnvironmentErrorTestCase, + EngineEnvironmentTestCase, + ExpectedOutcomeErrorTestCase, + ExpectedOutcomeTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineEnvironmentTestCase( + description="grouped_by_engine", + values=("native:PYTHONPATH=/tmp/hook", "native:ORDERS_REGION=a=b", "python:EMPTY="), + expected_environment={ + "native": {"PYTHONPATH": "/tmp/hook", "ORDERS_REGION": "a=b"}, + "python": {"EMPTY": ""}, + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_engine_assignments_when_parsing_then_values_group_by_engine( + test_case: EngineEnvironmentTestCase, +) -> None: + assert parse_engine_environment(list(test_case.values)) == test_case.expected_environment + + +@pytest.mark.parametrize( + "test_case", + [ + EngineEnvironmentErrorTestCase( + description=value.replace(":", "_").replace("=", "_") or "empty", + value=value, + expected_message="ENGINE:NAME=VALUE", + ) + for value in ("native", "native:NAME", ":NAME=1", "native:=1") + ], + ids=lambda case: case.description, +) +def test_given_malformed_assignment_when_parsing_then_usage_error_is_raised( + test_case: EngineEnvironmentErrorTestCase, +) -> None: + with pytest.raises(DifferentialUsageError, match=test_case.expected_message): + _ = parse_engine_environment([test_case.value]) + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedOutcomeTestCase( + description="success", value="success", expected_outcome=ExpectedOutcome() + ), + ExpectedOutcomeTestCase( + description="failure_code", + value="failure:P001", + expected_outcome=ExpectedOutcome(error_code="P001"), + ), + ExpectedOutcomeTestCase( + description="rule_code", + value="failure:SQBRSQL005", + expected_outcome=ExpectedOutcome(error_code="SQBRSQL005"), + ), + ], + ids=lambda case: case.description, +) +def test_given_expect_value_when_parsing_then_outcome_matches( + test_case: ExpectedOutcomeTestCase, +) -> None: + assert parse_expected_outcome(test_case.value) == test_case.expected_outcome + + +@pytest.mark.parametrize( + "test_case", + [ + ExpectedOutcomeErrorTestCase( + description=value.replace(":", "_") or "empty", + value=value, + expected_message="success or failure:", + ) + for value in ("", "failure", "failure:", "failure:p001", "P001", "ok") + ], + ids=lambda case: case.description, +) +def test_given_malformed_expect_value_when_parsing_then_argument_error_is_raised( + test_case: ExpectedOutcomeErrorTestCase, +) -> None: + with pytest.raises(argparse.ArgumentTypeError, match=test_case.expected_message): + _ = parse_expected_outcome(test_case.value) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/main/__init__.py b/tests/unit/scripts/compiler_differential/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compiler_differential/main/_test_types.py b/tests/unit/scripts/compiler_differential/main/_test_types.py new file mode 100644 index 0000000000..78f387505f --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/_test_types.py @@ -0,0 +1,47 @@ +"""Test case types for the generator and failure corpus entry points.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from scripts.compiler_differential.models import FailureCase + + +@dataclass(frozen=True) +class SeedDeterminismTestCase: + """One seed generated twice.""" + + description: str + seed: int + expected_identical: bool + + +@dataclass(frozen=True) +class SeedRangeTestCase: + """A seed range and what the generated projects must cover together.""" + + description: str + seeds: range + expected_features: frozenset[str] + expected_distinct_projects: int + expected_min_invalid: int + expected_max_invalid: int + + +@dataclass(frozen=True) +class FailureCorpusTestCase: + """The failure corpus and the breadth it must keep.""" + + description: str + expected_min_cases: int + expected_min_codes: int + expected_min_families: int + + +@dataclass(frozen=True) +class FailureCaseWriteTestCase: + """One failure case and the files its written project must contain.""" + + description: str + case: FailureCase + expected_files: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/main/helpers.py b/tests/unit/scripts/compiler_differential/main/helpers.py new file mode 100644 index 0000000000..daff4ea8e3 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/helpers.py @@ -0,0 +1,27 @@ +"""Helpers for generator and failure corpus tests.""" + +from __future__ import annotations + +from scripts.compiler_differential.main.generate_project import generate_project +from scripts.compiler_differential.models import GeneratedProject + + +def generated_projects(seeds: range) -> list[GeneratedProject]: + """Generate one project per seed.""" + + return [generate_project(seed=seed) for seed in seeds] + + +def project_features(projects: list[GeneratedProject]) -> frozenset[str]: + """Return every feature any project exercises.""" + + features: set[str] = set() + for project in projects: + features.update(project.features) + return frozenset(features) + + +def invalid_count(projects: list[GeneratedProject]) -> int: + """Count projects generated to fail with a known code.""" + + return sum(project.expected_error_code is not None for project in projects) diff --git a/tests/unit/scripts/compiler_differential/main/test_failure_cases.py b/tests/unit/scripts/compiler_differential/main/test_failure_cases.py new file mode 100644 index 0000000000..e8076319c3 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/test_failure_cases.py @@ -0,0 +1,63 @@ +"""The failure corpus stays broad, uniquely named, and writes complete projects.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.compiler_differential.main.failure_cases import failure_cases +from scripts.compiler_differential.models import FailureCase +from tests.unit.scripts.compiler_differential.main._test_types import ( + FailureCaseWriteTestCase, + FailureCorpusTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + FailureCorpusTestCase( + description="breadth", + expected_min_cases=45, + expected_min_codes=30, + expected_min_families=7, + ) + ], + ids=lambda case: case.description, +) +def test_given_failure_corpus_when_listing_then_names_are_unique_and_codes_are_broad( + test_case: FailureCorpusTestCase, +) -> None: + cases: tuple[FailureCase, ...] = failure_cases() + codes: set[str] = {case.expected_code for case in cases} + + assert len({case.name for case in cases}) == len(cases) + assert len(cases) >= test_case.expected_min_cases + assert len(codes) >= test_case.expected_min_codes + assert len({code.rstrip("0123456789") for code in codes}) >= test_case.expected_min_families + + +@pytest.mark.parametrize( + "test_case", + [ + FailureCaseWriteTestCase( + description=case.name, case=case, expected_files=tuple(sorted(case.files)) + ) + for case in failure_cases() + ], + ids=lambda case: case.description, +) +def test_given_failure_case_when_writing_then_every_file_exists( + test_case: FailureCaseWriteTestCase, tmp_path: Path +) -> None: + test_case.case.write(tmp_path) + + assert ( + tuple(sorted(path.relative_to(tmp_path).as_posix() for path in tmp_path.rglob("*.*"))) + == test_case.expected_files + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/main/test_generate_project.py b/tests/unit/scripts/compiler_differential/main/test_generate_project.py new file mode 100644 index 0000000000..bde93149c8 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/main/test_generate_project.py @@ -0,0 +1,90 @@ +"""Generated projects are deterministic per seed and cover the intended feature space.""" + +from __future__ import annotations + +import pytest + +from scripts.compiler_differential.main.generate_project import generate_project +from scripts.compiler_differential.models import GeneratedProject +from tests.unit.scripts.compiler_differential.main._test_types import ( + SeedDeterminismTestCase, + SeedRangeTestCase, +) +from tests.unit.scripts.compiler_differential.main.helpers import ( + generated_projects, + invalid_count, + project_features, +) + + +@pytest.mark.parametrize( + "test_case", + [ + SeedDeterminismTestCase(description=f"seed_{seed}", seed=seed, expected_identical=True) + for seed in (0, 7, 41) + ], + ids=lambda case: case.description, +) +def test_given_same_seed_when_generating_twice_then_projects_are_identical( + test_case: SeedDeterminismTestCase, +) -> None: + first: GeneratedProject = generate_project(seed=test_case.seed) + second: GeneratedProject = generate_project(seed=test_case.seed) + + assert (first == second) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + SeedRangeTestCase( + description="first_sixty_seeds", + seeds=range(60), + expected_features=frozenset( + { + "exact_owner_scope", + "tree_scope", + "project_scope", + "model_private_value", + "relationship_grant", + "name_shared_across_namespaces", + "non_ascii_constant", + "float_constant", + "decimal_constant", + "integer_enum", + "macro_reads_ctx", + "macro_reads_env", + "nested_macro", + "path_defaults_literal", + "path_defaults_wildcard", + "sql_hooks", + "python_nodes", + "unit_test", + "scenario", + "singular_audit", + "column_audits", + } + ), + expected_distinct_projects=60, + expected_min_invalid=5, + expected_max_invalid=30, + ) + ], + ids=lambda case: case.description, +) +def test_given_seed_range_when_generating_then_features_and_failures_are_covered( + test_case: SeedRangeTestCase, +) -> None: + projects: list[GeneratedProject] = generated_projects(test_case.seeds) + + assert test_case.expected_features <= project_features(projects) + assert len({tuple(project.files.items()) for project in projects}) == ( + test_case.expected_distinct_projects + ) + assert ( + test_case.expected_min_invalid <= invalid_count(projects) <= test_case.expected_max_invalid + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py index 51a4626013..3b871fc8f4 100644 --- a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/_test_types.py @@ -103,3 +103,33 @@ class UntypedCursorParsingTestCase: argv: tuple[str, ...] expected_start: str | None expected_end: str | None + + +@dataclass(frozen=True) +class CompilerEngineOptionTestCase: + """One compiler engine selection through the hidden flag and the environment.""" + + description: str + argv: tuple[str, ...] + environment_value: str + expected_engine: str | None + + +@dataclass(frozen=True) +class RejectedCompilerEngineTestCase: + """An unsupported engine selection and the usage error that names accepted values.""" + + description: str + argv: tuple[str, ...] + environment_value: str + expected_exit_code: int + expected_error: str + + +@dataclass(frozen=True) +class HiddenOptionTestCase: + """A help page and an option that must not appear on it.""" + + description: str + option: str + expected_listed: bool diff --git a/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py new file mode 100644 index 0000000000..273ba2609d --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/commands/_helpers/entry/test_compiler_engine_option.py @@ -0,0 +1,113 @@ +"""The hidden --compiler-engine flag overrides SQLBUILD_COMPILER_ENGINE and both are validated.""" + +from __future__ import annotations + +import pytest + +from sqlbuild.cli.commands._helpers.entry.parser import build_cli_parser +from sqlbuild.cli.commands._helpers.entry.parsing import parse_cli_invocation +from sqlbuild.cli.entry.models import ParsedCliInvocation +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from tests.unit.src.sqlbuild.cli.commands._helpers.entry._test_types import ( + CompilerEngineOptionTestCase, + HiddenOptionTestCase, + RejectedCompilerEngineTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + CompilerEngineOptionTestCase( + description="default_leaves_engine_to_environment", + argv=("compile",), + environment_value="", + expected_engine=None, + ), + CompilerEngineOptionTestCase( + description="flag_before_subcommand", + argv=("--compiler-engine", "native", "compile"), + environment_value="", + expected_engine="native", + ), + CompilerEngineOptionTestCase( + description="flag_after_subcommand", + argv=("plan", "--compiler-engine", "python"), + environment_value="", + expected_engine="python", + ), + CompilerEngineOptionTestCase( + description="flag_overrides_invalid_environment", + argv=("compile", "--compiler-engine", "python"), + environment_value="rust", + expected_engine="python", + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_selection_when_parsing_then_flag_value_is_recorded( + test_case: CompilerEngineOptionTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.environment_value) + + parsed: ParsedCliInvocation = parse_cli_invocation( + argv=test_case.argv, parser=build_cli_parser() + ) + + assert parsed.args is not None + assert parsed.args.compiler_engine == test_case.expected_engine + + +@pytest.mark.parametrize( + "test_case", + [ + RejectedCompilerEngineTestCase( + description="unknown_flag_value", + argv=("compile", "--compiler-engine", "rust"), + environment_value="", + expected_exit_code=2, + expected_error="argument --compiler-engine: invalid choice: 'rust'", + ), + RejectedCompilerEngineTestCase( + description="unknown_environment_value", + argv=("compile",), + environment_value="rust", + expected_exit_code=2, + expected_error="SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'rust')", + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_engine_when_parsing_then_usage_error_names_accepted_values( + test_case: RejectedCompilerEngineTestCase, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.environment_value) + + parsed: ParsedCliInvocation = parse_cli_invocation( + argv=test_case.argv, parser=build_cli_parser() + ) + + assert parsed.args is None + assert parsed.exit_code == test_case.expected_exit_code + assert test_case.expected_error in capsys.readouterr().err + + +@pytest.mark.parametrize( + "test_case", + [ + HiddenOptionTestCase( + description="root_help", option="--compiler-engine", expected_listed=False + ) + ], + ids=lambda case: case.description, +) +def test_given_root_help_when_rendering_then_compiler_engine_flag_stays_hidden( + test_case: HiddenOptionTestCase, +) -> None: + assert (test_case.option in build_cli_parser().format_help()) is test_case.expected_listed + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py index ea3686abfc..90ea98eb84 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/_test_types.py @@ -91,3 +91,22 @@ class ChangedPathsTestCase: change: Callable[[Path], object] expected_paths: tuple[str, ...] store: Callable[[Path], dict[str, StoredProjectFile]] = digested_project_files + + +@dataclass(frozen=True) +class EngineReuseDigestTestCase: + """Two engine environments for one invocation and whether their reuse keys may match.""" + + description: str + first_environment: dict[str, str] + second_environment: dict[str, str] + expected_same_digest: bool + + +@dataclass(frozen=True) +class EngineReuseStoreTestCase: + """One engine and the store directory that holds its compile reuse entry.""" + + description: str + engine: str + expected_entry_directory: str diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py index efacd4c1c0..dd03ca35e2 100644 --- a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/helpers.py @@ -16,13 +16,20 @@ is_racy, snapshot_project_files, ) +from sqlbuild.cli.compile_reuse._helpers.runtime_identity import ( + environment_digest, + invocation_digest, + tracked_environment_names, +) from sqlbuild.cli.compile_reuse.constants import RACY_WINDOW_NS from sqlbuild.cli.compile_reuse.models import ( + CompileReuseRequest, SettingsEnvironmentInputs, SettingsInputsResult, StoredProjectFile, ) from sqlbuild.cli.compile_reuse.types import FileStamp +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR _LATER_SNAPSHOT_NS: int = 10 * RACY_WINDOW_NS @@ -266,3 +273,42 @@ def described_settings_inputs( def _path_names(paths: tuple[str, ...]) -> tuple[str, ...]: return tuple(Path(path).name for path in paths) + + +def orders_reuse_request(project_dir: Path) -> CompileReuseRequest: + """Return a plain JSON compile reuse request for a project.""" + + return CompileReuseRequest( + project_dir=project_dir, + selected_target=None, + json_output=True, + no_color=True, + manifest=False, + dag_path=None, + defer_to=None, + no_cache=False, + no_sql_validation=False, + lineage_mode="fast", + select=(), + exclude=(), + cli_vars=None, + profiling=False, + debug=False, + ) + + +def engine_reuse_key( + *, environment: dict[str, str], project_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[str, str]: + """Return the invocation and environment digests computed under one engine environment.""" + + monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) + monkeypatch.delenv(STAGE_CAPTURE_DIR_ENV_VAR, raising=False) + for name, value in environment.items(): + monkeypatch.setenv(name, value) + return ( + invocation_digest( + request=orders_reuse_request(project_dir), project_dir=str(project_dir), use_color=False + ), + environment_digest(names=tracked_environment_names(template_names=())), + ) diff --git a/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py new file mode 100644 index 0000000000..a8cd940fe7 --- /dev/null +++ b/tests/unit/src/sqlbuild/cli/compile_reuse/_helpers/test_compiler_engine_reuse_key.py @@ -0,0 +1,105 @@ +"""Compile reuse never replays a compile recorded by the other compiler engine.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.cli.compile_reuse._helpers.attempt import attempt_reuse +from sqlbuild.cli.compile_reuse.constants import REUSE_DISABLE_ENV_VAR +from sqlbuild.cli.compile_reuse.models import CompileReuseAttempt +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers._test_types import ( + EngineReuseDigestTestCase, + EngineReuseStoreTestCase, +) +from tests.unit.src.sqlbuild.cli.compile_reuse._helpers.helpers import ( + engine_reuse_key, + orders_reuse_request, + write_file, +) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineReuseDigestTestCase( + description="python_then_native", + first_environment={COMPILER_ENGINE_ENV_VAR: "python"}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + expected_same_digest=False, + ), + EngineReuseDigestTestCase( + description="unset_then_native", + first_environment={}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + expected_same_digest=False, + ), + EngineReuseDigestTestCase( + description="unset_then_explicit_python", + first_environment={}, + second_environment={COMPILER_ENGINE_ENV_VAR: "python"}, + expected_same_digest=True, + ), + EngineReuseDigestTestCase( + description="native_twice", + first_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + second_environment={COMPILER_ENGINE_ENV_VAR: "native"}, + expected_same_digest=True, + ), + EngineReuseDigestTestCase( + description="stage_capture_directory_is_not_a_compile_input", + first_environment={}, + second_environment={STAGE_CAPTURE_DIR_ENV_VAR: "/tmp/captures"}, + expected_same_digest=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_same_invocation_when_engine_environment_changes_then_reuse_key_follows_engine( + test_case: EngineReuseDigestTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + first: tuple[str, str] = engine_reuse_key( + environment=test_case.first_environment, project_dir=tmp_path, monkeypatch=monkeypatch + ) + second: tuple[str, str] = engine_reuse_key( + environment=test_case.second_environment, project_dir=tmp_path, monkeypatch=monkeypatch + ) + + assert (first == second) is test_case.expected_same_digest + + +@pytest.mark.parametrize( + "test_case", + [ + EngineReuseStoreTestCase( + description="python", + engine="python", + expected_entry_directory="target/cache/compiler/project-reuse-v1", + ), + EngineReuseStoreTestCase( + description="native", + engine="native", + expected_entry_directory="target/cache/compiler-native-v1/project-reuse-v1", + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_attempting_reuse_then_entry_lives_in_that_engine_store( + test_case: EngineReuseStoreTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + write_file(tmp_path / "sqlbuild_project.toml", 'name = "orders"\nadapter = "duckdb"\n') + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine) + monkeypatch.setenv(REUSE_DISABLE_ENV_VAR, "0") + + attempt: CompileReuseAttempt = attempt_reuse(request=orders_reuse_request(tmp_path)) + + assert attempt.entry_path is not None + assert attempt.entry_path.parent.relative_to(tmp_path).as_posix() == ( + test_case.expected_entry_directory + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/__init__.py b/tests/unit/src/sqlbuild/compiler/frontier/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py new file mode 100644 index 0000000000..7049921840 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/_test_types.py @@ -0,0 +1,98 @@ +"""Test case types for compiler engine selection, store namespaces, and stage captures.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage + + +@dataclass(frozen=True) +class EngineResolutionTestCase: + """One SQLBUILD_COMPILER_ENGINE value and the engine it selects.""" + + description: str + raw_value: str + expected_engine: CompilerEngine + + +@dataclass(frozen=True) +class EngineErrorTestCase: + """One unsupported engine value and the exact error naming accepted values.""" + + description: str + raw_value: str + expected_message: str + + +@dataclass(frozen=True) +class EngineOverrideTestCase: + """An engine variable before an override, and what the override shows and restores.""" + + description: str + previous: str + override: CompilerEngine + expected_inside: CompilerEngine + expected_after: str + + +@dataclass(frozen=True) +class EngineCacheNameTestCase: + """One engine and the store directory name it owns.""" + + description: str + engine: CompilerEngine + base: str + expected_name: str + + +@dataclass(frozen=True) +class EngineStorePathsTestCase: + """One engine and the project-relative compiler and Rules store paths it uses.""" + + description: str + engine: CompilerEngine + expected_paths: tuple[str, ...] + + +@dataclass(frozen=True) +class StageCaptureTestCase: + """One value whose canonical capture must match an exact JSON-compatible shape.""" + + description: str + value: Callable[[], object] + expected_capture: object + + +@dataclass(frozen=True) +class StageCaptureOrderTestCase: + """Two values that differ only in ordering and whether their captures must be identical.""" + + description: str + first: Callable[[], object] + second: Callable[[], object] + expected_identical: bool + + +@dataclass(frozen=True) +class UnorderedAttributeTestCase: + """Two fill orders for a memo map and an ordered map, and whether captures match.""" + + description: str + first_memo_keys: tuple[str, ...] + second_memo_keys: tuple[str, ...] + first_ordered_keys: tuple[str, ...] + second_ordered_keys: tuple[str, ...] + expected_identical: bool + + +@dataclass(frozen=True) +class FrontierCaptureTestCase: + """Frontier stages compiled in order on one engine and the capture files they write.""" + + description: str + engine: CompilerEngine + stages: tuple[CompilerStage, ...] + expected_results: tuple[object, ...] + expected_files: tuple[str, ...] diff --git a/tests/unit/src/sqlbuild/compiler/frontier/helpers.py b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py new file mode 100644 index 0000000000..5259371578 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/helpers.py @@ -0,0 +1,87 @@ +"""Shared values and helpers for compiler frontier tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path, PurePosixPath + +from sqlbuild.compiler.fact_cache.main._compile_cache_root import compile_cache_root +from sqlbuild.compiler.frontier.main.compiler_cache_directory import compiler_cache_directory +from sqlbuild.compiler.scopes.constants import SCOPE_CACHE_DIRECTORY_NAME +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path + +HELPERS_MODULE: str = "tests.unit.src.sqlbuild.compiler.frontier.helpers" + + +class OrderStatus(StrEnum): + """A neutral enum captured by name.""" + + PLACED = "placed" + + +@dataclass(frozen=True) +class OrderLine: + """A neutral dataclass captured field by field.""" + + order_id: int + status: OrderStatus + path: PurePosixPath + + +def order_total(amount_cents: int) -> int: + """A neutral callable captured by its qualified name.""" + + return amount_cents + + +class LinkedCustomer: + """A plain object that can point back at itself.""" + + def __init__(self) -> None: + self.name: str = "customer" + self.peer: LinkedCustomer | None = None + + +def linked_customer() -> LinkedCustomer: + """Return a customer whose peer is itself.""" + + customer: LinkedCustomer = LinkedCustomer() + customer.peer = customer + return customer + + +class MemoCatalog: + """A plain object with one memo map and one ordered map.""" + + def __init__(self, *, memo: dict[str, int], ordered: dict[str, int]) -> None: + self.memo: dict[str, int] = memo + self.ordered: dict[str, int] = ordered + + +def memo_catalog(*, memo_keys: tuple[str, ...], ordered_keys: tuple[str, ...]) -> MemoCatalog: + """Return a catalog whose maps were filled in the given key orders.""" + + return MemoCatalog( + memo={key: len(key) for key in memo_keys}, + ordered={key: len(key) for key in ordered_keys}, + ) + + +def order_line() -> OrderLine: + """Return one neutral order line.""" + + return OrderLine(order_id=7, status=OrderStatus.PLACED, path=PurePosixPath("models/a.sql")) + + +def store_paths(project_dir: Path) -> tuple[str, ...]: + """Return every engine-owned store path relative to the project directory.""" + + paths: tuple[Path, ...] = ( + compiler_cache_directory(project_dir), + compile_cache_root(project_dir=project_dir, target_config=None, no_cache=False) + or project_dir, + compiler_cache_directory(project_dir) / SCOPE_CACHE_DIRECTORY_NAME, + rules_bulk_cache_path(project_dir=project_dir, file_name="sql.json"), + ) + return tuple(path.relative_to(project_dir).as_posix() for path in paths) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py b/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py new file mode 100644 index 0000000000..cf5b751b56 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_cache_namespace.py @@ -0,0 +1,86 @@ +"""On-disk compiler stores are namespaced by engine; Python keeps its existing paths.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.main.engine_cache_name import engine_cache_name +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( + EngineCacheNameTestCase, + EngineStorePathsTestCase, +) +from tests.unit.src.sqlbuild.compiler.frontier.helpers import store_paths + + +@pytest.mark.parametrize( + "test_case", + [ + EngineCacheNameTestCase( + description="python_keeps_name", + engine=CompilerEngine.PYTHON, + base="compiler", + expected_name="compiler", + ), + EngineCacheNameTestCase( + description="native_gets_versioned_suffix", + engine=CompilerEngine.NATIVE, + base="compiler", + expected_name="compiler-native-v1", + ), + EngineCacheNameTestCase( + description="native_rules_cache", + engine=CompilerEngine.NATIVE, + base="rules-cache", + expected_name="rules-cache-native-v1", + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_naming_store_then_only_native_is_suffixed( + test_case: EngineCacheNameTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + + assert engine_cache_name(test_case.base) == test_case.expected_name + + +@pytest.mark.parametrize( + "test_case", + [ + EngineStorePathsTestCase( + description="python_paths_unchanged", + engine=CompilerEngine.PYTHON, + expected_paths=( + "target/cache/compiler", + "target/cache/compiler", + "target/cache/compiler/declaration-scopes-v2", + "target/rules-cache/bulk/sql.json", + ), + ), + EngineStorePathsTestCase( + description="native_paths_separate", + engine=CompilerEngine.NATIVE, + expected_paths=( + "target/cache/compiler-native-v1", + "target/cache/compiler-native-v1", + "target/cache/compiler-native-v1/declaration-scopes-v2", + "target/rules-cache-native-v1/bulk/sql.json", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_when_resolving_stores_then_each_engine_owns_its_paths( + test_case: EngineStorePathsTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + + assert store_paths(tmp_path) == test_case.expected_paths + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py new file mode 100644 index 0000000000..a4c9d19334 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_resolve_compiler_engine.py @@ -0,0 +1,155 @@ +"""The compiler engine switch reads SQLBUILD_COMPILER_ENGINE and rejects unknown values.""" + +from __future__ import annotations + +import os + +import pytest + +from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR +from sqlbuild.compiler.frontier.exceptions import CompilerEngineError +from sqlbuild.compiler.frontier.main.compiler_engine_override import compiler_engine_override +from sqlbuild.compiler.frontier.main.resolve_compiler_engine import resolve_compiler_engine +from sqlbuild.compiler.frontier.types import CompilerEngine +from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( + EngineErrorTestCase, + EngineOverrideTestCase, + EngineResolutionTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + EngineResolutionTestCase( + description="empty", raw_value="", expected_engine=CompilerEngine.PYTHON + ), + EngineResolutionTestCase( + description="python", raw_value="python", expected_engine=CompilerEngine.PYTHON + ), + EngineResolutionTestCase( + description="native", raw_value="native", expected_engine=CompilerEngine.NATIVE + ), + ], + ids=lambda case: case.description, +) +def test_given_engine_variable_when_resolving_then_returns_selected_engine( + test_case: EngineResolutionTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.raw_value) + + assert resolve_compiler_engine() is test_case.expected_engine + + +@pytest.mark.parametrize( + "test_case", + [ + EngineResolutionTestCase( + description="unset", raw_value="", expected_engine=CompilerEngine.PYTHON + ) + ], + ids=lambda case: case.description, +) +def test_given_no_engine_variable_when_resolving_then_python_is_the_default( + test_case: EngineResolutionTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) + + assert resolve_compiler_engine() is test_case.expected_engine + + +@pytest.mark.parametrize( + "test_case", + [ + EngineErrorTestCase( + description="unknown", + raw_value="rust", + expected_message="SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'rust')", + ), + EngineErrorTestCase( + description="wrong_case", + raw_value="Native", + expected_message=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native (got 'Native')" + ), + ), + EngineErrorTestCase( + description="padded", + raw_value=" python", + expected_message=( + "SQLBUILD_COMPILER_ENGINE must be one of python, native (got ' python')" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_unknown_engine_when_resolving_then_error_names_accepted_and_current_values( + test_case: EngineErrorTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.raw_value) + + with pytest.raises(CompilerEngineError) as error: + _ = resolve_compiler_engine() + + assert str(error.value) == test_case.expected_message + + +@pytest.mark.parametrize( + "test_case", + [ + EngineOverrideTestCase( + description="python_restored", + previous="python", + override=CompilerEngine.NATIVE, + expected_inside=CompilerEngine.NATIVE, + expected_after="python", + ), + EngineOverrideTestCase( + description="empty_restored", + previous="", + override=CompilerEngine.NATIVE, + expected_inside=CompilerEngine.NATIVE, + expected_after="", + ), + ], + ids=lambda case: case.description, +) +def test_given_override_when_leaving_then_engine_variable_is_restored( + test_case: EngineOverrideTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.previous) + + with compiler_engine_override(test_case.override): + inside: CompilerEngine = resolve_compiler_engine() + + assert inside is test_case.expected_inside + assert os.environ[COMPILER_ENGINE_ENV_VAR] == test_case.expected_after + + +@pytest.mark.parametrize( + "test_case", + [ + EngineOverrideTestCase( + description="unset_removed", + previous="", + override=CompilerEngine.NATIVE, + expected_inside=CompilerEngine.NATIVE, + expected_after="", + ) + ], + ids=lambda case: case.description, +) +def test_given_unset_variable_when_override_ends_then_variable_is_removed( + test_case: EngineOverrideTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv(COMPILER_ENGINE_ENV_VAR, raising=False) + + with compiler_engine_override(test_case.override): + inside: CompilerEngine = resolve_compiler_engine() + + assert inside is test_case.expected_inside + assert os.environ.get(COMPILER_ENGINE_ENV_VAR, "") == test_case.expected_after + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py new file mode 100644 index 0000000000..6b2c2384f9 --- /dev/null +++ b/tests/unit/src/sqlbuild/compiler/frontier/test_stage_capture.py @@ -0,0 +1,205 @@ +"""Stage captures render frontier objects as deterministic, comparable JSON.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from sqlbuild.compiler.frontier._helpers.stage_capture import render_stage_capture +from sqlbuild.compiler.frontier.constants import ( + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, + STAGE_CAPTURE_UNORDERED_ATTRIBUTES, +) +from sqlbuild.compiler.frontier.main._compile_frontier import compile_frontier +from sqlbuild.compiler.frontier.types import CompilerEngine, CompilerStage +from tests.unit.src.sqlbuild.compiler.frontier._test_types import ( + FrontierCaptureTestCase, + StageCaptureOrderTestCase, + StageCaptureTestCase, + UnorderedAttributeTestCase, +) +from tests.unit.src.sqlbuild.compiler.frontier.helpers import ( + HELPERS_MODULE, + linked_customer, + memo_catalog, + order_line, + order_total, +) + + +@pytest.mark.parametrize( + "test_case", + [ + StageCaptureTestCase( + description="dataclass_enum_and_path", + value=order_line, + expected_capture={ + "__type__": f"{HELPERS_MODULE}:OrderLine", + "order_id": 7, + "status": {"__enum__": f"{HELPERS_MODULE}:OrderStatus.PLACED"}, + "path": {"__path__": "models/a.sql"}, + }, + ), + StageCaptureTestCase( + description="callable_becomes_qualified_name", + value=lambda: {"total": order_total}, + expected_capture={"total": {"__callable__": f"{HELPERS_MODULE}:order_total"}}, + ), + StageCaptureTestCase( + description="sets_are_sorted", + value=lambda: frozenset({"orders", "customers", "products"}), + expected_capture={"__set__": ["customers", "orders", "products"]}, + ), + StageCaptureTestCase( + description="non_string_keys_are_pairs_in_insertion_order", + value=lambda: {2: "second", 1: "first"}, + expected_capture={"__mapping__": [[2, "second"], [1, "first"]]}, + ), + StageCaptureTestCase( + description="cycles_are_cut", + value=linked_customer, + expected_capture={ + "__type__": f"{HELPERS_MODULE}:LinkedCustomer", + "name": "customer", + "peer": {"__cycle__": f"{HELPERS_MODULE}:LinkedCustomer"}, + }, + ), + StageCaptureTestCase( + description="non_finite_float_and_bytes", + value=lambda: (float("inf"), b"orders"), + expected_capture=[ + {"__float__": "inf"}, + { + "__bytes__": 6, + "sha256": "1c168adb00d208e42f93314529f1fa9c0427eb63233ceda95a5db52b7012a719", + }, + ], + ), + ], + ids=lambda case: case.description, +) +def test_given_frontier_value_when_rendering_capture_then_json_is_canonical( + test_case: StageCaptureTestCase, +) -> None: + assert json.loads(render_stage_capture(test_case.value())) == test_case.expected_capture + + +@pytest.mark.parametrize( + "test_case", + [ + StageCaptureOrderTestCase( + description="mapping_insertion_order_is_visible", + first=lambda: {"orders": 1, "customers": 2}, + second=lambda: {"customers": 2, "orders": 1}, + expected_identical=False, + ), + StageCaptureOrderTestCase( + description="non_string_key_insertion_order_is_visible", + first=lambda: {1: "orders", 2: "customers"}, + second=lambda: {2: "customers", 1: "orders"}, + expected_identical=False, + ), + StageCaptureOrderTestCase( + description="tuple_order_is_visible", + first=lambda: ("orders", "customers"), + second=lambda: ("customers", "orders"), + expected_identical=False, + ), + StageCaptureOrderTestCase( + description="set_order_is_not_visible", + first=lambda: frozenset(("orders", "customers", "products")), + second=lambda: frozenset(("products", "customers", "orders")), + expected_identical=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_reordered_value_when_rendering_capture_then_only_unordered_collections_match( + test_case: StageCaptureOrderTestCase, +) -> None: + assert ( + render_stage_capture(test_case.first()) == render_stage_capture(test_case.second()) + ) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + UnorderedAttributeTestCase( + description="memo_fill_order_is_not_visible", + first_memo_keys=("orders", "customers"), + second_memo_keys=("customers", "orders"), + first_ordered_keys=("orders", "customers"), + second_ordered_keys=("orders", "customers"), + expected_identical=True, + ), + UnorderedAttributeTestCase( + description="other_attribute_order_stays_visible", + first_memo_keys=("orders", "customers"), + second_memo_keys=("orders", "customers"), + first_ordered_keys=("orders", "customers"), + second_ordered_keys=("customers", "orders"), + expected_identical=False, + ), + ], + ids=lambda case: case.description, +) +def test_given_declared_memo_attribute_when_rendering_capture_then_only_it_is_sorted( + test_case: UnorderedAttributeTestCase, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem( + STAGE_CAPTURE_UNORDERED_ATTRIBUTES, f"{HELPERS_MODULE}:MemoCatalog", frozenset({"memo"}) + ) + first: str = render_stage_capture( + memo_catalog(memo_keys=test_case.first_memo_keys, ordered_keys=test_case.first_ordered_keys) + ) + second: str = render_stage_capture( + memo_catalog( + memo_keys=test_case.second_memo_keys, ordered_keys=test_case.second_ordered_keys + ) + ) + + assert (first == second) is test_case.expected_identical + + +@pytest.mark.parametrize( + "test_case", + [ + FrontierCaptureTestCase( + description=engine.value, + engine=engine, + stages=(CompilerStage.DISCOVERED_PROJECT_INPUTS, CompilerStage.COMPILED_PROJECT), + expected_results=( + "discovered_project_inputs-result", + "compiled_project-result", + ), + expected_files=("discovered_project_inputs.json", "compiled_project.json"), + ) + for engine in CompilerEngine + ], + ids=lambda case: case.description, +) +def test_given_capture_directory_when_compiling_frontier_then_each_stage_is_captured_in_order( + test_case: FrontierCaptureTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(COMPILER_ENGINE_ENV_VAR, test_case.engine.value) + monkeypatch.setenv(STAGE_CAPTURE_DIR_ENV_VAR, str(tmp_path)) + + results: tuple[object, ...] = tuple( + compile_frontier(until=stage, python_stage=lambda stage=stage: f"{stage.value}-result") + for stage in test_case.stages + ) + + captures: list[Path] = sorted(tmp_path.iterdir()) + assert results == test_case.expected_results + assert tuple(path.name.split("-", 1)[1] for path in captures) == test_case.expected_files + assert tuple(json.loads(path.read_text(encoding="utf-8")) for path in captures) == ( + test_case.expected_results + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py index 819d317d9d..6738ebdc7d 100644 --- a/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py +++ b/tests/unit/src/sqlbuild/rule_engine/_helpers/host/test_custom_host_pool.py @@ -7,6 +7,7 @@ from sqlbuild.compiler.compile.models import CompiledProject from sqlbuild.rule_engine._helpers.host import custom_host, custom_host_pool +from sqlbuild.rule_engine._helpers.run.cache_paths import rules_bulk_cache_path from sqlbuild.rule_engine.constants import CUSTOM_RULES_CACHE_FILE from sqlbuild.rule_engine.exceptions import HostCancelledError, RulesError from sqlbuild.rule_engine.models import CustomRulesOutcome @@ -75,8 +76,10 @@ def test_given_partitioned_hosts_when_evaluating_then_findings_and_cache_match_o detects_duplicates=test_case.detects_duplicates, ) project: CompiledProject = orders_project(model_count=test_case.model_count) - cache_path: Path = tmp_path / CUSTOM_RULES_CACHE_FILE - cache_glob: str = CUSTOM_RULES_CACHE_FILE + cache_path: Path = rules_bulk_cache_path( + project_dir=tmp_path, file_name=CUSTOM_RULES_CACHE_FILE + ) + cache_glob: str = cache_path.relative_to(tmp_path).as_posix() cache_enabled: bool = test_case.cache_enabled single_launches: list[str] = use_hosts(monkeypatch=monkeypatch, hosts=1)