diff --git a/Cargo.lock b/Cargo.lock index 1db1abd076..d10f81b95f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,8 +310,8 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "polyglot-sql" -version = "0.12.1" -source = "git+https://github.com/chio-labs/polyglot.git?rev=cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b#cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b" +version = "0.12.2" +source = "git+https://github.com/chio-labs/polyglot.git?rev=dd7154035566933a0f9acae17a2c102afb4dbfde#dd7154035566933a0f9acae17a2c102afb4dbfde" dependencies = [ "polyglot-sql-ast-derive", "serde", @@ -323,8 +323,8 @@ dependencies = [ [[package]] name = "polyglot-sql-ast-derive" -version = "0.12.1" -source = "git+https://github.com/chio-labs/polyglot.git?rev=cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b#cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b" +version = "0.12.2" +source = "git+https://github.com/chio-labs/polyglot.git?rev=dd7154035566933a0f9acae17a2c102afb4dbfde#dd7154035566933a0f9acae17a2c102afb4dbfde" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 5a1970bbee..e39097ff7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ fensu-policy = "=0.22.0" globset = "0.4" proc-macro2 = { version = "1", features = ["span-locations"] } pyo3 = { version = "0.26", features = ["abi3-py312"] } -polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "cd599007a1ba6c4cc1f2d99938fc0e6d3b376a9b", default-features = false, features = [ +polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "dd7154035566933a0f9acae17a2c102afb4dbfde", default-features = false, features = [ "ast-tools", "builder", "dialect-bigquery", diff --git a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs b/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs index aaf19f56f1..021a098b98 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/_helpers/engine.rs @@ -122,6 +122,8 @@ struct ProjectAnalysis { has_star: bool, #[serde(skip)] requires_legacy_fallback: bool, + #[serde(skip)] + preserve_fallback_lineage: bool, } struct ProjectAnalysisInputs<'a> { @@ -611,19 +613,18 @@ fn try_borrowed_query( upstream_columns, }); } + let requires_legacy_fallback = columns.iter().any(|column| column.data_type.is_none()); projections.push(( *index, Ok(ProjectAnalysis { columns, lineage_columns, has_star: false, - requires_legacy_fallback: false, + requires_legacy_fallback, + preserve_fallback_lineage: true, }), )); } - if projections.iter().any(|(_, result)| matches!(result, Ok(analysis) if analysis.columns.iter().any(|column| column.data_type.is_none()))) { - return Err(Box::new(work)); - } Ok(CompiledQueryWorkResult { projections, validation: if work.query.binding_schema.is_some() { @@ -933,6 +934,7 @@ struct CompactProjectBatch { #[serde(untagged)] enum CompactProjectResponse { Success((Vec, bool)), + LegacyTypeRecovery((Vec, bool, &'static str)), Failure(String), } @@ -985,7 +987,7 @@ impl CompactProjectAccumulator { return Ok(()); } }; - if analysis.requires_legacy_fallback { + if analysis.requires_legacy_fallback && !analysis.preserve_fallback_lineage { self.set_template( projection_index, CompactProjectResponse::Failure( @@ -1024,7 +1026,15 @@ impl CompactProjectAccumulator { } self.set_template( projection_index, - CompactProjectResponse::Success((columns, analysis.has_star)), + if analysis.requires_legacy_fallback { + CompactProjectResponse::LegacyTypeRecovery(( + columns, + analysis.has_star, + "native project type recovery requires legacy fallback", + )) + } else { + CompactProjectResponse::Success((columns, analysis.has_star)) + }, ) } @@ -1194,6 +1204,7 @@ fn project_analysis(inputs: ProjectAnalysisInputs<'_>) -> ProjectAnalysis { lineage_columns, has_star: inputs.analysis.has_root_star, requires_legacy_fallback, + preserve_fallback_lineage: false, } } diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs b/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs index cdfd37eea8..bbc44b8051 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/tests/helpers.rs @@ -164,6 +164,38 @@ pub(crate) fn binding_warnings_preserve_declared_relationship_evidence() -> bool true } +pub(crate) fn bound_lineage_survives_type_recovery() -> bool { + for operator in ["UNION ALL", "UNION ALL BY NAME"] { + let response: Value = serde_json::from_str( + &analyze_project_compact_json(&json!({ + "queries": [{ + "sql": format!("WITH combined AS (SELECT CAST(amount AS DOUBLE) AS amount FROM orders {operator} SELECT list_extract([amount], 1) AS amount FROM refunds) SELECT amount FROM combined"), + "dialect": "duckdb", + "schema": {"strict": true, "tables": [ + {"name": "orders", "columns": [{"name": "amount", "type": "DOUBLE"}]}, + {"name": "refunds", "columns": [{"name": "amount", "type": "DOUBLE"}]} + ]} + }], + "templates": [{"queryIndex": 0, "recoverCteFacts": true, "richTypeInference": false, "references": { + "orders": {"resourceType": "model", "resourceName": "orders"}, + "refunds": {"resourceType": "model", "resourceName": "refunds"} + }}], + "projections": [{"templateIndex": 0}] + }).to_string()).expect("query should analyze"), + ).expect("response should decode"); + let index = response["templates"][0][0][0] + .as_u64() + .expect("successful fact") as usize; + let fact = &response["facts"][index]; + assert_eq!( + response["templates"][0][2], + "native project type recovery requires legacy fallback" + ); + assert_eq!(fact[5].as_array().expect("upstream columns").len(), 2); + } + true +} + pub(crate) fn widening_aggregates_require_compatibility_recovery() -> bool { for expression in [ "AVG(order_id)", diff --git a/crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs b/crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs index d12bf69441..4438c97314 100644 --- a/crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs +++ b/crates/sqlbuild-rules-native/src/query_analysis/tests/test_query_analysis.rs @@ -4,7 +4,7 @@ use crate::query_analysis::tests::helpers::{ binding_warnings_preserve_declared_relationship_evidence, borrowed_facts_preserve_named_outputs_and_terminal_sources, borrowed_facts_preserve_union_dependencies_and_join_nullability, - canonical_queries_reuse_semantics_and_project_resources, + bound_lineage_survives_type_recovery, canonical_queries_reuse_semantics_and_project_resources, combined_queries_preserve_standalone_binding, compact_project_query_interns_repeated_lineage_strings, interleaved_query_templates_preserve_template_order, @@ -56,6 +56,11 @@ fn given_compact_query_cases_when_analyzing_projects_then_expected_behavior_hold run: widening_aggregates_require_compatibility_recovery, expected_success: true, }, + CompactQueryAnalysisTestCase { + description: "resolved bound lineage survives type recovery", + run: bound_lineage_survives_type_recovery, + expected_success: true, + }, CompactQueryAnalysisTestCase { description: "compact project queries intern repeated lineage strings", run: compact_project_query_interns_repeated_lineage_strings, diff --git a/pyproject.toml b/pyproject.toml index 1f4b702e33..fdc23d94e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ dependencies = [ "duckdb>=1.3.2", "filelock>=3.20.3", "orjson>=3.10.0", - "polyglot-sql-chio==0.12.1", + "polyglot-sql-chio==0.12.2", "pydantic-settings>=2,<3", "PyYAML>=6.0.2", "rich>=15.0.0", diff --git a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py index 55f4c28324..c531648972 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/sql_test_artifact_cache.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import cast +import orjson + from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.cli.compile.models import ( SqlTestArtifactCacheRecord, @@ -25,7 +27,7 @@ ) _CACHE_VERSION: int = 3 -_ALGORITHM_FINGERPRINT: str = "sql-test-artifact-v3-polyglot-scope-fix" +_ALGORITHM_FINGERPRINT: str = "sql-test-artifact-v4-orjson-identity" _CACHE_FILE_NAME: str = "sql-test-artifacts.json" _MAX_CACHE_BYTES: int = 10_000_000 _MAX_CACHE_RECORDS: int = 100_000 @@ -276,8 +278,7 @@ def _is_safe_relative_sql_path(path: Path) -> bool: def _payload_digest(payload: object) -> str: - encoded: str = json.dumps(payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True) - return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + return hashlib.sha256(orjson.dumps(payload, option=orjson.OPT_SORT_KEYS)).hexdigest() def _package_version(package_name: str) -> str: diff --git a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py index b1dddd963c..35cf8435e8 100644 --- a/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py +++ b/src/sqlbuild/cli/commands/_helpers/compile/target_writer.py @@ -499,20 +499,20 @@ def _remove_stale_compiled_files(*, target_dir: Path, managed_paths: set[Path]) compiled_dir: Path = target_dir / _COMPILED_DIR if not compiled_dir.is_dir(): return + managed_names: set[str] = {os.fspath(path) for path in managed_paths} for root, directories, filenames in os.walk(compiled_dir, topdown=False): - root_path: Path = Path(root) for filename in filenames: - path: Path = root_path / filename - if path not in managed_paths: - path.unlink() + path: str = os.path.join(root, filename) + if path not in managed_names: + os.unlink(path) for name in directories: - _remove_empty_directory(root_path / name) + _remove_empty_directory(os.path.join(root, name)) _remove_empty_directory(compiled_dir) -def _remove_empty_directory(directory: Path) -> None: +def _remove_empty_directory(directory: str | Path) -> None: try: - directory.rmdir() + os.rmdir(directory) except OSError: pass diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py b/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py index 2a65186b3b..ddfaf5e0f5 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/cache.py @@ -10,6 +10,7 @@ import sqlite3 import time from collections import defaultdict, deque +from collections.abc import Sequence from importlib.metadata import PackageNotFoundError, version from pathlib import Path from string import hexdigits @@ -26,13 +27,14 @@ CompactAnalysisCacheModel, CompactAnalysisCachePlan, CompactBatchPreparation, + CompactLineageFacts, CompiledLineageColumnFact, CompiledLineageSourceFact, CompileSqlReference, InferredColumn, PolyglotAnalysisResult, ) -from sqlbuild.compiler.compile.types import CompactBatchResponseCallback +from sqlbuild.compiler.compile.types import CompactBatchResponseCallback, CompiledResourceType from sqlbuild.compiler.lineage.types import ( ColumnLineageConfidence, ColumnTransformKind, @@ -44,7 +46,19 @@ from sqlbuild.compiler.sql_analysis.models import SqlBindingDiagnostic _ANALYSIS_CACHE_VERSION: int = 10 -_ANALYSIS_ALGORITHM_FINGERPRINT: str = "model-sql-analysis-v12-bound-borrowed-output-graph" +_ANALYSIS_ALGORITHM_FINGERPRINT: str = "model-sql-analysis-v13-bound-lineage-type-recovery" +_LINEAGE_COLUMN_VALUE_COUNT: int = 4 +_LINEAGE_SOURCE_VALUE_COUNT: int = 3 +_COMPACT_TRANSFORM_CODES: dict[str, int] = { + CompactLineageFacts.transform_kind(code).value: code for code in range(6) +} +_COMPACT_CONFIDENCE_CODES: dict[str, int] = { + CompactLineageFacts.confidence(code).value: code for code in range(3) +} +_RESOURCE_TYPE_VALUES: frozenset[str] = frozenset(item.value for item in CompiledResourceType) +_NULLABILITY_BY_VALUE: dict[str, InferredNullability] = { + item.value: item for item in InferredNullability +} _MAX_CACHE_ENTRY_BYTES: int = 10_000_000 _SHA256_HEX_LENGTH: int = 64 _CACHE_ENTRY_SEPARATOR: str = "\n" @@ -162,7 +176,7 @@ def model_analysis_cache_key( } if recover_cte_facts: payload["recover_cte_facts"] = True - return _payload_digest(payload) + return _json_value_digest(payload) def compact_analysis_batch_cache_key( @@ -185,7 +199,7 @@ def compact_analysis_model_reuse_key( ) -> str: """Return one model key that changes with every transitive model dependency.""" - return _payload_digest( + return _json_value_digest( { "cache_key": cache_key, "upstream": upstream_reuse_keys, @@ -488,6 +502,7 @@ def read_model_analyses( analyses: dict[str, PolyglotAnalysisResult] = {} signatures: dict[str, str] = {} output_signatures_by_key: dict[str, str] = {} + sources: _LineageSourceInterner = _LineageSourceInterner() try: connection_uri: str = f"file:{database_path}?mode=ro" with sqlite3.connect( @@ -548,6 +563,7 @@ def read_model_analyses( _analysis_from_contents( contents=contents, expected_cache_key=cache_key, + sources=sources, ) ) if cached_result is not None: @@ -644,7 +660,7 @@ def _expected_dependencies( def model_analysis_output_signature(analysis: PolyglotAnalysisResult) -> str: """Return the exported column signature relevant to downstream analysis.""" - return _payload_digest( + return _json_value_digest( { "columns": ( None @@ -664,7 +680,10 @@ def model_analysis_output_signature(analysis: PolyglotAnalysisResult) -> str: def _analysis_from_contents( - *, contents: object, expected_cache_key: str + *, + contents: object, + expected_cache_key: str, + sources: _LineageSourceInterner | None = None, ) -> tuple[PolyglotAnalysisResult, str] | None: if not isinstance(contents, str): return None @@ -680,9 +699,11 @@ def _analysis_from_contents( ), ): return None - payload: object = json.loads(serialized_payload) - return _analysis_from_payload(payload=payload, expected_cache_key=expected_cache_key) - except (ValueError, TypeError, KeyError, RecursionError, json.JSONDecodeError): + payload: object = orjson.loads(serialized_payload) + return _analysis_from_payload( + payload=payload, expected_cache_key=expected_cache_key, sources=sources + ) + except (ValueError, TypeError, KeyError, RecursionError, orjson.JSONDecodeError): return None @@ -813,7 +834,10 @@ def _lineage_column_payload(column: CompiledLineageColumnFact) -> list[object]: def _analysis_from_payload( - *, payload: object, expected_cache_key: str + *, + payload: object, + expected_cache_key: str, + sources: _LineageSourceInterner | None = None, ) -> tuple[PolyglotAnalysisResult, str]: if not isinstance(payload, dict): raise AnalysisCacheEntryError("analysis cache entry must be an object") @@ -839,9 +863,16 @@ def _analysis_from_payload( if columns_payload is None else tuple(_column_from_payload(column) for column in _value_lists(columns_payload)) ) - lineage_columns: tuple[CompiledLineageColumnFact, ...] = tuple( - _lineage_column_from_payload(column) for column in _value_lists(values["l"]) + lineage_payload: list[list[object]] = _value_lists(values["l"]) + lineage_columns: Sequence[CompiledLineageColumnFact] | None = _compact_lineage_from_payload( + payload=lineage_payload ) + if lineage_columns is None: + interner: _LineageSourceInterner = _LineageSourceInterner() if sources is None else sources + lineage_columns = tuple( + _lineage_column_from_payload(payload=column, sources=interner) + for column in lineage_payload + ) has_star: object = values["h"] if not isinstance(has_star, bool): raise AnalysisCacheEntryError("analysis cache has_star must be a boolean") @@ -904,11 +935,93 @@ def _column_from_payload(payload: list[object]) -> InferredColumn: return InferredColumn( name=name, type=column_type, - nullability=InferredNullability(nullability), + nullability=_NULLABILITY_BY_VALUE.get(nullability) or InferredNullability(nullability), ) -def _lineage_column_from_payload(payload: list[object]) -> CompiledLineageColumnFact: +def _compact_lineage_from_payload(*, payload: list[list[object]]) -> CompactLineageFacts | None: + """Validate cached lineage into lazy indexed rows; None means use the eager decoder.""" + + pool: dict[str, int] = {} + rows: list[tuple[int, int, int, tuple[tuple[int, int, int], ...]]] = [] + column: list[object] + for column in payload: + if len(column) != _LINEAGE_COLUMN_VALUE_COUNT: + raise AnalysisCacheEntryError("analysis cache lineage column must contain four values") + output_column, transform_kind, confidence, upstream_columns = column + if not ( + type(output_column) is str and type(transform_kind) is str and type(confidence) is str + ): + raise AnalysisCacheEntryError("analysis cache lineage attributes must be strings") + transform_code: int | None = _COMPACT_TRANSFORM_CODES.get(transform_kind) + confidence_code: int | None = _COMPACT_CONFIDENCE_CODES.get(confidence) + if transform_code is None or confidence_code is None: + return None + if type(upstream_columns) is not list: + raise AnalysisCacheEntryError("analysis cache collection must contain arrays") + source_rows: list[tuple[int, int, int]] = [] + source: object + for source in upstream_columns: + if not ( + type(source) is list + and len(source) == _LINEAGE_SOURCE_VALUE_COUNT + and type(source[0]) is str + and type(source[1]) is str + and type(source[2]) is str + ): + raise AnalysisCacheEntryError( + "analysis cache lineage source must contain three strings" + ) + if source[0] not in _RESOURCE_TYPE_VALUES: + raise AnalysisCacheEntryError("analysis cache lineage resource type is invalid") + source_rows.append( + ( + pool.setdefault(source[0], len(pool)), + pool.setdefault(source[1], len(pool)), + pool.setdefault(source[2], len(pool)), + ) + ) + rows.append( + ( + pool.setdefault(output_column, len(pool)), + transform_code, + confidence_code, + tuple(source_rows), + ) + ) + return CompactLineageFacts(string_pool=tuple(pool), rows=tuple(rows)) + + +class _LineageSourceInterner: + """Share identical immutable lineage source facts across one cache read.""" + + def __init__(self) -> None: + self._facts: dict[tuple[str, str, str], CompiledLineageSourceFact] = {} + + def decode(self, *, payload: list[object]) -> CompiledLineageSourceFact: + source_value_count: int = 3 + if len(payload) != source_value_count or not all( + isinstance(value, str) for value in payload + ): + raise AnalysisCacheEntryError( + "analysis cache lineage source must contain three strings" + ) + resource_type, resource_name, column_name = cast(list[str], payload) + key: tuple[str, str, str] = (resource_type, resource_name, column_name) + fact: CompiledLineageSourceFact | None = self._facts.get(key) + if fact is None: + fact = CompiledLineageSourceFact( + resource_type=resource_type, + resource_name=resource_name, + column_name=column_name, + ) + self._facts[key] = fact + return fact + + +def _lineage_column_from_payload( + *, payload: list[object], sources: _LineageSourceInterner +) -> CompiledLineageColumnFact: lineage_value_count: int = 4 if len(payload) != lineage_value_count: raise AnalysisCacheEntryError("analysis cache lineage column must contain four values") @@ -920,23 +1033,11 @@ def _lineage_column_from_payload(payload: list[object]) -> CompiledLineageColumn transform_kind=ColumnTransformKind(cast(str, transform_kind)), confidence=ColumnLineageConfidence(cast(str, confidence)), upstream_columns=tuple( - _lineage_source_from_payload(source) for source in _value_lists(upstream_columns) + sources.decode(payload=source) for source in _value_lists(upstream_columns) ), ) -def _lineage_source_from_payload(payload: list[object]) -> CompiledLineageSourceFact: - source_value_count: int = 3 - if len(payload) != source_value_count or not all(isinstance(value, str) for value in payload): - raise AnalysisCacheEntryError("analysis cache lineage source must contain three strings") - resource_type, resource_name, column_name = cast(list[str], payload) - return CompiledLineageSourceFact( - resource_type=resource_type, - resource_name=resource_name, - column_name=column_name, - ) - - def _cache_database_path(*, context: AnalysisCacheContext) -> Path: return context.root / f"v{_ANALYSIS_CACHE_VERSION}" / _CACHE_DATABASE_NAME @@ -971,6 +1072,12 @@ def _payload_digest(payload: object) -> str: return hashlib.sha256(encoded).hexdigest() +def _json_value_digest(payload: dict[str, object]) -> str: + """Digest string-keyed JSON scalars exactly as _payload_digest does, via orjson.""" + + return hashlib.sha256(orjson.dumps(payload, option=orjson.OPT_SORT_KEYS)).hexdigest() + + def _package_version(package: str) -> str: try: return version(package) diff --git a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py index c30951ec00..569ca031a9 100644 --- a/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py +++ b/src/sqlbuild/compiler/compile/_helpers/analysis/compact.py @@ -4,6 +4,7 @@ import logging import re +from collections.abc import Sequence from dataclasses import replace from typing import Any, cast @@ -30,6 +31,7 @@ ) from sqlbuild.compiler.compile.constants import ( COMPACT_ANALYSIS_FACT_LENGTH, + COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH, COMPACT_ANALYSIS_RESPONSE_LENGTH, COMPACT_ANALYSIS_SOURCE_LENGTH, RESOLVED_SOURCE_CONFIDENCE, @@ -353,7 +355,7 @@ def analyze_columns_and_lineage_with_polyglot( sqlbuild_error=str(error), ) return PolyglotAnalysisResult(analysis_succeeded=False) - columns, lineage_columns, has_star = _analyze_columns_and_lineage_from_polyglot_ast( + columns, legacy_lineage_columns, has_star = _analyze_columns_and_lineage_from_polyglot_ast( parsed=parsed, references=references, column_nullability_by_table=column_nullability_by_table or {}, @@ -361,6 +363,25 @@ def analyze_columns_and_lineage_with_polyglot( inference_profile=profile, recover_cte_facts=recover_cte_facts, ) + lineage_columns: Sequence[CompiledLineageColumnFact] = legacy_lineage_columns + if precomputed is not None and precomputed.compact_rows is not None: + lineage_columns = _compact_projected_analysis_result( + request=ProjectedAnalysisRequest( + analysis=None, + compact_rows=precomputed.compact_rows, + compact_fact_rows=precomputed.compact_fact_rows, + string_pool=precomputed.string_pool, + caches=CompactProjectionCaches( + columns=precomputed.compact_column_cache, + facts=precomputed.compact_fact_cache, + decoded_facts=precomputed.compact_decoded_fact_cache, + ), + template_index=precomputed.compact_template_index, + resource_name_indexes=precomputed.resource_name_indexes, + binding_diagnostics=(), + binding_validated=False, + ) + ).lineage_columns return PolyglotAnalysisResult( analysis_succeeded=True, columns=columns, @@ -742,9 +763,14 @@ def _project_compact_analysis_batch( ) ) continue + legacy_type_recovery: bool = ( + isinstance(template, list) + and len(template) == COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH + and template[2] == _NATIVE_LEGACY_FALLBACK + ) if not ( isinstance(template, list) - and len(template) == COMPACT_ANALYSIS_RESPONSE_LENGTH + and (len(template) == COMPACT_ANALYSIS_RESPONSE_LENGTH or legacy_type_recovery) and isinstance(template[0], list) and isinstance(template[1], bool) ): @@ -776,8 +802,8 @@ def _project_compact_analysis_batch( results.append( NativeCompactAnalysis( cleaned_sql=cleaned_sql, - analysis={"hasStar": template[1]}, - projected=True, + analysis=None if legacy_type_recovery else {"hasStar": template[1]}, + projected=not legacy_type_recovery, compact_rows=cast(list[object], template[0]), compact_fact_rows=cast(list[object], raw_facts), string_pool=string_pool, diff --git a/src/sqlbuild/compiler/compile/_helpers/render/macros.py b/src/sqlbuild/compiler/compile/_helpers/render/macros.py index 57a163c67f..3a531d37fc 100644 --- a/src/sqlbuild/compiler/compile/_helpers/render/macros.py +++ b/src/sqlbuild/compiler/compile/_helpers/render/macros.py @@ -3,7 +3,6 @@ from __future__ import annotations import ast -import copy import hashlib import heapq import inspect @@ -1046,16 +1045,27 @@ def _load_macro_modules_locked( def _rewrite_absolute_project_imports( *, tree: ast.Module, module_names: frozenset[str], root: str ) -> ast.Module: - class Rewriter(ast.NodeTransformer): - def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.ImportFrom: # noqa: N802 - if node.level == 0 and node.module in module_names: - return ast.copy_location( - ast.ImportFrom(module=f"{root}.{node.module}", names=node.names, level=0), - node, - ) - return node + """Retarget module-level project imports; analysis already rejects nested ones.""" - return ast.fix_missing_locations(Rewriter().visit(copy.deepcopy(tree))) + body: list[ast.stmt] = [] + statement: ast.stmt + for statement in tree.body: + if ( + isinstance(statement, ast.ImportFrom) + and statement.level == 0 + and statement.module in module_names + ): + body.append( + ast.copy_location( + ast.ImportFrom( + module=f"{root}.{statement.module}", names=statement.names, level=0 + ), + statement, + ) + ) + else: + body.append(statement) + return ast.Module(body=body, type_ignores=tree.type_ignores) def _matches_macro_module(*, imported_name: str, module_names: frozenset[str]) -> bool: diff --git a/src/sqlbuild/compiler/compile/constants.py b/src/sqlbuild/compiler/compile/constants.py index 2bf88a5daa..494ef088f9 100644 --- a/src/sqlbuild/compiler/compile/constants.py +++ b/src/sqlbuild/compiler/compile/constants.py @@ -134,6 +134,7 @@ COMPILE_CACHE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_CACHE" COMPILE_CACHE_DISABLE_VALUE: str = "1" COMPACT_ANALYSIS_RESPONSE_LENGTH: int = 2 +COMPACT_ANALYSIS_LEGACY_RESPONSE_LENGTH: int = 3 COMPACT_ANALYSIS_FACT_LENGTH: int = 6 COMPACT_ANALYSIS_SOURCE_LENGTH: int = 3 MODEL_SCHEMA_CONFIG_KEY: str = "schema" diff --git a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py index fcf057f385..7a954a3968 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py +++ b/src/sqlbuild/compiler/discovery/_helpers/filesystem/core.py @@ -225,7 +225,9 @@ def _discover_declaration_file_facts( directory_kind, directory_scope_kind = DECLARATION_DIRECTORY_FACTS[directory.name] if declaration_kind is not None and directory_kind is not declaration_kind: continue - relative_directory: Path = directory.relative_to(project_dir) + relative_directory: Path = _project_relative_path( + path=directory, project_dir=project_dir + ) descendants: tuple[str, ...] = relative_directory.parts[len(root_components) :] if any(part in _SCOPED_DECLARATION_DIRECTORIES for part in descendants[:-1]): raise DeclarationParseError( @@ -276,7 +278,8 @@ def _validate_declaration_groups(*, project_dir: Path) -> None: ) if unsupported: rendered: str = ", ".join( - path.relative_to(project_dir).as_posix() for path in unsupported + _project_relative_path(path=path, project_dir=project_dir).as_posix() + for path in unsupported ) raise DeclarationParseError( f"Declaration group {group.relative_to(project_dir).as_posix()}/ contains " @@ -298,7 +301,9 @@ def _declaration_files_under_root( for path in declaration_root.rglob("*") if path.is_dir() and path.name in _SCOPED_DECLARATION_DIRECTORIES ): - relative_nested_root: str = nested_root.relative_to(project_dir).as_posix() + relative_nested_root: str = _project_relative_path( + path=nested_root, project_dir=project_dir + ).as_posix() raise DeclarationParseError( f"Declaration root {relative_nested_root}/ is nested inside another declaration tree" ) @@ -312,19 +317,23 @@ def _declaration_files_under_root( results.append( _DeclarationFileFacts( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), declaration_kind=declaration_kind, scope_kind=scope_kind, ownership_root=ownership_root, owning_path=owning_path, - declaration_root=declaration_root.relative_to(project_dir), + declaration_root=_project_relative_path( + path=declaration_root, project_dir=project_dir + ), ) ) return results def _is_in_scoped_declaration_tree(*, file_path: Path, project_dir: Path) -> bool: - relative_parts: tuple[str, ...] = file_path.relative_to(project_dir).parts + relative_parts: tuple[str, ...] = _project_relative_path( + path=file_path, project_dir=project_dir + ).parts root_components: tuple[str, ...] for root_components in CANONICAL_AUTHORED_ROOTS: if relative_parts[: len(root_components)] != root_components: @@ -414,7 +423,7 @@ def _discover_model_file( extract_output_column_locations: bool, ) -> DiscoveredSqlModelFile: header_values, query_sql = parse_model_sql(contents=contents, file_path=file_path) - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) return DiscoveredSqlModelFile( file_path=file_path, relative_path=relative_path, @@ -544,7 +553,7 @@ def discover_model_schema_files(*, project_dir: Path) -> tuple[DiscoveredModelSc file_path: Path for file_path in sorted(schema_root.rglob("*.sql")): contents: str = file_path.read_text(encoding="utf-8") - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) discovered_files.append( DiscoveredModelSchemaFile( file_path=file_path, @@ -587,7 +596,7 @@ def discover_sql_function_files( discovered_function_files.append( DiscoveredSqlFunctionFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_values=header_values, body_sql=body_sql, @@ -618,7 +627,7 @@ def discover_python_function_files( discovered_function_files.append( DiscoveredPythonFunctionFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), contents=contents, header_values=header_values, entry_point=entry_point, @@ -661,7 +670,7 @@ def discover_schema_files(*, project_dir: Path) -> tuple[DiscoveredSchemaFile, . discovered_schema_files.append( DiscoveredSchemaFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), contents=contents, model_entries=model_entries, seed_entries=seed_entries, @@ -698,7 +707,7 @@ def discover_source_files( discovered_source_files.append( DiscoveredSourceFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), contents=contents, source_entries=source_entries, ) @@ -716,7 +725,7 @@ def discover_seed_files(*, project_dir: Path) -> tuple[DiscoveredSeedFile, ...]: return tuple( DiscoveredSeedFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), ) for file_path in sorted(seeds_root.rglob("*")) if file_path.is_file() and file_path.suffix == SEED_FILE_SUFFIX @@ -767,7 +776,7 @@ def discover_scenario_files( scenario: DiscoveredSqlScenarioFile = parse_sql_scenario_file( contents=contents, file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), ) except (OSError, UnicodeError, ValueError, SyntaxError) as error: if on_fault is None: @@ -797,7 +806,7 @@ def discover_audit_files( discovered_audit_files.append( DiscoveredAuditFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), contents=contents, blocks=parse_sql_audit_file(contents=contents, file_path=file_path), ) @@ -835,7 +844,7 @@ def discover_macro_files( def _discovery_fault(*, project_dir: Path, path: Path, error: Exception) -> DiscoveryFileFault: try: - relative_path: Path | None = path.relative_to(project_dir) + relative_path: Path | None = _project_relative_path(path=path, project_dir=project_dir) except ValueError: relative_path = None message: str = str(error).replace(str(project_dir), ".") @@ -864,7 +873,7 @@ def discover_materialization_files( discovered_files.append( DiscoveredMaterializationFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=file_path.stem, provider_usages=( _provider_usages(function=materialize_fn, provider_by_name=provider_by_name) @@ -972,7 +981,7 @@ def discover_hook_functions( discovered_hooks.append( DiscoveredHookFunction( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=hook_definition.name, function=value, description=hook_definition.description, @@ -1002,7 +1011,7 @@ def discover_sql_hook_files( ): continue try: - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) contents: str = file_path.read_text(encoding="utf-8") discovered_hooks.append( parse_sql_hook_file( @@ -1067,7 +1076,7 @@ def discover_provider_classes(*, project_dir: Path) -> tuple[DiscoveredProvider, discovered_providers.append( DiscoveredProvider( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=provider_name, provider_class=provider_class, settings=_provider_instance( @@ -1156,7 +1165,7 @@ def discover_event_exporter_declarations( discovered.append( DiscoveredEventExporterDeclaration( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=definition.name, function=value, event_kinds=definition.event_kinds, @@ -1205,7 +1214,7 @@ def _validate_event_exporter_declaration_signature( file_path: Path, project_dir: Path, ) -> None: - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) if inspect.iscoroutinefunction(function): raise EventExporterDiscoveryError( f"Event exporter '{exporter_name}' in {relative_path} must be synchronous" @@ -1261,7 +1270,7 @@ def _bind_event_exporter_provider_usages( project_dir: Path, provider_by_name: dict[str, DiscoveredProvider], ) -> tuple[DiscoveredProviderUsage, ...]: - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) parameters: tuple[inspect.Parameter, ...] = tuple( inspect.signature(function).parameters.values() ) @@ -1296,7 +1305,7 @@ def _public_python_files(*, root: Path) -> tuple[Path, ...]: def _provider_name(*, provider_class: type[Provider], file_path: Path, project_dir: Path) -> str: - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) explicit_name: str | None = provider_class.provider_name if explicit_name is not None: validate_resource_identity( @@ -1325,7 +1334,7 @@ def _provider_instance( try: return provider_class() except ValidationError as error: - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) raise ProviderDiscoveryError( f"Provider '{provider_name}' in {relative_path} has invalid settings:\n" f"{_format_provider_validation_error(error)}" @@ -1497,7 +1506,7 @@ def _append_module_audit_factories( definition: AuditFactoryDefinition | None = read_audit_factory_definition(value) if definition is None: continue - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) if node_folder != PYTHON_FACTORY_FOLDER: raise PythonNodeDiscoveryError( f"Audit factory '{definition.name}' in {relative_path} must live under factories/" @@ -1552,7 +1561,7 @@ def _call_audit_factory( file_path: Path, project_dir: Path, ) -> tuple[AuditCase, ...]: - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) if inspect.signature(factory).parameters: raise PythonNodeDiscoveryError( f"Audit factory '{definition.name}' in {relative_path} must not require arguments" @@ -1603,7 +1612,7 @@ def _append_python_node_function( bucket.add_loader( DiscoveredLoaderFunction( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=loader_definition.name, function=function, depends_on=loader_definition.depends_on, @@ -1633,7 +1642,7 @@ def _append_python_node_function( bucket.add_task( DiscoveredTaskFunction( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=task_definition.name, function=function, depends_on=task_definition.depends_on, @@ -1662,7 +1671,7 @@ def _append_python_node_function( bucket.add_asset( DiscoveredAssetFunction( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=asset_definition.name, function=function, depends_on=asset_definition.depends_on, @@ -1693,7 +1702,7 @@ def _append_python_node_function( bucket.add_check( DiscoveredCheckFunction( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), name=check_definition.name, function=function, depends_on=check_definition.depends_on, @@ -1723,7 +1732,7 @@ def _validate_python_node_kind( ) -> None: if expected_kind is None or actual_kind == expected_kind: return - relative_path: Path = file_path.relative_to(project_dir) + relative_path: Path = _project_relative_path(path=file_path, project_dir=project_dir) folder: str = relative_path.parts[0] if factory_definition is None: article: str = "an" if actual_kind[0] in PYTHON_NODE_KIND_VOWELS else "a" @@ -1796,7 +1805,7 @@ def _normalize_factory_result( def _load_loader_module(*, file_path: Path, project_dir: Path) -> ModuleType: module_name: str = "sqlbuild_project_loader_" + "_".join( - file_path.relative_to(project_dir).with_suffix("").parts + _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ) spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) if spec is None or spec.loader is None: @@ -1818,7 +1827,7 @@ def _load_loader_module(*, file_path: Path, project_dir: Path) -> ModuleType: def _load_python_node_module(*, file_path: Path, project_dir: Path, node_folder: str) -> ModuleType: module_name: str = "sqlbuild_project_python_node_" + "_".join( - file_path.relative_to(project_dir).with_suffix("").parts + _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts ) spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) if spec is None or spec.loader is None: @@ -1839,7 +1848,9 @@ def _load_python_node_module(*, file_path: Path, project_dir: Path, node_folder: def _load_provider_module(*, file_path: Path, project_dir: Path) -> ModuleType: - module_name: str = ".".join(file_path.relative_to(project_dir).with_suffix("").parts) + module_name: str = ".".join( + _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + ) _evict_stale_project_package_modules( root_module=module_name.split(".", maxsplit=1)[0], project_dir=project_dir, @@ -1869,7 +1880,9 @@ def _load_provider_module(*, file_path: Path, project_dir: Path) -> ModuleType: def _load_sink_module(*, file_path: Path, project_dir: Path) -> ModuleType: - module_name: str = ".".join(file_path.relative_to(project_dir).with_suffix("").parts) + module_name: str = ".".join( + _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + ) _evict_stale_project_package_modules( root_module=module_name.split(".", maxsplit=1)[0], project_dir=project_dir, @@ -1923,7 +1936,9 @@ def _evict_stale_project_package_modules(*, root_module: str, project_dir: Path) def _load_materialization_module(*, file_path: Path, project_dir: Path) -> ModuleType: - module_name: str = ".".join(file_path.relative_to(project_dir).with_suffix("").parts) + module_name: str = ".".join( + _project_relative_path(path=file_path, project_dir=project_dir).with_suffix("").parts + ) spec: ModuleSpec | None = importlib.util.spec_from_file_location(module_name, file_path) if spec is None or spec.loader is None: raise PythonNodeDiscoveryError(f"Could not load materialization file {file_path}") @@ -1951,10 +1966,23 @@ def discover_adapter_file(*, project_dir: Path) -> DiscoveredAdapterFile | None: return DiscoveredAdapterFile( file_path=file_path, - relative_path=file_path.relative_to(project_dir), + relative_path=_project_relative_path(path=file_path, project_dir=project_dir), ) +def _project_relative_path(*, path: Path, project_dir: Path) -> Path: + """Return path.relative_to(project_dir) without pathlib's per-parent comparison scan.""" + + root_parts: tuple[str, ...] = project_dir.parts + path_parts: tuple[str, ...] = path.parts + if ( + path.is_absolute() is not project_dir.is_absolute() + or path_parts[: len(root_parts)] != root_parts + ): + return path.relative_to(project_dir) + return path.with_segments(*path_parts[len(root_parts) :]) + + def _is_relative_to(*, path: Path, parent: Path) -> bool: try: path.relative_to(parent) diff --git a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py index 0e7d8b249d..92063a11a1 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py +++ b/src/sqlbuild/compiler/discovery/_helpers/validation/discovery.py @@ -804,15 +804,6 @@ def _validate_path_defaults_match_models( str(model_file.relative_path).replace("\\", "/").removeprefix("models/") for model_file in model_files ) - model_folders: tuple[str, ...] = tuple( - sorted( - { - str(Path(model_path).parent) - for model_path in model_paths - if str(Path(model_path).parent) != CURRENT_DIRECTORY_PATH - } - ) - ) matched_keys: set[str] = set() model_path: str for model_path in model_paths: @@ -826,6 +817,15 @@ def _validate_path_defaults_match_models( for path_key in path_defaults: if path_key in matched_keys: continue + model_folders: tuple[str, ...] = tuple( + sorted( + { + str(Path(model_path).parent) + for model_path in model_paths + if str(Path(model_path).parent) != CURRENT_DIRECTORY_PATH + } + ) + ) known_folders: str = ", ".join(model_folders[:5]) if model_folders else "" raise DiscoveryConflictError( f"path_defaults['{path_key}'] does not match any model paths. Known model folders " diff --git a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py index ab9163129e..45447c5501 100644 --- a/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py +++ b/src/sqlbuild/compiler/discovery/_helpers/yml/schema.py @@ -63,7 +63,8 @@ def parse_schema_yml( def _load_schema_payload(*, contents: str, file_path: Path) -> dict[str, object]: try: - payload: object = yaml.safe_load(contents) + loader: type[yaml.SafeLoader] = getattr(yaml, "CSafeLoader", yaml.SafeLoader) + payload: object = yaml.load(contents, Loader=loader) except YAMLError as error: raise SchemaParseError(f"{file_path} contains invalid YAML: {error}") from error if payload is None: diff --git a/src/sqlbuild/rule_engine/_helpers/run/rules.py b/src/sqlbuild/rule_engine/_helpers/run/rules.py index 7e419e4806..5b8cdece29 100644 --- a/src/sqlbuild/rule_engine/_helpers/run/rules.py +++ b/src/sqlbuild/rule_engine/_helpers/run/rules.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Any +from sqlbuild.compiler.compile.constants import MACRO_TOKEN from sqlbuild.compiler.compile.models import ( CompiledModel, CompiledObjectKey, @@ -23,6 +24,7 @@ from sqlbuild.compiler.compile.types import CompiledResourceType from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.pipeline.models import ProjectGraph +from sqlbuild.lint.constants import TEMPLATE_INTERPOLATION_START from sqlbuild.lint.main.build_expansion_context import build_expansion_context from sqlbuild.lint.main.collect_project_files import collect_project_files from sqlbuild.lint.main.run_lint import run_lint @@ -337,13 +339,36 @@ def _run_sql_rules( else: findings.extend(cached) selected_paths: set[Path] = {project_dir / path for path in misses} + file_identities: dict[str, str] = {} if selected_model_paths is None: model_paths: frozenset[Path] = frozenset(project_dir / path for path in models_by_path) - selected_paths.update( - path - for path in collect_project_files(project_dir=project_dir, selected_paths=None) - if path not in model_paths - ) + file_path: Path + contents: str + for file_path, contents in collect_project_files( + project_dir=project_dir, selected_paths=None + ).items(): + if file_path in model_paths: + continue + relative_path: str = file_path.relative_to(project_dir).as_posix() + file_identity: str | None = _sql_file_rule_identity( + file_path=file_path, + relative_path=relative_path, + contents=contents, + codes=codes, + dialect=dialect, + sql_expansions=project.sql_expansions, + ) + cached_file: tuple[Finding, ...] | None = ( + None + if file_identity is None + else _cached_sql_findings(entry=bucket.get(relative_path), identity=file_identity) + ) + if cached_file is None: + selected_paths.add(file_path) + if file_identity is not None: + file_identities[relative_path] = file_identity + else: + findings.extend(cached_file) result: LintRunResult | None = ( _run_prepared_lint( project_dir=project_dir, @@ -371,15 +396,19 @@ def _run_sql_rules( if violation.code in selected_codes ) findings.extend(evaluated) - if cache_enabled and misses: - by_path: dict[str, list[Finding]] = {path: [] for path in misses} + written_identities: dict[str, str] = { + **{path: identities[path] for path in misses}, + **file_identities, + } + if cache_enabled and written_identities: + by_path: dict[str, list[Finding]] = {path: [] for path in written_identities} for finding in evaluated: path: str = finding.path.as_posix() if path in by_path: by_path[path].append(finding) - for path in misses: + for path, identity in written_identities.items(): bucket[path] = { - "identity": identities[path], + "identity": identity, "findings": [_finding_cache_payload(item) for item in by_path[path]], } _write_sql_rule_cache(project_dir=project_dir, bucket=bucket) @@ -424,6 +453,38 @@ def _sql_rule_identity(*, model: CompiledModel, codes: tuple[str, ...], dialect: return digest.hexdigest() +def _sql_file_rule_identity( + *, + file_path: Path, + relative_path: str, + contents: str, + codes: tuple[str, ...], + dialect: str, + sql_expansions: dict[Path, CompiledSqlExpansion], +) -> str | None: + """Identify non-model SQL whose lint depends only on its text; expansions return None.""" + + if ( + MACRO_TOKEN in contents + or TEMPLATE_INTERPOLATION_START in contents + or file_path in sql_expansions + ): + return None + digest: Any = hashlib.sha256() + for value in ( + _SQL_RULE_CACHE_VERSION, + "file", + _SQLBUILD_VERSION, + dialect, + "\0".join(codes), + relative_path, + contents, + ): + digest.update(value.encode()) + digest.update(b"\0") + return digest.hexdigest() + + def _sql_rule_cache_path(project_dir: Path) -> Path: return project_dir / "target" / "rules-cache" / "bulk" / "sql.json" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index 055e8d7753..f84c13deae 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -137,6 +137,7 @@ class FreshProcessCompileCachePerformanceGuardTestCase: expected_cold_max_wall_seconds: float expected_warm_max_wall_seconds: float expected_edit_max_wall_seconds: float + expected_max_warm_to_cold_ratio: float expected_max_rss_bytes: int expected_max_cache_bytes: int expected_cold_fingerprint: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index e803bb6d5c..d0423828f0 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -25,6 +25,7 @@ _LOGGER: logging.Logger = logging.getLogger(__name__) _GIB: int = 1024 * 1024 * 1024 _MIB: int = 1024 * 1024 +_MAX_WARM_TO_COLD_RATIO: float = 0.67 @pytest.mark.performance @@ -44,6 +45,7 @@ expected_cold_max_wall_seconds=18.0, expected_warm_max_wall_seconds=10.0, expected_edit_max_wall_seconds=11.0, + expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( @@ -71,6 +73,7 @@ expected_cold_max_wall_seconds=28.0, expected_warm_max_wall_seconds=15.0, expected_edit_max_wall_seconds=16.0, + expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( @@ -98,6 +101,7 @@ expected_cold_max_wall_seconds=50.0, expected_warm_max_wall_seconds=28.0, expected_edit_max_wall_seconds=30.0, + expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_rss_bytes=2 * _GIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( @@ -166,6 +170,10 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in result.after_project_config_edit, ): assert measurement.elapsed_seconds < test_case.expected_warm_max_wall_seconds + assert ( + measurement.elapsed_seconds / result.cold.elapsed_seconds + <= test_case.expected_max_warm_to_cold_ratio + ) for measurement in (result.leaf_edit, result.macro_edit): assert measurement.elapsed_seconds < test_case.expected_edit_max_wall_seconds assert result.project_config_edit.elapsed_seconds < test_case.expected_cold_max_wall_seconds diff --git a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py index 3edfb27989..2d66509057 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/helpers.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/helpers.py @@ -1,9 +1,14 @@ from __future__ import annotations +import json from pathlib import Path from tempfile import TemporaryDirectory +from typing import cast import duckdb +import pytest + +from sqlbuild.cli.commands.main.entrypoint.entry import main def unavailable_artifact_directory(*, prefix: str) -> TemporaryDirectory[str]: @@ -89,3 +94,12 @@ def add_second_contract_source(*, project_dir: Path, database: Path) -> None: with duckdb.connect(str(database)) as connection: connection.execute("ALTER TABLE raw.orders ADD COLUMN generated_at TIMESTAMP") connection.execute("CREATE TABLE raw.customers(customer_id BIGINT, email VARCHAR)") + + +def compile_finding_keys(*, project_dir: Path, capsys: pytest.CaptureFixture[str]) -> set[str]: + """Compile through the CLI and return diagnostics as path:code keys.""" + + _ = main(["--project-dir", str(project_dir), "compile", "--json"]) + payload: dict[str, object] = json.loads(capsys.readouterr().out) + diagnostics: list[dict[str, object]] = cast(list[dict[str, object]], payload["diagnostics"]) + return {f"{item['path']}:{item['code']}" for item in diagnostics} diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py index 7d9c69d8e4..71020110f0 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_rules.py @@ -17,6 +17,7 @@ RulesIntegrationTestCase, TypedContractRuleIntegrationTestCase, ) +from tests.integration.src.sqlbuild.cli.commands.main.helpers import compile_finding_keys @pytest.mark.parametrize( @@ -555,6 +556,57 @@ def test_given_selected_sql_rule_when_compiling_then_authored_diagnostic_blocks_ assert (tmp_path / "target" / "manifest.json").is_file() +@pytest.mark.parametrize( + "test_case", + [RulesIntegrationTestCase("warm SQL rules track non-model file edits", 1, "SQBRSQL004")], + ids=lambda case: case.description, +) +def test_given_cached_sql_test_rules_when_test_or_macro_changes_then_findings_are_current( + test_case: RulesIntegrationTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text( + f'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["{test_case.expected_code}"]\n', + encoding="utf-8", + ) + model: Path = tmp_path / "models" / "orders.sql" + model.parent.mkdir() + model.write_text('MODEL (description "Orders");\nSELECT 1 AS order_id\n', encoding="utf-8") + tests_dir: Path = tmp_path / "tests" / "unit" + macro: Path = tests_dir / "_macros" / "row_limit.py" + macro.parent.mkdir(parents=True) + macro.write_text('def row_limit() -> str:\n return "LIMIT 1"\n', encoding="utf-8") + plain: Path = tests_dir / "test_plain_orders.sql" + plain_body: str = ( + "TEST ();\nWITH\n__ref__orders AS (SELECT 1 AS order_id),\n" + "__expected__orders AS (SELECT order_id FROM __ref__orders {limit})\n" + "SELECT 1\n" + ) + plain.write_text(plain_body.format(limit="LIMIT 1"), encoding="utf-8") + (tests_dir / "test_macro_orders.sql").write_text( + plain_body.format(limit="@row_limit()"), encoding="utf-8" + ) + plain_finding: str = f"tests/unit/test_plain_orders.sql:{test_case.expected_code}" + macro_finding: str = f"tests/unit/test_macro_orders.sql:{test_case.expected_code}" + + cold: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + warm: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + plain.write_text(plain_body.format(limit="ORDER BY order_id LIMIT 1"), encoding="utf-8") + after_test_edit: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + macro.write_text( + 'def row_limit() -> str:\n return "ORDER BY order_id LIMIT 1"\n', encoding="utf-8" + ) + after_macro_edit: set[str] = compile_finding_keys(project_dir=tmp_path, capsys=capsys) + + assert {plain_finding, macro_finding} <= cold + assert warm == cold + assert plain_finding not in after_test_edit + assert macro_finding in after_test_edit + assert macro_finding not in after_macro_edit + assert (tmp_path / "target" / "rules-cache" / "bulk" / "sql.json").is_file() + + @pytest.mark.parametrize( "test_case", [RulesIntegrationTestCase("custom rule is enforced and cached", 1, "XSQBRARCH001")], diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py index 5bd9a3b55b..88c03a9ad1 100644 --- a/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/pipeline/_test_types.py @@ -217,3 +217,17 @@ class SnowflakeTargetValidationIntegrationTestCase: class ExpectedCountTestCase: description: str expected_count: int + + +@dataclass(frozen=True) +class SetOperationTypeIntegrationTestCase: + description: str + query_sql: str + allowed_inferred_types: frozenset[str | None] + expected_rows: tuple[tuple[float], ...] + + +@dataclass(frozen=True) +class SetOperationLineageIntegrationTestCase: + description: str + expected_sources: frozenset[tuple[str, str]] diff --git a/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py new file mode 100644 index 0000000000..b2b49e39e4 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/pipeline/test_set_operation_types.py @@ -0,0 +1,129 @@ +"""Compile and execute widened set-operation outputs across dependency upgrades.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +import duckdb +import pytest +from duckdb import sqltypes + +from sqlbuild.adapters.duckdb.classes.duckdb_adapter import DuckDbAdapter +from sqlbuild.compiler.compile.models import ( + CompiledLineageColumnFact, + CompiledModel, + CompiledProject, +) +from sqlbuild.compiler.discovery.main.discover import discover_project_inputs +from sqlbuild.compiler.pipeline.main.project import compile_project +from tests.integration.src.sqlbuild.compiler.pipeline._test_types import ( + SetOperationLineageIntegrationTestCase, + SetOperationTypeIntegrationTestCase, +) + +_PROJECT_CONFIG: str = 'name = "orders"\nadapter = "duckdb"\n[rules]\nselect = []\n' +_WIDENED_TYPES: frozenset[str | None] = frozenset({None, "DOUBLE"}) + + +@pytest.mark.parametrize( + "test_case", + ( + SetOperationTypeIntegrationTestCase( + description="union all never narrows to the first branch", + query_sql=( + "SELECT CAST(1 AS INTEGER) AS amount UNION ALL SELECT CAST(2.5 AS DOUBLE) AS amount" + ), + allowed_inferred_types=_WIDENED_TYPES, + expected_rows=((1.0,), (2.5,)), + ), + SetOperationTypeIntegrationTestCase( + description="union all by name in a cte never narrows to the first branch", + query_sql=( + "WITH combined AS (SELECT CAST(1 AS INTEGER) AS amount UNION ALL BY NAME " + "SELECT CAST(2.5 AS DOUBLE) AS amount) SELECT amount FROM combined" + ), + allowed_inferred_types=_WIDENED_TYPES, + expected_rows=((1.0,), (2.5,)), + ), + ), + ids=lambda case: case.description, +) +def test_given_mixed_branch_types_when_compiling_then_known_types_match_duckdb( + test_case: SetOperationTypeIntegrationTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_CONFIG, encoding="utf-8") + models: Path = tmp_path / "models" + models.mkdir() + (models / "orders.sql").write_text( + f"MODEL (materialized table);\n{test_case.query_sql}", encoding="utf-8" + ) + + project: CompiledProject = compile_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + ) + (model,) = project.models + with duckdb.connect() as connection: + result: duckdb.DuckDBPyConnection = connection.execute(model.query_sql) + result_type: object = result.description[0][1] + rows: list[tuple[object, ...]] = result.fetchall() + + assert project.diagnostics == () + assert model.inferred_columns is not None + assert model.inferred_columns[0].type in test_case.allowed_inferred_types + assert result_type == sqltypes.DOUBLE + assert tuple(rows) == test_case.expected_rows + + +@pytest.mark.parametrize( + "test_case", + ( + SetOperationLineageIntegrationTestCase( + description="type recovery keeps resolved cte lineage", + expected_sources=frozenset({("orders", "amount"), ("refunds", "amount")}), + ), + ), + ids=lambda case: case.description, +) +def test_given_union_type_recovery_when_compiling_then_cte_lineage_is_preserved( + test_case: SetOperationLineageIntegrationTestCase, + tmp_path: Path, +) -> None: + (tmp_path / "sqlbuild_project.toml").write_text(_PROJECT_CONFIG, encoding="utf-8") + models: Path = tmp_path / "models" + models.mkdir() + (models / "orders.sql").write_text( + "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "SELECT CAST(2.5 AS DOUBLE) AS amount", + encoding="utf-8", + ) + (models / "combined_orders.sql").write_text( + "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + 'WITH combined AS (SELECT CAST(amount AS DOUBLE) AS amount FROM __ref("orders") ' + 'UNION ALL SELECT list_extract([amount], 1) AS amount FROM __ref("refunds")) ' + "SELECT amount FROM combined", + encoding="utf-8", + ) + (models / "refunds.sql").write_text( + "MODEL (materialized table, contract enforced, columns (amount (type DOUBLE)));\n" + "SELECT CAST(1.25 AS DOUBLE) AS amount", + encoding="utf-8", + ) + + project: CompiledProject = compile_project( + discovered_inputs=discover_project_inputs(project_dir=tmp_path), + adapter=DuckDbAdapter(), + ) + models_by_name: dict[str, CompiledModel] = {model.name: model for model in project.models} + lineage: Sequence[CompiledLineageColumnFact] | None = models_by_name[ + "combined_orders" + ].fast_lineage_columns + assert lineage is not None + (amount,) = lineage + + assert project.diagnostics == () + assert frozenset( + (source.resource_name, source.column_name) for source in amount.upstream_columns + ) == (test_case.expected_sources) diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py index 4e05eff095..1f5c1c2482 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/_test_types.py @@ -11,7 +11,11 @@ InferredColumn, ) from sqlbuild.compiler.compile.types import AttachedAuditTargetKind, SqlTestMode -from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.lineage.types import ( + ColumnLineageConfidence, + ColumnTransformKind, + InferredNullability, +) from sqlbuild.compiler.references.types import SqlReferenceKind from sqlbuild.compiler.scopes.models import UsageRecord from sqlbuild.compiler.scopes.types import ScopeKind @@ -872,3 +876,11 @@ class ExpectedBooleanTestCase: class ExpectedCountTestCase: description: str expected_count: int + + +@dataclass(frozen=True) +class CachedLineageRoundTripTestCase: + description: str + transform_kind: ColumnTransformKind + confidence: ColumnLineageConfidence + expected_compact: bool diff --git a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py index d1fe68db7f..a526cb8946 100644 --- a/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py +++ b/tests/unit/src/sqlbuild/compiler/compile/_helpers/test_analysis_cache.py @@ -22,17 +22,25 @@ from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.compile.models import ( AnalysisCacheContext, + CompactLineageFacts, CompileAnalysisSelection, + CompiledLineageColumnFact, + CompiledLineageSourceFact, CompiledModel, CompiledProject, CompileSqlReference, PolyglotAnalysisResult, ) -from sqlbuild.compiler.lineage.types import InferredNullability +from sqlbuild.compiler.lineage.types import ( + ColumnLineageConfidence, + ColumnTransformKind, + InferredNullability, +) from sqlbuild.compiler.profiling.main.collect import collect_compile_timings from sqlbuild.compiler.references.types import SqlReferenceKind from tests.unit.src.sqlbuild.compiler.compile._helpers._test_types import ( AnalysisCacheTestCase, + CachedLineageRoundTripTestCase, ) from tests.unit.src.sqlbuild.compiler.compile._helpers.helpers import ( compile_project_with_cache, @@ -462,6 +470,101 @@ def test_given_unsuccessful_analysis_when_writing_then_reuses_deterministic_resu assert analyses[cache_key].analysis_succeeded is False +@pytest.mark.parametrize( + "test_case", + ( + CachedLineageRoundTripTestCase( + description="indexed lineage decodes lazily", + transform_kind=ColumnTransformKind.CAST, + confidence=ColumnLineageConfidence.HIGH, + expected_compact=True, + ), + CachedLineageRoundTripTestCase( + description="unindexed lineage values decode eagerly", + transform_kind=ColumnTransformKind.UNKNOWN, + confidence=ColumnLineageConfidence.LOW, + expected_compact=False, + ), + ), + ids=lambda case: case.description, +) +def test_given_cached_lineage_when_reading_then_facts_round_trip_exactly( + test_case: CachedLineageRoundTripTestCase, + tmp_path: Path, +) -> None: + cache_key: str = "b" * 64 + context: AnalysisCacheContext = AnalysisCacheContext(root=tmp_path, shared_fingerprint="shared") + shared_source: CompiledLineageSourceFact = CompiledLineageSourceFact( + resource_type="source", resource_name="raw_orders", column_name="order_id" + ) + analysis: PolyglotAnalysisResult = PolyglotAnalysisResult( + analysis_succeeded=True, + lineage_columns=( + CompiledLineageColumnFact( + output_column="order_id", + upstream_columns=(shared_source,), + transform_kind=test_case.transform_kind, + confidence=test_case.confidence, + ), + CompiledLineageColumnFact( + output_column="order_key", + upstream_columns=( + shared_source, + CompiledLineageSourceFact( + resource_type="model", resource_name="orders", column_name="order_key" + ), + ), + transform_kind=ColumnTransformKind.EXPRESSION, + confidence=ColumnLineageConfidence.MEDIUM, + ), + ), + ) + write_model_analyses( + context=context, + analyses_by_key={cache_key: analysis}, + latest_analyses_by_model={"orders": analysis}, + ) + + analyses, _, _ = read_model_analyses( + context=context, + cache_keys=(cache_key,), + model_names=("orders",), + upstream_model_names_by_key={cache_key: ()}, + ) + + assert analyses[cache_key] == analysis + assert tuple(analyses[cache_key].lineage_columns) == analysis.lineage_columns + assert ( + isinstance(analyses[cache_key].lineage_columns, CompactLineageFacts) + is test_case.expected_compact + ) + + +@pytest.mark.parametrize( + "test_case", + (AnalysisCacheTestCase(description="stable per-model digests", expected_count=1),), + ids=lambda case: case.description, +) +def test_given_model_payload_when_digesting_then_fast_digest_matches_stable_encoding( + test_case: AnalysisCacheTestCase, +) -> None: + payload: dict[str, object] = { + "query_sql": "SELECT \"caf\u00e9\" AS name, '\\n\t\x01\u2028' AS note", + "references": [{"kind": "ref", "name": "orders", "package": None, "count": 2}], + "placeholders": {"b": "2", "a": "1"}, + "upstream": (("orders", "a" * 64),), + "binding_schema": None, + "has_star": False, + } + + digests: set[str] = { + analysis_cache._json_value_digest(payload), + analysis_cache._payload_digest(payload), + } + + assert len(digests) == test_case.expected_count + + @pytest.mark.parametrize( "test_case", (AnalysisCacheTestCase(description="semantic cache identities", expected_count=7),), diff --git a/uv.lock b/uv.lock index a4f3746d0f..ed62a1985b 100644 --- a/uv.lock +++ b/uv.lock @@ -2866,29 +2866,29 @@ wheels = [ [[package]] name = "polyglot-sql-chio" -version = "0.12.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/05/02a13098fe3bc4cbedcf483269775225d86111cab2f40fdd9322c5bd19de/polyglot_sql_chio-0.12.1.tar.gz", hash = "sha256:f0393e8a3675d81c2e48efa7cd83957d4a1ef1b0731a5876d67cc4ff7eacf429", size = 1977093, upload-time = "2026-09-20T12:33:52.58Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/d6/3ec58903c47db1a5f3cd1feea4981897e83312aff9c9858fae7969f9fe56/polyglot_sql_chio-0.12.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:94cf06b27a779e7b219b2bb10083076cf369f313c9970327807ba2020358f45d", size = 12002358, upload-time = "2026-09-20T12:32:49.635Z" }, - { url = "https://files.pythonhosted.org/packages/90/eb/a2b4b5bc53b2b6fdfb2baed721f5d9d5f011bfce02c8415a71c12572efdf/polyglot_sql_chio-0.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f13ad5a5d0c6379727f8da83825a6aa2e269d238bc29228451debd5147d64594", size = 11299245, upload-time = "2026-09-20T12:32:52.376Z" }, - { url = "https://files.pythonhosted.org/packages/ff/64/384b0fdb37caa65cb4914b8db6c915130f891e926d7d7a1172e39d6ff097/polyglot_sql_chio-0.12.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94cca6fd6e38a3b438bcb11db7e85073308606cb424b8295bcab0f5f606e14ea", size = 12155709, upload-time = "2026-09-20T12:32:55.279Z" }, - { url = "https://files.pythonhosted.org/packages/b7/03/f074ba45e674f2a626852299574576450026d0efa6960d9bdde0886b87f0/polyglot_sql_chio-0.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fabe0963d682d8d2bf9f531c1fe4f5cbe2f64eb9983449759716f16a0a1f15f6", size = 11199437, upload-time = "2026-09-20T12:32:58.189Z" }, - { url = "https://files.pythonhosted.org/packages/c3/9b/875f815c9b166b1120e0063d557bafa9809471394b48ece8a68917ef34c4/polyglot_sql_chio-0.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:8e447fc3980daa0b3c346fe1ddb718d062d648a5408abca4f4d41b867dd4d1fb", size = 10956608, upload-time = "2026-09-20T12:33:00.936Z" }, - { url = "https://files.pythonhosted.org/packages/86/62/de7a3b10923455316c089908066eb4e77cc65a1137b4d74bdaa4e3fe0ee8/polyglot_sql_chio-0.12.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a815196f1d3f35f2c6fa2aa77c47188e205b9561db4468ffc44e9b3fe9f1e2f4", size = 12002641, upload-time = "2026-09-20T12:33:03.987Z" }, - { url = "https://files.pythonhosted.org/packages/9d/8a/3f81d87129e29b0954d31e1eb7f13e442c71709f3a10073ac44b3b87063f/polyglot_sql_chio-0.12.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3da29758ee9eb7b86cf985aa05aa1b98e6e5e048ff30f265d14f192129c5f36f", size = 11296760, upload-time = "2026-09-20T12:33:06.922Z" }, - { url = "https://files.pythonhosted.org/packages/b1/fc/65784d8db9649bfafdffc5a85efad282ee726e0f556535fae91eda976bf8/polyglot_sql_chio-0.12.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be9f40e911aee632b0789fd955c126563941479018eef7b26587932ec22a09ed", size = 12156186, upload-time = "2026-09-20T12:33:09.434Z" }, - { url = "https://files.pythonhosted.org/packages/13/65/9a868fd86b2e4fff77abf1c6e5693550c38ca8e43489bc7b81811c2ea1aa/polyglot_sql_chio-0.12.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99a2d0cbf107e0071ee6f5cd5e33481e0752509cb8681fe251c112693bb8db7d", size = 11199514, upload-time = "2026-09-20T12:33:12.144Z" }, - { url = "https://files.pythonhosted.org/packages/f0/63/55e9ee7b304ec4ad2882beab69ef8fc4c5c82950aa6c998b1935fa74474c/polyglot_sql_chio-0.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:c3e8a4705db0dc4e311122cabad96915f2c367141ab42f09cae8db9bbc60ad20", size = 10956508, upload-time = "2026-09-20T12:33:15.009Z" }, - { url = "https://files.pythonhosted.org/packages/6b/22/0e24a2e7f92c9c8831ba860c23d54af7c23c57a3c19f280d3e7b85afa6f2/polyglot_sql_chio-0.12.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:fbddfb7a1fc6beeb6a02c82b9e842a99f83ebbd40da2ed602878907061fd0f2f", size = 12003657, upload-time = "2026-09-20T12:33:17.944Z" }, - { url = "https://files.pythonhosted.org/packages/47/89/4643e4295eae01a81e36214cba44b040245fef89724b85013531153d7701/polyglot_sql_chio-0.12.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b95391aa0551ed31b2ebbbd257c4161f4273c8be25f4682401fa474490dee7e2", size = 11300172, upload-time = "2026-09-20T12:33:20.601Z" }, - { url = "https://files.pythonhosted.org/packages/94/2a/eb1e4e7da29bf8fbf9834cc806e62dd9a27b4f7c81f04ba5d3e2d0dceb6a/polyglot_sql_chio-0.12.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a3c758e03da4178f7f6ef412306f48cbfab8f79c31e5ee6c2b7c0cc8ee35b42", size = 12157262, upload-time = "2026-09-20T12:33:23.3Z" }, - { url = "https://files.pythonhosted.org/packages/ba/09/910b2387dc016bcd5c364ec6d45b937ef1d02042a3c965249d0e01b0f67c/polyglot_sql_chio-0.12.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b2ed2b5bb72ba8bfebd121148464c0fc14b41aba453cbf9ab89b261113a5657", size = 11200962, upload-time = "2026-09-20T12:33:25.992Z" }, - { url = "https://files.pythonhosted.org/packages/a5/27/9183ebfb13177202f8c27923f139ed6dcf3877a729dcaabc310e976d0527/polyglot_sql_chio-0.12.1-cp314-cp314-win_amd64.whl", hash = "sha256:4e27bc7ea0a4145995b490b87739d65ea59a70239ddb0a832d9d35ffaca4cbc4", size = 10958367, upload-time = "2026-09-20T12:33:28.53Z" }, - { url = "https://files.pythonhosted.org/packages/d0/27/e15c9919e9c2503d57d5c75ca5eb5fa6b3371d667ac823bd51cafbf4ca69/polyglot_sql_chio-0.12.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19691b5f75c281fae3715730fa20a576ffb900cd1977119d55a7aede1e248acb", size = 12254920, upload-time = "2026-09-20T12:33:31.121Z" }, - { url = "https://files.pythonhosted.org/packages/c4/06/86beb292ae50c96173caf4683d04b142a8a2b7c3250b3652b681662e9a02/polyglot_sql_chio-0.12.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:447ca73aa0b5c96ec1346f879fbcaa181744768c927d84208ddfcabfbdc85a4f", size = 11270604, upload-time = "2026-09-20T12:33:33.736Z" }, - { url = "https://files.pythonhosted.org/packages/63/9c/affd9968e06c54bdb49df5204f3fba87b1a027cf559d27a9ff84a0be1cfe/polyglot_sql_chio-0.12.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49b324eaa8718039a3569a3a41777dd38a0fd7e4e09df1a3eb98caa717fe45aa", size = 11200962, upload-time = "2026-09-20T12:33:36.498Z" }, - { url = "https://files.pythonhosted.org/packages/02/98/002106d489c43c9bb3593d36192e530530c40b6158e2bc9ee059503ca5d9/polyglot_sql_chio-0.12.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d78c762fc6a64c785870360f3552ccfc1ed40626701c62370d710b592e6c0725", size = 11270603, upload-time = "2026-09-20T12:33:39.276Z" }, +version = "0.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/78/ff/d2e3e00fbe48a9fdce820af813e59e778a6e9873a9c64b48fa9b0caf6095/polyglot_sql_chio-0.12.2.tar.gz", hash = "sha256:e927175444ae5a0dbd1b5caa945cac5df1cb77f27cb9545275b9eec066d4ff7b", size = 2002986, upload-time = "2026-09-22T23:27:09.887Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/a7/95727b3fe0e6e6e3a4fb3c31526462be04a17e5869b293a114c0d0de6ad9/polyglot_sql_chio-0.12.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:815897873c7d141517520884206970f4985db6b257e9b88bac9bad409fbc847a", size = 12059902, upload-time = "2026-09-22T23:26:19.476Z" }, + { url = "https://files.pythonhosted.org/packages/ce/a2/ae0af0b2b8a91c9072207950fee5bb68cb294160fca22f01bfed61065a79/polyglot_sql_chio-0.12.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6acb8e9301b691166af8d47ef1bda870894c54cfda29cee986cabff039943552", size = 11350476, upload-time = "2026-09-22T23:26:21.737Z" }, + { url = "https://files.pythonhosted.org/packages/eb/00/47c9d01b7a05f28cc4efdf24d796af7406e08981f247c7c1157b374f6a01/polyglot_sql_chio-0.12.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d0a465759b2c5c1f6e3e894e41198267f09aa5b518e1717767ee8536fb945e6", size = 12213376, upload-time = "2026-09-22T23:26:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/c7/6e/3b8455526d64bfe8c7df50af34551b20539af2fd71e63de7a263bc142d1e/polyglot_sql_chio-0.12.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:faaee830b405d3cc5667543f63ebd00c5089f16a2474d9aed0f33c0e3071422f", size = 11257956, upload-time = "2026-09-22T23:26:26.116Z" }, + { url = "https://files.pythonhosted.org/packages/bb/27/802c3bb71d94210781afc6cc3e4ab9fddfdeddd51ddf6d8e03b6360229a1/polyglot_sql_chio-0.12.2-cp312-cp312-win_amd64.whl", hash = "sha256:097a8c6445fae0b3fb144ad1be1d8aebbc572bb5e94a34d9f169e918697fb593", size = 11013413, upload-time = "2026-09-22T23:26:28.425Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d9/64e08ab2c6758caaeb5d2341b549cea4481f7b9f517400ad642b54607c21/polyglot_sql_chio-0.12.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:85cba6c35b7b003e571ee4b4dd4f94a402f499a23a509587992abfb4f7bbfa1e", size = 12059437, upload-time = "2026-09-22T23:26:30.495Z" }, + { url = "https://files.pythonhosted.org/packages/93/41/1f229475a410f86fa711993c88214229868a1a6d5b2d0e37d867291edff2/polyglot_sql_chio-0.12.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a4d30ae9b88057844c2b369608a8248c6b00dd60d9db67300bca259afee4ddc2", size = 11350666, upload-time = "2026-09-22T23:26:32.631Z" }, + { url = "https://files.pythonhosted.org/packages/44/92/ede4c6d033bf8e567ecc39bff32fd444022dc28b2b590165808cd2859eae/polyglot_sql_chio-0.12.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b971424b63e01926e9ed2b074b844da01b375a13589157f716075793db31d6ed", size = 12211101, upload-time = "2026-09-22T23:26:35.135Z" }, + { url = "https://files.pythonhosted.org/packages/21/f8/08e4ca37ea5dab73d6113fe13802328161b87eafcd9d09fb46a556ee0c56/polyglot_sql_chio-0.12.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8173c06ff17c8797d8d4f2cae44792bb85275f91a8bbe899b58a94ce027d959", size = 11257651, upload-time = "2026-09-22T23:26:37.103Z" }, + { url = "https://files.pythonhosted.org/packages/55/a8/6bcdd68a978fac0115cc9bbc8c06f41a8cbb5455eaf9f135e7a6dca34065/polyglot_sql_chio-0.12.2-cp313-cp313-win_amd64.whl", hash = "sha256:a5631bd05e02b81993874f659454f0e78977ad037928cce5a4c13695a08c6d74", size = 11013192, upload-time = "2026-09-22T23:26:39.294Z" }, + { url = "https://files.pythonhosted.org/packages/6c/65/8008b6428b042280196d33e5762e4d130b7a0c4ea0c1082d6513cbc1a940/polyglot_sql_chio-0.12.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:06dab9afaf264a5d0172dcf8b6425c503986c71a3ff678e03493aa79f82ca75a", size = 12060516, upload-time = "2026-09-22T23:26:41.429Z" }, + { url = "https://files.pythonhosted.org/packages/94/ff/b43706fbd4d486004cbad1f7d54858c877a52125736c3c6340416625b901/polyglot_sql_chio-0.12.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:500f75f1fdaffd63f010380d9c9a3bb14270cfe30c778ef6d20b89d07be8df82", size = 11351238, upload-time = "2026-09-22T23:26:43.465Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b1/7345974d67a86e1f006ab1cdd90b1f2b334cfc47f870d6b7412650e940b0/polyglot_sql_chio-0.12.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d833909639d97f0319cd9ef09a064538eaea7c469cb7805b018cc3713f225244", size = 12214602, upload-time = "2026-09-22T23:26:45.778Z" }, + { url = "https://files.pythonhosted.org/packages/29/c7/91efe51051e5a30764e3e8719ab64aacadadb6fe8391e7a7ce8663a4d5fc/polyglot_sql_chio-0.12.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:267ebbc19a3f77b1877b20cba01ffa5ffa5391c91136087987a4d85bf9cb3da6", size = 11258677, upload-time = "2026-09-22T23:26:47.951Z" }, + { url = "https://files.pythonhosted.org/packages/be/c3/92a3651af6ffc60f1978ba9b89422bff1e505c1ec725cf9b499cef93b47b/polyglot_sql_chio-0.12.2-cp314-cp314-win_amd64.whl", hash = "sha256:daa618d58a8223e82e72521b9fee6d21f4245b28b61c66759470970acc0c0f0f", size = 11014391, upload-time = "2026-09-22T23:26:49.873Z" }, + { url = "https://files.pythonhosted.org/packages/52/51/75939ed4ce1e0745c6b595c931c91da823572fa8578459df41efd6ffa993/polyglot_sql_chio-0.12.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f189d445739ac52a4ae83728d660eee12f9b0394ba1f55be48f7ea0917426541", size = 12311577, upload-time = "2026-09-22T23:26:52.3Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a4/e74a3d3e2879f7cd55c28762faa17ced26026dbcc5b5b9f2b846bac3afab/polyglot_sql_chio-0.12.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:57e7316fafd6774784563a3242ced836326620c7ccea5134835f2b4013f0d75c", size = 11331490, upload-time = "2026-09-22T23:26:54.608Z" }, + { url = "https://files.pythonhosted.org/packages/b6/da/737c3fcdba8d291bdad510b586421916c4838dd84761818a480cc2834a37/polyglot_sql_chio-0.12.2-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1a1f21da77c9a652299eac3732cbe0d141f7bd48c2148b9e33cb054c6eae343", size = 11258677, upload-time = "2026-09-22T23:26:56.893Z" }, + { url = "https://files.pythonhosted.org/packages/01/39/e0cddc4bd970e08cce0c29e7dace3de5d0d2140d3e786c39abccb3dd6b40/polyglot_sql_chio-0.12.2-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59df75b9cbc1b0d4f32ae7d10015413c6d51cbe8ff323cbe694325fe533bc94b", size = 11331496, upload-time = "2026-09-22T23:26:59.097Z" }, ] [[package]] @@ -4232,7 +4232,7 @@ requires-dist = [ { name = "ingestr", marker = "extra == 'ingestr'", specifier = ">=1.0.0" }, { name = "jinja2", marker = "extra == 'dbt'", specifier = ">=3.1" }, { name = "orjson", specifier = ">=3.10.0" }, - { name = "polyglot-sql-chio", specifier = "==0.12.1" }, + { name = "polyglot-sql-chio", specifier = "==0.12.2" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2.0" }, { name = "pydantic-settings", specifier = ">=2,<3" }, { name = "pymssql", marker = "extra == 'sqlserver'", specifier = ">=2.3.0" },