diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 86e9f59e05..27e8cb57cc 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -294,7 +294,16 @@ jobs: run: >- make compiler-differential COMPILER_DIFFERENTIAL_ARGS="--stage-captures --require-discovery-coverage - --require-render-coverage" + --require-render-coverage --require-analysis-coverage + --wheel-site-report ${{ runner.temp }}/compiler-differential-wheel-sites.json" + - name: Upload the Polyglot wheel-site and deferral report + if: always() + uses: actions/upload-artifact@v4 + with: + name: compiler-differential-wheel-sites-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/compiler-differential-wheel-sites.json + if-no-files-found: warn + retention-days: 30 - name: Compare generated seeds and the failure corpus (python vs shipped native) run: make compiler-differential-shipped diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/__init__.py b/scripts/compiler_differential/_helpers/analysis_corpus/__init__.py new file mode 100644 index 0000000000..934b344cf3 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/__init__.py @@ -0,0 +1 @@ +"""Analysis-stage failure cases, one module per native analysis lane.""" diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py b/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py new file mode 100644 index 0000000000..9faaa527da --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/analysis_session.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native model analysis session lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def analysis_session_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py new file mode 100644 index 0000000000..7cb668e3bb --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/assembly.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native project assembly and target validation lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def assembly_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/contracts.py b/scripts/compiler_differential/_helpers/analysis_corpus/contracts.py new file mode 100644 index 0000000000..db6cbc7038 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/contracts.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native contracts and promotion conflicts lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def contracts_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py new file mode 100644 index 0000000000..e95a90f5fd --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/failure_cases.py @@ -0,0 +1,567 @@ +"""Minimal projects that fail SQL analysis, semantic, contract, or target validation.""" + +from scripts.compiler_differential._helpers.analysis_corpus.analysis_session import ( + analysis_session_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.assembly import ( + assembly_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.contracts import ( + contracts_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.lineage import ( + lineage_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.semantic_checks import ( + semantic_checks_failure_cases, +) +from scripts.compiler_differential._helpers.analysis_corpus.sql_test_glue import ( + sql_test_glue_failure_cases, +) +from scripts.compiler_differential._helpers.corpus.case_builder import ( + failure_case, + mart_body_files, + staging_files, +) +from scripts.compiler_differential.constants import ( + FAILURE_BASE_CONFIG, + FAILURE_BASE_STAGING, + FAILURE_CONFIG_PATH, + FAILURE_MART_PATH, + FAILURE_SOURCES_PATH, +) +from scripts.compiler_differential.models import FailureCase + +_MART_HEADER: str = '"Order totals per customer",' +_ORDERS: str = '__ref("stg_orders")' +_SNOWFLAKE_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "snowflake"\n\n' + '[connection]\naccount = "example"\nuser = "builder"\n' +) +_POSTGRES_CONFIG: str = ( + 'name = "failure_corpus"\nadapter = "postgres"\n\n' + '[connection]\nhost = "localhost"\ndatabase = "failure"\nschema = "public"\n' +) +_FUNCTION_PATH: str = "functions/sql/scaled_amount.sql" +_SCALED_AMOUNT: str = ( + "FUNCTION (\n" + ' description "Scale an amount by an integer factor",\n' + " arguments (p_amount DOUBLE, p_factor INTEGER),\n" + " returns DOUBLE,\n);\n\n" + "p_amount * p_factor\n" +) +_HOOK_PATH: str = "hooks/sql/record_refresh.sql" +_EXPRESSION_AUDIT: str = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' +_EXPRESSION_AUDIT_PATH: str = "models/marts/_sqlbuild/_audits/generic/expression_is_true.sql" +_TEST_PATH: str = "tests/unit/test_stg_orders.sql" +_ORDER_ROW: str = ( + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount, 'placed' AS status\n" +) + + +def _mart(body: str) -> dict[str, str]: + return mart_body_files(body) + + +def _mart_header(*, extra: str, body: str | None = None) -> dict[str, str]: + files: dict[str, str] = mart_body_files( + body + or 'SELECT customer_id, SUM(amount) AS total_amount\nFROM __ref("stg_orders")\n' + "GROUP BY customer_id\n" + ) + return { + FAILURE_MART_PATH: files[FAILURE_MART_PATH].replace( + _MART_HEADER, f"{_MART_HEADER}\n{extra}" + ) + } + + +def analysis_failure_cases() -> tuple[FailureCase, ...]: + """Return the analysis-stage cases, then every analysis lane's own cases.""" + + return ( + *_binding_cases(), + *_type_cases(), + *_grouping_cases(), + *_metadata_cases(), + *_contract_cases(), + *_target_cases(), + *_syntax_cases(), + *_ordering_cases(), + *analysis_session_failure_cases(), + *semantic_checks_failure_cases(), + *contracts_failure_cases(), + *lineage_failure_cases(), + *sql_test_glue_failure_cases(), + *assembly_failure_cases(), + ) + + +def _binding_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-ambiguous-column", + expected_code="B003", + files=_mart( + "SELECT order_id\n" + f"FROM {_ORDERS} a\nJOIN {_ORDERS} b ON a.customer_id = b.customer_id\n" + ), + ), + failure_case( + name="analysis-unknown-qualifier", + expected_code="B004", + expected_message="Unknown table or alias 'x' referenced by column 'order_id'", + files=_mart(f"SELECT x.order_id\nFROM {_ORDERS} o\n"), + ), + failure_case( + name="analysis-alias-column-list-width", + expected_code="B005", + files=_mart( + "SELECT o.a AS first_column\n" + f"FROM (SELECT order_id, customer_id FROM {_ORDERS}) AS o(a, b, c)\n" + ), + ), + failure_case( + name="analysis-unknown-function", + expected_code="B101", + files=_mart( + f"SELECT customer_id, not_a_function(amount) AS total_amount\nFROM {_ORDERS}\n" + ), + ), + failure_case( + name="analysis-function-argument-count", + expected_code="B102", + files=_mart(f"SELECT customer_id, SUBSTRING(status) AS total_amount\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-udf-argument-count", + expected_code="B102", + files={ + _FUNCTION_PATH: _SCALED_AMOUNT, + **_mart( + 'SELECT customer_id, __udf("scaled_amount")(amount) AS total_amount\n' + f"FROM {_ORDERS}\n" + ), + }, + ), + ) + + +def _type_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-non-boolean-predicate", + expected_code="B211", + files=_mart( + f"SELECT customer_id, amount AS total_amount\nFROM {_ORDERS}\nWHERE status\n" + ), + ), + failure_case( + name="analysis-arithmetic-operands", + expected_code="B212", + files=_mart(f"SELECT customer_id, status + 1 AS total_amount\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-function-argument-type", + expected_code="B213", + files=_mart(f"SELECT customer_id, ABS(status) AS total_amount\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-set-operation-branch-types", + expected_code="B215", + files=_mart( + f"SELECT customer_id, amount FROM {_ORDERS}\n" + f"UNION ALL\nSELECT customer_id, DATE '2026-01-01' FROM {_ORDERS}\n" + ), + ), + failure_case( + name="analysis-unsupported-cast", + expected_code="B218", + files=_mart(f"SELECT customer_id, DATE '2026-13-45' AS placed_on\nFROM {_ORDERS}\n"), + ), + ) + + +def _grouping_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-aggregate-in-where", + expected_code="B231", + files=_mart( + f"SELECT customer_id\nFROM {_ORDERS}\nWHERE SUM(amount) > 1\nGROUP BY customer_id\n" + ), + ), + failure_case( + name="analysis-window-without-over", + expected_code="B232", + files=_mart(f"SELECT customer_id, ROW_NUMBER() AS position\nFROM {_ORDERS}\n"), + ), + failure_case( + name="analysis-duplicate-cte-name", + expected_code="B233", + files=_mart( + f"WITH totals AS (SELECT customer_id FROM {_ORDERS}),\n" + f"totals AS (SELECT customer_id FROM {_ORDERS})\n" + "SELECT customer_id FROM totals\n" + ), + ), + failure_case( + name="analysis-non-constant-limit", + expected_code="B234", + files=_mart(f"SELECT customer_id\nFROM {_ORDERS}\nLIMIT customer_id\n"), + ), + ) + + +def _metadata_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="analysis-unique-key-unknown-column", + expected_code="B300", + expected_message="unique_key references unknown column 'order_key'", + files=_mart_header(extra=" materialized table,\n unique_key [order_key],"), + ), + failure_case( + name="analysis-relationships-unknown-field", + expected_code="B300", + expected_message="relationships target has no column 'customer_key'", + files=_mart_header( + extra=" columns (\n" + ' customer_id (audits [relationships (to __ref("stg_orders"), ' + "field customer_key)]),\n )," + ), + ), + failure_case( + name="analysis-accepted-values-type", + expected_code="B301", + expected_message="accepted_values", + files=_mart_header( + extra=" columns (\n" + " customer_id (audits [accepted_values (values [DATE '2026-01-01'])]),\n )," + ), + ), + failure_case( + name="analysis-cursor-type-family", + expected_code="B301", + expected_message="does not match column 'customer_id' type INT", + files={ + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n' + " materialized incremental,\n incremental_strategy delete_insert,\n" + " unique_key [customer_id],\n" + " cursor customer_id,\n cursor_type timestamp,\n cursor_grain day,\n" + " cursor_inputs (\n stg_orders customer_id,\n ),\n);\n\n" + "SELECT customer_id, SUM(amount) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ) + }, + ), + failure_case( + name="analysis-udf-argument-family", + expected_code="B301", + expected_message="Function 'scaled_amount' argument 'p_amount' expects DOUBLE", + files={ + _FUNCTION_PATH: _SCALED_AMOUNT, + **_mart( + 'SELECT customer_id, __udf("scaled_amount")(status, 2) AS total_amount\n' + f"FROM {_ORDERS}\n" + ), + }, + ), + failure_case( + name="analysis-expression-audit-unknown-column", + expected_code="B002", + expected_message="Unknown column 'discount' in output", + files={ + _EXPRESSION_AUDIT_PATH: _EXPRESSION_AUDIT, + **_mart_header(extra=' audits [expression_is_true (expression "discount >= 0")],'), + }, + ), + failure_case( + name="analysis-test-fixture-unknown-column", + expected_code="B302", + expected_message="SQL test '__source__raw_orders' names unknown column 'discount'", + files={ + _TEST_PATH: ( + "TEST();\n\nWITH\n__source__raw_orders AS (\n" + " SELECT 1 AS order_id, 10 AS customer_id, CAST(5 AS DOUBLE) AS amount," + " 'placed' AS status, 2 AS discount\n),\n" + "__expected__stg_orders AS (\n SELECT 1 AS order_id\n)\nSELECT 1\n" + ) + }, + ), + ) + + +def _contract_cases() -> tuple[FailureCase, ...]: + contract_header: str = ' description "Staged orders",\n contract enforced,\n' + query: str = 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + return ( + failure_case( + name="contract-nullability", + expected_code="K004", + expected_message="column 'customer_id' is declared non-null but may be nullable", + files=staging_files( + f"MODEL (\n{contract_header}" + " columns (\n order_id (type INTEGER),\n" + " customer_id (type INTEGER, nullable false),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" + "SELECT order_id, CAST(NULL AS INTEGER) AS customer_id, amount, status\n" + 'FROM __source("raw_orders")\n' + ), + ), + failure_case( + name="contract-without-columns", + expected_code="K006", + expected_message="model 'stg_orders' has contract enforced but declares no columns", + files=staging_files(f"MODEL (\n{contract_header});\n\n{query}"), + ), + failure_case( + name="contract-immediate-promotion", + expected_code="K011", + expected_message="contract enforced requires staged table promotion", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[settings]\ntable_promotion_mode = "immediate"\n', + **staging_files( + f"MODEL (\n{contract_header} materialized table,\n" + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n ),\n);\n\n" + f"{query}" + ), + }, + ), + failure_case( + name="contract-dynamic-columns-unproven", + expected_code="K011", + expected_message="dynamic column contract is not proven", + files=_mart_header( + extra=" materialized table,\n columns (customer_id (type INTEGER)),\n" + " dynamic_columns (\n status_amounts (\n pivot_column status,\n" + " value_column amount,\n aggregate MAX,\n type DOUBLE\n )\n ),", + body=f"PIVOT {_ORDERS}\nON status\nUSING MAX(amount)\nGROUP BY customer_id\n", + ), + ), + failure_case( + name="contract-dynamic-columns-unsupported-dialect", + expected_code="K011", + expected_message="does not support compiler-proven", + files={ + FAILURE_CONFIG_PATH: _POSTGRES_CONFIG, + **_mart_header( + extra=" materialized table,\n columns (customer_id (type INTEGER)),\n" + " dynamic_columns (\n status_amounts (\n pivot_column status,\n" + " value_column amount,\n aggregate MAX,\n type DOUBLE\n" + " )\n ),", + body=( + "SELECT customer_id, MAX(amount) AS placed\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + }, + ), + ) + + +def _target_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="target-missing-database", + expected_code="S101", + expected_message="snowflake execution requires explicit target database", + expected_help="my_database", + files={FAILURE_CONFIG_PATH: _SNOWFLAKE_CONFIG + '\n[defaults]\nschema = "ANALYTICS"\n'}, + ), + failure_case( + name="target-managed-loader-collision", + expected_code="S102", + expected_message="Managed loader write collision", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + '\n[targets.dev]\nschema = "dev"\nloader_schema = "raw"\n' + + '\n[targets.prod]\nschema = "prod"\nloader_schema = "raw"\n', + "python/loaders/orders.py": ( + "from sqlbuild.loaders import loader\n\n\n" + "@loader\n" + "def loaded_orders(ctx):\n" + ' """Load raw orders."""\n' + ' return [{"order_id": 1}]\n' + ), + "sources/loaded.yml": ( + "sources:\n - name: loaded_orders\n description: Loaded orders.\n" + " managed: true\n write_strategy: table\n" + " columns:\n - name: order_id\n type: INTEGER\n" + ), + }, + ), + failure_case( + name="target-missing-write-schema", + expected_code="S103", + expected_message="has no explicitly resolved physical write schema", + files={FAILURE_CONFIG_PATH: _SNOWFLAKE_CONFIG + 'database = "ANALYTICS"\n'}, + ), + failure_case( + name="target-named-without-schema", + expected_code="S104", + expected_message="Named target 'dev' must explicitly set schema", + files={ + FAILURE_CONFIG_PATH: _SNOWFLAKE_CONFIG.replace( + 'adapter = "snowflake"\n', 'adapter = "snowflake"\ndefault_target = "dev"\n' + ) + + 'database = "ANALYTICS"\n\n[targets.dev]\ndatabase = "ANALYTICS"\n' + }, + ), + ) + + +def _syntax_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="syntax-hook", + expected_code="P001", + expected_message="record_refresh", + files={ + _HOOK_PATH: ('HOOK (\n description "Record a refresh"\n);\n\nSELECT FROM WHERE\n'), + **_mart_header(extra=' post_hooks [sql("record_refresh")],'), + }, + ), + failure_case( + name="syntax-inline-hook", + expected_code="P001", + expected_message="hook", + files=_mart_header(extra=' post_hooks [inline_sql("SELECT FROM WHERE")],'), + ), + failure_case( + name="syntax-function", + expected_code="P001", + expected_message="scaled_amount", + files={ + _FUNCTION_PATH: _SCALED_AMOUNT.replace( + "p_amount * p_factor", "p_amount * (p_factor" + ), + **_mart( + 'SELECT customer_id, __udf("scaled_amount")(amount, 2) AS total_amount\n' + f"FROM {_ORDERS}\n" + ), + }, + ), + failure_case( + name="syntax-source-expression", + expected_code="P001", + expected_message="raw_orders", + files={ + FAILURE_SOURCES_PATH: ( + "sources:\n - name: raw_orders\n description: Orders feed.\n" + " expression: >-\n" + " (SELECT 1 AS order_id,, 10 AS customer_id,\n" + " CAST(5 AS DOUBLE) AS amount, 'placed' AS status)\n" + ) + }, + ), + failure_case( + name="analysis-rejected-opt-out-hides-findings", + expected_code="P009", + expected_message="`sql_analysis false` is not needed for model 'customer_totals'", + files={ + FAILURE_CONFIG_PATH: FAILURE_BASE_CONFIG + + "\n[settings]\nrequire_sql_analysis = true\n", + **_mart_header( + extra=" sql_analysis false,", + body=( + "SELECT customer_id, SUM(refund) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + }, + ), + failure_case( + name="analysis-opt-out-conflicts-with-legacy-key", + expected_code="P003", + expected_message="MODEL sql_analysis conflicts with legacy sql_validation", + files=_mart_header(extra=" sql_analysis false,\n sql_validation true,"), + ), + ) + + +def _ordering_cases() -> tuple[FailureCase, ...]: + return ( + failure_case( + name="order-two-failing-models", + expected_code="B002", + expected_message="Unknown column 'missing_column' in stg_orders", + expected_codes=("B002", "B002"), + files={ + **staging_files( + FAILURE_BASE_STAGING.replace("amount, status", "amount, discount, status") + ), + "models/marts/order_counts.sql": ( + 'MODEL (\n description "Order counts per customer",\n);\n\n' + "SELECT customer_id, COUNT(missing_column) AS order_count\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + }, + ), + failure_case( + name="order-binding-metadata-contract", + expected_code="B002", + expected_codes=("B002", "B300", "K001"), + files={ + **staging_files( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n customer_id (type INTEGER),\n" + " amount (type DOUBLE),\n status (type VARCHAR),\n" + " discount (type DOUBLE),\n ),\n);\n\n" + 'SELECT order_id, customer_id, amount, status\nFROM __source("raw_orders")\n' + ), + **_mart_header( + extra=" materialized table,\n unique_key [order_key],", + body=( + "SELECT customer_id, SUM(refund) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + }, + ), + failure_case( + name="order-poisoned-downstream-uses", + expected_code="B002", + expected_message="Unknown column 'refund' in raw_orders", + expected_notes=( + "1 downstream output uses were not type-checked because of this error", + ), + files={ + **staging_files( + FAILURE_BASE_STAGING.replace("amount, status", "amount, refund, status") + ), + FAILURE_MART_PATH: ( + 'MODEL (\n description "Order totals per customer",\n);\n\n' + "SELECT customer_id, SUM(refund) AS total_refund\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + }, + ), + failure_case( + name="order-contract-error-then-warning", + expected_code="K005", + expected_warning_code="K003", + expected_codes=("K005", "K003"), + files=staging_files( + 'MODEL (\n description "Staged orders",\n contract enforced,\n' + " columns (\n order_id (type INTEGER),\n note (type VARCHAR),\n" + " customer_id (type INTEGER),\n amount (type DOUBLE),\n ),\n);\n\n" + "SELECT order_id, NULL AS note, customer_id, amount, status\n" + 'FROM __source("raw_orders")\n' + ), + ), + failure_case( + name="order-closest-column-help", + expected_code="B002", + expected_help="did you mean 'amount'?", + expected_notes=("stg_orders has: amount, status, customer_id, order_id",), + expected_location=(5, 25), + files=_mart( + "SELECT customer_id, SUM(amonut) AS total_amount\n" + f"FROM {_ORDERS}\nGROUP BY customer_id\n" + ), + ), + ) diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/lineage.py b/scripts/compiler_differential/_helpers/analysis_corpus/lineage.py new file mode 100644 index 0000000000..fe6729e247 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/lineage.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native FAST lineage facts lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def lineage_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py new file mode 100644 index 0000000000..069b7d2986 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/semantic_checks.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native semantic completion lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def semantic_checks_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py new file mode 100644 index 0000000000..456d975490 --- /dev/null +++ b/scripts/compiler_differential/_helpers/analysis_corpus/sql_test_glue.py @@ -0,0 +1,9 @@ +"""Failure cases owned by the native SQL-test planning glue lane.""" + +from scripts.compiler_differential.models import FailureCase + + +def sql_test_glue_failure_cases() -> tuple[FailureCase, ...]: + """Return this lane's failure cases; the lane appends here without editing shared lists.""" + + return () diff --git a/scripts/compiler_differential/_helpers/comparing/comparison.py b/scripts/compiler_differential/_helpers/comparing/comparison.py index f10225a405..53c5976c0b 100644 --- a/scripts/compiler_differential/_helpers/comparing/comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/comparison.py @@ -86,6 +86,13 @@ def first_diagnostic_message(*, outcome: CommandOutcome, severity: str) -> str | return str(diagnostics[0].get("message")) if diagnostics else None +def first_diagnostic(*, outcome: CommandOutcome, severity: str) -> dict[str, object]: + """Return a JSON compile report's first diagnostic of one severity, or an empty mapping.""" + + diagnostics: list[dict[str, object]] = _diagnostics(outcome=outcome, severity=severity) + return diagnostics[0] if diagnostics else {} + + def _diagnostics(*, outcome: CommandOutcome, severity: str | None) -> list[dict[str, object]]: payload: dict[str, object] = as_json_object(_loads(outcome.stdout)) or {} diagnostics: object = payload.get("diagnostics") diff --git a/scripts/compiler_differential/_helpers/comparing/project_comparison.py b/scripts/compiler_differential/_helpers/comparing/project_comparison.py index 7e5c1397c4..ac14b3d1fd 100644 --- a/scripts/compiler_differential/_helpers/comparing/project_comparison.py +++ b/scripts/compiler_differential/_helpers/comparing/project_comparison.py @@ -10,6 +10,7 @@ compare_engine_runs, diagnostic_codes, ) +from scripts.compiler_differential._helpers.coverage.analysis import project_analysis_kinds from scripts.compiler_differential._helpers.coverage.discovery import project_discovery_kinds from scripts.compiler_differential._helpers.coverage.render import project_render_kinds from scripts.compiler_differential._helpers.running.execution import run_engine @@ -74,6 +75,8 @@ def compare_project(*, project: CorpusProject, options: DifferentialOptions) -> else frozenset() ), rendered_kinds=project_render_kinds(left.captures) if counted else frozenset(), + analysed_kinds=project_analysis_kinds(left.captures) if counted else frozenset(), + records=(left.records, right.records) if options.analysis_records else None, ) diff --git a/scripts/compiler_differential/_helpers/corpus/case_builder.py b/scripts/compiler_differential/_helpers/corpus/case_builder.py index 7e3bc2886e..bd6f445942 100644 --- a/scripts/compiler_differential/_helpers/corpus/case_builder.py +++ b/scripts/compiler_differential/_helpers/corpus/case_builder.py @@ -19,8 +19,12 @@ def failure_case( files: dict[str, str], expected_warning_code: str | None = None, expected_message: str | None = None, + expected_help: str | None = None, + expected_notes: tuple[str, ...] = (), + expected_location: tuple[int, int] | None = None, + expected_codes: tuple[str, ...] | None = None, ) -> FailureCase: - """Return the base project with `files` added or replaced.""" + """Return the base project with `files` added or replaced; notes are whole note lines.""" return FailureCase( files={**FAILURE_BASE_FILES, **files}, @@ -28,6 +32,10 @@ def failure_case( expected_code=expected_code, expected_warning_code=expected_warning_code, expected_message=expected_message, + expected_help=expected_help, + expected_notes=expected_notes, + expected_location=expected_location, + expected_codes=expected_codes, ) diff --git a/scripts/compiler_differential/_helpers/corpus/corpus.py b/scripts/compiler_differential/_helpers/corpus/corpus.py index b217c7f0c0..9fecf60b33 100644 --- a/scripts/compiler_differential/_helpers/corpus/corpus.py +++ b/scripts/compiler_differential/_helpers/corpus/corpus.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools import tomllib from pathlib import Path @@ -9,6 +10,7 @@ from scripts.compiler_differential.classes.dense_project import DenseProject from scripts.compiler_differential.classes.project_builder import ProjectBuilder from scripts.compiler_differential.constants import ( + ANALYSIS_DIALECT_VARIANTS, COLD_COMPILE, CORPUS_DENSE, CORPUS_EXAMPLES, @@ -21,6 +23,7 @@ FIXTURE_EXPECTED_OUTCOMES, FIXTURE_PROJECT_SUBDIRECTORIES, FIXTURE_ROOT, + GENERATOR_DIALECT_BLOCKS, PLAN, PROJECT_CONFIG_FILE, STORE_WARM_COMPILE, @@ -76,6 +79,10 @@ def build_corpus( ) if CORPUS_SEEDS in corpora: entries.extend(_seed_project(seed) for seed in seeds) + entries.extend( + _dialect_seed_project(seed=seed, dialect=dialect) + for seed, dialect in itertools.product(seeds[:1], ANALYSIS_DIALECT_VARIANTS) + ) if CORPUS_FAILURES in corpora: entries.extend( CorpusProject( @@ -138,7 +145,7 @@ def _seed_project(seed: int) -> CorpusProject: generated: GeneratedProject = ProjectBuilder(seed).build() return CorpusProject( name=f"seed/{seed}", - commands=(COLD_COMPILE, STORE_WARM_COMPILE, PLAN), + commands=(COLD_COMPILE, STORE_WARM_COMPILE, PLAN, *generated.extra_commands), expected=ExpectedOutcome( error_code=generated.expected_error_code, succeeding_commands=generated.succeeding_commands, @@ -146,3 +153,18 @@ def _seed_project(seed: int) -> CorpusProject: writer=generated.write, seed_coverage=True, ) + + +def _dialect_seed_project(*, seed: int, dialect: str) -> CorpusProject: + """Compile a seed's analysis blocks under another adapter's dialect; plan needs a warehouse.""" + + generated: GeneratedProject = ProjectBuilder( + seed, blocks=GENERATOR_DIALECT_BLOCKS, dialect=dialect + ).build() + return CorpusProject( + name=f"seed/{seed}-{dialect}", + commands=(COLD_COMPILE,), + expected=EXPECT_SUCCESS, + writer=generated.write, + seed_coverage=True, + ) diff --git a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py index 90b28c7d88..c7c2af8612 100644 --- a/scripts/compiler_differential/_helpers/corpus/emitted_codes.py +++ b/scripts/compiler_differential/_helpers/corpus/emitted_codes.py @@ -8,11 +8,13 @@ from scripts.compiler_differential._helpers.comparing.comparison import ( diagnostic_codes, + first_diagnostic, first_diagnostic_message, ) from scripts.compiler_differential._helpers.corpus.corpus import build_corpus from scripts.compiler_differential._helpers.running.execution import run_engine from scripts.compiler_differential.constants import ( + ANALYSIS_CODE_SCAN_ROOTS, CORPUS_FAILURES, DIAGNOSTIC_CODE_PATTERN, ERROR_SEVERITY, @@ -30,6 +32,7 @@ ) from sqlbuild.compiler.auditing.constants import BUILT_IN_AUDIT_SHADOW_CODE from sqlbuild.compiler.compile import constants as compile_constants +from sqlbuild.compiler.compile._helpers.diagnostics.help import semantic_help_catalogue from sqlbuild.compiler.compile.exceptions import CompileInputError from sqlbuild.compiler.discovery.exceptions import DiscoveryError from sqlbuild.compiler.resource_names.exceptions import ResourceIdentityError @@ -91,6 +94,17 @@ def render_error_codes() -> frozenset[str]: ) +def analysis_error_codes(compiler_root: Path) -> frozenset[str]: + """Return the help-catalogue codes plus every code literal in the analysis-stage modules.""" + + found: set[str] = set(semantic_help_catalogue()) + for root in ANALYSIS_CODE_SCAN_ROOTS: + path: Path = compiler_root / root + for module in sorted(path.rglob("*.py")) if path.is_dir() else (path,): + found.update(_module_codes(module)) + return frozenset(found) + + def inline_compile_codes(compile_root: Path) -> dict[str, tuple[str, ...]]: """Map each `code="X###"` literal in render-stage compile modules to where it is written.""" @@ -118,6 +132,30 @@ def unaccounted_inline_codes( } +def _module_codes(path: Path) -> set[str]: + codes: set[str] = set() + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + literal: str | None = _code_literal(node) or _code_constant(node) + if literal is not None: + codes.add(literal) + return codes + + +def _code_constant(node: ast.AST) -> str | None: + if not isinstance(node, (ast.Assign, ast.AnnAssign)): + return None + targets: list[ast.expr] = node.targets if isinstance(node, ast.Assign) else [node.target] + value: ast.expr | None = node.value + if not ( + any(isinstance(target, ast.Name) and target.id.endswith("_CODE") for target in targets) + and isinstance(value, ast.Constant) + and isinstance(value.value, str) + and DIAGNOSTIC_CODE_PATTERN.fullmatch(value.value) + ): + return None + return value.value + + def _code_literal(node: ast.AST) -> str | None: if not ( isinstance(node, ast.keyword) @@ -143,10 +181,20 @@ def _emitted_codes(*, project: CorpusProject, options: DifferentialOptions) -> E side=LEFT_SIDE, options=options, ) + first: dict[str, object] = first_diagnostic(outcome=run.outcomes[0], severity=ERROR_SEVERITY) + notes: object = first.get("notes") + line: object = first.get("line") + column: object = first.get("column") return EmittedCodes( errors=diagnostic_codes(outcome=run.outcomes[0], severity=ERROR_SEVERITY), warnings=diagnostic_codes(outcome=run.outcomes[0], severity=WARNING_SEVERITY), first_error_message=first_diagnostic_message( outcome=run.outcomes[0], severity=ERROR_SEVERITY ), + first_error_help=None if first.get("help") is None else str(first.get("help")), + first_error_notes=tuple(str(note) for note in notes) if isinstance(notes, list) else (), + first_error_location=( + (line, column) if isinstance(line, int) and isinstance(column, int) else None + ), + codes=diagnostic_codes(outcome=run.outcomes[0], severity=None), ) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 4ff02ce71e..f041e96d90 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -1,5 +1,8 @@ """Minimal failing projects, each derived from the shared base project.""" +from scripts.compiler_differential._helpers.analysis_corpus.failure_cases import ( + analysis_failure_cases, +) from scripts.compiler_differential._helpers.corpus.case_builder import ( failure_case, mart_body_files, @@ -45,6 +48,7 @@ def all_failure_cases() -> tuple[FailureCase, ...]: *render_failure_cases(), *scope_failure_cases(), *reference_failure_cases(), + *analysis_failure_cases(), ) diff --git a/scripts/compiler_differential/_helpers/coverage/analysis.py b/scripts/compiler_differential/_helpers/coverage/analysis.py new file mode 100644 index 0000000000..c7e4576ccd --- /dev/null +++ b/scripts/compiler_differential/_helpers/coverage/analysis.py @@ -0,0 +1,575 @@ +"""Classify `CompiledProject` captures into analysis kinds; only analysed evidence counts.""" + +from __future__ import annotations + +import dataclasses +import re +from collections.abc import Iterator, Sequence +from pathlib import Path + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential._helpers.coverage.capture_values import ( + capture_problems, + decoded, + enum_name, + has_content, + records, +) +from scripts.compiler_differential.classes.capture_file import expand_capture_text +from scripts.compiler_differential.constants import ( + ANALYSIS_CALLABLE_CAPTURE_FIELDS, + ANALYSIS_DETAIL_KINDS, + ANALYSIS_EXPLICIT_WRITE_SCHEMA_DIALECTS, + ANALYSIS_NON_COLLECTION_FIELDS, + ANALYSIS_OPAQUE_CAPTURE_VALUES, + ANALYSIS_STAGE_CAPTURE_SUFFIX, + ANALYSIS_UPSTREAM_CHAIN_HOPS, + COMPILED_PROJECT_CAPTURE_TYPE, +) +from sqlbuild.compiler.compile.models import CompiledProject +from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES + +_SET_OPERATION: re.Pattern[str] = re.compile(r"\b(?:UNION|INTERSECT|EXCEPT)\b", re.IGNORECASE) +_LEADING_CTE: re.Pattern[str] = re.compile(r"^\s*WITH\s+\w+\s+AS\s*\(", re.IGNORECASE) +_QUOTED_IDENTIFIER: re.Pattern[str] = re.compile(r'"([^"]+)"') +_SIZED_TYPE: re.Pattern[str] = re.compile(r"\((\d+)\)\s*$") +_RUNTIME_PLACEHOLDER: re.Pattern[str] = re.compile(r"@@@([A-Za-z_]\w*)") +_CURSOR_INTRINSIC: str = "__cursor_start()" +_HOOK_KEYS: tuple[str, ...] = ("pre_hooks", "post_hooks") +_EXPECTED_CTE_PREFIX: str = "__expected__" +_SQL_HOOK_TYPE: str = "SqlHookEntry" +_NON_NULL: str = "non_null" +_ENFORCED: str = "enforced" +_TABLE: str = "table" +_INCREMENTAL: str = "incremental" +_STAGED: str = "staged" +_MODEL_REFERENCE: str = "ref" +_SOURCE_REFERENCE: str = "source" +_SEED_REFERENCE: str = "seed" +_UDF_REFERENCE: str = "udf" +_TABLE_FUNCTION_REFERENCE: str = "table_function" +_AUDIT_KINDS: dict[str, str] = { + "accepted_values": "accepted_values_audit", + "relationships": "relationships_audit", + "expression_is_true": "expression_audit", +} +_METADATA_COLUMN_KEYS: tuple[str, ...] = ("unique_key", "cursor", "partition_column") + + +@dataclasses.dataclass(frozen=True) +class _Project: + """The parts of one `CompiledProject` capture the analysis kinds read.""" + + capture: dict[str, object] + models: dict[str, dict[str, object]] + sources: dict[str, dict[str, object]] + seeds: dict[str, dict[str, object]] + functions: dict[str, dict[str, object]] + + +def required_analysis_kinds() -> tuple[str, ...]: + """Return every `CompiledProject` collection field plus the finer required kinds.""" + + return (*analysis_collection_kinds(), *ANALYSIS_DETAIL_KINDS) + + +def analysis_collection_kinds() -> tuple[str, ...]: + """Return the `CompiledProject` fields that hold compiled resources or facts.""" + + return tuple( + field.name + for field in dataclasses.fields(CompiledProject) + if field.name not in ANALYSIS_NON_COLLECTION_FIELDS + ) + + +def project_analysis_kinds(captures: dict[str, dict[str, Path]]) -> frozenset[str]: + """Return the analysis kinds every `CompiledProject` capture of one project proves.""" + + kinds: set[str] = set() + for files in captures.values(): + for name, path in files.items(): + if name.endswith(ANALYSIS_STAGE_CAPTURE_SUFFIX): + kinds.update(analysed_input_kinds(path.read_text(encoding="utf-8"))) + return frozenset(kinds) + + +def analysis_capture_problems(capture_text: str) -> tuple[str, ...]: + """Return why a compiled-project capture is unsound; the catalog's native handle is allowed.""" + + return capture_problems( + capture_text=ANALYSIS_OPAQUE_CAPTURE_VALUES.sub("null", capture_text), + fields=tuple( + field.name + for field in dataclasses.fields(CompiledProject) + if field.name not in _omitted_fields() + ), + callable_fields=ANALYSIS_CALLABLE_CAPTURE_FIELDS, + ) + + +def analysed_input_kinds(capture_text: str) -> frozenset[str]: + """Return the analysis input kinds one canonical `CompiledProject` capture proves.""" + + capture: dict[str, object] = as_json_object(expand_capture_text(capture_text)) or {} + project: _Project = _Project( + capture=capture, + models=_by_name(capture.get("models")), + sources={ + str((as_json_object(source.get("source_entry")) or {}).get("name")): source + for source in records(capture.get("sources")) + }, + seeds=_by_name(capture.get("seeds")), + functions=_by_name(capture.get("functions")), + ) + kinds: set[str] = { + kind for kind in analysis_collection_kinds() if has_content(capture.get(kind)) + } + kinds.update(_inference_kinds(project)) + kinds.update(_binding_path_kinds(project)) + kinds.update(_shape_publication_kinds(project)) + kinds.update(_input_resource_kinds(project)) + kinds.update(_configuration_kinds(project)) + kinds.update(_metadata_kinds(project)) + kinds.update(_attachment_kinds(project)) + dialect: object = capture.get("sql_analysis_dialect") + if isinstance(dialect, str) and any(_analysed(model) for model in project.models.values()): + kinds.add(f"dialect_{dialect}") + return frozenset(kinds) + + +def _omitted_fields() -> frozenset[str]: + return STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(COMPILED_PROJECT_CAPTURE_TYPE, frozenset()) + + +def _by_name(value: object) -> dict[str, dict[str, object]]: + return {str(record.get("name")): record for record in records(value)} + + +def _analysed(model: dict[str, object]) -> bool: + """Whether binding validation ran on the model's SQL.""" + + return model.get("binding_validated") is True + + +def _columns(model: dict[str, object]) -> list[dict[str, object]]: + return records(model.get("inferred_columns")) + + +def _values(resource: dict[str, object]) -> dict[str, object]: + return as_json_object((as_json_object(resource.get("config")) or {}).get("values")) or {} + + +def _references(model: dict[str, object]) -> Iterator[tuple[str, str]]: + for reference in records(model.get("references")): + yield str(enum_name(reference.get("ref_kind"))), str(reference.get("ref_name")) + + +def _referenced(*, model: dict[str, object], kind: str) -> list[str]: + return [name for ref_kind, name in _references(model) if ref_kind == kind] + + +def _contract_enforced(model: dict[str, object]) -> bool: + return _values(model).get("contract") == _ENFORCED + + +def _schema_columns(resource: dict[str, object]) -> list[dict[str, object]]: + return records((as_json_object(resource.get("schema_entry")) or {}).get("columns")) + + +def _nullability(column: dict[str, object]) -> str | None: + return enum_name(column.get("nullability")) + + +def _inference_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + for model in project.models.values(): + if not _analysed(model): + continue + columns: list[dict[str, object]] = _columns(model) + if any(column.get("type") is not None for column in columns): + kinds.add("typed_column") + if any(column.get("type") is None for column in columns): + kinds.add("untyped_column") + upstream_models: list[str] = [ + name + for name in _referenced(model=model, kind=_MODEL_REFERENCE) + if name in project.models + ] + if model.get("fast_lineage_has_star") is True: + if model.get("fast_lineage_star_resolved") is not True: + if model.get("dynamic_column_contract") is None: + kinds.add("star_unresolved") + elif upstream_models: + kinds.add("star_over_published_shape") + else: + kinds.add("star_over_complete_input") + query: str = str(model.get("query_sql", "")) + if _SET_OPERATION.search(query) and upstream_models and columns: + kinds.add("set_operation") + if _LEADING_CTE.search(query) and columns: + kinds.add("cte") + if _passes_cte_type_through(project=project, model=model, upstream=upstream_models): + kinds.add("cte_passthrough_type") + if _QUOTED_IDENTIFIER.search(query) and _quoted_case_folded( + project=project, model=model, upstream=upstream_models + ): + kinds.add("quoted_identifier_case") + proof: dict[str, object] = as_json_object(model.get("dynamic_column_contract")) or {} + if proof.get("output_proven") is True and records(proof.get("families")): + kinds.add("dynamic_column_proof") + values: dict[str, object] = _values(model) + if ( + _CURSOR_INTRINSIC in query + and values.get("materialized") == _INCREMENTAL + and values.get("cursor") is not None + ): + kinds.add("cursor_intrinsic") + placeholders: set[str] = set(_RUNTIME_PLACEHOLDER.findall(query)) + defaults: dict[str, object] = as_json_object(values.get("placeholders")) or {} + if placeholders and placeholders <= set(defaults): + kinds.add("runtime_placeholder") + return kinds + + +def _passes_cte_type_through( + *, project: _Project, model: dict[str, object], upstream: list[str] +) -> bool: + """Whether a CTE model keeps an upstream column's exact inferred type.""" + + upstream_types: set[tuple[str, str]] = set().union( + *(_typed_columns(project.models[name]) for name in upstream) + ) + return bool(_typed_columns(model) & upstream_types) + + +def _typed_columns(model: dict[str, object]) -> set[tuple[str, str]]: + """Return the model's `(name, type)` pairs for every inferred column with a type.""" + + return { + (str(column.get("name")), str(column.get("type"))) + for column in _columns(model) + if column.get("type") is not None + } + + +def _quoted_case_folded( + *, project: _Project, model: dict[str, object], upstream: list[str] +) -> bool: + """Whether a quoted identifier bound to an upstream column only by ignoring its case.""" + + names: set[str] = set().union(*(_column_names_of(project.models[name]) for name in upstream)) + folded: set[str] = {name.casefold() for name in names} + return not records(model.get("binding_diagnostics")) and any( + quoted not in names and quoted.casefold() in folded + for quoted in _QUOTED_IDENTIFIER.findall(str(model.get("query_sql", ""))) + ) + + +def _column_names_of(model: dict[str, object]) -> set[str]: + return {str(column.get("name")) for column in _columns(model)} + + +def _binding_path_kinds(project: _Project) -> set[str]: + """Distinguish the dependency-ordered binding dataflow from the one-batch path.""" + + kinds: set[str] = set() + analysed: list[dict[str, object]] = [ + model for model in project.models.values() if _analysed(model) + ] + complete: list[bool] = [ + _reads_complete_inputs(project=project, model=model) for model in analysed + ] + if analysed and all(complete): + kinds.add("batch_binding") + if not all(complete): + kinds.add("dataflow_binding") + analysable: list[dict[str, object]] = [ + model + for model in project.models.values() + if _values(model).get("sql_analysis") is not False and records(model.get("references")) + ] + if analysed and any( + model.get("inferred_columns") is None and not _analysed(model) for model in analysable + ): + kinds.add("select_limited_analysis") + if ( + project.models + and not analysed + and all( + model.get("inferred_columns") is None + for model in analysable + if model.get("dynamic_column_contract") is None + ) + ): + kinds.add("sql_analysis_disabled") + if _longest_analysed_chain(project) >= ANALYSIS_UPSTREAM_CHAIN_HOPS: + kinds.add("upstream_chain") + return kinds + + +def _reads_complete_inputs(*, project: _Project, model: dict[str, object]) -> bool: + """Mirror `build_complete_binding_schemas` plus resolved expression-source shapes.""" + + resolved_expressions: frozenset[str] = _resolved_expression_sources(project) + for kind, name in _references(model): + if kind == _MODEL_REFERENCE: + upstream: dict[str, object] | None = project.models.get(name) + if upstream is None or not _declares_complete_schema(upstream): + return False + elif kind == _SOURCE_REFERENCE: + entry: dict[str, object] = ( + as_json_object((project.sources.get(name) or {}).get("source_entry")) or {} + ) + if str(entry.get("expression")) not in resolved_expressions and not ( + entry.get("contract") == _ENFORCED and records(entry.get("columns")) + ): + return False + elif kind == _SEED_REFERENCE: + if not _schema_columns(project.seeds.get(name) or {}): + return False + elif kind == _TABLE_FUNCTION_REFERENCE: + if not records((project.functions.get(name) or {}).get("return_columns")): + return False + return True + + +def _declares_complete_schema(model: dict[str, object]) -> bool: + schema_entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + return ( + _contract_enforced(model) + and bool(_schema_columns(model)) + and not has_content(schema_entry.get("dynamic_columns")) + ) + + +def _resolved_expression_sources(project: _Project) -> frozenset[str]: + """Return the source expressions whose shape compile resolved before analysis.""" + + shapes: object = decoded( + (as_json_object(project.capture.get("binding_catalog")) or {}).get("expression_shapes") + ) + pairs: Sequence[object] = ( + list(shapes.items()) + if isinstance(shapes, dict) + else shapes + if isinstance(shapes, list) + else [] + ) + resolved: set[str] = set() + for pair in pairs: + match pair: + case [expression, shape] if shape is not None: + resolved.add(str(expression)) + case _: + pass + return frozenset(resolved) + + +def _longest_analysed_chain(project: _Project) -> int: + """Return the most model-to-model hops along analysed models, ignoring reference cycles.""" + + hops: dict[str, int] = dict.fromkeys( + (name for name, model in project.models.items() if _analysed(model)), 0 + ) + for _ in range(len(hops)): + hops = {name: _upstream_hops(project=project, name=name, hops=hops) for name in hops} + return max(hops.values(), default=0) + + +def _upstream_hops(*, project: _Project, name: str, hops: dict[str, int]) -> int: + """Return one more hop than the deepest analysed upstream model, capped by the model count.""" + + upstream: list[int] = [ + hops[upstream_name] + for upstream_name in _referenced(model=project.models[name], kind=_MODEL_REFERENCE) + if upstream_name in hops + ] + return min(max(upstream, default=-1) + 1, len(hops)) + + +def _shape_publication_kinds(project: _Project) -> set[str]: + """Prove a contract's declared types and nullability reached a downstream model's shape.""" + + kinds: set[str] = set() + for model in project.models.values(): + if not _analysed(model): + continue + downstream: dict[str, dict[str, object]] = { + str(column.get("name")): column for column in _columns(model) + } + for name in _referenced(model=model, kind=_MODEL_REFERENCE): + upstream: dict[str, object] | None = project.models.get(name) + if upstream is None or not _contract_enforced(upstream): + continue + inferred: dict[str, dict[str, object]] = { + str(column.get("name")): column for column in _columns(upstream) + } + for declared in _schema_columns(upstream): + column_name: str = str(declared.get("name")) + published: dict[str, object] | None = downstream.get(column_name) + if published is None: + continue + size: re.Match[str] | None = _SIZED_TYPE.search(str(declared.get("type") or "")) + published_size: re.Match[str] | None = _SIZED_TYPE.search( + str(published.get("type") or "") + ) + if size and published_size and size.group(1) == published_size.group(1): + kinds.add("sized_contract_shape") + if ( + declared.get("nullable") is False + and _nullability(published) == _NON_NULL + and _nullability(inferred.get(column_name) or {}) != _NON_NULL + ): + kinds.add("contract_nullability_shape") + return kinds + + +def _input_resource_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + expression_shapes: object = (as_json_object(project.capture.get("binding_catalog")) or {}).get( + "expression_shapes" + ) + for model in project.models.values(): + if not _analysed(model): + continue + for kind, name in _references(model): + if kind == _SOURCE_REFERENCE and has_content(expression_shapes): + entry: dict[str, object] = ( + as_json_object((project.sources.get(name) or {}).get("source_entry")) or {} + ) + if entry.get("expression"): + kinds.add("source_expression_shape") + elif kind == _SEED_REFERENCE and any( + column.get("type") for column in _schema_columns(project.seeds.get(name) or {}) + ): + kinds.add("seed_column_types") + elif kind in (_UDF_REFERENCE, _TABLE_FUNCTION_REFERENCE): + kinds.update(_function_kinds(function=project.functions.get(name), kind=kind)) + return kinds + + +def _function_kinds(*, function: dict[str, object] | None, kind: str) -> set[str]: + if function is None: + return set() + arguments: list[dict[str, object]] = records(function.get("arguments")) + kinds: set[str] = set() + if arguments and all(argument.get("type") for argument in arguments): + kinds.add("udf_argument_types") + if kind == _TABLE_FUNCTION_REFERENCE and records(function.get("return_columns")): + kinds.add("table_function") + elif kind == _UDF_REFERENCE and function.get("returns"): + kinds.add("scalar_udf") + return kinds + + +def _configuration_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + settings: dict[str, object] = as_json_object(project.capture.get("settings")) or {} + for model in project.models.values(): + values: dict[str, object] = _values(model) + if ( + settings.get("require_sql_analysis") is True + and values.get("sql_analysis") is False + and model.get("inferred_columns") is None + and model.get("rejected_sql_analysis_opt_out") is None + ): + kinds.add("sql_analysis_opt_out") + if model.get("inferred_columns") is None: + continue + for key in _HOOK_KEYS: + hooks: list[dict[str, object]] = [ + hook + for hook in records(values.get(key)) + if str(hook.get("__type__", "")).endswith(_SQL_HOOK_TYPE) + ] + kinds.update( + "named_sql_hook" if hook.get("definition_sql") else "inline_sql_hook" + for hook in hooks + ) + if len(hooks) > 1: + kinds.add("hook_list") + if ( + settings.get("table_promotion_mode") == _STAGED + and _contract_enforced(model) + and values.get("materialized", _TABLE) == _TABLE + ): + kinds.add("explicit_promotion_mode") + if project.capture.get("enforce_explicit_references") is True and has_content( + project.capture.get("unmatched_literal_sql_relations") + ): + kinds.add("python_sql_literal_relation") + destinations: list[dict[str, object]] = [ + as_json_object(resource.get("destination")) or {} + for resource in (*project.models.values(), *project.seeds.values()) + ] + if ( + str(project.capture.get("sql_analysis_dialect")) in ANALYSIS_EXPLICIT_WRITE_SCHEMA_DIALECTS + and destinations + and all(destination.get("schema") for destination in destinations) + ): + kinds.add("managed_write_schema") + return kinds + + +def _metadata_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + for model in project.models.values(): + if not _analysed(model): + continue + entry: dict[str, object] = as_json_object(model.get("schema_entry")) or {} + audits: list[dict[str, object]] = records(entry.get("audits")) + for column in records(entry.get("columns")): + audits.extend(records(column.get("audits"))) + kinds.update( + _AUDIT_KINDS[name] + for audit in audits + if (name := str(audit.get("definition_name"))) in _AUDIT_KINDS + ) + values: dict[str, object] = _values(model) + output: set[str] = {str(column.get("name")).casefold() for column in _columns(model)} + if any( + (names := _column_names(values.get(key))) and names <= output + for key in _METADATA_COLUMN_KEYS + ): + kinds.add("metadata_column_reference") + return kinds + + +def _column_names(value: object) -> set[str]: + """Return the case-folded column names a metadata key such as `unique_key` names.""" + + if isinstance(value, str): + return {value.casefold()} + if isinstance(value, list): + return {str(name).casefold() for name in value} + return set() + + +def _attachment_kinds(project: _Project) -> set[str]: + kinds: set[str] = set() + for test in records(project.capture.get("sql_tests")): + payload: dict[str, object] = as_json_object(test.get("payload")) or {} + expected: list[str] = [ + str(cte.get("name", "")).removeprefix(_EXPECTED_CTE_PREFIX) + for cte in records(payload.get("expected_ctes")) + ] + if any(_analysed(project.models.get(name) or {}) for name in expected): + kinds.add("sql_test_expected_columns") + mocks: list[str] = [ + str(mock) for mock in (as_json_object(payload.get("macro_mocks")) or {}).values() + ] + overrides: dict[str, object] = as_json_object(payload.get("model_query_overrides")) or {} + if any( + _mock_rendered(project=project, mocks=mocks, name=name, override=str(override)) + for name, override in overrides.items() + ): + kinds.add("sql_test_macro_mock") + return kinds + + +def _mock_rendered(*, project: _Project, mocks: list[str], name: str, override: str) -> bool: + """Whether a test's re-expanded query holds a mocked macro value the model's own SQL lacks.""" + + compiled: str = str((project.models.get(name) or {}).get("query_sql", "")) + return any(mock in override and mock not in compiled for mock in mocks) diff --git a/scripts/compiler_differential/_helpers/running/execution.py b/scripts/compiler_differential/_helpers/running/execution.py index 3c4b283cb0..c58f5d32a8 100644 --- a/scripts/compiler_differential/_helpers/running/execution.py +++ b/scripts/compiler_differential/_helpers/running/execution.py @@ -7,7 +7,9 @@ import subprocess from pathlib import Path +from scripts.compiler_differential._helpers.running.records import read_analysis_records from scripts.compiler_differential.constants import ( + ANALYSIS_RECORD_ENV_VAR, CAPTURES_DIRECTORY, COMMAND_TIMEOUT_SECONDS, COMPILED_DIRECTORY, @@ -18,6 +20,7 @@ GENERATOR_ENVIRONMENT, MANIFEST_FILE, PROJECT_DIRECTORY, + RECORDS_DIRECTORY, SQB_ENTRY, STAGE_CAPTURE_ENV_VAR, ) @@ -60,17 +63,27 @@ def run_engine( ) capture_root: Path = case_dir / f"{side}-{engine}-{CAPTURES_DIRECTORY}" shutil.rmtree(capture_root, ignore_errors=True) + record_root: Path = case_dir / f"{side}-{engine}-{RECORDS_DIRECTORY}" + shutil.rmtree(record_root, ignore_errors=True) outcomes: list[CommandOutcome] = [] for index, command in enumerate(project.commands): capture_dir: Path | None = ( capture_root / f"{index}-{command.label}" if options.stage_captures else None ) + record_dir: Path | None = ( + record_root / f"{index}-{command.label}" if options.analysis_records else None + ) outcomes.append( _run_command( command=command, project_dir=project_dir, environment={ - **_command_environment(engine=engine, options=options, capture_dir=capture_dir), + **_command_environment( + engine=engine, + options=options, + capture_dir=capture_dir, + record_dir=record_dir, + ), **dict(command.environment), }, python=options.python, @@ -83,6 +96,7 @@ def run_engine( manifest=_optional_text(project_dir / MANIFEST_FILE), dag=_optional_text(project_dir / DAG_FILE), captures=_captures(capture_root), + records=read_analysis_records(record_root) if options.analysis_records else None, ) retained: Path = case_dir / engine shutil.rmtree(retained, ignore_errors=True) @@ -105,7 +119,11 @@ def harness_environment() -> dict[str, str]: def _command_environment( - *, engine: str, options: DifferentialOptions, capture_dir: Path | None + *, + engine: str, + options: DifferentialOptions, + capture_dir: Path | None, + record_dir: Path | None, ) -> dict[str, str]: environment: dict[str, str] = { **harness_environment(), @@ -114,6 +132,8 @@ def _command_environment( } if capture_dir is not None: environment[STAGE_CAPTURE_ENV_VAR] = str(capture_dir) + if record_dir is not None: + environment[ANALYSIS_RECORD_ENV_VAR] = str(record_dir) return environment diff --git a/scripts/compiler_differential/_helpers/running/records.py b/scripts/compiler_differential/_helpers/running/records.py new file mode 100644 index 0000000000..b1ab9e0145 --- /dev/null +++ b/scripts/compiler_differential/_helpers/running/records.py @@ -0,0 +1,84 @@ +"""Read the debug analysis records one engine's processes wrote: wheel call sites and deferrals.""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import cast + +from scripts.compiler_differential._helpers.comparing.compare import as_json_object +from scripts.compiler_differential.constants import ( + ANALYSIS_DEFERRAL_RECORD_PREFIX, + WHEEL_SITE_RECORD_PREFIX, +) +from scripts.compiler_differential.models import AnalysisRecords, ProjectComparison + + +def read_analysis_records(root: Path) -> AnalysisRecords: + """Sum every process's wheel-site counts and `{"kind", "site"}` deferral lines below `root`.""" + + sites: Counter[tuple[str, str]] = Counter() + deferrals: Counter[tuple[str, str]] = Counter() + for path in sorted(root.rglob(f"{WHEEL_SITE_RECORD_PREFIX}*.json")) if root.is_dir() else (): + payload: dict[str, object] = as_json_object(json.loads(path.read_text("utf-8"))) or {} + rows: object = payload.get("calls") + for row in rows if isinstance(rows, list) else (): + site, api, count = cast(list[object], row) + sites[(str(site), str(api))] += int(str(count)) + for path in ( + sorted(root.rglob(f"{ANALYSIS_DEFERRAL_RECORD_PREFIX}*.jsonl")) if root.is_dir() else () + ): + for line in path.read_text("utf-8").splitlines(): + record: dict[str, object] = as_json_object(json.loads(line)) or {} + deferrals[(str(record.get("kind")), str(record.get("site")))] += 1 + return AnalysisRecords(wheel_sites=dict(sites), deferrals=dict(deferrals)) + + +def wheel_site_report( + *, comparisons: list[ProjectComparison], engines: tuple[str, str] +) -> dict[str, object]: + """Return wheel calls per engine, site and API, split by corpus, plus deferral counts.""" + + report: dict[str, object] = {} + for side, engine in enumerate(engines): + sites: dict[str, dict[str, int]] = {} + deferrals: dict[str, dict[str, int]] = {} + for comparison in comparisons: + records: AnalysisRecords | None = ( + comparison.records[side] if comparison.records else None + ) + if records is None: + continue + corpus: str = comparison.project.split("/", 1)[0] + for (site, api), count in records.wheel_sites.items(): + by_corpus: dict[str, int] = sites.setdefault(f"{site} {api}", {}) + by_corpus[corpus] = by_corpus.get(corpus, 0) + count + for (kind, site), count in records.deferrals.items(): + by_corpus = deferrals.setdefault(f"{kind} {site}", {}) + by_corpus[corpus] = by_corpus.get(corpus, 0) + count + report[engine] = { + "wheel_sites": dict(sorted(sites.items())), + "deferrals": dict(sorted(deferrals.items())), + } + return report + + +def format_wheel_site_report(report: dict[str, object]) -> str: + """Render the wheel-site and deferral counts for the run log; nothing here gates.""" + + lines: list[str] = [] + for engine, raw in report.items(): + entry: dict[str, object] = as_json_object(raw) or {} + for label, key in ( + ("Polyglot wheel calls", "wheel_sites"), + ("Analysis deferrals", "deferrals"), + ): + counts: dict[str, object] = as_json_object(entry.get(key)) or {} + lines.append(f"{label} ({engine}): {'none recorded' if not counts else ''}".rstrip()) + for name, by_corpus in counts.items(): + corpora: dict[str, object] = as_json_object(by_corpus) or {} + split: str = ", ".join(f"{corpus} {count}" for corpus, count in corpora.items()) + total: int = sum(int(str(count)) for count in corpora.values()) + lines.append(f" {total:>7} {name} ({split})") + return "\n".join(lines) diff --git a/scripts/compiler_differential/_helpers/running/report.py b/scripts/compiler_differential/_helpers/running/report.py index d8e835850d..ba7d523fbb 100644 --- a/scripts/compiler_differential/_helpers/running/report.py +++ b/scripts/compiler_differential/_helpers/running/report.py @@ -3,6 +3,7 @@ from __future__ import annotations from scripts.compiler_differential.constants import ( + ANALYSIS_INDIRECT_KINDS, CONFIG_ONLY_KINDS, RENDER_CONFIG_ONLY_KINDS, RENDER_INDIRECT_KINDS, @@ -81,6 +82,17 @@ def format_render_coverage(*, covered: frozenset[str], required: tuple[str, ...] ) +def format_analysis_coverage(*, covered: frozenset[str], required: tuple[str, ...]) -> str: + """Report how many required analysis input kinds the seed corpus exercised.""" + + return _format_coverage( + stage="Analysis", + covered=covered, + required=required, + notes=(("indirect kinds", "credited because", ANALYSIS_INDIRECT_KINDS),), + ) + + def _format_coverage( *, stage: str, diff --git a/scripts/compiler_differential/_helpers/running/run.py b/scripts/compiler_differential/_helpers/running/run.py index 44ce33fec8..d6dbbb668f 100644 --- a/scripts/compiler_differential/_helpers/running/run.py +++ b/scripts/compiler_differential/_helpers/running/run.py @@ -30,6 +30,8 @@ def differential_options(*, args: argparse.Namespace, work_dir: Path) -> Differe engine_environment=parse_engine_environment(args.engine_env), require_discovery_coverage=args.require_discovery_coverage, require_render_coverage=args.require_render_coverage, + require_analysis_coverage=args.require_analysis_coverage, + analysis_records=args.wheel_site_report is not None, ) diff --git a/scripts/compiler_differential/classes/analysis_features.py b/scripts/compiler_differential/classes/analysis_features.py new file mode 100644 index 0000000000..e45a65edaa --- /dev/null +++ b/scripts/compiler_differential/classes/analysis_features.py @@ -0,0 +1,348 @@ +"""Write the optional project inputs that make generated seeds cover every analysis input kind.""" + +from __future__ import annotations + +import random +from collections.abc import Callable + +from scripts.compiler_differential.constants import ( + GENERATOR_ANALYSIS_FOLDER, + GENERATOR_ANALYSIS_MODE_COMMANDS, + GENERATOR_ANALYSIS_TEST_FOLDER, + GENERATOR_OPEN_SOURCE, + GENERATOR_SELECTED_MODEL, +) +from scripts.compiler_differential.models import DifferentialCommand, ModelPlan + + +class AnalysisFeatureWriter: + """Add self-contained SQL analysis blocks to one generated project, chosen per seed.""" + + def __init__( + self, + *, + blocks: tuple[str, ...], + rng: random.Random, + files: dict[str, str], + features: set[str], + staging: list[ModelPlan], + sources: list[str], + ) -> None: + self._blocks: tuple[str, ...] = blocks + self._random: random.Random = rng + self._files: dict[str, str] = files + self._features: set[str] = features + self._staging: list[ModelPlan] = staging + self._sources: list[str] = sources + self.config_lines: list[str] = [] + self.settings_lines: list[str] = [] + self.commands: tuple[DifferentialCommand, ...] = () + + def write(self) -> None: + """Write the selected analysis blocks in their canonical order.""" + + writers: dict[str, Callable[[], None]] = { + "column_shapes": self._column_shapes, + "contract_chain": self._contract_chain, + "udf_signatures": self._udf_signatures, + "dynamic_columns": self._dynamic_columns, + "quoted_identifiers": self._quoted_identifiers, + "metadata_checks": self._metadata_checks, + "sql_test_mocks": self._sql_test_mocks, + "analysis_opt_out": self._analysis_opt_out, + "python_sql": self._python_sql, + "analysis_modes": self._analysis_modes, + } + for block in self._blocks: + writer: Callable[[], None] | None = writers.get(block) + if writer is not None: + writer() + + def _base(self) -> ModelPlan: + return self._random.choice(self._staging) + + def _model(self, *, name: str, header: str, body: str, description: str) -> None: + self._files[f"{GENERATOR_ANALYSIS_FOLDER}/{name}.sql"] = ( + f'MODEL (\n description "{description}",\n{header});\n\n{body}' + ) + + def _column_shapes(self) -> None: + self._features.add("column_shapes") + base: ModelPlan = self._base() + other: ModelPlan = self._base() + source: str = self._random.choice(self._sources) + self._model( + name=f"typed_{base.name}", + description=f"Typed and untyped outputs of {base.name}", + header="", + body=( + "SELECT id, amount * 2 AS doubled_amount, CAST(status AS VARCHAR) AS status_text,\n" + " created_at, NULL AS missing_note\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._model( + name=f"every_{source}", + description=f"Every column of {source}", + header="", + body=f'SELECT *\nFROM __source("{source}")\n', + ) + self._model( + name=f"every_{base.name}", + description=f"Every column of {base.name}", + header="", + body=f'SELECT *\nFROM __ref("{base.name}")\n', + ) + self._files["sources/open.yml"] = ( + "sources:\n" + f" - name: {GENERATOR_OPEN_SOURCE}\n" + " description: Events whose columns are only known to the warehouse.\n" + ) + self._model( + name="every_open_event", + description="Every column of an open source", + header="", + body=f'SELECT *\nFROM __source("{GENERATOR_OPEN_SOURCE}")\n', + ) + self._model( + name=f"combined_{base.name}", + description=f"Recent and older rows of {base.name} and {other.name}", + header="", + body=( + "WITH recent AS (\n" + f' SELECT id, created_at FROM __ref("{base.name}") WHERE amount > 0\n' + "),\nolder AS (\n" + f' SELECT id, created_at FROM __ref("{other.name}")\n' + ")\n" + "SELECT id, created_at FROM recent\nUNION ALL\nSELECT id, created_at FROM older\n" + ), + ) + self._model( + name=f"passed_{base.name}", + description=f"Columns of {base.name} passed through a CTE", + header="", + body=( + f'WITH passthrough AS (\n SELECT id, created_at FROM __ref("{base.name}")\n)\n' + "SELECT id, created_at\nFROM passthrough\n" + ), + ) + + def _contract_chain(self) -> None: + self._features.add("contract_chain") + base: ModelPlan = self._base() + width: int = self._random.randint(16, 40) + source: str = base.inputs[0].split(":", 1)[1] + contracted: str = f"contracted_{base.name}" + self._model( + name=contracted, + description=f"Contracted copy of {source}", + header=( + " materialized table,\n" + " contract enforced,\n" + " columns (\n" + " id (type INTEGER, nullable false),\n" + f" label (type VARCHAR({width})),\n" + " amount (type DOUBLE),\n" + " ),\n" + ), + body=( + f"SELECT CAST(id AS INTEGER) AS id, CAST(status AS VARCHAR({width})) AS label,\n" + " amount\n" + f'FROM __source("{source}")\n' + ), + ) + upstream: str = contracted + for level, expression in enumerate(("label", "UPPER(label) AS label", "label"), start=1): + name: str = f"chain_{level}_{base.name}" + self._model( + name=name, + description=f"Level {level} below the contracted copy of {base.name}", + header="", + body=f'SELECT id, {expression}, amount\nFROM __ref("{upstream}")\n', + ) + upstream = name + self.settings_lines.append('table_promotion_mode = "staged"') + + def _udf_signatures(self) -> None: + self._features.add("udf_signatures") + base: ModelPlan = self._base() + factor: int = self._random.randint(2, 9) + self._files["functions/sql/scaled_amount.sql"] = ( + "FUNCTION (\n" + ' description "Scale an amount by an integer factor",\n' + " arguments (p_amount DOUBLE, p_factor INTEGER),\n" + " returns DOUBLE,\n);\n\n" + "p_amount * p_factor\n" + ) + self._files["functions/sql/table_fn__amount_bands.sql"] = ( + "FUNCTION (\n" + ' description "Band staged amounts above a floor",\n' + " arguments (p_floor DOUBLE),\n" + " returns table (\n id INTEGER,\n band VARCHAR\n ),\n);\n\n" + "SELECT CAST(id AS INTEGER) AS id,\n" + " CASE WHEN amount > p_floor THEN 'high' ELSE 'low' END AS band\n" + f'FROM __ref("{base.name}")\n' + ) + self._model( + name=f"scaled_{base.name}", + description=f"Scaled amounts of {base.name}", + header="", + body=( + f'SELECT id, __udf("scaled_amount")(amount, {factor}) AS scaled_amount\n' + f'FROM __ref("{base.name}")\n' + ), + ) + self._model( + name="banded_amounts", + description="Amount bands from a table function", + header="", + body=( + "SELECT bands.id, bands.band\n" + f'FROM __table_fn("table_fn__amount_bands")({factor}.5) AS bands\n' + ), + ) + + def _dynamic_columns(self) -> None: + self._features.add("dynamic_columns") + base: ModelPlan = self._base() + pivot_input: str = f"pivot_input_{base.name}" + self._model( + name=pivot_input, + description=f"Contracted pivot input from {base.name}", + header=( + " materialized table,\n" + " contract enforced,\n" + " columns (\n" + " id (type INTEGER),\n" + " status (type VARCHAR),\n" + " amount (type DOUBLE),\n" + " ),\n" + ), + body=( + "SELECT CAST(id AS INTEGER) AS id, CAST(status AS VARCHAR) AS status, amount\n" + f'FROM __ref("{base.name}")\n' + ), + ) + self._model( + name=f"pivoted_{base.name}", + description=f"Amounts of {base.name} pivoted by status", + header=( + " materialized table,\n" + " columns (id (type INTEGER)),\n" + " dynamic_columns (\n" + " status_amounts (\n" + " pivot_column status,\n" + " value_column amount,\n" + " aggregate MAX,\n" + " type DOUBLE\n" + " )\n" + " ),\n" + ), + body=f'PIVOT __ref("{pivot_input}")\nON status\nUSING MAX(amount)\nGROUP BY id\n', + ) + + def _quoted_identifiers(self) -> None: + self._features.add("quoted_identifiers") + base: ModelPlan = self._base() + self._model( + name=f"quoted_{base.name}", + description=f"Quoted identifiers over {base.name}", + header="", + body=( + 'SELECT "ID" AS "OrderKey", "Amount", "Status" AS "Status"\n' + f'FROM __ref("{base.name}")\n' + ), + ) + + def _metadata_checks(self) -> None: + self._features.add("metadata_checks") + base: ModelPlan = self._base() + self._files[ + f"{GENERATOR_ANALYSIS_FOLDER}/_sqlbuild/_audits/generic/expression_is_true.sql" + ] = 'AUDIT ();\n\nSELECT *\nFROM __ref("@model")\nWHERE NOT (@expression)\n' + self._model( + name=f"checked_{base.name}", + description=f"Metadata-checked copy of {base.name}", + header=( + " materialized table,\n" + " unique_key [id],\n" + " audits [\n" + ' expression_is_true (expression "amount >= -1000", severity warn),\n' + " ],\n" + " columns (\n" + f' id (audits [relationships (to __ref("{base.name}"), field id)]),\n' + " status (audits [accepted_values (values " + '["placed", "expédié", "取消", "returned ✓"], severity warn)]),\n' + " ),\n" + ), + body=f'SELECT id, amount, status\nFROM __ref("{base.name}")\n', + ) + + def _sql_test_mocks(self) -> None: + self._features.add("sql_test_mocks") + base: ModelPlan = self._base() + region: str = self._random.choice(("north", "south", "east")) + self._files[f"{GENERATOR_ANALYSIS_FOLDER}/_sqlbuild/_macros/regions.py"] = ( + "def analysis_region() -> str:\n" + ' """Return the region rows are assigned to."""\n' + f" return \"'{region}'\"\n" + ) + model: str = f"regional_{base.name}" + self._model( + name=model, + description=f"Regional copy of {base.name}", + header="", + body=f'SELECT id, amount, @analysis_region() AS region\nFROM __ref("{base.name}")\n', + ) + self._files[f"{GENERATOR_ANALYSIS_TEST_FOLDER}/test_{model}.sql"] = ( + "TEST();\n\nWITH\n" + "__macro__analysis_region AS (\n SELECT '''west'''\n),\n" + f"__ref__{base.name} AS (\n" + " SELECT 1 AS id, CAST(4 AS DOUBLE) AS amount, 'placed' AS status,\n" + " TIMESTAMP '2026-01-01 00:00:00' AS created_at\n" + "),\n" + f"__expected__{model} AS (\n" + " SELECT 1 AS id, CAST(4 AS DOUBLE) AS amount, 'west' AS region\n" + ")\nSELECT 1\n" + ) + + def _analysis_opt_out(self) -> None: + self._features.add("analysis_opt_out") + base: ModelPlan = self._base() + self._model( + name=f"unanalysed_{base.name}", + description=f"Copy of {base.name} that SQL analysis cannot parse", + header=" sql_analysis false,\n", + body=f'SELECT id, amount\nFROM __ref("{base.name}")\nWHERE id = ANY [1, 2]\n', + ) + self.settings_lines.append("require_sql_analysis = true") + + def _python_sql(self) -> None: + self._features.add("python_sql") + self._files["python/tasks/flag_exports.py"] = ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\n" + "def count_flagged_events(ctx: TaskContext):\n" + ' """Count externally flagged events."""\n' + ' ctx.query("SELECT COUNT(*) FROM main.external_event_flags")\n' + " return ctx.result()\n" + ) + + def _analysis_modes(self) -> None: + self._features.add("analysis_modes") + base: ModelPlan = self._base() + self.commands = tuple( + _selected_command(label=label, arguments=arguments, model=base.name) + for label, arguments in GENERATOR_ANALYSIS_MODE_COMMANDS + ) + + +def _selected_command(*, label: str, arguments: tuple[str, ...], model: str) -> DifferentialCommand: + """Return one analysis-mode command with the selected model substituted.""" + + return DifferentialCommand( + label=label, + arguments=tuple( + model if argument == GENERATOR_SELECTED_MODEL else argument for argument in arguments + ), + ) diff --git a/scripts/compiler_differential/classes/project_builder.py b/scripts/compiler_differential/classes/project_builder.py index 03b776ddbf..6827d59e50 100644 --- a/scripts/compiler_differential/classes/project_builder.py +++ b/scripts/compiler_differential/classes/project_builder.py @@ -6,6 +6,7 @@ from collections.abc import Callable from dataclasses import replace +from scripts.compiler_differential.classes.analysis_features import AnalysisFeatureWriter from scripts.compiler_differential.classes.discovery_features import ( DiscoveryFeatureWriter, feature_blocks_for_seed, @@ -15,6 +16,8 @@ DUCKDB_ADAPTER, GENERATOR_CONSTANT_KIND, GENERATOR_CROSS_DOMAIN_SHARE, + GENERATOR_DIALECT_CONNECTIONS, + GENERATOR_DIALECT_TARGETS, GENERATOR_DOMAINS, GENERATOR_ENUM_KIND, GENERATOR_FLOAT_VALUES, @@ -44,9 +47,12 @@ class ProjectBuilder: """Accumulate the files of one generated project from a seeded random source.""" - def __init__(self, seed: int, *, blocks: tuple[str, ...] | None = None) -> None: + def __init__( + self, seed: int, *, blocks: tuple[str, ...] | None = None, dialect: str | None = None + ) -> None: self._seed: int = seed self._forced_blocks: tuple[str, ...] | None = blocks + self._dialect: str | None = dialect self._random: random.Random = random.Random(seed) self._files: dict[str, str] = {} self._features: set[str] = set() @@ -80,6 +86,7 @@ def build(self) -> GeneratedProject: self._random.choice(sorted(GENERATOR_INVALID_CODES)) if self._random.random() < GENERATOR_INVALID_SHARE and not set(blocks) & set(GENERATOR_RARE_FEATURE_BLOCKS) + and self._dialect is None else None ) self._write_declarations() @@ -107,10 +114,20 @@ def build(self) -> GeneratedProject: label=self._label, ) rendering.write() + analysis: AnalysisFeatureWriter = AnalysisFeatureWriter( + blocks=blocks, + rng=self._random, + files=self._files, + features=self._features, + staging=[model for model in self._models if model.layer == _STAGING], + sources=self._sources, + ) + analysis.write() self._write_config( domains=domains, adapter=extras.adapter, - extra_lines=[*extras.config_lines, *rendering.config_lines], + extra_lines=[*extras.config_lines, *rendering.config_lines, *analysis.config_lines], + settings_lines=analysis.settings_lines, templated_target=rendering.templated_target, ) if invalid_kind is not None: @@ -125,6 +142,7 @@ def build(self) -> GeneratedProject: ), features=tuple(sorted(self._features)), succeeding_commands=extras.succeeding_commands, + extra_commands=analysis.commands, ) def _target_schema(self, *, schema: str, templated: bool) -> str: @@ -144,15 +162,18 @@ def _write_config( domains: list[str], adapter: str | None, extra_lines: list[str], + settings_lines: list[str], templated_target: bool, ) -> None: lines: list[str] = [ f'name = "generated_{self._seed}"', - f'adapter = "{adapter or DUCKDB_ADAPTER}"', + f'adapter = "{self._dialect or adapter or DUCKDB_ADAPTER}"', 'default_target = "dev"', "", "[connection]", - 'database = "generated.duckdb"', + *GENERATOR_DIALECT_CONNECTIONS.get( + self._dialect or "", ('database = "generated.duckdb"',) + ), "", "[vars]", f'region = "{self._random.choice(("north", "south", "east"))}"', @@ -165,9 +186,11 @@ def _write_config( schema=self._random.choice(("analytics", "main")), templated=templated_target ) ), + *GENERATOR_DIALECT_TARGETS.get(self._dialect or "", ()), "", "[settings]", f'default_audit_severity = "{self._random.choice(("warn", "error"))}"', + *settings_lines, "", "[defaults]", f'materialized = "{self._random.choice(("table", "view"))}"', diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index de07f6da70..d67572e0b5 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -7,11 +7,15 @@ ENGINE_ENV_VAR: str = "SQLBUILD_COMPILER_ENGINE" STAGE_CAPTURE_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +ANALYSIS_RECORD_ENV_VAR: str = "SQLBUILD_ANALYSIS_RECORD_DIR" +WHEEL_SITE_RECORD_PREFIX: str = "polyglot-sites-" +ANALYSIS_DEFERRAL_RECORD_PREFIX: str = "analysis-deferrals-" +RECORDS_DIRECTORY: str = "records" ENGINE_NAMES: tuple[str, ...] = ("python", "native", "native-preview") DEFAULT_ENGINES: tuple[str, str] = ("python", "native-preview") SQB_ENTRY: str = "import sys; from sqlbuild.cli.entry.main.entry import main; sys.exit(main())" EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset( - {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR} + {"VIRTUAL_ENV", ENGINE_ENV_VAR, STAGE_CAPTURE_ENV_VAR, ANALYSIS_RECORD_ENV_VAR} ) EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" PROJECT_CONFIG_FILE: str = "sqlbuild_project.toml" @@ -198,6 +202,18 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "resource_audits", "model_config", ) +GENERATOR_ANALYSIS_BLOCKS: tuple[str, ...] = ( + "column_shapes", + "contract_chain", + "udf_signatures", + "dynamic_columns", + "quoted_identifiers", + "metadata_checks", + "sql_test_mocks", + "analysis_opt_out", + "python_sql", + "analysis_modes", +) GENERATOR_FEATURE_BLOCKS: tuple[str, ...] = ( "incremental_append", "incremental_delete_insert", @@ -227,6 +243,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "dbt_ref", "line_endings", *GENERATOR_RENDER_BLOCKS, + *GENERATOR_ANALYSIS_BLOCKS, ) GENERATOR_FEATURE_STRIDE: int = 4 GENERATOR_OPTIONAL_FEATURE_SHARE: float = 0.2 @@ -236,6 +253,26 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: GENERATOR_FEATURE_FOLDER: str = "models/features" GENERATOR_RENDER_FOLDER: str = "models/rendering" GENERATOR_RENDER_TEST_FOLDER: str = "tests/unit/rendering" +GENERATOR_ANALYSIS_FOLDER: str = "models/analysis" +GENERATOR_ANALYSIS_TEST_FOLDER: str = "tests/unit/analysis" +GENERATOR_OPEN_SOURCE: str = "open_events" +GENERATOR_DIALECT_CONNECTIONS: dict[str, tuple[str, ...]] = { + "postgres": ('host = "localhost"', 'database = "generated"'), + "snowflake": ('account = "example"', 'user = "builder"', 'database = "GENERATED"'), +} +GENERATOR_DIALECT_TARGETS: dict[str, tuple[str, ...]] = {"snowflake": ('database = "GENERATED"',)} +GENERATOR_DIALECT_EXCLUDED_BLOCKS: frozenset[str] = frozenset( + {"dynamic_columns", "quoted_identifiers", "analysis_modes"} +) +GENERATOR_DIALECT_BLOCKS: tuple[str, ...] = tuple( + block for block in GENERATOR_ANALYSIS_BLOCKS if block not in GENERATOR_DIALECT_EXCLUDED_BLOCKS +) +GENERATOR_SELECTED_MODEL: str = "{model}" +GENERATOR_ANALYSIS_MODE_COMMANDS: tuple[tuple[str, tuple[str, ...]], ...] = ( + ("compile-select", ("compile", "--json", "--select", GENERATOR_SELECTED_MODEL)), + ("compile-no-analysis", ("compile", "--json", "--no-sql-analysis")), + ("compile-rich-lineage", ("compile", "--json", "--lineage-mode", "rich")), +) GENERATOR_CHANNEL_ENV_VAR: str = "SQB_DIFFERENTIAL_CHANNEL" GENERATOR_MISSING_ENV_VAR: str = "SQB_DIFFERENTIAL_UNSET" GENERATOR_ENVIRONMENT: dict[str, str] = {GENERATOR_CHANNEL_ENV_VAR: "web-orders"} @@ -506,3 +543,105 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "S022": _TOLERANT_SCOPE_ONLY, "S023": _TOLERANT_SCOPE_ONLY, } +ANALYSIS_STAGE_CAPTURE_SUFFIX: str = "-compiled_project.json" +COMPILED_PROJECT_CAPTURE_TYPE: str = "sqlbuild.compiler.compile.models:CompiledProject" +ANALYSIS_NON_COLLECTION_FIELDS: frozenset[str] = frozenset( + { + "run_id", + "effective_target_name", + "effective_connection", + "effective_vars", + "binding_catalog", + "effective_target_database", + "effective_target_schema", + "sql_analysis_dialect", + "sql_lexical_syntax", + "compile_cache_dir", + "settings", + "scenario", + "enforce_explicit_references", + "diagnostics", + "external_sql_reference_resolver", + "scope_index", + "native_session", + } +) +ANALYSIS_CALLABLE_CAPTURE_FIELDS: frozenset[str] = CALLABLE_CAPTURE_FIELDS +ANALYSIS_OPAQUE_CAPTURE_VALUES: re.Pattern[str] = re.compile( + r'\{"__opaque__":\s*"sqlbuild\._native:ProjectCatalog"\}' +) +ANALYSIS_EXPLICIT_WRITE_SCHEMA_DIALECTS: frozenset[str] = frozenset({"snowflake"}) +ANALYSIS_DIALECT_VARIANTS: tuple[str, ...] = ("postgres", "snowflake") +ANALYSIS_UPSTREAM_CHAIN_HOPS: int = 3 +ANALYSIS_DETAIL_KINDS: tuple[str, ...] = ( + "typed_column", + "untyped_column", + "star_over_complete_input", + "star_over_published_shape", + "star_unresolved", + "set_operation", + "cte", + "cte_passthrough_type", + "quoted_identifier_case", + "dynamic_column_proof", + "cursor_intrinsic", + "runtime_placeholder", + "batch_binding", + "dataflow_binding", + "select_limited_analysis", + "sql_analysis_disabled", + "upstream_chain", + "sized_contract_shape", + "contract_nullability_shape", + "source_expression_shape", + "seed_column_types", + "scalar_udf", + "table_function", + "udf_argument_types", + "sql_analysis_opt_out", + "inline_sql_hook", + "named_sql_hook", + "hook_list", + "explicit_promotion_mode", + "python_sql_literal_relation", + "managed_write_schema", + "accepted_values_audit", + "relationships_audit", + "expression_audit", + "metadata_column_reference", + "sql_test_expected_columns", + "sql_test_macro_mock", + "dialect_duckdb", + "dialect_postgresql", + "dialect_snowflake", +) +ANALYSIS_INDIRECT_KINDS: dict[str, str] = { + "managed_write_schema": ( + "every destination resolved an explicit schema under an adapter that requires one; " + "the capture does not record the write-schema check itself" + ), +} +ANALYSIS_CODE_SCAN_ROOTS: tuple[str, ...] = ( + "compile/_helpers/assembly", + "compile/_helpers/analysis", + "compile/_helpers/attachment/model_config.py", + "compile/_helpers/diagnostics", + "contracts", + "pipeline/_helpers/target_validation.py", + "sql_analysis", +) +_NEVER_REPORTED_BY_POLYGLOT: str = ( + "polyglot defines the finding but never reports it, and no SQLBuild check maps to it" +) +ANALYSIS_UNREACHABLE_CODES: dict[str, str] = { + "B000": ( + "the binding layer's internal unknown-relation code; compile drops it because an " + "unknown relation is an open input" + ), + "B210": _NEVER_REPORTED_BY_POLYGLOT, + "B214": ( + "polyglot reports assignment types only for INSERT and UPDATE statements; compiled " + "models, tests and audits are queries" + ), + "B219": _NEVER_REPORTED_BY_POLYGLOT, +} diff --git a/scripts/compiler_differential/main/differential.py b/scripts/compiler_differential/main/differential.py index 314fc3ceab..446b25b09e 100644 --- a/scripts/compiler_differential/main/differential.py +++ b/scripts/compiler_differential/main/differential.py @@ -3,16 +3,23 @@ from __future__ import annotations import argparse +import json import os import sys import tempfile import time from pathlib import Path +from scripts.compiler_differential._helpers.coverage.analysis import required_analysis_kinds from scripts.compiler_differential._helpers.coverage.discovery import required_discovery_kinds from scripts.compiler_differential._helpers.coverage.render import required_render_kinds from scripts.compiler_differential._helpers.running.options import parse_expected_outcome +from scripts.compiler_differential._helpers.running.records import ( + format_wheel_site_report, + wheel_site_report, +) from scripts.compiler_differential._helpers.running.report import ( + format_analysis_coverage, format_discovery_coverage, format_render_coverage, format_summary, @@ -52,6 +59,15 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: if options.stage_captures and CORPUS_SEEDS in args.corpus else {} ) + if args.wheel_site_report is not None: + report: dict[str, object] = wheel_site_report( + comparisons=comparisons, engines=options.engines + ) + args.wheel_site_report.parent.mkdir(parents=True, exist_ok=True) + _ = args.wheel_site_report.write_text( + json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + print(format_wheel_site_report(report)) print( format_summary( comparisons=comparisons, @@ -71,7 +87,7 @@ def run_compiler_differential(argv: list[str] | None = None) -> int: def _seed_coverage( *, comparisons: list[ProjectComparison], options: DifferentialOptions ) -> dict[str, tuple[str, ...]]: - """Print discovery and render coverage; return the missing kinds each requirement enforces.""" + """Print each stage's coverage; return the missing kinds each requirement enforces.""" discovered: frozenset[str] = frozenset().union( *(comparison.discovered_kinds for comparison in comparisons) @@ -80,14 +96,21 @@ def _seed_coverage( *(comparison.rendered_kinds for comparison in comparisons) ) required_discovery: tuple[str, ...] = required_discovery_kinds() + analysed: frozenset[str] = frozenset().union( + *(comparison.analysed_kinds for comparison in comparisons) + ) required_render: tuple[str, ...] = required_render_kinds() + required_analysis: tuple[str, ...] = required_analysis_kinds() print(format_discovery_coverage(covered=discovered, required=required_discovery)) print(format_render_coverage(covered=rendered, required=required_render)) + print(format_analysis_coverage(covered=analysed, required=required_analysis)) missing: dict[str, tuple[str, ...]] = {} if options.require_discovery_coverage: missing["discovery"] = tuple(kind for kind in required_discovery if kind not in discovered) if options.require_render_coverage: missing["render"] = tuple(kind for kind in required_render if kind not in rendered) + if options.require_analysis_coverage: + missing["analysis"] = tuple(kind for kind in required_analysis if kind not in analysed) return missing @@ -141,6 +164,21 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="fail unless the seed corpus exercises every render input kind (needs captures)", ) + parser.add_argument( + "--require-analysis-coverage", + action="store_true", + help="fail unless the seed corpus exercises every analysis input kind (needs captures)", + ) + parser.add_argument( + "--wheel-site-report", + type=Path, + default=None, + metavar="PATH", + help=( + "record Polyglot wheel calls by calling site and analysis deferrals in every engine " + "process, print them and write them to PATH as JSON (reported, never gated)" + ), + ) parser.add_argument( "--engine-env", action="append", @@ -154,6 +192,7 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: for flag, required in ( ("--require-discovery-coverage", args.require_discovery_coverage), ("--require-render-coverage", args.require_render_coverage), + ("--require-analysis-coverage", args.require_analysis_coverage), ): if required and not (args.stage_captures and CORPUS_SEEDS in args.corpus): parser.error(f"{flag} needs --stage-captures and the seeds corpus") diff --git a/scripts/compiler_differential/main/generate_project.py b/scripts/compiler_differential/main/generate_project.py index 76f581b12a..15587490e3 100644 --- a/scripts/compiler_differential/main/generate_project.py +++ b/scripts/compiler_differential/main/generate_project.py @@ -4,7 +4,9 @@ from scripts.compiler_differential.models import GeneratedProject -def generate_project(*, seed: int, blocks: tuple[str, ...] | None = None) -> GeneratedProject: - """Return the seed's deterministic project; `blocks` overrides its optional feature blocks.""" +def generate_project( + *, seed: int, blocks: tuple[str, ...] | None = None, dialect: str | None = None +) -> GeneratedProject: + """Return the seed's project; `blocks` overrides its blocks, `dialect` its adapter.""" - return ProjectBuilder(seed, blocks=blocks).build() + return ProjectBuilder(seed, blocks=blocks, dialect=dialect).build() diff --git a/scripts/compiler_differential/models.py b/scripts/compiler_differential/models.py index 3af73b087b..0a9cd20d14 100644 --- a/scripts/compiler_differential/models.py +++ b/scripts/compiler_differential/models.py @@ -49,6 +49,14 @@ class CommandOutcome: timed_out: bool = False +@dataclass(frozen=True) +class AnalysisRecords: + """Wheel calls by `(site, api)` and analysis deferrals by `(kind, site)` for one run.""" + + wheel_sites: dict[tuple[str, str], int] = field(default_factory=dict) + deferrals: dict[tuple[str, str], int] = field(default_factory=dict) + + @dataclass(frozen=True) class EngineRun: """Everything one engine produced for one corpus project.""" @@ -59,6 +67,7 @@ class EngineRun: manifest: str | None dag: str | None captures: dict[str, dict[str, Path]] = field(default_factory=dict) + records: AnalysisRecords | None = None @dataclass(frozen=True) @@ -91,6 +100,8 @@ class ProjectComparison: seconds: float discovered_kinds: frozenset[str] = frozenset() rendered_kinds: frozenset[str] = frozenset() + analysed_kinds: frozenset[str] = frozenset() + records: tuple[AnalysisRecords | None, AnalysisRecords | None] | None = None @dataclass(frozen=True) @@ -105,6 +116,8 @@ class DifferentialOptions: engine_environment: dict[str, dict[str, str]] require_discovery_coverage: bool = False require_render_coverage: bool = False + require_analysis_coverage: bool = False + analysis_records: bool = False @dataclass(frozen=True) @@ -130,6 +143,7 @@ class GeneratedProject(WritableProject): expected_error_code: str | None features: tuple[str, ...] succeeding_commands: tuple[str, ...] = () + extra_commands: tuple[DifferentialCommand, ...] = () @dataclass(frozen=True) @@ -140,6 +154,10 @@ class FailureCase(WritableProject): expected_code: str expected_warning_code: str | None = None expected_message: str | None = None + expected_help: str | None = None + expected_notes: tuple[str, ...] = () + expected_location: tuple[int, int] | None = None + expected_codes: tuple[str, ...] | None = None @dataclass(frozen=True) @@ -177,6 +195,10 @@ class EmittedCodes: errors: tuple[str, ...] warnings: tuple[str, ...] first_error_message: str | None = None + first_error_help: str | None = None + first_error_notes: tuple[str, ...] = () + first_error_location: tuple[int, int] | None = None + codes: tuple[str, ...] = () @property def first_error(self) -> str | None: diff --git a/src/sqlbuild/cli/compile_reuse/constants.py b/src/sqlbuild/cli/compile_reuse/constants.py index 0f9e81a3b0..2bd4a76bac 100644 --- a/src/sqlbuild/cli/compile_reuse/constants.py +++ b/src/sqlbuild/cli/compile_reuse/constants.py @@ -8,9 +8,11 @@ CompileArtifactWriteRegistry, ) from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR, COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR, ) +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR REUSE_FORMAT_VERSION: int = 1 REUSE_ENTRY_DIRECTORY_NAME: str = "project-reuse-v1" @@ -25,12 +27,17 @@ REUSE_ENTRY_BYTE_ORDER: Literal["big"] = "big" REUSE_MAX_ENTRY_BYTES: int = 512 * 1024 * 1024 REUSE_MAX_STORED_ENTRIES: int = 8 -REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" +REUSE_DISABLE_ENV_VAR: str = COMPILE_REUSE_DISABLE_ENV_VAR REUSE_DISABLE_VALUE: str = "1" REUSE_LOGGER_NAME: str = "sqlbuild.compile.reuse" TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} + { + ANALYSIS_RECORD_DIR_ENV_VAR, + COMPILE_REUSE_DISABLE_ENV_VAR, + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, + } ) MISSING_ENVIRONMENT_VALUE: str = "\0missing" MISSING_FILE_DIGEST: str = "\0missing" diff --git a/src/sqlbuild/compiler/frontier/constants.py b/src/sqlbuild/compiler/frontier/constants.py index b10c7aab0c..4e0ddf0b0e 100644 --- a/src/sqlbuild/compiler/frontier/constants.py +++ b/src/sqlbuild/compiler/frontier/constants.py @@ -34,6 +34,7 @@ CACHE_DIRECTORY_NAME: str = "cache" COMPILER_CACHE_DIRECTORY_NAME: str = "compiler" STAGE_CAPTURE_DIR_ENV_VAR: str = "SQLBUILD_COMPILER_STAGE_CAPTURE_DIR" +COMPILE_REUSE_DISABLE_ENV_VAR: str = "SQLBUILD_DISABLE_COMPILE_REUSE" STAGE_CAPTURE_SUFFIX: str = ".json" STAGE_CAPTURE_SHARED_MARKER: str = "__shared__" STAGE_CAPTURE_ROOT_KEY: str = "__capture__" @@ -57,6 +58,7 @@ ), "sqlbuild.compiler.discovery.models:DiscoveredProjectInputs": frozenset({"native_session"}), "sqlbuild.compiler.compile.models:DeclarationScopeResolver": frozenset({"native_contexts"}), + "sqlbuild.compiler.compile.models:CompiledProject": frozenset({"native_session"}), } STAGE_CAPTURE_DECODED_SEQUENCES: frozenset[str] = frozenset( {"sqlbuild.compiler.compile.models:CompactLineageFacts"} diff --git a/src/sqlbuild/compiler/macro_bridge/constants.py b/src/sqlbuild/compiler/macro_bridge/constants.py index 50b84b4929..af15940fe6 100644 --- a/src/sqlbuild/compiler/macro_bridge/constants.py +++ b/src/sqlbuild/compiler/macro_bridge/constants.py @@ -5,8 +5,13 @@ import itertools from contextvars import ContextVar -from sqlbuild.compiler.frontier.constants import COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR +from sqlbuild.compiler.frontier.constants import ( + COMPILE_REUSE_DISABLE_ENV_VAR, + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, +) from sqlbuild.compiler.references.types import SqlReferenceKind +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR MACRO_USE_EVENT: int = 0 DECLARATION_READ_EVENT: int = 1 @@ -31,5 +36,10 @@ INTERPRETER_MODULE_ORIGINS: frozenset[str] = frozenset({"built-in", "frozen"}) STORE_TRACKED_ENVIRONMENT_PREFIXES: tuple[str, ...] = ("SQLBUILD_", "SQB_") STORE_UNTRACKED_ENVIRONMENT_NAMES: frozenset[str] = frozenset( - {COMPILER_ENGINE_ENV_VAR, STAGE_CAPTURE_DIR_ENV_VAR} + { + ANALYSIS_RECORD_DIR_ENV_VAR, + COMPILE_REUSE_DISABLE_ENV_VAR, + COMPILER_ENGINE_ENV_VAR, + STAGE_CAPTURE_DIR_ENV_VAR, + } ) diff --git a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py index 2252e43013..d6afc038ed 100644 --- a/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py +++ b/src/sqlbuild/compiler/sql_analysis/_helpers/polyglot.py @@ -2,11 +2,20 @@ from __future__ import annotations +import atexit +import inspect +import json +import os +import sys +import threading +from collections import Counter +from pathlib import Path from typing import Any import polyglot_sql from sqlbuild.compiler.sql_analysis.constants import ( + ANALYSIS_RECORD_DIR_ENV_VAR, POLYGLOT_ANALYZE_QUERY_ENTRY_POINT, POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_COUNT, POLYGLOT_ANALYZE_QUERY_OPTIONS_ARGUMENT_INDEX, @@ -14,6 +23,7 @@ POLYGLOT_COMPLEXITY_GUARD_PARAMETER, POLYGLOT_MAX_FUNCTION_CALL_DEPTH, POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION, + POLYGLOT_SITE_RECORD_PREFIX, ) _COMPLEXITY_GUARD: dict[str, int] = { @@ -57,19 +67,81 @@ def _has_explicit_options_guard(self, *, args: tuple[Any, ...], kwargs: dict[str return isinstance(options, dict) and POLYGLOT_COMPLEXITY_GUARD_OPTION in options +class _SiteRecorder: + """Debug-only count of Polyglot wheel calls by calling `file:function`, written at exit.""" + + def __init__(self, *, directory: Path) -> None: + self._directory: Path = directory + self._package_root: Path = Path(__file__).resolve().parents[3] + self._calls: Counter[tuple[str, str]] = Counter() + self._lock: threading.Lock = threading.Lock() + _ = atexit.register(self._write) + + @classmethod + def from_environment(cls) -> _SiteRecorder | None: + """Return a recorder when the debug record directory is set, otherwise None.""" + + directory: str | None = os.environ.get(ANALYSIS_RECORD_DIR_ENV_VAR) + return cls(directory=Path(directory)) if directory else None + + def record(self, *, api: str, caller: Any) -> None: + code: Any = caller.f_code + path: Path = Path(code.co_filename).resolve() + site: str = ( + path.relative_to(self._package_root).as_posix() + if path.is_relative_to(self._package_root) + else path.name + ) + with self._lock: + self._calls[(f"{site}:{code.co_qualname}", api)] += 1 + + def _write(self) -> None: + with self._lock: + calls: Counter[tuple[str, str]] = self._calls.copy() + if not calls: + return + self._directory.mkdir(parents=True, exist_ok=True) + rows: list[list[object]] = [ + [site, api, count] for (site, api), count in sorted(calls.items()) + ] + target: Path = self._directory / f"{POLYGLOT_SITE_RECORD_PREFIX}{os.getpid()}.json" + _ = target.write_text(json.dumps({"calls": rows}) + "\n", encoding="utf-8") + + +class _RecordedEntryPoint: + """One Polyglot callable that records its calling site before running.""" + + def __init__(self, *, name: str, function: Any, recorder: _SiteRecorder) -> None: + self._name: str = name + self._function: Any = function + self._recorder: _SiteRecorder = recorder + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + self._recorder.record(api=self._name, caller=sys._getframe(1)) + return self._function(*args, **kwargs) + + class _PolyglotSqlModule: """Proxy Polyglot parsing through SQLBuild's trusted-SQL policy.""" + def __init__(self, *, recorder: _SiteRecorder | None) -> None: + self._recorder: _SiteRecorder | None = recorder + def __getattr__(self, name: str) -> Any: attribute: Any = getattr(polyglot_sql, name) - return ( + entry_point: Any = ( _GuardedEntryPoint(name=name, function=attribute) if name in _GUARDED_ENTRY_POINTS else attribute ) + if self._recorder is None or not callable(attribute) or inspect.isclass(attribute): + return entry_point + return _RecordedEntryPoint(name=name, function=entry_point, recorder=self._recorder) -_POLYGLOT_SQL_MODULE: _PolyglotSqlModule = _PolyglotSqlModule() +_POLYGLOT_SQL_MODULE: _PolyglotSqlModule = _PolyglotSqlModule( + recorder=_SiteRecorder.from_environment() +) def import_polyglot() -> Any: diff --git a/src/sqlbuild/compiler/sql_analysis/constants.py b/src/sqlbuild/compiler/sql_analysis/constants.py index 87feca62c3..d40fe049bf 100644 --- a/src/sqlbuild/compiler/sql_analysis/constants.py +++ b/src/sqlbuild/compiler/sql_analysis/constants.py @@ -31,6 +31,8 @@ POLYGLOT_COMPLEXITY_GUARD_OPTION: str = "complexityGuard" POLYGLOT_MAX_FUNCTION_CALL_DEPTH_OPTION: str = "maxFunctionCallDepth" POLYGLOT_MAX_FUNCTION_CALL_DEPTH: int = 512 +ANALYSIS_RECORD_DIR_ENV_VAR: str = "SQLBUILD_ANALYSIS_RECORD_DIR" +POLYGLOT_SITE_RECORD_PREFIX: str = "polyglot-sites-" BINDING_UNKNOWN_TABLE_INTERNAL_CODE: str = "B000" POLYGLOT_KIND_ALIAS: str = "alias" diff --git a/tests/e2e/scripts/compiler_differential/_test_types.py b/tests/e2e/scripts/compiler_differential/_test_types.py index 9c434d9283..e488fe198a 100644 --- a/tests/e2e/scripts/compiler_differential/_test_types.py +++ b/tests/e2e/scripts/compiler_differential/_test_types.py @@ -61,6 +61,12 @@ class FailureCorpusCodesTestCase: expected_discovery_codes: frozenset[str] expected_render_codes: frozenset[str] unreachable_render_codes: frozenset[str] + expected_first_helps: dict[str, str] + expected_first_notes: dict[str, frozenset[str]] + expected_first_locations: dict[str, tuple[int, int]] + expected_code_orders: dict[str, tuple[str, ...]] + expected_analysis_codes: frozenset[str] + unreachable_analysis_codes: frozenset[str] @dataclass(frozen=True) @@ -83,3 +89,28 @@ class RenderCaptureTestCase: blocks: tuple[str, ...] command: tuple[str, ...] expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class AnalysisCaptureTestCase: + """A generated project, the commands it runs, and the analysis kinds their captures prove.""" + + description: str + seed: int + blocks: tuple[str, ...] + dialect: str | None + commands: tuple[tuple[str, ...], ...] + generated_command_count: int + expected_kinds: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class WheelSiteReportTestCase: + """A harness run that records wheel sites, optionally with an injected native deferral.""" + + description: str + perturbation: str + expected_lines: tuple[str, ...] + expected_sites: frozenset[str] + expected_deferrals: frozenset[str] diff --git a/tests/e2e/scripts/compiler_differential/helpers.py b/tests/e2e/scripts/compiler_differential/helpers.py index 61cb287cc4..dff1c5eb7f 100644 --- a/tests/e2e/scripts/compiler_differential/helpers.py +++ b/tests/e2e/scripts/compiler_differential/helpers.py @@ -73,6 +73,18 @@ def _perturbed(*, until, **stages): frontier.native_frontier = _perturbed """ +DEFERRAL_PERTURBATION: str = """ +import json +import os +from pathlib import Path + +_directory = os.environ.get("SQLBUILD_ANALYSIS_RECORD_DIR") +if _directory: + Path(_directory).mkdir(parents=True, exist_ok=True) + with open(Path(_directory) / f"analysis-deferrals-{os.getpid()}.jsonl", "a") as _record: + _record.write(json.dumps({"kind": "legacy_fallback", "site": "orders.sql"}) + "\\n") +""" + _BROKEN_REF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "broken_orders"\nadapter = "duckdb"\n\n' diff --git a/tests/e2e/scripts/compiler_differential/test_analysis_capture.py b/tests/e2e/scripts/compiler_differential/test_analysis_capture.py new file mode 100644 index 0000000000..0cd6b1209a --- /dev/null +++ b/tests/e2e/scripts/compiler_differential/test_analysis_capture.py @@ -0,0 +1,114 @@ +"""Real compiles capture every compiled-project field canonically and prove every analysis kind.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.coverage.analysis import ( + analysed_input_kinds, + analysis_capture_problems, + required_analysis_kinds, +) +from scripts.compiler_differential._helpers.running.execution import harness_environment +from scripts.compiler_differential.constants import ( + ANALYSIS_STAGE_CAPTURE_SUFFIX, + GENERATOR_DIALECT_BLOCKS, + GENERATOR_FEATURE_BLOCKS, + SQB_ENTRY, + STAGE_CAPTURE_ENV_VAR, +) +from scripts.compiler_differential.main.generate_project import generate_project +from scripts.compiler_differential.models import GeneratedProject +from tests.e2e.scripts.compiler_differential._test_types import AnalysisCaptureTestCase + +# dbt_ref fails compile without its manifest; macro_generated_reference turns off the explicit +# references that Python SQL relation checks need. +_CONFLICTING_BLOCKS: frozenset[str] = frozenset({"dbt_ref", "macro_generated_reference"}) +_OTHER_DIALECT_KINDS: frozenset[str] = frozenset( + {"dialect_postgresql", "dialect_snowflake", "managed_write_schema"} +) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCaptureTestCase( + description="every_compiling_feature_block_in_every_analysis_mode", + seed=11, + blocks=tuple( + block for block in GENERATOR_FEATURE_BLOCKS if block not in _CONFLICTING_BLOCKS + ), + dialect=None, + commands=(("compile", "--json"), ("plan", "--json")), + generated_command_count=3, + expected_kinds=frozenset(required_analysis_kinds()) - _OTHER_DIALECT_KINDS, + expected_absent=_OTHER_DIALECT_KINDS, + ), + AnalysisCaptureTestCase( + description="analysis_blocks_compiled_under_snowflake", + seed=11, + blocks=GENERATOR_DIALECT_BLOCKS, + dialect="snowflake", + commands=(("compile", "--json"),), + generated_command_count=0, + expected_kinds=frozenset( + {"dialect_snowflake", "managed_write_schema", "upstream_chain", "typed_column"} + ), + expected_absent=frozenset({"dialect_duckdb", "dialect_postgresql"}), + ), + AnalysisCaptureTestCase( + description="analysis_blocks_compiled_under_postgres", + seed=11, + blocks=GENERATOR_DIALECT_BLOCKS, + dialect="postgres", + commands=(("compile", "--json"),), + generated_command_count=0, + expected_kinds=frozenset({"dialect_postgresql", "sized_contract_shape", "scalar_udf"}), + expected_absent=frozenset({"dialect_duckdb", "managed_write_schema"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_project_with_every_analysis_input_when_capturing_then_capture_is_complete( + test_case: AnalysisCaptureTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + generated: GeneratedProject = generate_project( + seed=test_case.seed, blocks=test_case.blocks, dialect=test_case.dialect + ) + generated.write(project_dir) + generated_commands: tuple[tuple[str, ...], ...] = tuple( + command.arguments for command in generated.extra_commands + )[: test_case.generated_command_count] + commands: tuple[tuple[str, ...], ...] = (*test_case.commands, *generated_commands) + + kinds: set[str] = set() + for index, command in enumerate(commands): + capture_dir: Path = tmp_path / f"captures-{index}" + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", SQB_ENTRY, "--no-color", *command], + cwd=project_dir, + env={**harness_environment(), STAGE_CAPTURE_ENV_VAR: str(capture_dir)}, + capture_output=True, + text=True, + check=False, + timeout=600, + ) + assert completed.returncode == 0, (command, completed.stdout + completed.stderr) + text: str = next(capture_dir.glob(f"*{ANALYSIS_STAGE_CAPTURE_SUFFIX}")).read_text( + encoding="utf-8" + ) + assert analysis_capture_problems(text) == (), command + kinds.update(analysed_input_kinds(text)) + + assert len(generated_commands) == test_case.generated_command_count + assert test_case.expected_kinds <= kinds, test_case.expected_kinds - kinds + assert not test_case.expected_absent & kinds, test_case.expected_absent & kinds + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py index 42665b68ca..9ab1471e8e 100644 --- a/tests/e2e/scripts/compiler_differential/test_compiler_differential.py +++ b/tests/e2e/scripts/compiler_differential/test_compiler_differential.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import re import shutil from pathlib import Path @@ -14,9 +15,11 @@ HarnessRunTestCase, ProjectExpectationTestCase, SharedAnalysisSeedTestCase, + WheelSiteReportTestCase, ) from tests.e2e.scripts.compiler_differential.helpers import ( COMPILED_PROJECT_CAPTURE, + DEFERRAL_PERTURBATION, DISCOVERY_PERTURBATION, NATIVE_ONLY_STDERR_LINE, RENDER_PERTURBATION, @@ -240,7 +243,7 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f expected_exit_code=1, expected_lines=( "OK seed/0", - "Compiler differential FAILED: 0 of 1 projects differ", + "Compiler differential FAILED: 0 of 3 projects differ", "Required discovery coverage missing: ", ), expected_absent=("passed",), @@ -259,11 +262,32 @@ def test_given_both_engines_fail_alike_when_outcome_is_unexpected_then_harness_f expected_lines=( "OK seed/0", "Render coverage: ", - "Compiler differential FAILED: 0 of 1 projects differ", + "Compiler differential FAILED: 0 of 3 projects differ", "Required render coverage missing: ", ), expected_absent=("passed", "Required discovery coverage missing"), ), + CoverageFailureTestCase( + description="one_seed_cannot_cover_every_analysis_kind", + extra_arguments=( + "--corpus", + "seeds", + "--seeds", + "1", + "--stage-captures", + "--require-analysis-coverage", + ), + expected_exit_code=1, + expected_lines=( + "OK seed/0", + "OK seed/0-postgres", + "OK seed/0-snowflake", + "Analysis coverage: ", + "Compiler differential FAILED: 0 of 3 projects differ", + "Required analysis coverage missing: ", + ), + expected_absent=("passed", "Required render coverage missing"), + ), ], ids=lambda case: case.description, ) @@ -288,7 +312,7 @@ def test_given_required_coverage_missing_when_comparing_then_harness_fails_and_s seed=SHARED_ANALYSIS_SEED, expected_lines=( f"OK seed/{SHARED_ANALYSIS_SEED}", - "Compiler differential passed: 1 projects identical (python vs python)", + "Compiler differential passed: 3 projects identical (python vs python)", ), expected_capture_sides=("left-python-captures", "right-python-captures"), expected_compile_exit_code=0, @@ -374,5 +398,68 @@ def test_given_failure_expectation_when_code_is_not_the_first_error_then_harness assert all(line in output for line in test_case.expected_lines), output +@pytest.mark.parametrize( + "test_case", + [ + WheelSiteReportTestCase( + description="python_and_preview_wheel_calls_without_deferrals", + perturbation="", + expected_lines=( + "Polyglot wheel calls (python):", + "Polyglot wheel calls (native-preview):", + "Analysis deferrals (python): none recorded", + "Analysis deferrals (native-preview): none recorded", + "Compiler differential passed: 1 projects identical", + ), + expected_sites=frozenset( + { + "compiler/compile/_helpers/analysis/validation.py:" + "_validate_sql_syntax_with_message parse_one", + } + ), + expected_deferrals=frozenset(), + ), + WheelSiteReportTestCase( + description="preview_deferrals_are_counted_per_kind_and_site", + perturbation=DEFERRAL_PERTURBATION, + expected_lines=( + "Analysis deferrals (python): none recorded", + "Analysis deferrals (native-preview):", + " legacy_fallback orders.sql (project ", + ), + expected_sites=frozenset(), + expected_deferrals=frozenset({"legacy_fallback orders.sql"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_wheel_site_report_when_comparing_then_sites_and_deferrals_are_reported( + test_case: WheelSiteReportTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + report_path: Path = tmp_path / "report" / "wheel-sites.json" + exit_code: int = run_compiler_differential( + harness_arguments( + work_dir=tmp_path / "work", + extra=( + *perturbation_arguments(tmp_path / "perturbation", source=test_case.perturbation), + "--wheel-site-report", + str(report_path), + ), + ) + ) + + output: str = capsys.readouterr().out + report: dict[str, dict[str, dict[str, dict[str, int]]]] = json.loads( + report_path.read_text(encoding="utf-8") + ) + preview: dict[str, dict[str, dict[str, int]]] = report["native-preview"] + assert exit_code == 0, output + assert all(line in output for line in test_case.expected_lines), output + assert test_case.expected_sites <= set(report["python"]["wheel_sites"]), report + assert set(preview["deferrals"]) == test_case.expected_deferrals + assert all(sum(by_corpus.values()) > 0 for by_corpus in preview["deferrals"].values()) + assert report["python"]["deferrals"] == {} + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py index 8f07f996bd..172a53588d 100644 --- a/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py +++ b/tests/e2e/scripts/compiler_differential/test_failure_corpus_codes.py @@ -1,4 +1,5 @@ -"""Every failure case emits its declared code first, reaching every discovery and render code.""" +"""Every failure case emits its declared first error, reaching every discovery, render and +analysis code.""" from __future__ import annotations @@ -8,12 +9,15 @@ import pytest +import sqlbuild.compiler as compiler_package from scripts.compiler_differential._helpers.corpus.emitted_codes import ( + analysis_error_codes, discovery_error_codes, emitted_failure_codes, render_error_codes, ) from scripts.compiler_differential.constants import ( + ANALYSIS_UNREACHABLE_CODES, DISCOVERY_UNREACHABLE_CODES, RENDER_UNREACHABLE_CODES, ) @@ -21,6 +25,8 @@ from scripts.compiler_differential.models import DifferentialOptions, EmittedCodes from tests.e2e.scripts.compiler_differential._test_types import FailureCorpusCodesTestCase +_COMPILER_ROOT: Path = Path(compiler_package.__file__).parent + @pytest.mark.parametrize( "test_case", @@ -44,6 +50,29 @@ - frozenset(DISCOVERY_UNREACHABLE_CODES), expected_render_codes=render_error_codes() - frozenset(RENDER_UNREACHABLE_CODES), unreachable_render_codes=frozenset(RENDER_UNREACHABLE_CODES), + expected_first_helps={ + f"failure/{case.name}": case.expected_help + for case in failure_cases() + if case.expected_help is not None + }, + expected_first_notes={ + f"failure/{case.name}": frozenset(case.expected_notes) + for case in failure_cases() + if case.expected_notes + }, + expected_first_locations={ + f"failure/{case.name}": case.expected_location + for case in failure_cases() + if case.expected_location is not None + }, + expected_code_orders={ + f"failure/{case.name}": case.expected_codes + for case in failure_cases() + if case.expected_codes is not None + }, + expected_analysis_codes=analysis_error_codes(_COMPILER_ROOT) + - frozenset(ANALYSIS_UNREACHABLE_CODES), + unreachable_analysis_codes=frozenset(ANALYSIS_UNREACHABLE_CODES), ) ], ids=lambda case: case.description, @@ -84,6 +113,23 @@ def test_given_failure_corpus_when_compiling_then_first_errors_match_and_cover_e assert test_case.expected_render_codes <= reached, test_case.expected_render_codes - reached assert not test_case.unreachable_render_codes & set(first_errors.values()) assert test_case.unreachable_render_codes <= render_error_codes() + assert all( + fragment in (emitted[name].first_error_help or "") + for name, fragment in test_case.expected_first_helps.items() + ), {name: emitted[name].first_error_help for name in test_case.expected_first_helps} + assert all( + notes <= set(emitted[name].first_error_notes) + for name, notes in test_case.expected_first_notes.items() + ), {name: emitted[name].first_error_notes for name in test_case.expected_first_notes} + assert { + name: emitted[name].first_error_location for name in test_case.expected_first_locations + } == test_case.expected_first_locations + assert { + name: emitted[name].codes for name in test_case.expected_code_orders + } == test_case.expected_code_orders + assert test_case.expected_analysis_codes <= reached, test_case.expected_analysis_codes - reached + assert not test_case.unreachable_analysis_codes & set(first_errors.values()) + assert test_case.unreachable_analysis_codes <= analysis_error_codes(_COMPILER_ROOT) if __name__ == "__main__": diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 59ba8d67f5..7efeec91b1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3965,6 +3965,39 @@ def fact_comment(root: Path) -> None: replace_project_text(root, FACT_ORDERS_MODEL, "FROM __ref", "-- leaf\nFROM __ref") +def payment_expression_type_change(root: Path) -> None: + """Widen one column of the payments source expression, which changes its inferred shape.""" + + replace_project_text( + root, + "sources/raw.yml", + "SELECT 1 AS id, 1 AS order_id,", + "SELECT 1 AS id, CAST(1 AS BIGINT) AS order_id,", + ) + + +def completed_order_udf_signature_change(root: Path) -> None: + """Change the completed-order UDF's declared return type, which its callers' shapes read.""" + + replace_project_text( + root, + "functions/sql/udf__is_completed_order.sql", + "returns BOOLEAN,\n);\n\norder_status = 'completed'", + "returns INTEGER,\n);\n\nCAST(order_status = 'completed' AS INTEGER)", + ) + + +def order_status_nullability_change(root: Path) -> None: + """Declare one raw orders source column non-null in its schema entry.""" + + replace_project_text( + root, + "sources/raw.yml", + " - name: status\n type: VARCHAR\n", + " - name: status\n type: VARCHAR\n nullable: false\n", + ) + + def staging_column_removed(root: Path) -> None: """Drop the quantity column from staging orders, which downstream models and tests read.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py index b4100aaac1..90bd684ca7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_incremental_edits.py @@ -24,6 +24,7 @@ compile_in_process, compile_in_process_output, compiled_text, + completed_order_udf_signature_change, disable_change_detection, edit_and_compile, enable_compile_reuse, @@ -37,6 +38,8 @@ json_report_keys, model_edit_step, move_project_file, + order_status_nullability_change, + payment_expression_type_change, prime_render_store, prime_render_store_in_process, random_edit_plan, @@ -50,6 +53,7 @@ staging_header_change, staging_new_column, staging_type_change, + star_chain_added, write_external_flavor, write_generated_edit_models, write_project_file, @@ -208,6 +212,18 @@ model_edit_step("edit_after_resources", fact_comment), ), ), + IncrementalEditSequenceTestCase( + description="analysis_input_edits", + steps=( + full_edit_step("star_chain_added", star_chain_added), + model_edit_step("upstream_type_flips_downstream_shapes", staging_type_change), + model_edit_step("contract_type_edit", staging_contract_change), + full_edit_step("source_expression_edit", payment_expression_type_change), + full_edit_step("udf_signature_edit", completed_order_udf_signature_change), + full_edit_step("schema_entry_nullability_edit", order_status_nullability_change), + model_edit_step("edit_after_analysis_inputs", fact_comment), + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py index 01bd824022..daf0d72598 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_compile_reuse.py @@ -9,6 +9,7 @@ import pytest +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( CompileReuseBypassTestCase, CompileReuseConcurrentWriterTestCase, @@ -279,6 +280,11 @@ def test_given_reused_compile_when_input_changes_then_it_misses_and_matches_unca edit=lambda _root: None, edited_env={"ORDERS_UNRELATED_SETTING": "1"}, ), + CompileReuseHitTestCase( + description="analysis_record_directory", + edit=lambda _root: None, + edited_env={ANALYSIS_RECORD_DIR_ENV_VAR: "target/analysis-records"}, + ), CompileReuseHitTestCase( description="editor_settings_added", edit=lambda root: write_project_file(root, ".vscode/settings.json", "{}\n"), diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py new file mode 100644 index 0000000000..5618f65ea9 --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/__init__.py @@ -0,0 +1 @@ +"""SQL analysis integration tests.""" diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py new file mode 100644 index 0000000000..5af0584a8f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/_test_types.py @@ -0,0 +1,15 @@ +"""Test case types for SQL analysis integration tests.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SiteRecorderTestCase: + """One process calling the wheel through a product site, with or without the record dir.""" + + description: str + record_dir: str + expected_rows: tuple[tuple[str, str, int], ...] + expected_entry_point: str diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py new file mode 100644 index 0000000000..6f91e00bbe --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/helpers.py @@ -0,0 +1,18 @@ +"""Read the wheel-site records a test process wrote.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from sqlbuild.compiler.sql_analysis.constants import POLYGLOT_SITE_RECORD_PREFIX + + +def recorded_rows(root: Path) -> tuple[tuple[str, str, int], ...]: + """Return every `(site, api, calls)` row the site recorder wrote below `root`.""" + + rows: list[tuple[str, str, int]] = [] + for path in sorted(root.rglob(f"{POLYGLOT_SITE_RECORD_PREFIX}*.json")): + for site, api, count in json.loads(path.read_text(encoding="utf-8"))["calls"]: + rows.append((str(site), str(api), int(count))) + return tuple(rows) diff --git a/tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py b/tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py new file mode 100644 index 0000000000..de009bcc7f --- /dev/null +++ b/tests/integration/src/sqlbuild/compiler/sql_analysis/test_polyglot_site_recorder.py @@ -0,0 +1,65 @@ +"""The debug site recorder counts wheel calls by product site only when its directory is set.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from sqlbuild.compiler.sql_analysis.constants import ANALYSIS_RECORD_DIR_ENV_VAR +from tests.integration.src.sqlbuild.compiler.sql_analysis._test_types import SiteRecorderTestCase +from tests.integration.src.sqlbuild.compiler.sql_analysis.helpers import recorded_rows + +_PROCESS: str = ( + "from sqlbuild.compiler.references.main.extract_relation_names import " + "extract_relation_names\n" + "from sqlbuild.compiler.sql_analysis.main.import_polyglot_sql import import_polyglot_sql\n" + "for _ in range(2):\n" + " assert extract_relation_names(sql='SELECT 1 FROM main.orders', dialect='duckdb')\n" + "module = import_polyglot_sql()\n" + "print(type(module.parse_one).__name__, isinstance(module.PolyglotError, type))\n" +) +_SITE: str = "compiler/references/_helpers/relation_names.py:extract_relation_names_impl" + + +@pytest.mark.parametrize( + "test_case", + [ + SiteRecorderTestCase( + description="recording", + record_dir="records", + expected_rows=((_SITE, "parse", 2),), + expected_entry_point="_RecordedEntryPoint", + ), + SiteRecorderTestCase( + description="disabled", + record_dir="", + expected_rows=(), + expected_entry_point="_GuardedEntryPoint", + ), + ], + ids=lambda case: case.description, +) +def test_given_record_dir_when_process_calls_the_wheel_then_sites_are_counted_at_exit( + test_case: SiteRecorderTestCase, tmp_path: Path +) -> None: + completed: subprocess.CompletedProcess[str] = subprocess.run( + [sys.executable, "-c", _PROCESS], + cwd=tmp_path, + env={**os.environ, ANALYSIS_RECORD_DIR_ENV_VAR: test_case.record_dir}, + capture_output=True, + text=True, + check=False, + timeout=120, + ) + + assert completed.returncode == 0, completed.stderr + assert recorded_rows(tmp_path) == test_case.expected_rows + assert completed.stdout.split() == [test_case.expected_entry_point, "True"] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py index 48f6053e2b..5f250d89bd 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/_test_types.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from pathlib import Path -from scripts.compiler_differential.models import ExpectedOutcome +from scripts.compiler_differential.models import DifferentialCommand, ExpectedOutcome @dataclass(frozen=True) @@ -25,3 +25,22 @@ class InlineCompileCodesTestCase: compile_root: Path accounted_codes: frozenset[str] expected_unaccounted: dict[str, tuple[str, ...]] + + +@dataclass(frozen=True) +class DialectVariantTestCase: + """A seed range and the dialect variants its first seed must add to the corpus.""" + + description: str + seeds: range + expected_names: tuple[str, ...] + expected_variant_commands: tuple[DifferentialCommand, ...] + + +@dataclass(frozen=True) +class AnalysisCodesTestCase: + """The compiler package and the analysis codes its scan must find and document.""" + + description: str + compiler_root: Path + expected_codes: frozenset[str] diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py new file mode 100644 index 0000000000..f774280585 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_analysis_codes.py @@ -0,0 +1,71 @@ +"""The analysis code scan finds every B, K, S and syntax code, and documents each unreachable one.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +import sqlbuild.compiler as compiler_package +from scripts.compiler_differential._helpers.analysis_corpus.failure_cases import ( + analysis_failure_cases, +) +from scripts.compiler_differential._helpers.corpus.emitted_codes import analysis_error_codes +from scripts.compiler_differential.constants import ANALYSIS_UNREACHABLE_CODES +from scripts.compiler_differential.main.failure_cases import failure_cases +from scripts.compiler_differential.models import FailureCase +from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( + AnalysisCodesTestCase, +) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCodesTestCase( + description="compiler_package", + compiler_root=Path(compiler_package.__file__).parent, + expected_codes=frozenset( + { + *(f"B00{number}" for number in (0, 2, 3, 4, 5)), + "B101", + "B102", + *(f"B2{number}" for number in range(10, 20)), + *(f"B23{number}" for number in range(5)), + "B300", + "B301", + "B302", + *(f"K00{number}" for number in range(1, 7)), + "K011", + "P001", + "P003", + "S101", + "S102", + "S103", + "S104", + } + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_compiler_when_scanning_analysis_codes_then_each_is_cased_or_unreachable( + test_case: AnalysisCodesTestCase, +) -> None: + codes: frozenset[str] = analysis_error_codes(test_case.compiler_root) + cases: tuple[FailureCase, ...] = (*failure_cases(), *analysis_failure_cases()) + cased: set[str | None] = {case.expected_code for case in cases} | { + case.expected_warning_code for case in cases + } + + assert test_case.expected_codes <= codes, test_case.expected_codes - codes + assert set(ANALYSIS_UNREACHABLE_CODES) <= codes + assert all(reason.strip() for reason in ANALYSIS_UNREACHABLE_CODES.values()) + assert codes - set(ANALYSIS_UNREACHABLE_CODES) <= cased, ( + codes - set(ANALYSIS_UNREACHABLE_CODES) - cased + ) + assert not set(ANALYSIS_UNREACHABLE_CODES) & {case.expected_code for case in cases} + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py index e1f4c0299b..3bba3bdc8a 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py +++ b/tests/unit/scripts/compiler_differential/_helpers/corpus/test_corpus.py @@ -7,9 +7,11 @@ import pytest from scripts.compiler_differential._helpers.corpus.corpus import build_corpus +from scripts.compiler_differential.constants import COLD_COMPILE from scripts.compiler_differential.models import CorpusProject, ExpectedOutcome from tests.unit.scripts.compiler_differential._helpers.corpus._test_types import ( CorpusExpectationTestCase, + DialectVariantTestCase, ) _REPO_ROOT: Path = Path(__file__).resolve().parents[6] @@ -65,5 +67,41 @@ def test_given_corpus_selection_when_building_then_entries_declare_expected_outc } == test_case.expected_outcomes +@pytest.mark.parametrize( + "test_case", + [ + DialectVariantTestCase( + description="first_seed_compiles_under_each_dialect", + seeds=range(5, 7), + expected_names=( + "seed/5", + "seed/6", + "seed/5-postgres", + "seed/5-snowflake", + ), + expected_variant_commands=(COLD_COMPILE,), + ) + ], + ids=lambda case: case.description, +) +def test_given_seed_range_when_building_then_first_seed_adds_compile_only_dialect_variants( + test_case: DialectVariantTestCase, +) -> None: + corpus: list[CorpusProject] = build_corpus( + repo_root=_REPO_ROOT, + corpora=("seeds",), + seeds=test_case.seeds, + dense_models=20, + projects=(), + project_expectation=ExpectedOutcome(), + ) + variants: list[CorpusProject] = corpus[len(test_case.seeds) :] + + assert tuple(entry.name for entry in corpus) == test_case.expected_names + assert all(entry.seed_coverage for entry in corpus) + assert {entry.commands for entry in variants} == {test_case.expected_variant_commands} + assert all(entry.expected == ExpectedOutcome() for entry in variants) + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py index a9e3ca07af..d873fc67d9 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/_test_types.py @@ -71,3 +71,22 @@ class RenderEncodedInputsTestCase: description: str inputs: CompileProjectInputs expected_kinds: frozenset[str] + + +@dataclass(frozen=True) +class AnalysisCaptureKindsTestCase: + """One compiled-project capture fragment and the analysis kinds it must and must not prove.""" + + description: str + capture: dict[str, object] + expected_present: frozenset[str] + expected_absent: frozenset[str] + + +@dataclass(frozen=True) +class AnalysisCaptureProblemsTestCase: + """A compiled-project capture text and how many problems make it unsound.""" + + description: str + capture_text: str + expected_problem_count: int diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py index 0c0d785c4d..9a71c56c7a 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/helpers.py @@ -1,4 +1,4 @@ -"""Capture fragments for discovery coverage tests.""" +"""Capture fragments for discovery, render and analysis coverage tests.""" from __future__ import annotations @@ -6,7 +6,7 @@ from itertools import filterfalse from typing import cast -from sqlbuild.compiler.compile.models import CompileProjectInputs +from sqlbuild.compiler.compile.models import CompiledProject, CompileProjectInputs from sqlbuild.compiler.discovery.models import DiscoveredProjectInputs from sqlbuild.compiler.frontier.constants import STAGE_CAPTURE_OMITTED_ATTRIBUTES @@ -106,3 +106,91 @@ def audit_input(*, definition: str, template: str, rendered: str) -> dict[str, o "attached_target_name": "orders", "attached_column_name": "amount", } + + +_NULLABILITY: str = "sqlbuild.compiler.lineage.types:InferredNullability" + + +def empty_compiled_capture(**overrides: object) -> dict[str, object]: + """Return a capture with every captured `CompiledProject` field, in order, then overrides.""" + + type_name: str = "sqlbuild.compiler.compile.models:CompiledProject" + omitted: frozenset[str] = STAGE_CAPTURE_OMITTED_ATTRIBUTES.get(type_name, frozenset()) + names: list[str] = [field.name for field in dataclasses.fields(CompiledProject)] + return { + "__type__": type_name, + **dict.fromkeys(filterfalse(omitted.__contains__, names), []), + **overrides, + } + + +def inferred_column(*, name: str, type_sql: str | None, nullability: str = "UNKNOWN") -> object: + """Return one captured inferred output column.""" + + return { + "name": name, + "type": type_sql, + "nullability": {"__enum__": f"{_NULLABILITY}.{nullability}"}, + } + + +def compiled_model( + *, + name: str, + query_sql: str = "SELECT 1 AS id", + columns: list[object] | None = None, + references: list[dict[str, object]] | None = None, + validated: bool = True, + values: dict[str, object] | None = None, + schema_columns: list[dict[str, object]] | None = None, + dynamic_columns: list[dict[str, object]] | None = None, + star: tuple[bool, bool] = (False, False), +) -> dict[str, object]: + """Return one captured compiled model; `columns=None` means it was not analysed.""" + + return { + "name": name, + "query_sql": query_sql, + "config": {"values": values or {}}, + "references": references or [], + "schema_entry": { + "columns": schema_columns or [], + "dynamic_columns": dynamic_columns or [], + "audits": [], + }, + "inferred_columns": columns, + "binding_validated": validated, + "binding_diagnostics": [], + "fast_lineage_has_star": star[0], + "fast_lineage_star_resolved": star[1], + "dynamic_column_contract": None, + "rejected_sql_analysis_opt_out": None, + "destination": {"schema": "analytics"}, + } + + +def compiled_source(*, name: str, expression: str | None) -> dict[str, object]: + """Return one captured compiled source with an optional SQL expression.""" + + return {"source_entry": {"name": name, "expression": expression, "columns": []}} + + +def hooked_compiled_capture(*, columns: list[object] | None) -> dict[str, object]: + """Return a staged-promotion capture whose contracted model has inline and named SQL hooks.""" + + hooks: list[dict[str, object]] = [ + {"__type__": "sqlbuild:SqlHookEntry", "definition_sql": None}, + {"__type__": "sqlbuild:SqlHookEntry", "definition_sql": "SELECT 1"}, + ] + return empty_compiled_capture( + settings={"table_promotion_mode": "staged"}, + models=[ + compiled_model( + name="contracted", + columns=columns, + validated=columns is not None, + values={"contract": "enforced", "materialized": "table", "post_hooks": hooks}, + schema_columns=[{"name": "id", "type": "INTEGER", "nullable": False}], + ) + ], + ) diff --git a/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py new file mode 100644 index 0000000000..b118c6a866 --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/coverage/test_analysis.py @@ -0,0 +1,351 @@ +"""Compiled-project captures are classified into the analysis kinds the seed corpus must exercise.""" + +from __future__ import annotations + +import json + +import pytest + +from scripts.compiler_differential._helpers.coverage.analysis import ( + analysed_input_kinds, + analysis_capture_problems, + required_analysis_kinds, +) +from scripts.compiler_differential.constants import ANALYSIS_NON_COLLECTION_FIELDS +from tests.unit.scripts.compiler_differential._helpers.coverage._test_types import ( + AnalysisCaptureKindsTestCase, + AnalysisCaptureProblemsTestCase, + RequiredKindsTestCase, +) +from tests.unit.scripts.compiler_differential._helpers.coverage.helpers import ( + compiled_model, + compiled_source, + empty_compiled_capture, + hooked_compiled_capture, + inferred_column, + reference, +) + +_TYPED_ORDERS: list[object] = [ + inferred_column(name="id", type_sql="INT"), + inferred_column(name="placed_at", type_sql="TIMESTAMP"), +] +_CONTRACT_COLUMNS: list[dict[str, object]] = [ + {"name": "id", "type": "INTEGER", "nullable": False}, + {"name": "label", "type": "VARCHAR(24)", "nullable": None}, +] +_CONTRACT: dict[str, object] = {"contract": "enforced", "materialized": "table"} +_CONFIG_KINDS: tuple[str, ...] = ( + "explicit_promotion_mode", + "inline_sql_hook", + "named_sql_hook", + "hook_list", +) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCaptureKindsTestCase( + description="analysed_star_over_a_source_expression_and_a_model", + capture=empty_compiled_capture( + sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + binding_catalog={"expression_shapes": {"(SELECT 1 AS id)": {"id": "INT"}}}, + sql_analysis_dialect="duckdb", + models=[ + compiled_model( + name="stg_orders", + columns=[*_TYPED_ORDERS, inferred_column(name="note", type_sql=None)], + references=[reference(kind="SOURCE", name="raw_orders")], + star=(True, True), + ), + compiled_model( + name="every_order", + columns=_TYPED_ORDERS, + references=[reference(kind="REF", name="stg_orders")], + star=(True, True), + ), + ], + ), + expected_present=frozenset( + { + "models", + "sources", + "typed_column", + "untyped_column", + "star_over_complete_input", + "star_over_published_shape", + "source_expression_shape", + "dataflow_binding", + "dialect_duckdb", + } + ), + expected_absent=frozenset({"star_unresolved", "batch_binding", "upstream_chain"}), + ), + AnalysisCaptureKindsTestCase( + description="columns_without_binding_validation_prove_nothing", + capture=empty_compiled_capture( + sql_analysis_dialect="duckdb", + unchecked_output_columns={"__set__": []}, + models=[ + compiled_model( + name="stg_orders", + query_sql="WITH a AS (SELECT 1 AS id) SELECT id FROM a UNION SELECT 2", + columns=_TYPED_ORDERS, + references=[reference(kind="REF", name="upstream")], + validated=False, + star=(True, False), + ), + compiled_model(name="upstream", columns=None, validated=False), + ], + ), + expected_present=frozenset({"models"}), + expected_absent=frozenset( + { + "sql_analysis_disabled", + "typed_column", + "star_unresolved", + "set_operation", + "cte", + "dialect_duckdb", + "unchecked_output_columns", + } + ), + ), + AnalysisCaptureKindsTestCase( + description="contract_types_and_nullability_published_downstream", + capture=empty_compiled_capture( + models=[ + compiled_model( + name="contracted", + columns=[ + inferred_column(name="id", type_sql="INT"), + inferred_column(name="label", type_sql="VARCHAR(24)"), + ], + values=_CONTRACT, + schema_columns=_CONTRACT_COLUMNS, + ), + compiled_model( + name="below", + columns=[ + inferred_column(name="id", type_sql="INT", nullability="NON_NULL"), + inferred_column(name="label", type_sql="TEXT(24)"), + ], + references=[reference(kind="REF", name="contracted")], + ), + ], + settings={"table_promotion_mode": "staged"}, + ), + expected_present=frozenset( + { + "sized_contract_shape", + "contract_nullability_shape", + "batch_binding", + "explicit_promotion_mode", + } + ), + expected_absent=frozenset({"dataflow_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="nullability_already_inferred_upstream_is_not_contract_evidence", + capture=empty_compiled_capture( + models=[ + compiled_model( + name="contracted", + columns=[ + inferred_column(name="id", type_sql="INT", nullability="NON_NULL") + ], + values=_CONTRACT, + schema_columns=_CONTRACT_COLUMNS, + ), + compiled_model( + name="below", + columns=[ + inferred_column(name="id", type_sql="INT", nullability="NON_NULL") + ], + references=[reference(kind="REF", name="contracted")], + ), + ], + ), + expected_present=frozenset({"typed_column"}), + expected_absent=frozenset({"contract_nullability_shape", "sized_contract_shape"}), + ), + AnalysisCaptureKindsTestCase( + description="selection_leaves_unselected_models_unanalysed", + capture=empty_compiled_capture( + sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + binding_catalog={ + "expression_shapes": { + "__unordered_mapping__": [["(SELECT 1 AS id)", {"id": "INT"}]] + } + }, + models=[ + compiled_model( + name="stg_orders", + columns=_TYPED_ORDERS, + references=[reference(kind="SOURCE", name="raw_orders")], + ), + compiled_model( + name="totals", + columns=None, + references=[reference(kind="REF", name="stg_orders")], + validated=False, + ), + ], + ), + expected_present=frozenset({"select_limited_analysis", "batch_binding"}), + expected_absent=frozenset({"sql_analysis_disabled", "dataflow_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="unresolved_expression_source_is_not_a_complete_input", + capture=empty_compiled_capture( + sources=[compiled_source(name="raw_orders", expression="(SELECT 1 AS id)")], + binding_catalog={ + "expression_shapes": {"__unordered_mapping__": [["(SELECT 1 AS id)", None]]} + }, + models=[ + compiled_model( + name="stg_orders", + columns=_TYPED_ORDERS, + references=[reference(kind="SOURCE", name="raw_orders")], + ) + ], + ), + expected_present=frozenset({"dataflow_binding"}), + expected_absent=frozenset({"batch_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="dynamic_column_contract_is_not_a_complete_input", + capture=empty_compiled_capture( + models=[ + compiled_model( + name="contracted", + columns=[inferred_column(name="id", type_sql="INT")], + values=_CONTRACT, + schema_columns=_CONTRACT_COLUMNS, + dynamic_columns=[{"name": "status_amounts"}], + ), + compiled_model( + name="below", + columns=[inferred_column(name="id", type_sql="INT")], + references=[reference(kind="REF", name="contracted")], + ), + ], + ), + expected_present=frozenset({"dataflow_binding"}), + expected_absent=frozenset({"batch_binding"}), + ), + AnalysisCaptureKindsTestCase( + description="hooks_and_promotion_on_an_analysed_model", + capture=hooked_compiled_capture(columns=_TYPED_ORDERS), + expected_present=frozenset(_CONFIG_KINDS), + expected_absent=frozenset(), + ), + AnalysisCaptureKindsTestCase( + description="hooks_and_promotion_on_an_unanalysed_model_are_not_credited", + capture=hooked_compiled_capture(columns=None), + expected_present=frozenset({"models"}), + expected_absent=frozenset(_CONFIG_KINDS), + ), + AnalysisCaptureKindsTestCase( + description="literal_sql_relations_need_explicit_references", + capture=empty_compiled_capture( + enforce_explicit_references=False, + unmatched_literal_sql_relations=[{"owner_label": "task:export"}], + ), + expected_present=frozenset({"unmatched_literal_sql_relations"}), + expected_absent=frozenset({"python_sql_literal_relation"}), + ), + AnalysisCaptureKindsTestCase( + description="quoted_identifier_bound_only_by_ignoring_case", + capture=empty_compiled_capture( + models=[ + compiled_model(name="stg_orders", columns=_TYPED_ORDERS), + compiled_model( + name="quoted", + query_sql='SELECT "ID" AS "OrderKey" FROM __ref("stg_orders")', + columns=[inferred_column(name="OrderKey", type_sql="INT")], + references=[reference(kind="REF", name="stg_orders")], + ), + ], + ), + expected_present=frozenset({"quoted_identifier_case"}), + expected_absent=frozenset({"cte", "set_operation"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_compiled_capture_when_classifying_then_only_analysed_evidence_counts( + test_case: AnalysisCaptureKindsTestCase, +) -> None: + kinds: frozenset[str] = analysed_input_kinds(json.dumps(test_case.capture)) + + assert test_case.expected_present <= kinds, test_case.expected_present - kinds + assert not test_case.expected_absent & kinds, test_case.expected_absent & kinds + + +@pytest.mark.parametrize( + "test_case", + [ + RequiredKindsTestCase( + description="compiled_collections_and_detail_kinds", + expected_required=frozenset( + {"models", "sql_tests", "sql_expansions", "typed_column", "dialect_snowflake"} + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_compiled_project_fields_when_listing_required_kinds_then_settings_are_excluded( + test_case: RequiredKindsTestCase, +) -> None: + required: tuple[str, ...] = required_analysis_kinds() + + assert test_case.expected_required <= set(required) + assert not ANALYSIS_NON_COLLECTION_FIELDS & set(required) + assert len(required) == len(set(required)) + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisCaptureProblemsTestCase( + description="binding_catalog_native_handle_is_expected", + capture_text=json.dumps( + empty_compiled_capture( + binding_catalog={"native": {"__opaque__": "sqlbuild._native:ProjectCatalog"}} + ) + ), + expected_problem_count=0, + ), + AnalysisCaptureProblemsTestCase( + description="any_other_opaque_value_is_a_problem", + capture_text=json.dumps( + empty_compiled_capture(models=[{"__opaque__": "sqlbuild._native:Unexpected"}]) + ), + expected_problem_count=1, + ), + AnalysisCaptureProblemsTestCase( + description="missing_field_is_a_problem", + capture_text=json.dumps( + { + key: value + for key, value in empty_compiled_capture().items() + if key != "sql_scenarios" + } + ), + expected_problem_count=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_compiled_capture_when_checking_soundness_then_problems_are_counted( + test_case: AnalysisCaptureProblemsTestCase, +) -> None: + assert len(analysis_capture_problems(test_case.capture_text)) == ( + test_case.expected_problem_count + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py index 87744bb7f6..6233c40c85 100644 --- a/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py +++ b/tests/unit/scripts/compiler_differential/_helpers/running/_test_types.py @@ -64,3 +64,14 @@ class SummaryTestCase: missing_coverage: dict[str, tuple[str, ...]] expected_lines: tuple[str, ...] expected_absent: tuple[str, ...] + + +@dataclass(frozen=True) +class AnalysisRecordsTestCase: + """Record files two engine processes wrote and the report lines they must produce.""" + + description: str + files: dict[str, str] + expected_wheel_sites: dict[tuple[str, str], int] + expected_deferrals: dict[tuple[str, str], int] + expected_lines: tuple[str, ...] diff --git a/tests/unit/scripts/compiler_differential/_helpers/running/test_records.py b/tests/unit/scripts/compiler_differential/_helpers/running/test_records.py new file mode 100644 index 0000000000..2f3a00bccd --- /dev/null +++ b/tests/unit/scripts/compiler_differential/_helpers/running/test_records.py @@ -0,0 +1,82 @@ +"""Wheel-site and deferral records are summed per engine, site and corpus, never gated.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.compiler_differential._helpers.running.records import ( + format_wheel_site_report, + read_analysis_records, + wheel_site_report, +) +from scripts.compiler_differential.models import AnalysisRecords, ProjectComparison +from tests.unit.scripts.compiler_differential._helpers.running._test_types import ( + AnalysisRecordsTestCase, +) + +_SITE: str = "compiler/compile/_helpers/analysis/columns.py:_infer_columns_with_polyglot" + + +@pytest.mark.parametrize( + "test_case", + [ + AnalysisRecordsTestCase( + description="two_processes_and_one_deferral", + files={ + "0-compile/polyglot-sites-11.json": json.dumps( + {"calls": [[_SITE, "parse_one", 2]]} + ), + "2-plan/polyglot-sites-12.json": json.dumps({"calls": [[_SITE, "parse_one", 3]]}), + "2-plan/analysis-deferrals-12.jsonl": ( + json.dumps({"kind": "legacy_fallback", "site": "compact.py"}) + "\n" + ), + }, + expected_wheel_sites={(_SITE, "parse_one"): 5}, + expected_deferrals={("legacy_fallback", "compact.py"): 1}, + expected_lines=( + "Polyglot wheel calls (python):", + f" 5 {_SITE} parse_one (seed 5)", + "Analysis deferrals (python):", + " 1 legacy_fallback compact.py (seed 1)", + "Polyglot wheel calls (native-preview): none recorded", + "Analysis deferrals (native-preview): none recorded", + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_record_files_when_reporting_then_counts_are_summed_per_engine_and_corpus( + test_case: AnalysisRecordsTestCase, tmp_path: Path +) -> None: + for relative_path, contents in test_case.files.items(): + path: Path = tmp_path / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + records: AnalysisRecords = read_analysis_records(tmp_path) + report: str = format_wheel_site_report( + wheel_site_report( + comparisons=[ + ProjectComparison( + project="seed/3", + differences=(), + seconds=1.0, + records=(records, read_analysis_records(tmp_path / "missing")), + ) + ], + engines=("python", "native-preview"), + ) + ) + + assert (records.wheel_sites, records.deferrals) == ( + test_case.expected_wheel_sites, + test_case.expected_deferrals, + ) + assert report.splitlines() == list(test_case.expected_lines) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])